Refactor EXD parsing, begin adding write support

src/common.rs

@@ -9,7 +9,7 @@ use binrw::binrw;
 #[binrw]
 #[brw(repr(u8))]
 #[repr(u8)]
-#[derive(Copy, Clone)]
+#[derive(Debug, Copy, Clone)]
 /// The language the game data is written for. Some of these languages are supported in the Global region.
 pub enum Language {
     /// Used for data that is language-agnostic, such as item data.

src/exd.rs

@@ -1,13 +1,12 @@
 // SPDX-FileCopyrightText: 2023 Joshua Goins <josh@redstrate.com>
 // SPDX-License-Identifier: GPL-3.0-or-later
 
-use std::io::{Cursor, Seek, SeekFrom};
+use std::io::{BufWriter, Cursor, Read, Seek, SeekFrom, Write};
 
-use binrw::binrw;
+use binrw::{binrw, BinResult, BinWrite};
-use binrw::helpers::until_eof;
 use binrw::{BinRead, Endian};
 
-use crate::ByteSpan;
+use crate::{ByteBuffer, ByteSpan};
 use crate::common::Language;
 use crate::exh::{ColumnDataType, EXH, ExcelColumnDefinition, ExcelDataPagination};
 
@@ -15,16 +14,21 @@ use crate::exh::{ColumnDataType, EXH, ExcelColumnDefinition, ExcelDataPagination
 #[brw(magic = b"EXDF")]
 #[brw(big)]
 #[allow(dead_code)]
+#[derive(Debug)]
 struct EXDHeader {
     version: u16,
 
-    #[br(pad_before = 2)]
+    #[brw(pad_before = 2)] // empty
-    #[br(pad_after = 20)]
+    /// Size of the ExcelDataOffset array
     index_size: u32,
+    #[brw(pad_after = 16)] // empty
+    /// Total size of the string data?
+    data_section_size: u32,
 }
 
 #[binrw]
 #[brw(big)]
+#[derive(Debug)]
 struct ExcelDataOffset {
     row_id: u32,
     pub offset: u32,
@@ -38,17 +42,147 @@ struct ExcelDataRowHeader {
     row_count: u16,
 }
 
+#[binrw::parser(reader)]
+fn parse_rows(exh: &EXH, data_offsets: &Vec<ExcelDataOffset>) -> BinResult<Vec<ExcelRow>> {
+    let mut rows = Vec::new();
+
+    for offset in data_offsets {
+        reader.seek(SeekFrom::Start(offset.offset.into()))?;
+
+        let row_header = ExcelDataRowHeader::read(reader)?;
+
+        let data_offset = reader.stream_position().unwrap() as u32;
+
+        let mut read_row = |row_offset: u32| -> Option<ExcelSingleRow> {
+            let mut subrow = ExcelSingleRow {
+                columns: Vec::with_capacity(exh.column_definitions.len()),
+            };
+
+            for column in &exh.column_definitions {
+                reader
+                    .seek(SeekFrom::Start((row_offset + column.offset as u32).into()))
+                    .ok()?;
+
+                subrow
+                    .columns
+                    .push(EXD::read_column(reader, exh, column).unwrap());
+            }
+
+            Some(subrow)
+        };
+
+        let new_row = if row_header.row_count > 1 {
+            let mut rows = Vec::new();
+            for i in 0..row_header.row_count {
+                let subrow_offset =
+                    data_offset + (i * exh.header.data_offset + 2 * (i + 1)) as u32;
+
+                rows.push(read_row(subrow_offset).unwrap());
+            }
+            ExcelRowKind::SubRows(rows)
+        } else {
+            ExcelRowKind::SingleRow(read_row(data_offset).unwrap())
+        };
+        rows.push(ExcelRow {
+            row_id: offset.row_id,
+            kind: new_row
+        });
+    }
+
+    Ok(rows)
+}
+
+#[binrw::writer(writer)]
+fn write_rows(
+    rows: &Vec<ExcelRow>,
+    exh: &EXH,
+) -> BinResult<()> {
+    // seek past the data offsets, which we will write later
+    let data_offsets_pos = writer.stream_position().unwrap();
+    writer.seek(SeekFrom::Current((core::mem::size_of::<ExcelDataOffset>() * rows.len()) as i64)).unwrap();
+
+    let mut data_offsets = Vec::new();
+
+    for row in rows {
+        data_offsets.push(ExcelDataOffset {
+            row_id: row.row_id,
+            offset: writer.stream_position().unwrap() as u32
+        });
+
+        let row_header = ExcelDataRowHeader {
+            data_size: 0,
+            row_count: 0
+        };
+        row_header.write(writer).unwrap();
+
+        // write column data
+        {
+            let mut write_row = |row: &ExcelSingleRow| {
+                for (i, column) in row.columns.iter().enumerate() {
+                    EXD::write_column(writer, &column, &exh.column_definitions[i]);
+                }
+            };
+
+            match &row.kind {
+                ExcelRowKind::SingleRow(excel_single_row) => write_row(excel_single_row),
+                ExcelRowKind::SubRows(excel_single_rows) => {
+                    for row in excel_single_rows {
+                        write_row(row);
+                    }
+                },
+            }
+        }
+
+        // write strings at the end of column data
+        {
+            let mut write_row_strings = |row: &ExcelSingleRow| {
+                for column in &row.columns {
+                    match column {
+                        ColumnData::String(val) => {
+                            let bytes = val.as_bytes();
+                            bytes.write(writer).unwrap();
+                        },
+                        _ => {}
+                    }
+                }
+            };
+
+            match &row.kind {
+                ExcelRowKind::SingleRow(excel_single_row) => write_row_strings(excel_single_row),
+                ExcelRowKind::SubRows(excel_single_rows) => {
+                    for row in excel_single_rows {
+                        write_row_strings(row);
+                    }
+                },
+            }
+        }
+
+        // There's an empty byte between each row... for some reason
+        0u8.write_le(writer).unwrap();
+    }
+
+    // now write the data offsets
+    writer.seek(SeekFrom::Start(data_offsets_pos)).unwrap();
+    data_offsets.write(writer).unwrap();
+
+    Ok(())
+}
+
 #[binrw]
 #[brw(big)]
 #[allow(dead_code)]
+#[derive(Debug)]
+#[brw(import(exh: &EXH))]
 pub struct EXD {
     header: EXDHeader,
 
     #[br(count = header.index_size / core::mem::size_of::<ExcelDataOffset>() as u32)]
+    #[bw(ignore)]
     data_offsets: Vec<ExcelDataOffset>,
 
-    #[br(seek_before = SeekFrom::Start(0), parse_with = until_eof)]
+    #[br(parse_with = parse_rows, args(&exh, &data_offsets))]
-    data: Vec<u8>,
+    #[bw(write_with = write_rows, args(&exh))]
+    rows: Vec<ExcelRow>,
 }
 
 #[derive(Debug)]
@@ -66,71 +200,35 @@ pub enum ColumnData {
     UInt64(u64),
 }
 
+#[derive(Debug)]
+pub struct ExcelSingleRow {
+    pub columns: Vec<ColumnData>,
+}
+
+#[derive(Debug)]
+pub enum ExcelRowKind {
+    SingleRow(ExcelSingleRow),
+    SubRows(Vec<ExcelSingleRow>)
+}
+
 #[derive(Debug)]
 pub struct ExcelRow {
-    pub data: Vec<ColumnData>,
+    pub row_id: u32,
+    pub kind: ExcelRowKind,
 }
 
 impl EXD {
-    pub fn from_existing(buffer: ByteSpan) -> Option<EXD> {
+    pub fn from_existing(exh: &EXH, buffer: ByteSpan) -> Option<EXD> {
-        EXD::read(&mut Cursor::new(&buffer)).ok()
+        EXD::read_args(&mut Cursor::new(&buffer), (exh,)).ok()
     }
 
-    pub fn read_row(&self, exh: &EXH, id: u32) -> Option<Vec<ExcelRow>> {
+    fn read_data_raw<T: Read + Seek, Z: BinRead<Args<'static> = ()>>(cursor: &mut T) -> Option<Z> {
-        let mut cursor = Cursor::new(&self.data);
-
-        for offset in &self.data_offsets {
-            if offset.row_id == id {
-                cursor.seek(SeekFrom::Start(offset.offset.into())).ok()?;
-
-                let row_header = ExcelDataRowHeader::read(&mut cursor).ok()?;
-
-                let header_offset = offset.offset + 6; // std::mem::size_of::<ExcelDataRowHeader>() as u32;
-
-                let mut read_row = |row_offset: u32| -> Option<ExcelRow> {
-                    let mut subrow = ExcelRow {
-                        data: Vec::with_capacity(exh.column_definitions.len()),
-                    };
-
-                    for column in &exh.column_definitions {
-                        cursor
-                            .seek(SeekFrom::Start((row_offset + column.offset as u32).into()))
-                            .ok()?;
-
-                        subrow
-                            .data
-                            .push(Self::read_column(&mut cursor, exh, row_offset, column).unwrap());
-                    }
-
-                    Some(subrow)
-                };
-
-                return if row_header.row_count > 1 {
-                    let mut rows = Vec::new();
-                    for i in 0..row_header.row_count {
-                        let subrow_offset =
-                            header_offset + (i * exh.header.data_offset + 2 * (i + 1)) as u32;
-
-                        rows.push(read_row(subrow_offset).unwrap());
-                    }
-                    Some(rows)
-                } else {
-                    Some(vec![read_row(header_offset).unwrap()])
-                };
-            }
-        }
-
-        None
-    }
-
-    fn read_data_raw<Z: BinRead<Args<'static> = ()>>(cursor: &mut Cursor<&Vec<u8>>) -> Option<Z> {
         Z::read_options(cursor, Endian::Big, ()).ok()
     }
 
-    fn read_column(
+    fn read_column<T: Read + Seek>(
-        cursor: &mut Cursor<&Vec<u8>>,
+        cursor: &mut T,
         exh: &EXH,
-        row_offset: u32,
         column: &ExcelColumnDefinition,
     ) -> Option<ColumnData> {
         let mut read_packed_bool = |shift: i32| -> bool {
@@ -144,9 +242,12 @@ impl EXD {
             ColumnDataType::String => {
                 let string_offset: u32 = Self::read_data_raw(cursor).unwrap();
 
+                let old_pos = cursor.stream_position().unwrap();
+
+                // -4 to take into account reading string_offset
                 cursor
-                    .seek(SeekFrom::Start(
+                    .seek(SeekFrom::Current(
-                        (row_offset + exh.header.data_offset as u32 + string_offset).into(),
+                        (exh.header.data_offset as u32 + string_offset - 4).into(),
                     ))
                     .ok()?;
 
@@ -158,6 +259,8 @@ impl EXD {
                     byte = Self::read_data_raw(cursor).unwrap();
                 }
 
+                cursor.seek(SeekFrom::Start(old_pos)).unwrap();
+
                 Some(ColumnData::String(string))
             }
             ColumnDataType::Bool => {
@@ -169,20 +272,12 @@ impl EXD {
             ColumnDataType::Int8 => Some(ColumnData::Int8(Self::read_data_raw(cursor).unwrap())),
             ColumnDataType::UInt8 => Some(ColumnData::UInt8(Self::read_data_raw(cursor).unwrap())),
             ColumnDataType::Int16 => Some(ColumnData::Int16(Self::read_data_raw(cursor).unwrap())),
-            ColumnDataType::UInt16 => {
+            ColumnDataType::UInt16 => Some(ColumnData::UInt16(Self::read_data_raw(cursor).unwrap())),
-                Some(ColumnData::UInt16(Self::read_data_raw(cursor).unwrap()))
-            }
             ColumnDataType::Int32 => Some(ColumnData::Int32(Self::read_data_raw(cursor).unwrap())),
-            ColumnDataType::UInt32 => {
+            ColumnDataType::UInt32 => Some(ColumnData::UInt32(Self::read_data_raw(cursor).unwrap())),
-                Some(ColumnData::UInt32(Self::read_data_raw(cursor).unwrap()))
+            ColumnDataType::Float32 => Some(ColumnData::Float32(Self::read_data_raw(cursor).unwrap())),
-            }
-            ColumnDataType::Float32 => {
-                Some(ColumnData::Float32(Self::read_data_raw(cursor).unwrap()))
-            }
             ColumnDataType::Int64 => Some(ColumnData::Int64(Self::read_data_raw(cursor).unwrap())),
-            ColumnDataType::UInt64 => {
+            ColumnDataType::UInt64 => Some(ColumnData::UInt64(Self::read_data_raw(cursor).unwrap())),
-                Some(ColumnData::UInt64(Self::read_data_raw(cursor).unwrap()))
-            }
             ColumnDataType::PackedBool0 => Some(ColumnData::Bool(read_packed_bool(0))),
             ColumnDataType::PackedBool1 => Some(ColumnData::Bool(read_packed_bool(1))),
             ColumnDataType::PackedBool2 => Some(ColumnData::Bool(read_packed_bool(2))),
@@ -194,6 +289,51 @@ impl EXD {
         }
     }
 
+    fn write_data_raw<T: Write + Seek, Z: BinWrite<Args<'static> = ()>>(cursor: &mut T, value: &Z) {
+        value.write_options(cursor, Endian::Big, ()).unwrap()
+    }
+
+     fn write_column<T: Write + Seek>(
+        cursor: &mut T,
+        column: &ColumnData,
+        column_definition: &ExcelColumnDefinition,
+     ) {
+        let write_packed_bool = |cursor: &mut T, shift: i32, boolean: &bool| {
+            let val = 0i32; // TODO
+            Self::write_data_raw(cursor, &val);
+        };
+
+        match column {
+            ColumnData::String(_) => {
+                let string_offset = 0u32; // TODO, but 0 is fine for single string column data
+                Self::write_data_raw(cursor, &string_offset);
+            },
+            ColumnData::Bool(val) => {
+                match column_definition.data_type {
+                    ColumnDataType::Bool => todo!(),
+                    ColumnDataType::PackedBool0 => write_packed_bool(cursor, 0, val),
+                    ColumnDataType::PackedBool1 => write_packed_bool(cursor, 1, val),
+                    ColumnDataType::PackedBool2 => write_packed_bool(cursor, 2, val),
+                    ColumnDataType::PackedBool3 => write_packed_bool(cursor, 3, val),
+                    ColumnDataType::PackedBool4 => write_packed_bool(cursor, 4, val),
+                    ColumnDataType::PackedBool5 => write_packed_bool(cursor, 5, val),
+                    ColumnDataType::PackedBool6 => write_packed_bool(cursor, 6, val),
+                    ColumnDataType::PackedBool7 => write_packed_bool(cursor, 7, val),
+                    _ => panic!("This makes no sense!")
+                }
+            },
+            ColumnData::Int8(val) => Self::write_data_raw(cursor, val),
+            ColumnData::UInt8(val) => Self::write_data_raw(cursor, val),
+            ColumnData::Int16(val) => Self::write_data_raw(cursor, val),
+            ColumnData::UInt16(val) => Self::write_data_raw(cursor, val),
+            ColumnData::Int32(val) => Self::write_data_raw(cursor, val),
+            ColumnData::UInt32(val) => Self::write_data_raw(cursor, val),
+            ColumnData::Float32(val) => Self::write_data_raw(cursor, val),
+            ColumnData::Int64(val) => Self::write_data_raw(cursor, val),
+            ColumnData::UInt64(val) => Self::write_data_raw(cursor, val),
+        }
+    }
+
     pub fn calculate_filename(
         name: &str,
         language: Language,
@@ -210,6 +350,19 @@ impl EXD {
             }
         }
     }
+
+      pub fn write_to_buffer(&self, exh: &EXH) -> Option<ByteBuffer> {
+        let mut buffer = ByteBuffer::new();
+
+        {
+            let cursor = Cursor::new(&mut buffer);
+            let mut writer = BufWriter::new(cursor);
+
+            self.write_args(&mut writer, (exh,)).unwrap();
+        }
+
+        Some(buffer)
+    }
 }
 
 #[cfg(test)]
@@ -241,6 +394,6 @@ mod tests {
         };
 
         // Feeding it invalid data should not panic
-        EXD::from_existing(&read(d).unwrap());
+        EXD::from_existing(&exh, &read(d).unwrap());
     }
 }

src/exh.rs

@@ -15,6 +15,7 @@ use crate::common::Language;
 #[brw(magic = b"EXHF")]
 #[brw(big)]
 #[allow(dead_code)]
+#[derive(Debug)]
 pub struct EXHHeader {
     pub(crate) version: u16,
 
@@ -57,6 +58,7 @@ pub enum ColumnDataType {
 
 #[binrw]
 #[brw(big)]
+#[derive(Debug)]
 pub struct ExcelColumnDefinition {
     pub data_type: ColumnDataType,
     pub offset: u16,
@@ -65,6 +67,7 @@ pub struct ExcelColumnDefinition {
 #[binrw]
 #[brw(big)]
 #[allow(dead_code)]
+#[derive(Debug)]
 pub struct ExcelDataPagination {
     pub start_id: u32,
     pub row_count: u32,
@@ -73,6 +76,7 @@ pub struct ExcelDataPagination {
 #[binrw]
 #[brw(big)]
 #[allow(dead_code)]
+#[derive(Debug)]
 pub struct EXH {
     pub header: EXHHeader,
 

src/gamedata.rs

@@ -291,7 +291,7 @@ impl GameData {
 
         let exd_file = self.extract(&exd_path)?;
 
-        EXD::from_existing(&exd_file)
+        EXD::from_existing(&exh, &exd_file)
     }
 
     /// Applies the patch to game data and returns any errors it encounters. This function will not update the version in the GameData struct.