Support chunked dat/index files

Needed for grabbing later expansion content that's split between chunks.

src/dat.rs

@@ -201,9 +201,7 @@ impl DatFile {
     /// by the function.
     ///
     /// If the block of data is successfully parsed, it returns the file data - otherwise is None.
-    pub fn read_from_offset(&mut self, offset: u32) -> Option<ByteBuffer> {
+    pub fn read_from_offset(&mut self, offset: u64) -> Option<ByteBuffer> {
-        let offset = (offset * 0x80) as u64;
-
         self.file
             .seek(SeekFrom::Start(offset))
             .expect("Unable to find offset in file.");

src/gamedata.rs

@@ -13,7 +13,7 @@ use crate::dat::DatFile;
 use crate::exd::EXD;
 use crate::exh::EXH;
 use crate::exl::EXL;
-use crate::index::{Index2File, IndexFile, IndexHashBitfield};
+use crate::index::{Index2File, IndexEntry, IndexFile};
 use crate::patch::{apply_patch, PatchError};
 use crate::repository::{string_to_category, Category, Repository};
 use crate::ByteBuffer;
@@ -127,14 +127,14 @@ impl GameData {
         self.repositories.sort();
     }
 
-    fn get_dat_file(&self, path: &str, data_file_id: u32) -> Option<DatFile> {
+    fn get_dat_file(&self, path: &str, chunk: u8, data_file_id: u32) -> Option<DatFile> {
         let (repository, category) = self.parse_repository_category(path).unwrap();
 
         let dat_path: PathBuf = [
             self.game_directory.clone(),
             "sqpack".to_string(),
             repository.name.clone(),
-            repository.dat_filename(category, data_file_id),
+            repository.dat_filename(chunk, category, data_file_id),
         ]
         .iter()
         .collect();
@@ -161,17 +161,7 @@ impl GameData {
             return false;
         };
 
-        self.cache_index_file((&index_path.0, &index_path.1));
+        return self.find_entry(path).is_some();
-
-        if let Some(index_file) = self.get_index_file(&index_path.0) {
-            return index_file.exists(path);
-        }
-
-        if let Some(index2_file) = self.get_index2_file(&index_path.1) {
-            return index2_file.exists(path);
-        }
-
-        false
     }
 
     /// Extracts the file located at `path`. This is returned as an in-memory buffer, and will usually
@@ -194,10 +184,10 @@ impl GameData {
 
         let slice = self.find_entry(path);
         match slice {
-            Some(entry) => {
+            Some((entry, chunk)) => {
-                let mut dat_file = self.get_dat_file(path, entry.data_file_id().into())?;
+                let mut dat_file = self.get_dat_file(path, chunk, entry.data_file_id.into())?;
 
-                dat_file.read_from_offset(entry.offset())
+                dat_file.read_from_offset(entry.offset as u64)
             }
             None => None,
         }
@@ -206,45 +196,55 @@ impl GameData {
     /// Parses a path structure and spits out the corresponding category and repository.
     fn parse_repository_category(&self, path: &str) -> Option<(&Repository, Category)> {
         let tokens: Vec<&str> = path.split('/').collect(); // TODO: use split_once here
-        let repository_token = tokens[0];
 
         if tokens.len() < 2 {
             return None;
         }
 
+        let repository_token = tokens[1];
+
         for repository in &self.repositories {
             if repository.name == repository_token {
-                return Some((repository, string_to_category(tokens[1])?));
+                return Some((repository, string_to_category(tokens[0])?));
             }
         }
 
         Some((&self.repositories[0], string_to_category(tokens[0])?))
     }
 
-    fn get_index_filenames(&self, path: &str) -> Option<(String, String)> {
+    fn get_index_filenames(&self, path: &str) -> Option<(Vec<(String, u8)>, Vec<(String, u8)>)> {
         let (repository, category) = self.parse_repository_category(path)?;
 
-        let index_path: PathBuf = [
+        let mut index1_filenames = vec![];
-            &self.game_directory,
+        let mut index2_filenames = vec![];
-            "sqpack",
-            &repository.name,
-            &repository.index_filename(category),
-        ]
-        .iter()
-        .collect();
 
-        let index2_path: PathBuf = [
+        for chunk in 0..255 {
-            &self.game_directory,
+            let index_path: PathBuf = [
-            "sqpack",
+                &self.game_directory,
-            &repository.name,
+                "sqpack",
-            &repository.index2_filename(category),
+                &repository.name,
-        ]
+                &repository.index_filename(chunk, category),
-        .iter()
+            ]
-        .collect();
+                .iter()
+                .collect();
+
+            index1_filenames.push((index_path.into_os_string().into_string().unwrap(), chunk));
+
+            let index2_path: PathBuf = [
+                &self.game_directory,
+                "sqpack",
+                &repository.name,
+                &repository.index2_filename(chunk, category),
+            ]
+                .iter()
+                .collect();
+
+            index2_filenames.push((index2_path.into_os_string().into_string().unwrap(), chunk));
+        }
 
         Some((
-            index_path.into_os_string().into_string().unwrap(),
+            index1_filenames,
-            index2_path.into_os_string().into_string().unwrap(),
+            index2_filenames
         ))
     }
 
@@ -397,17 +397,19 @@ impl GameData {
         Ok(())
     }
 
-    fn cache_index_file(&mut self, filenames: (&str, &str)) {
+    fn cache_index_file(&mut self, filename: &str) {
-        if !self.index_files.contains_key(filenames.0) {
+        if !self.index_files.contains_key(filename) {
-            if let Some(index_file) = IndexFile::from_existing(filenames.0) {
+            if let Some(index_file) = IndexFile::from_existing(filename) {
-                self.index_files.insert(filenames.0.to_string(), index_file);
+                self.index_files.insert(filename.to_string(), index_file);
             }
         }
+    }
 
-        if !self.index2_files.contains_key(filenames.1) {
+    fn cache_index2_file(&mut self, filename: &str) {
-            if let Some(index_file) = Index2File::from_existing(filenames.1) {
+        if !self.index2_files.contains_key(filename) {
+            if let Some(index_file) = Index2File::from_existing(filename) {
                 self.index2_files
-                    .insert(filenames.1.to_string(), index_file);
+                    .insert(filename.to_string(), index_file);
             }
         }
     }
@@ -420,25 +422,26 @@ impl GameData {
         self.index2_files.get(filename)
     }
 
-    fn find_entry(&mut self, path: &str) -> Option<IndexHashBitfield> {
+    fn find_entry(&mut self, path: &str) -> Option<(IndexEntry, u8)> {
-        let index_path = self.get_index_filenames(path)?;
+        let (index_paths, index2_paths) = self.get_index_filenames(path)?;
-        debug!(
-            "Trying index files {index_path}, {index2_path}",
-            index_path = index_path.0,
-            index2_path = index_path.1
-        );
 
-        self.cache_index_file((&index_path.0, &index_path.1));
+        for (index_path, chunk) in index_paths {
+            self.cache_index_file(&index_path);
 
-        if let Some(index_file) = self.get_index_file(&index_path.0) {
+            if let Some(index_file) = self.get_index_file(&index_path) {
-            if let Some(entry) = index_file.find_entry(path) {
+                if let Some(entry) = index_file.find_entry(path) {
-                return Some(entry.bitfield);
+                    return Some((entry, chunk));
+                }
             }
         }
 
-        if let Some(index2_file) = self.get_index2_file(&index_path.1) {
+        for (index2_path, chunk) in index2_paths {
-            if let Some(entry) = index2_file.find_entry(path) {
+            self.cache_index2_file(&index2_path);
-                return Some(entry.bitfield);
+
+            if let Some(index_file) = self.get_index2_file(&index2_path) {
+                if let Some(entry) = index_file.find_entry(path) {
+                    return Some((entry, chunk));
+                }
             }
         }
 

src/index.rs

@@ -12,9 +12,8 @@ use binrw::BinRead;
 use modular_bitfield::prelude::*;
 
 #[binrw]
-#[br(magic = b"SqPack")]
+#[br(magic = b"SqPack\0\0")]
 pub struct SqPackHeader {
-    #[br(pad_before = 2)]
     platform_id: Platform,
     #[br(pad_before = 3)]
     size: u32,
@@ -25,26 +24,48 @@ pub struct SqPackHeader {
 #[binrw]
 pub struct SqPackIndexHeader {
     size: u32,
-    file_type: u32,
+    version: u32,
     index_data_offset: u32,
     index_data_size: u32,
-}
+    index_data_hash: [u8; 64],
-
+    number_of_data_file: u32,
-#[bitfield]
+    synonym_data_offset: u32,
-#[binrw]
+    synonym_data_size: u32,
-#[br(map = Self::from_bytes)]
+    synonym_data_hash: [u8; 64],
-#[derive(Clone, Copy, Debug)]
+    empty_block_data_offset: u32,
-pub struct IndexHashBitfield {
+    empty_block_data_size: u32,
-    pub size: B1,
+    empty_block_data_hash: [u8; 64],
-    pub data_file_id: B3,
+    dir_index_data_offset: u32,
-    pub offset: B28,
+    dir_index_data_size: u32,
+    dir_index_data_hash: [u8; 64],
+    index_type: u32,
+    #[br(pad_before = 656)]
+    self_hash: [u8; 64]
 }
 
 #[binrw]
 pub struct IndexHashTableEntry {
     pub hash: u64,
-    #[br(pad_after = 4)]
+
-    pub(crate) bitfield: IndexHashBitfield,
+    #[br(temp)]
+    #[bw(ignore)]
+    data: u32,
+
+    #[br(temp)]
+    #[bw(ignore)]
+    padding: u32,
+
+    #[br(calc = (data & 0b1) == 0b1)]
+    #[bw(ignore)]
+    pub is_synonym: bool,
+
+    #[br(calc = ((data & 0b1110) >> 1) as u8)]
+    #[bw(ignore)]
+    pub data_file_id: u8,
+
+    #[br(calc = (data & !0xF) * 0x08)]
+    #[bw(ignore)]
+    pub offset: u32,
 }
 
 // The only difference between index and index2 is how the path hash is stored.
@@ -54,7 +75,22 @@ pub struct IndexHashTableEntry {
 #[derive(Debug)]
 pub struct Index2HashTableEntry {
     pub hash: u32,
-    pub(crate) bitfield: IndexHashBitfield,
+
+    #[br(temp)]
+    #[bw(ignore)]
+    data: u32,
+
+    #[br(calc = (data & 0b1) == 0b1)]
+    #[bw(ignore)]
+    pub is_synonym: bool,
+
+    #[br(calc = ((data & 0b1110) >> 1) as u8)]
+    #[bw(ignore)]
+    pub data_file_id: u8,
+
+    #[br(calc = (data & !0xF) * 0x08)]
+    #[bw(ignore)]
+    pub offset: u32,
 }
 
 #[derive(Debug)]
@@ -73,8 +109,7 @@ pub struct IndexFile {
     index_header: SqPackIndexHeader,
 
     #[br(seek_before = SeekFrom::Start(index_header.index_data_offset.into()))]
-    // +4 because of padding
+    #[br(count = index_header.index_data_size / 16)]
-    #[br(count = index_header.index_data_size / core::mem::size_of::<IndexHashTableEntry>() as u32 + 4)]
     pub entries: Vec<IndexHashTableEntry>,
 }
 
@@ -87,7 +122,7 @@ pub struct Index2File {
     index_header: SqPackIndexHeader,
 
     #[br(seek_before = SeekFrom::Start(index_header.index_data_offset.into()))]
-    #[br(count = index_header.index_data_size / core::mem::size_of::<Index2HashTableEntry>() as u32)]
+    #[br(count = index_header.index_data_size / 8)]
     pub entries: Vec<Index2HashTableEntry>,
 }
 
@@ -130,9 +165,18 @@ impl IndexFile {
         self.entries.iter().any(|s| s.hash == hash)
     }
 
-    pub fn find_entry(&self, path: &str) -> Option<&IndexHashTableEntry> {
+    pub fn find_entry(&self, path: &str) -> Option<IndexEntry> {
         let hash = IndexFile::calculate_hash(path);
-        self.entries.iter().find(|s| s.hash == hash)
+
+        if let Some(entry) = self.entries.iter().find(|s| s.hash == hash) {
+            return Some(IndexEntry {
+                hash: entry.hash,
+                data_file_id: entry.data_file_id,
+                offset: entry.offset,
+            });
+        }
+
+        None
     }
 }
 
@@ -156,9 +200,18 @@ impl Index2File {
         self.entries.iter().any(|s| s.hash == hash)
     }
 
-    pub fn find_entry(&self, path: &str) -> Option<&Index2HashTableEntry> {
+    pub fn find_entry(&self, path: &str) -> Option<IndexEntry> {
         let hash = Index2File::calculate_hash(path);
-        self.entries.iter().find(|s| s.hash == hash)
+
+        if let Some(entry) = self.entries.iter().find(|s| s.hash == hash) {
+            return Some(IndexEntry {
+                hash: entry.hash as u64,
+                data_file_id: entry.data_file_id,
+                offset: entry.offset,
+            });
+        }
+
+        None
     }
 }
 

src/repository.rs

@@ -70,35 +70,35 @@ impl PartialOrd for Repository {
 #[derive(Debug, PartialEq, Eq, Copy, Clone)]
 pub enum Category {
     /// Common files such as game fonts, and other data that doesn't really fit anywhere else.
-    Common,
+    Common = 0x00,
     /// Shared data between game maps.
-    BackgroundCommon,
+    BackgroundCommon = 0x01,
     /// Game map data such as models, textures, and so on.
-    Background,
+    Background = 0x02,
     /// Cutscene content such as animations.
-    Cutscene,
+    Cutscene = 0x03,
     /// Character model files and more.
-    Character,
+    Character = 0x04,
     /// Compiled shaders used by the retail client.
-    Shader,
+    Shader = 0x05,
     /// UI layouts and textures.
-    UI,
+    UI = 0x06,
     /// Sound effects, basically anything not under `Music`.
-    Sound,
+    Sound = 0x07,
     /// This "VFX" means "visual effects", and contains textures and definitions for stuff like battle effects.
-    VFX,
+    VFX = 0x08,
     /// A leftover from 1.0, where the UI was primarily driven by LUA scripts.
-    UIScript,
+    UIScript = 0x09,
     /// Excel data.
-    EXD,
+    EXD = 0x0A,
     /// Many game events are driven by LUA scripts, such as cutscenes.
-    GameScript,
+    GameScript = 0x0B,
     /// Music!
-    Music,
+    Music = 0x0C,
     /// Unknown purpose, most likely to test SqPack functionality.
-    SqPackTest,
+    SqPackTest = 0x12,
     /// Unknown purpose, most likely debug files.
-    Debug,
+    Debug = 0x13,
 }
 
 pub fn string_to_category(string: &str) -> Option<Category> {
@@ -170,25 +170,24 @@ impl Repository {
     }
 
     /// Calculate an index filename for a specific category, like _"0a0000.win32.index"_.
-    pub fn index_filename(&self, category: Category) -> String {
+    pub fn index_filename(&self, chunk: u8, category: Category) -> String {
         format!(
             "{:02x}{:02}{:02}.{}.index",
             category as i32,
             self.expansion(),
-            0,
+            chunk,
             get_platform_string(&self.platform)
         )
     }
 
     /// Calculate an index2 filename for a specific category, like _"0a0000.win32.index"_.
-    pub fn index2_filename(&self, category: Category) -> String {
+    pub fn index2_filename(&self, chunk: u8, category: Category) -> String {
-        format!("{}2", self.index_filename(category))
+        format!("{}2", self.index_filename(chunk, category))
     }
 
     /// Calculate a dat filename given a category and a data file id, returns something like _"0a0000.win32.dat0"_.
-    pub fn dat_filename(&self, category: Category, data_file_id: u32) -> String {
+    pub fn dat_filename(&self, chunk: u8, category: Category, data_file_id: u32) -> String {
         let expansion = self.expansion();
-        let chunk = 0;
         let platform = get_platform_string(&self.platform);
 
         format!(