Support index2 files

Needed for extracting data from benchmarks, which only ship with this
index file type.

src/gamedata.rs

@@ -13,11 +13,10 @@ use crate::dat::DatFile;
 use crate::exd::EXD;
 use crate::exh::EXH;
 use crate::exl::EXL;
-use crate::index::IndexFile;
+use crate::index::{Index2File, IndexFile, IndexHashBitfield};
 use crate::ByteBuffer;
 use crate::patch::{apply_patch, PatchError};
 use crate::repository::{Category, Repository, string_to_category};
-use crate::sqpack::calculate_hash;
 
 /// Framework for operating on game data.
 pub struct GameData {
@@ -27,7 +26,8 @@ pub struct GameData {
     /// Repositories in the game directory.
     pub repositories: Vec<Repository>,
 
-    index_files: HashMap<String, IndexFile>
+    index_files: HashMap<String, IndexFile>,
+    index2_files: HashMap<String, Index2File>
 }
 
 fn is_valid(path: &str) -> bool {
@@ -78,7 +78,8 @@ impl GameData {
             true => Some(Self {
                 game_directory: String::from(directory),
                 repositories: vec![],
-                index_files: HashMap::new()
+                index_files: HashMap::new(),
+                index2_files: HashMap::new()
             }),
             false => {
                 warn!("Game data is not valid!");
@@ -157,15 +158,19 @@ impl GameData {
     /// }
     /// ```
     pub fn exists(&mut self, path: &str) -> bool {
-        let hash = calculate_hash(path);
+        let index_path = self.get_index_filenames(path);
-        let index_path = self.get_index_filename(path);
 
-        self.cache_index_file(&index_path);
+        self.cache_index_file((&index_path.0, &index_path.1));
-        if let Some(index_file) = self.get_index_file(&index_path) {
+
-            index_file.entries.iter().any(|s| s.hash == hash)
+        if let Some(index_file) = self.get_index_file(&index_path.0) {
-        } else {
+            return index_file.exists(path);
-            false
         }
+
+        if let Some(index2_file) = self.get_index2_file(&index_path.1) {
+            return index2_file.exists(path);
+        }
+
+        false
     }
 
     /// Extracts the file located at `path`. This is returned as an in-memory buffer, and will usually
@@ -185,18 +190,12 @@ impl GameData {
     pub fn extract(&mut self, path: &str) -> Option<ByteBuffer> {
         debug!(file=path, "Extracting file");
 
-        let hash = calculate_hash(path);
+        let slice = self.find_entry(path);
-        let index_path = self.get_index_filename(path);
-
-        self.cache_index_file(&index_path);
-        let index_file = self.get_index_file(&index_path)?;
-
-        let slice = index_file.entries.iter().find(|s| s.hash == hash);
         match slice {
             Some(entry) => {
-                let mut dat_file = self.get_dat_file(path, entry.bitfield.data_file_id().into())?;
+                let mut dat_file = self.get_dat_file(path, entry.data_file_id().into())?;
 
-                dat_file.read_from_offset(entry.bitfield.offset())
+                dat_file.read_from_offset(entry.offset())
             }
             None => None,
         }
@@ -220,7 +219,7 @@ impl GameData {
         Some((&self.repositories[0], string_to_category(tokens[0])?))
     }
 
-    fn get_index_filename(&self, path: &str) -> String {
+    fn get_index_filenames(&self, path: &str) -> (String, String) {
         let (repository, category) = self.parse_repository_category(path).unwrap();
 
         let index_path: PathBuf = [
@@ -232,7 +231,16 @@ impl GameData {
             .iter()
             .collect();
 
-        index_path.into_os_string().into_string().unwrap()
+        let index2_path: PathBuf = [
+            &self.game_directory,
+            "sqpack",
+            &repository.name,
+            &repository.index2_filename(category),
+        ]
+            .iter()
+            .collect();
+
+        (index_path.into_os_string().into_string().unwrap(), index2_path.into_os_string().into_string().unwrap())
     }
 
     /// Read an excel sheet by name (e.g. "Achievement")
@@ -384,10 +392,16 @@ impl GameData {
         Ok(())
     }
 
-    fn cache_index_file(&mut self, filename: &str)  {
+    fn cache_index_file(&mut self, filenames: (&str, &str))  {
-        if !self.index_files.contains_key(filename) {
+        if !self.index_files.contains_key(filenames.0) {
-            if let Some(index_file) = IndexFile::from_existing(filename) {
+            if let Some(index_file) = IndexFile::from_existing(filenames.0) {
-                self.index_files.insert(filename.to_string(), index_file);
+                self.index_files.insert(filenames.0.to_string(), index_file);
+            }
+        }
+
+        if !self.index_files.contains_key(filenames.1) {
+            if let Some(index_file) = Index2File::from_existing(filenames.1) {
+                self.index2_files.insert(filenames.1.to_string(), index_file);
             }
         }
     }
@@ -395,6 +409,31 @@ impl GameData {
     fn get_index_file(&self, filename: &str) -> Option<&IndexFile> {
         self.index_files.get(filename)
     }
+
+    fn get_index2_file(&self, filename: &str) -> Option<&Index2File> {
+        println!("Trying {}", filename);
+        self.index2_files.get(filename)
+    }
+
+    fn find_entry(&mut self, path: &str) -> Option<IndexHashBitfield> {
+        let index_path = self.get_index_filenames(path);
+
+        self.cache_index_file((&index_path.0, &index_path.1));
+
+        if let Some(index_file) = self.get_index_file(&index_path.0) {
+            if let Some(entry) = index_file.find_entry(path) {
+                return Some(entry.bitfield);
+            }
+        }
+
+        if let Some(index2_file) = self.get_index2_file(&index_path.1) {
+            if let Some(entry) = index2_file.find_entry(path) {
+                return Some(entry.bitfield);
+            }
+        }
+
+        None
+    }
 }
 
 #[cfg(test)]

src/index.rs

@@ -8,6 +8,7 @@ use std::io::SeekFrom;
 use binrw::BinRead;
 use binrw::binrw;
 use modular_bitfield::prelude::*;
+use crate::crc::Jamcrc;
 
 #[binrw]
 #[brw(repr = u8)]
@@ -40,6 +41,7 @@ pub struct SqPackIndexHeader {
 #[bitfield]
 #[binrw]
 #[br(map = Self::from_bytes)]
+#[derive(Clone, Copy, Debug)]
 pub struct IndexHashBitfield {
     pub size: B1,
     pub data_file_id: B3,
@@ -53,6 +55,16 @@ pub struct IndexHashTableEntry {
     pub(crate) bitfield: IndexHashBitfield,
 }
 
+// The only difference between index and index2 is how the path hash is stored.
+// The folder name and the filename are split in index1 (hence why it's 64-bits and not 32-bit)
+// But in index2, its both the file and folder name in one single CRC hash.
+#[binrw]
+#[derive(Debug)]
+pub struct Index2HashTableEntry {
+    pub hash: u32,
+    pub(crate) bitfield: IndexHashBitfield,
+}
+
 #[derive(Debug)]
 pub struct IndexEntry {
     pub hash: u64,
@@ -69,15 +81,89 @@ pub struct IndexFile {
     index_header: SqPackIndexHeader,
 
     #[br(seek_before = SeekFrom::Start(index_header.index_data_offset.into()))]
-    #[br(count = index_header.index_data_size / 16)]
+    // +4 because of padding
+    #[br(count = index_header.index_data_size / core::mem::size_of::<IndexHashTableEntry>() as u32 + 4)]
     pub entries: Vec<IndexHashTableEntry>,
 }
 
+#[binrw]
+#[br(little)]
+pub struct Index2File {
+    sqpack_header: SqPackHeader,
+
+    #[br(seek_before = SeekFrom::Start(sqpack_header.size.into()))]
+    index_header: SqPackIndexHeader,
+
+    #[br(seek_before = SeekFrom::Start(index_header.index_data_offset.into()))]
+    #[br(count = index_header.index_data_size / core::mem::size_of::<Index2HashTableEntry>() as u32)]
+    pub entries: Vec<Index2HashTableEntry>,
+}
+
+const CRC: Jamcrc = Jamcrc::new();
+
 impl IndexFile {
     /// Creates a new reference to an existing index file.
-    pub fn from_existing(path: &str) -> Option<IndexFile> {
+    pub fn from_existing(path: &str) -> Option<Self> {
         let mut index_file = std::fs::File::open(path).ok()?;
 
-        IndexFile::read(&mut index_file).ok()
+        Self::read(&mut index_file).ok()
+    }
+
+    /// Calculates a partial hash for a given path
+    pub fn calculate_partial_hash(path: &str) -> u32 {
+        let lowercase = path.to_lowercase();
+
+        CRC.checksum(lowercase.as_bytes())
+    }
+
+    /// Calculates a hash for `index` files from a game path.
+    pub fn calculate_hash(path: &str) -> u64 {
+        let lowercase = path.to_lowercase();
+
+        let pos = lowercase.rfind('/').unwrap();
+
+        let (directory, filename) = lowercase.split_at(pos);
+
+        let directory_crc = CRC.checksum(directory.as_bytes());
+        let filename_crc = CRC.checksum(filename[1..filename.len()].as_bytes());
+
+        (directory_crc as u64) << 32 | (filename_crc as u64)
+    }
+
+    // TODO: turn into traits?
+    pub fn exists(&self, path: &str) -> bool {
+        let hash = IndexFile::calculate_hash(path);
+        self.entries.iter().any(|s| s.hash == hash)
+    }
+
+    pub fn find_entry(&self, path: &str) -> Option<&IndexHashTableEntry> {
+        let hash = IndexFile::calculate_hash(path);
+        self.entries.iter().find(|s| s.hash == hash)
+    }
+}
+
+impl Index2File {
+    /// Creates a new reference to an existing index2 file.
+    pub fn from_existing(path: &str) -> Option<Self> {
+        let mut index_file = std::fs::File::open(path).ok()?;
+
+        Some(Self::read(&mut index_file).unwrap())
+    }
+
+    /// Calculates a hash for `index2` files from a game path.
+    pub fn calculate_hash(path: &str) -> u32 {
+        let lowercase = path.to_lowercase();
+
+        CRC.checksum(lowercase.as_bytes())
+    }
+
+    pub fn exists(&self, path: &str) -> bool {
+        let hash = Index2File::calculate_hash(path);
+        self.entries.iter().any(|s| s.hash == hash)
+    }
+
+    pub fn find_entry(&self, path: &str) -> Option<&Index2HashTableEntry> {
+        let hash = Index2File::calculate_hash(path);
+        self.entries.iter().find(|s| s.hash == hash)
     }
 }

src/repository.rs

@@ -177,15 +177,11 @@ impl Repository {
         d.push("ffxivgame.ver");
 
         let version = read_version(d.as_path());
-        if version.is_some() {
         Some(Repository {
             name: "ffxiv".parse().unwrap(),
             repo_type: Base,
             version,
         })
-        } else {
-            None
-        }
     }
 
     fn expansion(&self) -> i32 {
@@ -206,6 +202,14 @@ impl Repository {
         )
     }
     
+    /// Calculate an index2 filename for a specific category, like _"0a0000.win32.index"_.
+    pub fn index2_filename(&self, category: Category) -> String {
+        format!(
+            "{}2",
+            self.index_filename(category)
+        )
+    }
+
     /// Calculate a dat filename given a category and a data file id, returns something like _"0a0000.win32.dat0"_.
     pub fn dat_filename(&self, category: Category, data_file_id: u32) -> String {
         let expansion = self.expansion();

src/sqpack.rs

@@ -6,32 +6,8 @@ use std::io::{Read, Seek, SeekFrom};
 use binrw::BinRead;
 
 use crate::compression::no_header_decompress;
-use crate::crc::Jamcrc;
 use crate::dat::{BlockHeader, CompressionMode};
 
-const CRC: Jamcrc = Jamcrc::new();
-
-/// Calculates a partial hash for a given path
-pub fn calculate_partial_hash(path: &str) -> u32 {
-    let lowercase = path.to_lowercase();
-
-    CRC.checksum(lowercase.as_bytes())
-}
-
-/// Calculates a hash for `index` files from a game path.
-pub fn calculate_hash(path: &str) -> u64 {
-    let lowercase = path.to_lowercase();
-
-    let pos = lowercase.rfind('/').unwrap();
-
-    let (directory, filename) = lowercase.split_at(pos);
-
-    let directory_crc = CRC.checksum(directory.as_bytes());
-    let filename_crc = CRC.checksum(filename[1..filename.len()].as_bytes());
-
-    (directory_crc as u64) << 32 | (filename_crc as u64)
-}
-
 pub fn read_data_block<T: Read + Seek>(mut buf: T, starting_position: u64) -> Option<Vec<u8>> {
     buf.seek(SeekFrom::Start(starting_position)).ok()?;