Speed up the start-up of Data Explorer, especially with large databases

I ran it through Hotspot, and it takes a *while* going back and forth
with the database. The database is usually small enough to fit in
memory, so we just do that instead.

apps/sagasu/include/hashdatabase.h

@@ -25,5 +25,11 @@ public:
     QString getFilename(uint32_t i);
 
 private:
+    void cacheDatabase();
+
     QSqlDatabase m_db;
+
+    // Database transactions are super slow, so we keep a copy in memory
+    QHash<uint32_t, QString> m_fileHashes;
+    QHash<uint32_t, QString> m_folderHashes;
 };

apps/sagasu/src/hashdatabase.cpp

@@ -21,19 +21,24 @@ HashDatabase::HashDatabase(QObject *parent)
     QSqlQuery query;
     query.exec(QStringLiteral("CREATE TABLE IF NOT EXISTS folder_hashes (hash INTEGER PRIMARY KEY, name TEXT NOT NULL)"));
     query.exec(QStringLiteral("CREATE TABLE IF NOT EXISTS file_hashes (hash INTEGER PRIMARY KEY, name TEXT NOT NULL)"));
+
+    cacheDatabase();
 }
 
 void HashDatabase::addFolder(const QString &folder)
 {
-    std::string folderStd = folder.toStdString();
+    const std::string folderStd = folder.toStdString();
+    const auto hash = physis_generate_partial_hash(folderStd.c_str());
 
     QSqlQuery query;
     query.prepare(
         QStringLiteral("REPLACE INTO folder_hashes (hash, name) "
                        "VALUES (?, ?)"));
-    query.addBindValue(physis_generate_partial_hash(folderStd.c_str()));
+    query.addBindValue(hash);
     query.addBindValue(folder);
     query.exec();
+
+    m_folderHashes[hash] = folder;
 }
 
 void HashDatabase::addFile(const QString &file)
@@ -46,15 +51,18 @@ void HashDatabase::addFile(const QString &file)
 
     qInfo() << "Adding" << filename;
 
-    std::string folderStd = filename.toStdString();
+    const std::string folderStd = filename.toStdString();
+    const auto hash = physis_generate_partial_hash(folderStd.c_str());
 
     QSqlQuery query;
     query.prepare(
         QStringLiteral("REPLACE INTO file_hashes (hash, name) "
                        "VALUES (?, ?)"));
-    query.addBindValue(physis_generate_partial_hash(folderStd.c_str()));
+    query.addBindValue(hash);
     query.addBindValue(filename);
     query.exec();
+
+    m_fileHashes[hash] = filename;
 }
 
 QVector<QString> HashDatabase::getKnownFolders()
@@ -73,26 +81,12 @@ QVector<QString> HashDatabase::getKnownFolders()
 
 bool HashDatabase::knowsFile(const uint32_t i)
 {
-    QSqlQuery query;
+    return m_fileHashes.contains(i);
-    query.prepare(QStringLiteral("SELECT COUNT(1) FROM file_hashes WHERE hash = ?;"));
-    query.addBindValue(i);
-    query.exec();
-
-    query.next();
-
-    return query.value(0) == 1;
 }
 
 QString HashDatabase::getFilename(const uint32_t i)
 {
-    QSqlQuery query;
+    return m_fileHashes[i];
-    query.prepare(QStringLiteral("SELECT name FROM file_hashes WHERE hash = ?;"));
-    query.addBindValue(i);
-    query.exec();
-
-    query.next();
-
-    return query.value(0).toString();
 }
 
 void HashDatabase::importFileList(const QByteArray &file)
@@ -100,8 +94,6 @@ void HashDatabase::importFileList(const QByteArray &file)
     QVariantList folderNames, folderHashes;
     QVariantList fileNames, fileHashes;
 
-    m_db.transaction();
-
     QSqlQuery folderQuery;
     folderQuery.prepare(
         QStringLiteral("REPLACE INTO folder_hashes (hash, name) "
@@ -112,6 +104,14 @@ void HashDatabase::importFileList(const QByteArray &file)
         QStringLiteral("REPLACE INTO file_hashes (hash, name) "
                        "VALUES (?, ?)"));
 
+    struct PreparedRow {
+        uint folderHash;
+        QString folderName;
+        uint fileHash;
+        QString fileName;
+    };
+    std::vector<PreparedRow> preparedRows;
+
     QTextStream stream(file);
     stream.readLine(); // skip header
     while (!stream.atEnd()) {
@@ -131,19 +131,63 @@ void HashDatabase::importFileList(const QByteArray &file)
             filename = path;
         }
 
+        preparedRows.push_back(PreparedRow{
+            .folderHash = folderHash.toUInt(),
+            .folderName = foldername,
+            .fileHash = fileHash.toUInt(),
+            .fileName = filename,
+        });
+    }
+
+    qInfo() << "Finished preparing the rows! Now inserting into the database...";
+
+    m_db.transaction();
+    for (const auto &row : preparedRows) {
         // execBatch is too slow as the QSQLITE doesn't support batch operations
-        if (!foldername.isEmpty()) {
+        if (!row.folderName.isEmpty()) {
-            folderQuery.bindValue(0, folderHash.toUInt());
+            folderQuery.bindValue(0, row.folderHash);
-            folderQuery.bindValue(1, foldername);
+            folderQuery.bindValue(1, row.folderName);
             folderQuery.exec();
         }
 
-        fileQuery.bindValue(0, fileHash.toUInt());
+        fileQuery.bindValue(0, row.fileHash);
-        fileQuery.bindValue(1, filename);
+        fileQuery.bindValue(1, row.fileName);
         fileQuery.exec();
     }
-
     m_db.commit();
+
+    qInfo() << "Finished database import!";
+
+    // reload cache
+    cacheDatabase();
+}
+
+void HashDatabase::cacheDatabase()
+{
+    m_fileHashes.clear();
+    m_folderHashes.clear();
+
+    // file hashes
+    {
+        QSqlQuery query;
+        query.prepare(QStringLiteral("SELECT hash, name FROM file_hashes;"));
+        query.exec();
+
+        while (query.next()) {
+            m_fileHashes[query.value(0).toUInt()] = query.value(1).toString();
+        }
+    }
+
+    // folder hashes
+    {
+        QSqlQuery query;
+        query.prepare(QStringLiteral("SELECT hash, name FROM folder_hashes;"));
+        query.exec();
+
+        while (query.next()) {
+            m_folderHashes[query.value(0).toUInt()] = query.value(1).toString();
+        }
+    }
 }
 
 #include "moc_hashdatabase.cpp"