mirror of
https://github.com/redstrate/Astra.git
synced 2025-04-26 14:17:45 +00:00
Send a hash filelist up to the room to ensure we don't duplicate work
This commit is contained in:
parent
a382c07896
commit
0658f09995
5 changed files with 92 additions and 45 deletions
|
|
@ -159,7 +159,7 @@ private:
|
|||
|
||||
QCoro::Task<> fetchNews();
|
||||
|
||||
QCoro::Task<> handleGameExit(Profile *profile);
|
||||
QCoro::Task<> handleGameExit(const Profile *profile);
|
||||
|
||||
SteamAPI *m_steamApi = nullptr;
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
#include <Quotient/accountregistry.h>
|
||||
#include <Quotient/connection.h>
|
||||
#include <Task>
|
||||
#include <qcorotask.h>
|
||||
|
||||
/**
|
||||
* @brief Handles setting up the connection to Matrix and all of the fun things needed to do for that.
|
||||
|
|
@ -63,6 +63,7 @@ public:
|
|||
struct PreviousCharacterData {
|
||||
QString mxcUri;
|
||||
QString hostname;
|
||||
QMap<QString, QString> fileHashes;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -74,12 +75,12 @@ public:
|
|||
* @brief Uploads character data for @p id from @p path.
|
||||
* @return True if uploaded successfuly, false otherwise.
|
||||
*/
|
||||
QCoro::Task<bool> uploadedCharacterData(const QString &id, const QString &path);
|
||||
QCoro::Task<bool> uploadCharacterArchive(const QString &id, const QString &path, const QMap<QString, QString> &fileHashes);
|
||||
|
||||
/**
|
||||
* @brief Downloads the character data archive from @p mxcUri and extracts it in @p destPath.
|
||||
*/
|
||||
QCoro::Task<bool> downloadCharacterData(const QString &mxcUri, const QString &destPath);
|
||||
QCoro::Task<bool> downloadCharacterArchive(const QString &mxcUri, const QString &destPath);
|
||||
|
||||
/**
|
||||
* @brief Checks if there's a lock.
|
||||
|
|
|
|||
|
|
@ -5,13 +5,17 @@
|
|||
|
||||
#include <KLocalizedString>
|
||||
#include <KZip>
|
||||
#include <QCoro>
|
||||
#include <qcorosignal.h>
|
||||
#include <qcorotask.h>
|
||||
|
||||
#include "astra_log.h"
|
||||
#include "syncmanager.h"
|
||||
|
||||
const auto gearsetFilename = QStringLiteral("GEARSET.DAT");
|
||||
|
||||
CharacterSync::CharacterSync(Account &account, LauncherCore &launcher, QObject *parent)
|
||||
: launcher(launcher)
|
||||
: QObject(parent)
|
||||
, launcher(launcher)
|
||||
, m_account(account)
|
||||
{
|
||||
}
|
||||
|
|
@ -22,9 +26,8 @@ QCoro::Task<bool> CharacterSync::sync(const bool initialSync)
|
|||
co_return true;
|
||||
}
|
||||
|
||||
auto syncManager = launcher.syncManager();
|
||||
const auto syncManager = launcher.syncManager();
|
||||
if (!syncManager->connected()) {
|
||||
qInfo() << "B";
|
||||
// TODO: provide an option to continue in the UI
|
||||
Q_EMIT launcher.loginError(i18n("Failed to connect to sync server! Please check your sync settings."));
|
||||
co_return false;
|
||||
|
|
@ -78,21 +81,38 @@ QCoro::Task<bool> CharacterSync::sync(const bool initialSync)
|
|||
const QString id = dir.fileName(); // FFXIV_CHR0040000001000001 for example
|
||||
const auto previousData = co_await syncManager->getUploadedCharacterData(id);
|
||||
|
||||
// TODO: make this a little bit smarter. We shouldn't waste time re-uploading data that's exactly the same.
|
||||
if (!initialSync || !previousData.has_value()) {
|
||||
// if we didn't upload character data yet, upload it now
|
||||
co_await uploadCharacterData(dir.absoluteFilePath(), id);
|
||||
} else {
|
||||
// otherwise, download it
|
||||
|
||||
const bool exists = QFile::exists(dir.absoluteFilePath() + QStringLiteral("/GEARSET.DAT"));
|
||||
|
||||
// but check first if it's our hostname. only skip if it exists
|
||||
if (exists && QSysInfo::machineHostName() == previousData->hostname) {
|
||||
qCDebug(ASTRA_LOG) << "Skipping" << id << "We uploaded this data.";
|
||||
continue;
|
||||
// The files are packed into an archive. So if only one of the files doesn't exist or fails the hash check, download the whole thing and overwrite.
|
||||
bool areFilesDifferent = false;
|
||||
for (const auto &[file, hash] : previousData->fileHashes.asKeyValueRange()) {
|
||||
QFile existingFile(QDir(dir.absoluteFilePath()).absoluteFilePath(file));
|
||||
if (!existingFile.exists()) {
|
||||
areFilesDifferent = true;
|
||||
qCDebug(ASTRA_LOG) << id << "does not match locally, reason:" << existingFile.fileName() << "does not exist";
|
||||
break;
|
||||
}
|
||||
|
||||
existingFile.open(QIODevice::ReadOnly);
|
||||
const auto existingHash = QString::fromUtf8(QCryptographicHash::hash(existingFile.readAll(), QCryptographicHash::Algorithm::Sha256).toHex());
|
||||
if (existingHash != hash) {
|
||||
areFilesDifferent = true;
|
||||
qCDebug(ASTRA_LOG) << id << "does not match locally, reason: hashes do not match for" << file;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const bool hasPreviousUpload = !previousData.has_value();
|
||||
const bool isGameClosing = !initialSync;
|
||||
|
||||
// We want to upload if the files are truly different, or there is no existing data on the server.
|
||||
const bool needsUpload = (areFilesDifferent && isGameClosing) || hasPreviousUpload;
|
||||
|
||||
// We want to download if the files are different.
|
||||
const bool needsDownload = areFilesDifferent;
|
||||
|
||||
if (needsUpload) {
|
||||
// if we didn't upload character data yet, upload it now
|
||||
co_await uploadCharacterData(dir.absoluteFilePath(), id);
|
||||
} else if (needsDownload) {
|
||||
co_await downloadCharacterData(dir.absoluteFilePath(), id, previousData->mxcUri);
|
||||
}
|
||||
}
|
||||
|
|
@ -103,21 +123,26 @@ QCoro::Task<bool> CharacterSync::sync(const bool initialSync)
|
|||
QCoro::Task<void> CharacterSync::uploadCharacterData(const QDir &dir, const QString &id)
|
||||
{
|
||||
qCDebug(ASTRA_LOG) << "Uploading" << dir << id;
|
||||
QTemporaryDir tempDir;
|
||||
const QTemporaryDir tempDir;
|
||||
|
||||
auto tempZipPath = tempDir.filePath(QStringLiteral("%1.zip").arg(id));
|
||||
const auto tempZipPath = tempDir.filePath(QStringLiteral("%1.zip").arg(id));
|
||||
|
||||
KZip *zip = new KZip(tempZipPath);
|
||||
const auto zip = new KZip(tempZipPath);
|
||||
zip->setCompression(KZip::DeflateCompression);
|
||||
zip->open(QIODevice::WriteOnly);
|
||||
|
||||
QFile gearsetFile(dir.filePath(QStringLiteral("GEARSET.DAT")));
|
||||
QFile gearsetFile(dir.filePath(gearsetFilename));
|
||||
gearsetFile.open(QFile::ReadOnly);
|
||||
|
||||
zip->writeFile(QStringLiteral("GEARSET.DAT"), gearsetFile.readAll());
|
||||
const auto data = gearsetFile.readAll();
|
||||
|
||||
zip->writeFile(gearsetFilename, data);
|
||||
zip->close();
|
||||
|
||||
co_await launcher.syncManager()->uploadedCharacterData(id, tempZipPath);
|
||||
QMap<QString, QString> fileHashes;
|
||||
fileHashes[gearsetFilename] = QString::fromUtf8(QCryptographicHash::hash(data, QCryptographicHash::Algorithm::Sha256).toHex());
|
||||
|
||||
co_await launcher.syncManager()->uploadCharacterArchive(id, tempZipPath, fileHashes);
|
||||
// TODO: error handling
|
||||
|
||||
co_return;
|
||||
|
|
@ -125,19 +150,19 @@ QCoro::Task<void> CharacterSync::uploadCharacterData(const QDir &dir, const QStr
|
|||
|
||||
QCoro::Task<void> CharacterSync::downloadCharacterData(const QDir &dir, const QString &id, const QString &contentUri)
|
||||
{
|
||||
QTemporaryDir tempDir;
|
||||
const QTemporaryDir tempDir;
|
||||
|
||||
auto tempZipPath = tempDir.filePath(QStringLiteral("%1.zip").arg(id));
|
||||
const auto tempZipPath = tempDir.filePath(QStringLiteral("%1.zip").arg(id));
|
||||
|
||||
co_await launcher.syncManager()->downloadCharacterData(contentUri, tempZipPath);
|
||||
co_await launcher.syncManager()->downloadCharacterArchive(contentUri, tempZipPath);
|
||||
|
||||
KZip *zip = new KZip(tempZipPath);
|
||||
auto zip = new KZip(tempZipPath);
|
||||
zip->setCompression(KZip::DeflateCompression);
|
||||
zip->open(QIODevice::ReadOnly);
|
||||
|
||||
qCDebug(ASTRA_LOG) << "contents:" << zip->directory()->entries();
|
||||
|
||||
zip->directory()->file(QStringLiteral("GEARSET.DAT"))->copyTo(dir.absolutePath());
|
||||
Q_UNUSED(zip->directory()->file(gearsetFilename)->copyTo(dir.absolutePath()))
|
||||
|
||||
qCDebug(ASTRA_LOG) << "Extracted character data!";
|
||||
|
||||
|
|
|
|||
|
|
@ -515,9 +515,10 @@ QCoro::Task<> LauncherCore::fetchNews()
|
|||
Q_EMIT newsChanged();
|
||||
}
|
||||
|
||||
QCoro::Task<> LauncherCore::handleGameExit(Profile *profile)
|
||||
QCoro::Task<> LauncherCore::handleGameExit(const Profile *profile)
|
||||
{
|
||||
#ifdef BUILD_SYNC
|
||||
// TODO: once we have Steam API support we can tell Steam to delay putting the Deck to sleep until our upload is complete
|
||||
if (m_settings->enableSync()) {
|
||||
Q_EMIT showWindow();
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,12 @@ const auto roomType = QStringLiteral("zone.xiv.astra-sync");
|
|||
const auto syncEventType = QStringLiteral("zone.xiv.astra.sync");
|
||||
const auto lockEventType = QStringLiteral("zone.xiv.astra.lock");
|
||||
|
||||
const auto hostnameKey = QStringLiteral("hostname");
|
||||
const auto latestKey = QStringLiteral("latest");
|
||||
const auto noneKey = QStringLiteral("none");
|
||||
const auto filesKey = QStringLiteral("files");
|
||||
const auto contentUriKey = QStringLiteral("content-uri");
|
||||
|
||||
using namespace Quotient;
|
||||
|
||||
SyncManager::SyncManager(QObject *parent)
|
||||
|
|
@ -104,7 +110,10 @@ Quotient::Connection *SyncManager::connection() const
|
|||
|
||||
QCoro::Task<> SyncManager::sync()
|
||||
{
|
||||
// TODO: de-duplicate sync() calls. otherwise if they happen in quick succession, they wait on each other which is useless for our use case
|
||||
// We don't need two syncs running at once.
|
||||
if (connection()->syncJob()) {
|
||||
co_return;
|
||||
}
|
||||
|
||||
auto connection = m_accountRegistry.accounts().first();
|
||||
connection->sync();
|
||||
|
|
@ -209,12 +218,18 @@ QCoro::Task<std::optional<SyncManager::PreviousCharacterData>> SyncManager::getU
|
|||
co_return std::nullopt;
|
||||
} else {
|
||||
qCDebug(ASTRA_LOG) << "previous sync event:" << syncEvent;
|
||||
co_return PreviousCharacterData{.mxcUri = syncEvent[QStringLiteral("content-uri")].toString(),
|
||||
.hostname = syncEvent[QStringLiteral("hostname")].toString()};
|
||||
|
||||
auto filesVariantMap = syncEvent[filesKey].toVariant().toMap();
|
||||
QMap<QString, QString> fileHashes;
|
||||
for (const auto &[file, hashVariant] : filesVariantMap.asKeyValueRange()) {
|
||||
fileHashes[file] = hashVariant.toString();
|
||||
}
|
||||
|
||||
co_return PreviousCharacterData{.mxcUri = syncEvent[contentUriKey].toString(), .hostname = syncEvent[hostnameKey].toString(), .fileHashes = fileHashes};
|
||||
}
|
||||
}
|
||||
|
||||
QCoro::Task<bool> SyncManager::uploadedCharacterData(const QString &id, const QString &path)
|
||||
QCoro::Task<bool> SyncManager::uploadCharacterArchive(const QString &id, const QString &path, const QMap<QString, QString> &fileHashes)
|
||||
{
|
||||
Q_ASSERT(m_currentRoom);
|
||||
|
||||
|
|
@ -225,16 +240,22 @@ QCoro::Task<bool> SyncManager::uploadedCharacterData(const QString &id, const QS
|
|||
|
||||
const QUrl contentUri = uploadFileJob->contentUri();
|
||||
|
||||
QVariantMap fileHashesVariant;
|
||||
for (const auto &[file, hash] : fileHashes.asKeyValueRange()) {
|
||||
fileHashesVariant[file] = QVariant::fromValue(hash);
|
||||
}
|
||||
|
||||
auto syncSetState = m_currentRoom->setState(
|
||||
syncEventType,
|
||||
id,
|
||||
QJsonObject{{{QStringLiteral("content-uri"), contentUri.toString()}, {QStringLiteral("hostname"), QSysInfo::machineHostName()}}});
|
||||
QJsonObject{
|
||||
{{contentUriKey, contentUri.toString()}, {hostnameKey, QSysInfo::machineHostName()}, {filesKey, QJsonObject::fromVariantMap(fileHashesVariant)}}});
|
||||
co_await qCoro(syncSetState, &BaseJob::finished);
|
||||
|
||||
co_return true;
|
||||
}
|
||||
|
||||
QCoro::Task<bool> SyncManager::downloadCharacterData(const QString &mxcUri, const QString &destPath)
|
||||
QCoro::Task<bool> SyncManager::downloadCharacterArchive(const QString &mxcUri, const QString &destPath)
|
||||
{
|
||||
auto job = connection()->downloadFile(QUrl::fromUserInput(mxcUri), destPath);
|
||||
co_await qCoro(job, &BaseJob::finished);
|
||||
|
|
@ -246,14 +267,14 @@ QCoro::Task<bool> SyncManager::downloadCharacterData(const QString &mxcUri, cons
|
|||
|
||||
QCoro::Task<std::optional<QString>> SyncManager::checkLock()
|
||||
{
|
||||
const auto lockEvent = m_currentRoom->currentState().contentJson(syncEventType, QStringLiteral("latest"));
|
||||
const auto lockEvent = m_currentRoom->currentState().contentJson(syncEventType, latestKey);
|
||||
if (lockEvent.isEmpty()) {
|
||||
co_return std::nullopt;
|
||||
}
|
||||
|
||||
qCDebug(ASTRA_LOG) << "previous lock event:" << lockEvent;
|
||||
const QString hostname = lockEvent[QStringLiteral("hostname")].toString();
|
||||
if (hostname == QStringLiteral("none")) {
|
||||
const QString hostname = lockEvent[hostnameKey].toString();
|
||||
if (hostname == noneKey) {
|
||||
co_return std::nullopt;
|
||||
}
|
||||
|
||||
|
|
@ -262,15 +283,14 @@ QCoro::Task<std::optional<QString>> SyncManager::checkLock()
|
|||
|
||||
QCoro::Task<> SyncManager::setLock()
|
||||
{
|
||||
auto lockSetState =
|
||||
m_currentRoom->setState(syncEventType, QStringLiteral("latest"), QJsonObject{{QStringLiteral("hostname"), QSysInfo::machineHostName()}});
|
||||
auto lockSetState = m_currentRoom->setState(syncEventType, latestKey, QJsonObject{{hostnameKey, QSysInfo::machineHostName()}});
|
||||
co_await qCoro(lockSetState, &BaseJob::finished);
|
||||
co_return;
|
||||
}
|
||||
|
||||
QCoro::Task<> SyncManager::breakLock()
|
||||
{
|
||||
auto lockSetState = m_currentRoom->setState(syncEventType, QStringLiteral("latest"), QJsonObject{{QStringLiteral("hostname"), QStringLiteral("none")}});
|
||||
auto lockSetState = m_currentRoom->setState(syncEventType, latestKey, QJsonObject{{hostnameKey, noneKey}});
|
||||
co_await qCoro(lockSetState, &BaseJob::finished);
|
||||
co_return;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue