From 4e6978ff6f61777a2e2e989cba58a9f0c48d2782 Mon Sep 17 00:00:00 2001 From: flow Date: Thu, 16 Jun 2022 11:45:29 -0300 Subject: [PATCH] feat: improve metadata gen. networking and performance This makes the metadata generation code a lot messier and harder to use, but there's not really much else that can be done about it while preserving all it's capabilities :( At least we now have speed Signed-off-by: flow --- launcher/modplatform/CheckUpdateTask.h | 2 +- launcher/modplatform/EnsureMetadataTask.cpp | 604 +++++++++++++----- launcher/modplatform/EnsureMetadataTask.h | 35 +- .../modplatform/flame/FlameCheckUpdate.cpp | 209 +++--- .../modrinth/ModrinthCheckUpdate.cpp | 5 +- launcher/ui/dialogs/ModUpdateDialog.cpp | 113 +++- launcher/ui/dialogs/ModUpdateDialog.h | 8 +- 7 files changed, 641 insertions(+), 335 deletions(-) diff --git a/launcher/modplatform/CheckUpdateTask.h b/launcher/modplatform/CheckUpdateTask.h index 8c701e46..d96fc340 100644 --- a/launcher/modplatform/CheckUpdateTask.h +++ b/launcher/modplatform/CheckUpdateTask.h @@ -13,7 +13,7 @@ class CheckUpdateTask : public Task { public: CheckUpdateTask(std::list& mods, std::list& mcVersions, ModAPI::ModLoaderTypes loaders, std::shared_ptr mods_folder) - : m_mods(mods), m_game_versions(mcVersions), m_loaders(loaders), m_mods_folder(mods_folder) {}; + : Task(nullptr), m_mods(mods), m_game_versions(mcVersions), m_loaders(loaders), m_mods_folder(mods_folder) {}; struct UpdatableMod { QString name; diff --git a/launcher/modplatform/EnsureMetadataTask.cpp b/launcher/modplatform/EnsureMetadataTask.cpp index dc92d8ab..cf4e55b9 100644 --- a/launcher/modplatform/EnsureMetadataTask.cpp +++ b/launcher/modplatform/EnsureMetadataTask.cpp @@ -19,226 +19,502 @@ static ModPlatform::ProviderCapabilities ProviderCaps; static ModrinthAPI modrinth_api; static FlameAPI flame_api; -EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir& dir, bool try_all, ModPlatform::Provider prov) - : m_mod(mod), m_index_dir(dir), m_provider(prov), m_try_all(try_all) -{} +EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir dir, ModPlatform::Provider prov) : Task(nullptr), m_index_dir(dir), m_provider(prov) +{ + auto hash = getHash(mod); + if (hash.isEmpty()) + emitFail(mod); + else + m_mods.insert(hash, mod); +} + +EnsureMetadataTask::EnsureMetadataTask(std::list& mods, QDir dir, ModPlatform::Provider prov) + : Task(nullptr), m_index_dir(dir), m_provider(prov) +{ + for (auto& mod : mods) { + if (!mod.valid()) { + emitFail(mod); + continue; + } + + auto hash = getHash(mod); + if (hash.isEmpty()) { + emitFail(mod); + continue; + } + + m_mods.insert(hash, mod); + } +} + +QString EnsureMetadataTask::getHash(Mod& mod) +{ + /* Here we create a mapping hash -> mod, because we need that relationship to parse the API routes */ + QByteArray jar_data; + try { + jar_data = FS::read(mod.fileinfo().absoluteFilePath()); + } catch (FS::FileSystemException& e) { + qCritical() << QString("Failed to open / read JAR file of %1").arg(mod.name()); + qCritical() << QString("Reason: ") << e.cause(); + + return {}; + } + + switch (m_provider) { + case ModPlatform::Provider::MODRINTH: { + auto hash_type = ProviderCaps.hashType(ModPlatform::Provider::MODRINTH).first(); + + return QString(ProviderCaps.hash(ModPlatform::Provider::MODRINTH, jar_data, hash_type).toHex()); + } + case ModPlatform::Provider::FLAME: { + QByteArray jar_data_treated; + for (char c : jar_data) { + // CF-specific + if (!(c == 9 || c == 10 || c == 13 || c == 32)) + jar_data_treated.push_back(c); + } + + return QString::number(MurmurHash2(jar_data_treated, jar_data_treated.length())); + } + } + + return {}; +} bool EnsureMetadataTask::abort() { - return m_task_handler->abort(); + // Prevent sending signals to a dead object + disconnect(this, 0, 0, 0); + + if (m_current_task) + return m_current_task->abort(); + return true; } void EnsureMetadataTask::executeTask() { - // They already have the right metadata :o - if (m_mod.status() != ModStatus::NoMetadata && m_mod.metadata() && m_mod.metadata()->provider == m_provider) { - emitReady(); - return; + setStatus(tr("Checking if mods have metadata...")); + + for (auto mod : m_mods) { + if (!mod.valid()) + continue; + + // They already have the right metadata :o + if (mod.status() != ModStatus::NoMetadata && mod.metadata() && mod.metadata()->provider == m_provider) { + qDebug() << "Mod" << mod.name() << "already has metadata!"; + emitReady(mod); + return; + } + + // Folders don't have metadata + if (mod.type() == Mod::MOD_FOLDER) { + emitReady(mod); + return; + } } - // Folders don't have metadata - if (m_mod.type() == Mod::MOD_FOLDER) { - emitReady(); - return; - } - - setStatus(tr("Generating %1's metadata...").arg(m_mod.name())); - qDebug() << QString("Generating %1's metadata...").arg(m_mod.name()); - - QByteArray jar_data; - - try { - jar_data = FS::read(m_mod.fileinfo().absoluteFilePath()); - } catch (FS::FileSystemException& e) { - qCritical() << QString("Failed to open / read JAR file of %1").arg(m_mod.name()); - qCritical() << QString("Reason: ") << e.cause(); - - emitFail(); - return; - } - - auto tsk = new MultipleOptionsTask(nullptr, "GetMetadataTask"); + NetJob::Ptr version_task; switch (m_provider) { case (ModPlatform::Provider::MODRINTH): - modrinthEnsureMetadata(*tsk, jar_data); - if (m_try_all) - flameEnsureMetadata(*tsk, jar_data); - + version_task = modrinthVersionsTask(); break; case (ModPlatform::Provider::FLAME): - flameEnsureMetadata(*tsk, jar_data); - if (m_try_all) - modrinthEnsureMetadata(*tsk, jar_data); - + version_task = flameVersionsTask(); break; } - connect(tsk, &MultipleOptionsTask::finished, this, [tsk] { tsk->deleteLater(); }); - connect(tsk, &MultipleOptionsTask::failed, [this] { - qCritical() << QString("Download of %1's metadata failed").arg(m_mod.name()); + auto invalidade_leftover = [this] { + QMutableHashIterator mods_iter(m_mods); + while (mods_iter.hasNext()) { + auto mod = mods_iter.next(); + emitFail(mod.value()); + } - emitFail(); - }); - connect(tsk, &MultipleOptionsTask::succeeded, this, &EnsureMetadataTask::emitReady); + emitSucceeded(); + }; - m_task_handler = tsk; + connect(version_task.get(), &Task::finished, this, [this, invalidade_leftover] { + NetJob::Ptr project_task; - tsk->start(); -} + switch (m_provider) { + case (ModPlatform::Provider::MODRINTH): + project_task = modrinthProjectsTask(); + break; + case (ModPlatform::Provider::FLAME): + project_task = flameProjectsTask(); + break; + } -void EnsureMetadataTask::emitReady() -{ - emit metadataReady(); - emitSucceeded(); -} - -void EnsureMetadataTask::emitFail() -{ - qDebug() << QString("Failed to generate metadata for %1").arg(m_mod.name()); - emit metadataFailed(); - //emitFailed(tr("Failed to generate metadata for %1").arg(m_mod.name())); - emitSucceeded(); -} - -void EnsureMetadataTask::modrinthEnsureMetadata(SequentialTask& tsk, QByteArray& jar_data) -{ - // Modrinth currently garantees that some hash types will always be present. - // But let's be sure and cover all cases anyways :) - for (auto hash_type : ProviderCaps.hashType(ModPlatform::Provider::MODRINTH)) { - auto* response = new QByteArray(); - auto hash = QString(ProviderCaps.hash(ModPlatform::Provider::MODRINTH, jar_data, hash_type).toHex()); - auto ver_task = modrinth_api.currentVersion(hash, hash_type, response); - - // Prevents unfortunate timings when aborting the task - if (!ver_task) + if (!project_task) { + invalidade_leftover(); return; + } - connect(ver_task.get(), &NetJob::succeeded, this, [this, ver_task, response] { - QJsonParseError parse_error{}; - QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error); - if (parse_error.error != QJsonParseError::NoError) { - qWarning() << "Error while parsing JSON response from " << m_mod.name() << " at " << parse_error.offset - << " reason: " << parse_error.errorString(); - qWarning() << *response; - - ver_task->failed(parse_error.errorString()); - return; - } - - auto doc_obj = Json::requireObject(doc); - auto ver = Modrinth::loadIndexedPackVersion(doc_obj, {}, m_mod.fileinfo().fileName()); - - // Minimal IndexedPack to create the metadata - ModPlatform::IndexedPack pack; - pack.name = m_mod.name(); - pack.provider = ModPlatform::Provider::MODRINTH; - pack.addonId = ver.addonId; - - // Prevent file name mismatch - ver.fileName = m_mod.fileinfo().fileName(); - - QDir tmp_index_dir(m_index_dir); - - { - LocalModUpdateTask update_metadata(m_index_dir, pack, ver); - QEventLoop loop; - QTimer timeout; - - QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit); - QObject::connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit); - - update_metadata.start(); - timeout.start(100); - - loop.exec(); - } - - auto mod_name = m_mod.name(); - auto meta = new Metadata::ModStruct(Metadata::get(tmp_index_dir, mod_name)); - m_mod.setMetadata(meta); + connect(project_task.get(), &Task::finished, this, [=] { + invalidade_leftover(); + project_task->deleteLater(); + m_current_task = nullptr; }); - tsk.addTask(ver_task); - } + m_current_task = project_task.get(); + project_task->start(); + }); + + connect(version_task.get(), &Task::finished, [=] { + version_task->deleteLater(); + m_current_task = nullptr; + }); + + if (m_mods.size() > 1) + setStatus(tr("Requesting metadata information from %1...").arg(ProviderCaps.readableName(m_provider))); + else if (!m_mods.empty()) + setStatus(tr("Requesting metadata information from %1 for '%2'...") + .arg(ProviderCaps.readableName(m_provider), m_mods.begin().value().name())); + + m_current_task = version_task.get(); + version_task->start(); } -void EnsureMetadataTask::flameEnsureMetadata(SequentialTask& tsk, QByteArray& jar_data) +void EnsureMetadataTask::emitReady(Mod& m) { - QByteArray jar_data_treated; - for (char c : jar_data) { - // CF-specific - if (!(c == 9 || c == 10 || c == 13 || c == 32)) - jar_data_treated.push_back(c); - } + qDebug() << QString("Generated metadata for %1").arg(m.name()); + emit metadataReady(m); + + m_mods.remove(getHash(m)); +} + +void EnsureMetadataTask::emitFail(Mod& m) +{ + qDebug() << QString("Failed to generate metadata for %1").arg(m.name()); + emit metadataFailed(m); + + m_mods.remove(getHash(m)); +} + +// Modrinth + +NetJob::Ptr EnsureMetadataTask::modrinthVersionsTask() +{ + auto hash_type = ProviderCaps.hashType(ModPlatform::Provider::MODRINTH).first(); auto* response = new QByteArray(); + auto ver_task = modrinth_api.currentVersions(m_mods.keys(), hash_type, response); - std::list fingerprints; - auto murmur = MurmurHash2(jar_data_treated, jar_data_treated.length()); - fingerprints.push_back(murmur); - - auto ver_task = flame_api.matchFingerprints(fingerprints, response); - - connect(ver_task.get(), &Task::succeeded, this, [this, ver_task, response] { - QDir tmp_index_dir(m_index_dir); + // Prevents unfortunate timings when aborting the task + if (!ver_task) + return {}; + connect(ver_task.get(), &NetJob::succeeded, this, [this, response] { QJsonParseError parse_error{}; QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error); if (parse_error.error != QJsonParseError::NoError) { - qWarning() << "Error while parsing JSON response from " << m_mod.name() << " at " << parse_error.offset + qWarning() << "Error while parsing JSON response from Modrinth::CurrentVersions at " << parse_error.offset << " reason: " << parse_error.errorString(); qWarning() << *response; - ver_task->failed(parse_error.errorString()); + failed(parse_error.errorString()); + return; + } + + try { + auto entries = Json::requireObject(doc); + for (auto& hash : m_mods.keys()) { + auto mod = m_mods.find(hash).value(); + try { + auto entry = Json::requireObject(entries, hash); + + setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod.name())); + qDebug() << "Getting version for" << mod.name() << "from Modrinth"; + + m_temp_versions.insert(hash, Modrinth::loadIndexedPackVersion(entry)); + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << entries; + + emitFail(mod); + } + } + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << doc; + } + }); + + return ver_task; +} + +NetJob::Ptr EnsureMetadataTask::modrinthProjectsTask() +{ + QHash addonIds; + for (auto const& data : m_temp_versions) + addonIds.insert(data.addonId.toString(), data.hash); + + auto response = new QByteArray(); + NetJob::Ptr proj_task; + + if (addonIds.isEmpty()) { + qWarning() << "No addonId found!"; + } else if (addonIds.size() == 1) { + proj_task = modrinth_api.getProject(*addonIds.keyBegin(), response); + } else { + proj_task = modrinth_api.getProjects(addonIds.keys(), response); + } + + // Prevents unfortunate timings when aborting the task + if (!proj_task) + return {}; + + connect(proj_task.get(), &NetJob::succeeded, this, [this, response, addonIds] { + QJsonParseError parse_error{}; + auto doc = QJsonDocument::fromJson(*response, &parse_error); + if (parse_error.error != QJsonParseError::NoError) { + qWarning() << "Error while parsing JSON response from Modrinth projects task at " << parse_error.offset + << " reason: " << parse_error.errorString(); + qWarning() << *response; + return; + } + + try { + QJsonArray entries; + if (addonIds.size() == 1) + entries = { doc.object() }; + else + entries = Json::requireArray(doc); + + for (auto entry : entries) { + auto entry_obj = Json::requireObject(entry); + auto entry_id = Json::requireString(entry_obj, "id"); + + auto hash = addonIds.find(entry_id).value(); + + auto mod = m_mods.find(hash).value(); + + try { + setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod.name())); + + ModPlatform::IndexedPack pack; + Modrinth::loadIndexedPack(pack, entry_obj); + + modrinthCallback(pack, m_temp_versions.find(hash).value(), mod); + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << entries; + + emitFail(mod); + } + } + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << doc; + } + }); + + return proj_task; +} + +// Flame +NetJob::Ptr EnsureMetadataTask::flameVersionsTask() +{ + auto* response = new QByteArray(); + + std::list fingerprints; + for (auto& murmur : m_mods.keys()) { + fingerprints.push_back(murmur.toUInt()); + } + + auto ver_task = flame_api.matchFingerprints(fingerprints, response); + + connect(ver_task.get(), &Task::succeeded, this, [this, response] { + QJsonParseError parse_error{}; + QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error); + if (parse_error.error != QJsonParseError::NoError) { + qWarning() << "Error while parsing JSON response from Modrinth::CurrentVersions at " << parse_error.offset + << " reason: " << parse_error.errorString(); + qWarning() << *response; + + failed(parse_error.errorString()); return; } try { auto doc_obj = Json::requireObject(doc); - auto data_obj = Json::ensureObject(doc_obj, "data"); - auto match_obj = Json::ensureObject(Json::ensureArray(data_obj, "exactMatches")[0], {}); - if (match_obj.isEmpty()) { - qCritical() << "Fingerprint match is empty!"; + auto data_obj = Json::requireObject(doc_obj, "data"); + auto data_arr = Json::requireArray(data_obj, "exactMatches"); + + if (data_arr.isEmpty()) { + qWarning() << "No matches found for fingerprint search!"; - ver_task->failed(parse_error.errorString()); return; } - auto file_obj = Json::ensureObject(match_obj, "file"); + for (auto match : data_arr) { + auto match_obj = Json::ensureObject(match, {}); + auto file_obj = Json::ensureObject(match_obj, "file", {}); - ModPlatform::IndexedPack pack; - pack.name = m_mod.name(); - pack.provider = ModPlatform::Provider::FLAME; - pack.addonId = Json::requireInteger(file_obj, "modId"); + if (match_obj.isEmpty() || file_obj.isEmpty()) { + qWarning() << "Fingerprint match is empty!"; - ModPlatform::IndexedVersion ver = FlameMod::loadIndexedPackVersion(file_obj); + return; + } - // Prevent file name mismatch - ver.fileName = m_mod.fileinfo().fileName(); + auto fingerprint = QString::number(Json::ensureVariant(file_obj, "fileFingerprint").toUInt()); + auto mod = m_mods.find(fingerprint); + if (mod == m_mods.end()) { + qWarning() << "Invalid fingerprint from the API response."; + continue; + } - { - LocalModUpdateTask update_metadata(m_index_dir, pack, ver); - QEventLoop loop; - QTimer timeout; + setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod->name())); - QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit); - QObject::connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit); - - update_metadata.start(); - timeout.start(100); - - loop.exec(); + m_temp_versions.insert(fingerprint, FlameMod::loadIndexedPackVersion(file_obj)); } - auto mod_name = m_mod.name(); - auto meta = new Metadata::ModStruct(Metadata::get(tmp_index_dir, mod_name)); - m_mod.setMetadata(meta); - } catch (Json::JsonException& e) { - emitFailed(e.cause() + " : " + e.what()); + qDebug() << e.cause(); + qDebug() << doc; } }); - tsk.addTask(ver_task); + return ver_task; +} + +NetJob::Ptr EnsureMetadataTask::flameProjectsTask() +{ + QHash addonIds; + for (auto const& hash : m_mods.keys()) { + if (m_temp_versions.contains(hash)) { + auto const& data = m_temp_versions.find(hash).value(); + addonIds.insert(data.addonId.toString(), hash); + } + } + + auto response = new QByteArray(); + NetJob::Ptr proj_task; + + if (addonIds.isEmpty()) { + qWarning() << "No addonId found!"; + } else if (addonIds.size() == 1) { + proj_task = flame_api.getProject(*addonIds.keyBegin(), response); + } else { + proj_task = flame_api.getProjects(addonIds.keys(), response); + } + + // Prevents unfortunate timings when aborting the task + if (!proj_task) + return {}; + + connect(proj_task.get(), &NetJob::succeeded, this, [this, response, addonIds] { + QJsonParseError parse_error{}; + auto doc = QJsonDocument::fromJson(*response, &parse_error); + if (parse_error.error != QJsonParseError::NoError) { + qWarning() << "Error while parsing JSON response from Modrinth projects task at " << parse_error.offset + << " reason: " << parse_error.errorString(); + qWarning() << *response; + return; + } + + try { + QJsonArray entries; + if (addonIds.size() == 1) + entries = { Json::requireObject(Json::requireObject(doc), "data") }; + else + entries = Json::requireArray(Json::requireObject(doc), "data"); + + for (auto entry : entries) { + auto entry_obj = Json::requireObject(entry); + + auto id = QString::number(Json::requireInteger(entry_obj, "id")); + auto hash = addonIds.find(id).value(); + auto mod = m_mods.find(hash).value(); + + try { + setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod.name())); + + ModPlatform::IndexedPack pack; + FlameMod::loadIndexedPack(pack, entry_obj); + + flameCallback(pack, m_temp_versions.find(hash).value(), mod); + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << entries; + + emitFail(mod); + } + } + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << doc; + } + }); + + return proj_task; +} + +void EnsureMetadataTask::modrinthCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod& mod) +{ + // Prevent file name mismatch + ver.fileName = mod.fileinfo().fileName(); + + QDir tmp_index_dir(m_index_dir); + + { + LocalModUpdateTask update_metadata(m_index_dir, pack, ver); + QEventLoop loop; + + QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit); + + update_metadata.start(); + + if (!update_metadata.isFinished()) + loop.exec(); + } + + auto metadata = Metadata::get(tmp_index_dir, pack.slug); + if (!metadata.isValid()) { + qCritical() << "Failed to generate metadata at last step!"; + emitFail(mod); + return; + } + + mod.setMetadata(metadata); + + emitReady(mod); +} + +void EnsureMetadataTask::flameCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod& mod) +{ + try { + // Prevent file name mismatch + ver.fileName = mod.fileinfo().fileName(); + + QDir tmp_index_dir(m_index_dir); + + { + LocalModUpdateTask update_metadata(m_index_dir, pack, ver); + QEventLoop loop; + + QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit); + + update_metadata.start(); + + if (!update_metadata.isFinished()) + loop.exec(); + } + + auto metadata = Metadata::get(tmp_index_dir, pack.slug); + if (!metadata.isValid()) { + qCritical() << "Failed to generate metadata at last step!"; + emitFail(mod); + return; + } + + mod.setMetadata(metadata); + + emitReady(mod); + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + + emitFail(mod); + } } diff --git a/launcher/modplatform/EnsureMetadataTask.h b/launcher/modplatform/EnsureMetadataTask.h index 624e253a..880503b9 100644 --- a/launcher/modplatform/EnsureMetadataTask.h +++ b/launcher/modplatform/EnsureMetadataTask.h @@ -2,6 +2,7 @@ #include "ModIndex.h" #include "tasks/SequentialTask.h" +#include "net/NetJob.h" class Mod; class QDir; @@ -11,7 +12,10 @@ class EnsureMetadataTask : public Task { Q_OBJECT public: - EnsureMetadataTask(Mod&, QDir&, bool try_all, ModPlatform::Provider = ModPlatform::Provider::MODRINTH); + EnsureMetadataTask(Mod&, QDir, ModPlatform::Provider = ModPlatform::Provider::MODRINTH); + EnsureMetadataTask(std::list&, QDir, ModPlatform::Provider = ModPlatform::Provider::MODRINTH); + + ~EnsureMetadataTask() = default; public slots: bool abort() override; @@ -20,22 +24,31 @@ class EnsureMetadataTask : public Task { private: // FIXME: Move to their own namespace - void modrinthEnsureMetadata(SequentialTask&, QByteArray&); - void flameEnsureMetadata(SequentialTask&, QByteArray&); + auto modrinthVersionsTask() -> NetJob::Ptr; + auto modrinthProjectsTask() -> NetJob::Ptr; + + auto flameVersionsTask() -> NetJob::Ptr; + auto flameProjectsTask() -> NetJob::Ptr; // Helpers - void emitReady(); - void emitFail(); + void emitReady(Mod&); + void emitFail(Mod&); + + auto getHash(Mod&) -> QString; + + private slots: + void modrinthCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod&); + void flameCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod&); signals: - void metadataReady(); - void metadataFailed(); + void metadataReady(Mod&); + void metadataFailed(Mod&); private: - Mod& m_mod; - QDir& m_index_dir; + QHash m_mods; + QDir m_index_dir; ModPlatform::Provider m_provider; - bool m_try_all; - MultipleOptionsTask* m_task_handler = nullptr; + QHash m_temp_versions; + NetJob* m_current_task; }; diff --git a/launcher/modplatform/flame/FlameCheckUpdate.cpp b/launcher/modplatform/flame/FlameCheckUpdate.cpp index 12d99029..3658bf8d 100644 --- a/launcher/modplatform/flame/FlameCheckUpdate.cpp +++ b/launcher/modplatform/flame/FlameCheckUpdate.cpp @@ -10,7 +10,6 @@ #include "ModDownloadTask.h" static FlameAPI api; -static ModPlatform::ProviderCapabilities ProviderCaps; bool FlameCheckUpdate::abort() { @@ -64,6 +63,50 @@ ModPlatform::IndexedPack getProjectInfo(ModPlatform::IndexedVersion& ver_info) return pack; } +ModPlatform::IndexedVersion getFileInfo(int addonId, int fileId) +{ + ModPlatform::IndexedVersion ver; + + QEventLoop loop; + + auto get_file_info_job = new NetJob("Flame::GetFileInfoJob", APPLICATION->network()); + + auto response = new QByteArray(); + auto url = QString("https://api.curseforge.com/v1/mods/%1/files/%2").arg(QString::number(addonId), QString::number(fileId)); + auto dl = Net::Download::makeByteArray(url, response); + get_file_info_job->addNetAction(dl); + + QObject::connect(get_file_info_job, &NetJob::succeeded, [response, &ver]() { + QJsonParseError parse_error{}; + QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error); + if (parse_error.error != QJsonParseError::NoError) { + qWarning() << "Error while parsing JSON response from FlameCheckUpdate at " << parse_error.offset + << " reason: " << parse_error.errorString(); + qWarning() << *response; + return; + } + + try { + auto doc_obj = Json::requireObject(doc); + auto data_obj = Json::requireObject(doc_obj, "data"); + ver = FlameMod::loadIndexedPackVersion(data_obj); + } catch (Json::JsonException& e) { + qWarning() << e.cause(); + qDebug() << doc; + } + }); + + QObject::connect(get_file_info_job, &NetJob::finished, [&loop, get_file_info_job] { + get_file_info_job->deleteLater(); + loop.quit(); + }); + + get_file_info_job->start(); + loop.exec(); + + return ver; +} + /* Check for update: * - Get latest version available * - Compare hash of the latest version with the current hash @@ -72,148 +115,60 @@ ModPlatform::IndexedPack getProjectInfo(ModPlatform::IndexedVersion& ver_info) void FlameCheckUpdate::executeTask() { setStatus(tr("Preparing mods for CurseForge...")); - setProgress(0, 5); - QHash mappings; - - // Create all hashes - std::list murmur_hashes; - - auto best_hash_type = ProviderCaps.hashType(ModPlatform::Provider::FLAME).first(); + int i = 0; for (auto mod : m_mods) { - QByteArray jar_data; + setStatus(tr("Getting API response from CurseForge for '%1'").arg(mod.name())); + setProgress(i++, m_mods.size()); - try { - jar_data = FS::read(mod.fileinfo().absoluteFilePath()); - } catch (FS::FileSystemException& e) { - qCritical() << QString("Failed to open / read JAR file of %1").arg(mod.name()); - qCritical() << QString("Reason: ") << e.cause(); + auto latest_ver = api.getLatestVersion({ mod.metadata()->project_id.toString(), m_game_versions, m_loaders }); - failed(e.what()); + // Check if we were aborted while getting the latest version + if (m_was_aborted) { + aborted(); return; } - QByteArray jar_data_treated; - for (char c : jar_data) { - // CF-specific - if (!(c == 9 || c == 10 || c == 13 || c == 32)) - jar_data_treated.push_back(c); + setStatus(tr("Parsing the API response from CurseForge for '%1'...").arg(mod.name())); + + if (!latest_ver.addonId.isValid()) { + emit checkFailed(mod, tr("No valid version found for this mod. It's probably unavailable for the current game " + "version / mod loader.")); + continue; } - auto murmur_hash = MurmurHash2(jar_data_treated, jar_data_treated.length()); - murmur_hashes.emplace_back(murmur_hash); + if (latest_ver.downloadUrl.isEmpty() && latest_ver.fileId != mod.metadata()->file_id) { + auto pack = getProjectInfo(latest_ver); + auto recover_url = QString("%1/download/%2").arg(pack.websiteUrl, latest_ver.fileId.toString()); + emit checkFailed(mod, tr("Mod has a new update available, but is opted-out on CurseForge"), recover_url); - mappings.insert(mod.metadata()->mod_id().toInt(), mod); - } - - auto* response = new QByteArray(); - auto job = api.matchFingerprints(murmur_hashes, response); - - QEventLoop lock; - - connect(job.get(), &Task::succeeded, this, [this, response, &mappings] { - QJsonParseError parse_error{}; - QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error); - if (parse_error.error != QJsonParseError::NoError) { - qWarning() << "Error while parsing JSON response from FlameCheckUpdate at " << parse_error.offset - << " reason: " << parse_error.errorString(); - qWarning() << *response; - - failed(parse_error.errorString()); - return; + continue; } - setStatus(tr("Parsing the first API response from CurseForge...")); - setProgress(2, 5); + if (!latest_ver.hash.isEmpty() && (mod.metadata()->hash != latest_ver.hash || mod.status() == ModStatus::NotInstalled)) { + // Fake pack with the necessary info to pass to the download task :) + ModPlatform::IndexedPack pack; + pack.name = mod.name(); + pack.slug = mod.metadata()->slug; + pack.addonId = mod.metadata()->project_id; + pack.websiteUrl = mod.homeurl(); + for (auto& author : mod.authors()) + pack.authors.append({ author }); + pack.description = mod.description(); + pack.provider = ModPlatform::Provider::FLAME; - try { - auto doc_obj = Json::requireObject(doc); - auto data_obj = Json::ensureObject(doc_obj, "data"); - auto match_arr = Json::ensureArray(data_obj, "exactMatches"); - for (auto match : match_arr) { - auto match_obj = Json::ensureObject(match); - - ModPlatform::IndexedVersion current_ver; - try { - auto file_obj = Json::requireObject(match_obj, "file"); - current_ver = FlameMod::loadIndexedPackVersion(file_obj); - } catch (Json::JsonException& e) { - qCritical() << "Error while parsing Flame indexed version"; - qCritical() << e.what(); - failed(tr("An error occured while parsing a CurseForge indexed version!")); - return; - } - - auto mod_iter = mappings.find(current_ver.addonId.toInt()); - if (mod_iter == mappings.end()) { - qCritical() << "Failed to remap mod from Flame!"; - qDebug() << match_obj; - continue; - } - - auto mod = mod_iter.value(); - - setStatus(tr("Waiting for the API response from CurseForge for '%1'...").arg(mod.name())); - setProgress(3, 5); - - auto latest_ver = api.getLatestVersion({ current_ver.addonId.toString(), m_game_versions, m_loaders }); - - // Check if we were aborted while getting the latest version - if (m_was_aborted) { - aborted(); - return; - } - - setStatus(tr("Parsing the API response from CurseForge for '%1'...").arg(mod.name())); - setProgress(4, 5); - - if (!latest_ver.addonId.isValid()) { - emit checkFailed( - mod, - tr("No valid version found for this mod. It's probably unavailable for the current game version / mod loader.")); - continue; - } - - if (latest_ver.downloadUrl.isEmpty() && latest_ver.fileId != current_ver.fileId) { - auto pack = getProjectInfo(latest_ver); - auto recover_url = QString("%1/download/%2").arg(pack.websiteUrl, latest_ver.fileId.toString()); - emit checkFailed(mod, tr("Mod has a new update available, but is opted-out on CurseForge"), recover_url); - - continue; - } - - if (!latest_ver.hash.isEmpty() && (current_ver.hash != latest_ver.hash || mod.status() == ModStatus::NotInstalled)) { - // Fake pack with the necessary info to pass to the download task :) - ModPlatform::IndexedPack pack; - pack.name = mod.name(); - pack.addonId = mod.metadata()->project_id; - pack.websiteUrl = mod.homeurl(); - for (auto& author : mod.authors()) - pack.authors.append({ author }); - pack.description = mod.description(); - pack.provider = ModPlatform::Provider::FLAME; - - auto download_task = new ModDownloadTask(pack, latest_ver, m_mods_folder); - m_updatable.emplace_back(mod.name(), current_ver.hash, current_ver.version, latest_ver.version, - api.getModFileChangelog(latest_ver.addonId.toInt(), latest_ver.fileId.toInt()), - ModPlatform::Provider::FLAME, download_task); - } + auto old_version = mod.version(); + if (old_version.isEmpty() && mod.status() != ModStatus::NotInstalled) { + auto current_ver = getFileInfo(latest_ver.addonId.toInt(), mod.metadata()->file_id.toInt()); + old_version = current_ver.version; } - } catch (Json::JsonException& e) { - failed(e.cause() + " : " + e.what()); + auto download_task = new ModDownloadTask(pack, latest_ver, m_mods_folder); + m_updatable.emplace_back(mod.name(), mod.metadata()->hash, old_version, latest_ver.version, + api.getModFileChangelog(latest_ver.addonId.toInt(), latest_ver.fileId.toInt()), + ModPlatform::Provider::FLAME, download_task); } - }); - - connect(job.get(), &Task::finished, &lock, &QEventLoop::quit); - - setStatus(tr("Waiting for the first API response from CurseForge...")); - setProgress(1, 5); - - m_net_job = job.get(); - job->start(); - - lock.exec(); + } emitSucceeded(); } diff --git a/launcher/modplatform/modrinth/ModrinthCheckUpdate.cpp b/launcher/modplatform/modrinth/ModrinthCheckUpdate.cpp index 981c4216..78275cf0 100644 --- a/launcher/modplatform/modrinth/ModrinthCheckUpdate.cpp +++ b/launcher/modplatform/modrinth/ModrinthCheckUpdate.cpp @@ -88,7 +88,9 @@ void ModrinthCheckUpdate::executeTask() qDebug() << "Mod " << mappings.find(hash).value().name() << " got an empty response."; qDebug() << "Hash: " << hash; - emit checkFailed(mappings.find(hash).value(), tr("Couldn't find the latest version of this mod with the correct mod loader and game version.")); + emit checkFailed( + mappings.find(hash).value(), + tr("No valid version found for this mod. It's probably unavailable for the current game version / mod loader.")); continue; } @@ -134,6 +136,7 @@ void ModrinthCheckUpdate::executeTask() // Fake pack with the necessary info to pass to the download task :) ModPlatform::IndexedPack pack; pack.name = mod.name(); + pack.slug = mod.metadata()->slug; pack.addonId = mod.metadata()->project_id; pack.websiteUrl = mod.homeurl(); for (auto& author : mod.authors()) diff --git a/launcher/ui/dialogs/ModUpdateDialog.cpp b/launcher/ui/dialogs/ModUpdateDialog.cpp index a4d83483..7584621a 100644 --- a/launcher/ui/dialogs/ModUpdateDialog.cpp +++ b/launcher/ui/dialogs/ModUpdateDialog.cpp @@ -34,12 +34,13 @@ static ModAPI::ModLoaderTypes mcLoaders(BaseInstance* inst) ModUpdateDialog::ModUpdateDialog(QWidget* parent, BaseInstance* instance, - const std::shared_ptr& mods, + const std::shared_ptr mods, std::list& search_for) : ReviewMessageBox(parent, tr("Confirm mods to update"), "") , m_parent(parent) , m_mod_model(mods) , m_candidates(search_for) + , m_second_try_metadata(new SequentialTask()) , m_instance(instance) { ReviewMessageBox::setGeometry(0, 0, 800, 600); @@ -47,15 +48,6 @@ ModUpdateDialog::ModUpdateDialog(QWidget* parent, ui->explainLabel->setText(tr("You're about to update the following mods:")); ui->onlyCheckedLabel->setText(tr("Only mods with a check will be updated!")); - connect(&m_check_task, &Task::failed, this, - [&](QString reason) { CustomMessageBox::selectable(this, tr("Error"), reason, QMessageBox::Critical)->exec(); }); - - connect(&m_check_task, &Task::succeeded, this, [&]() { - QStringList warnings = m_check_task.warnings(); - if (warnings.count()) { - CustomMessageBox::selectable(this, tr("Warnings"), warnings.join('\n'), QMessageBox::Warning)->exec(); - } - }); } void ModUpdateDialog::checkCandidates() @@ -89,26 +81,39 @@ void ModUpdateDialog::checkCandidates() auto versions = mcVersions(m_instance); auto loaders = mcLoaders(m_instance); + SequentialTask check_task (m_parent, tr("Checking for updates")); + if (!m_modrinth_to_update.empty()) { m_modrinth_check_task = new ModrinthCheckUpdate(m_modrinth_to_update, versions, loaders, m_mod_model); connect(m_modrinth_check_task, &CheckUpdateTask::checkFailed, this, [this](Mod mod, QString reason, QUrl recover_url) { m_failed_check_update.emplace_back(mod, reason, recover_url); }); - m_check_task.addTask(m_modrinth_check_task); + check_task.addTask(m_modrinth_check_task); } if (!m_flame_to_update.empty()) { m_flame_check_task = new FlameCheckUpdate(m_flame_to_update, versions, loaders, m_mod_model); connect(m_flame_check_task, &CheckUpdateTask::checkFailed, this, [this](Mod mod, QString reason, QUrl recover_url) { m_failed_check_update.emplace_back(mod, reason, recover_url); }); - m_check_task.addTask(m_flame_check_task); + check_task.addTask(m_flame_check_task); } + connect(&check_task, &Task::failed, this, + [&](QString reason) { CustomMessageBox::selectable(this, tr("Error"), reason, QMessageBox::Critical)->exec(); }); + + connect(&check_task, &Task::succeeded, this, [&]() { + QStringList warnings = check_task.warnings(); + if (warnings.count()) { + CustomMessageBox::selectable(this, tr("Warnings"), warnings.join('\n'), QMessageBox::Warning)->exec(); + } + }); + // Check for updates + // FIXME: SOMEHOW THIS IS NOT MODAL??????? ProgressDialog progress_dialog(m_parent); progress_dialog.setSkipButton(true, tr("Abort")); progress_dialog.setVisible(true); progress_dialog.setWindowTitle(tr("Checking for updates...")); - auto ret = progress_dialog.execWithTask(&m_check_task); + auto ret = progress_dialog.execWithTask(&check_task); // If the dialog was skipped / some download error happened if (ret == QDialog::DialogCode::Rejected) { @@ -183,13 +188,29 @@ auto ModUpdateDialog::ensureMetadata() -> bool { auto index_dir = indexDir(); - auto* seq = new SequentialTask(m_parent, tr("Looking for metadata")); + SequentialTask seq(m_parent, tr("Looking for metadata")); + + // A better use of data structures here could remove the need for this QHash + QHash should_try_others; + std::list modrinth_tmp; + std::list flame_tmp; bool confirm_rest = false; bool try_others_rest = false; bool skip_rest = false; ModPlatform::Provider provider_rest = ModPlatform::Provider::MODRINTH; + auto addToTmp = [&](Mod& m, ModPlatform::Provider p) { + switch (p) { + case ModPlatform::Provider::MODRINTH: + modrinth_tmp.push_back(m); + break; + case ModPlatform::Provider::FLAME: + flame_tmp.push_back(m); + break; + } + }; + for (auto& candidate : m_candidates) { if (candidate.status() != ModStatus::NoMetadata) { onMetadataEnsured(candidate); @@ -200,10 +221,8 @@ auto ModUpdateDialog::ensureMetadata() -> bool continue; if (confirm_rest) { - auto* task = new EnsureMetadataTask(candidate, index_dir, try_others_rest, provider_rest); - connect(task, &EnsureMetadataTask::metadataReady, [this, &candidate] { onMetadataEnsured(candidate); }); - connect(task, &EnsureMetadataTask::metadataFailed, [this, &candidate] { onMetadataFailed(candidate); }); - seq->addTask(task); + addToTmp(candidate, provider_rest); + should_try_others.insert(candidate.internal_id(), try_others_rest); continue; } @@ -224,18 +243,36 @@ auto ModUpdateDialog::ensureMetadata() -> bool try_others_rest = response.try_others; } - if (confirmed) { - auto* task = new EnsureMetadataTask(candidate, index_dir, response.try_others, response.chosen); - connect(task, &EnsureMetadataTask::metadataReady, [this, &candidate] { onMetadataEnsured(candidate); }); - connect(task, &EnsureMetadataTask::metadataFailed, [this, &candidate] { onMetadataFailed(candidate); }); - seq->addTask(task); - } + should_try_others.insert(candidate.internal_id(), response.try_others); + + if (confirmed) + addToTmp(candidate, response.chosen); } + if (!modrinth_tmp.empty()) { + auto* modrinth_task = new EnsureMetadataTask(modrinth_tmp, index_dir, ModPlatform::Provider::MODRINTH); + connect(modrinth_task, &EnsureMetadataTask::metadataReady, [this](Mod& candidate) { onMetadataEnsured(candidate); }); + connect(modrinth_task, &EnsureMetadataTask::metadataFailed, [this, &should_try_others](Mod& candidate) { + onMetadataFailed(candidate, should_try_others.find(candidate.internal_id()).value(), ModPlatform::Provider::MODRINTH); + }); + seq.addTask(modrinth_task); + } + + if (!flame_tmp.empty()) { + auto* flame_task = new EnsureMetadataTask(flame_tmp, index_dir, ModPlatform::Provider::FLAME); + connect(flame_task, &EnsureMetadataTask::metadataReady, [this](Mod& candidate) { onMetadataEnsured(candidate); }); + connect(flame_task, &EnsureMetadataTask::metadataFailed, [this, &should_try_others](Mod& candidate) { + onMetadataFailed(candidate, should_try_others.find(candidate.internal_id()).value(), ModPlatform::Provider::FLAME); + }); + seq.addTask(flame_task); + } + + seq.addTask(m_second_try_metadata); + ProgressDialog checking_dialog(m_parent); checking_dialog.setSkipButton(true, tr("Abort")); checking_dialog.setWindowTitle(tr("Generating metadata...")); - auto ret_metadata = checking_dialog.execWithTask(seq); + auto ret_metadata = checking_dialog.execWithTask(&seq); return (ret_metadata != QDialog::DialogCode::Rejected); } @@ -256,9 +293,31 @@ void ModUpdateDialog::onMetadataEnsured(Mod& mod) } } -void ModUpdateDialog::onMetadataFailed(Mod& mod) +ModPlatform::Provider next(ModPlatform::Provider p) { - m_failed_metadata.push_back(mod); + switch (p) { + case ModPlatform::Provider::MODRINTH: + return ModPlatform::Provider::FLAME; + case ModPlatform::Provider::FLAME: + return ModPlatform::Provider::MODRINTH; + } + + return ModPlatform::Provider::FLAME; +} + +void ModUpdateDialog::onMetadataFailed(Mod& mod, bool try_others, ModPlatform::Provider first_choice) +{ + if (try_others) { + auto index_dir = indexDir(); + + auto* task = new EnsureMetadataTask(mod, index_dir, next(first_choice)); + connect(task, &EnsureMetadataTask::metadataReady, [this](Mod& candidate) { onMetadataEnsured(candidate); }); + connect(task, &EnsureMetadataTask::metadataFailed, [this](Mod& candidate) { onMetadataFailed(candidate, false); }); + + m_second_try_metadata->addTask(task); + } else { + m_failed_metadata.push_back(mod); + } } void ModUpdateDialog::appendMod(CheckUpdateTask::UpdatableMod const& info) diff --git a/launcher/ui/dialogs/ModUpdateDialog.h b/launcher/ui/dialogs/ModUpdateDialog.h index 30cd5cbd..f40fc594 100644 --- a/launcher/ui/dialogs/ModUpdateDialog.h +++ b/launcher/ui/dialogs/ModUpdateDialog.h @@ -17,7 +17,7 @@ class ModUpdateDialog final : public ReviewMessageBox { public: explicit ModUpdateDialog(QWidget* parent, BaseInstance* instance, - const std::shared_ptr& mod_model, + const std::shared_ptr mod_model, std::list& search_for); void checkCandidates(); @@ -35,21 +35,21 @@ class ModUpdateDialog final : public ReviewMessageBox { private slots: void onMetadataEnsured(Mod&); - void onMetadataFailed(Mod&); + void onMetadataFailed(Mod&, bool try_others = false, ModPlatform::Provider first_choice = ModPlatform::Provider::MODRINTH); private: QWidget* m_parent; - SequentialTask m_check_task; ModrinthCheckUpdate* m_modrinth_check_task = nullptr; FlameCheckUpdate* m_flame_check_task = nullptr; - const std::shared_ptr& m_mod_model; + const std::shared_ptr m_mod_model; std::list& m_candidates; std::list m_modrinth_to_update; std::list m_flame_to_update; + SequentialTask* m_second_try_metadata; std::list m_failed_metadata; std::list> m_failed_check_update;