Merge pull request #771 from flowln/modrinth_multiple_downloads

launcher/FileSystem.cpp

@@ -454,4 +454,47 @@ bool createShortCut(QString location, QString dest, QStringList args, QString na
     return false;
 #endif
 }
+
+QStringList listFolderPaths(QDir root)
+{
+    auto createAbsPath = [](QFileInfo const& entry) { return FS::PathCombine(entry.path(), entry.fileName()); };
+
+    QStringList entries;
+
+    root.refresh();
+    for (auto entry : root.entryInfoList(QDir::Filter::Files)) {
+        entries.append(createAbsPath(entry));
+    }
+
+    for (auto entry : root.entryInfoList(QDir::Filter::AllDirs | QDir::Filter::NoDotAndDotDot)) {
+        entries.append(listFolderPaths(createAbsPath(entry)));
+    }
+
+    return entries;
+}
+
+bool overrideFolder(QString overwritten_path, QString override_path)
+{
+    if (!FS::ensureFolderPathExists(overwritten_path))
+        return false;
+
+    QStringList paths_to_override;
+    QDir root_override (override_path);
+    for (auto file : listFolderPaths(root_override)) {
+        QString destination = file;
+        destination.replace(override_path, overwritten_path);
+
+        qDebug() << QString("Applying override %1 in %2").arg(file, destination);
+
+        if (QFile::exists(destination))
+            QFile::remove(destination);
+        if (!QFile::rename(file, destination)) {
+            qCritical() << QString("Failed to apply override from %1 to %2").arg(file, destination);
+            return false;
+        }
+    }
+
+    return true;
+}
+
 }

launcher/FileSystem.h

@@ -124,4 +124,8 @@ QString getDesktopDir();
 // call it *name* and assign it the icon *icon*
 // return true if operation succeeded
 bool createShortCut(QString location, QString dest, QStringList args, QString name, QString iconLocation);
+
+// Overrides one folder with the contents of another, preserving items exclusive to the first folder
+// Equivalent to doing QDir::rename, but allowing for overrides
+bool overrideFolder(QString overwritten_path, QString override_path);
 }

launcher/InstanceImportTask.cpp

@@ -582,10 +582,10 @@ void InstanceImportTask::processMultiMC()
     emitSucceeded();
 }
 
+// https://docs.modrinth.com/docs/modpacks/format_definition/
 void InstanceImportTask::processModrinth()
 {
     std::vector<Modrinth::File> files;
-    std::vector<Modrinth::File> non_whitelisted_files;
     QString minecraftVersion, fabricVersion, quiltVersion, forgeVersion;
     try {
         QString indexPath = FS::PathCombine(m_stagingPath, "modrinth.index.json");
@@ -600,11 +600,13 @@ void InstanceImportTask::processModrinth()
 
             auto jsonFiles = Json::requireIsArrayOf<QJsonObject>(obj, "files", "modrinth.index.json");
             bool had_optional = false;
-            for (auto& modInfo : jsonFiles) {
+            for (auto modInfo : jsonFiles) {
                 Modrinth::File file;
                 file.path = Json::requireString(modInfo, "path");
 
                 auto env = Json::ensureObject(modInfo, "env");
+                // 'env' field is optional
+                if (!env.isEmpty()) {
                     QString support = Json::ensureString(env, "client", "unsupported");
                     if (support == "unsupported") {
                         continue;
@@ -614,13 +616,15 @@ void InstanceImportTask::processModrinth()
                             had_optional = true;
                             auto info = CustomMessageBox::selectable(
                                 m_parent, tr("Optional mod detected!"),
-                            tr("One or more mods from this modpack are optional. They will be downloaded, but disabled by default!"), QMessageBox::Information);
+                                tr("One or more mods from this modpack are optional. They will be downloaded, but disabled by default!"),
+                                QMessageBox::Information);
                             info->exec();
                         }
 
                         if (file.path.endsWith(".jar"))
                             file.path += ".disabled";
                     }
+                }
 
                 QJsonObject hashes = Json::requireObject(modInfo, "hashes");
                 QString hash;
@@ -640,40 +644,31 @@ void InstanceImportTask::processModrinth()
                 }
                 file.hash = QByteArray::fromHex(hash.toLatin1());
                 file.hashAlgorithm = hashAlgorithm;
+                
                 // Do not use requireUrl, which uses StrictMode, instead use QUrl's default TolerantMode
                 // (as Modrinth seems to incorrectly handle spaces)
                 
-                file.download = Json::requireString(Json::ensureArray(modInfo, "downloads").first(), "Download URL for " + file.path);
+                auto download_arr = Json::ensureArray(modInfo, "downloads");
+                for(auto download : download_arr) {
+                    qWarning() << download.toString();
+                    bool is_last = download.toString() == download_arr.last().toString();
 
-                if (!file.download.isValid()) {
+                    auto download_url = QUrl(download.toString());
-                    qDebug() << QString("Download URL (%1) for %2 is not a correctly formatted URL").arg(file.download.toString(), file.path);
+
+                    if (!download_url.isValid()) {
+                        qDebug() << QString("Download URL (%1) for %2 is not a correctly formatted URL")
+                                        .arg(download_url.toString(), file.path);
+                        if(is_last && file.downloads.isEmpty())
                             throw JSONValidationError(tr("Download URL for %1 is not a correctly formatted URL").arg(file.path));
                     }
-                else if (!Modrinth::validateDownloadUrl(file.download)) {
+                    else {
-                    qDebug() << QString("Download URL (%1) for %2 is from a non-whitelisted by Modrinth domain").arg(file.download.toString(), file.path);
+                        file.downloads.push_back(download_url);
-                    non_whitelisted_files.push_back(file);
+                    }
                 }
 
                 files.push_back(file);
             }
 
-            if (!non_whitelisted_files.empty()) {
-                QString text;
-                for (const auto& file : non_whitelisted_files) {
-                    text += tr("Filepath: %1<br>URL: <a href='%2'>%2</a><br>").arg(file.path, file.download.toString());
-                }
-
-                auto message_dialog = new ScrollMessageBox(m_parent, tr("Non-whitelisted mods found"),
-                                                           tr("The following mods have URLs that are not whitelisted by Modrinth.\n"
-                                                              "Proceed with caution!"),
-                                                           text);
-                message_dialog->setModal(true);
-                if (message_dialog->exec() == QDialog::Rejected) {
-                    emitFailed("Aborted");
-                    return;
-                }
-            }
-
             auto dependencies = Json::requireObject(obj, "dependencies", "modrinth.index.json");
             for (auto it = dependencies.begin(), end = dependencies.end(); it != end; ++it) {
                 QString name = it.key();
@@ -702,15 +697,25 @@ void InstanceImportTask::processModrinth()
         return;
     }
     
-    QString overridePath = FS::PathCombine(m_stagingPath, "overrides");
+    auto mcPath = FS::PathCombine(m_stagingPath, ".minecraft");
-    if (QFile::exists(overridePath)) {
+
-        QString mcPath = FS::PathCombine(m_stagingPath, ".minecraft");
+    auto override_path = FS::PathCombine(m_stagingPath, "overrides");
-        if (!QFile::rename(overridePath, mcPath)) {
+    if (QFile::exists(override_path)) {
+        if (!QFile::rename(override_path, mcPath)) {
             emitFailed(tr("Could not rename the overrides folder:\n") + "overrides");
             return;
         }
     }
 
+    // Do client overrides
+    auto client_override_path = FS::PathCombine(m_stagingPath, "client-overrides");
+    if (QFile::exists(client_override_path)) {
+        if (!FS::overrideFolder(mcPath, client_override_path)) {
+            emitFailed(tr("Could not rename the client overrides folder:\n") + "client overrides");
+            return;
+        }
+    }
+
     QString configPath = FS::PathCombine(m_stagingPath, "instance.cfg");
     auto instanceSettings = std::make_shared<INISettingsObject>(configPath);
     MinecraftInstance instance(m_globalSettings, instanceSettings, m_stagingPath);
@@ -735,13 +740,24 @@ void InstanceImportTask::processModrinth()
     instance.saveNow();
 
     m_filesNetJob = new NetJob(tr("Mod download"), APPLICATION->network());
-    for (auto &file : files)
+    for (auto file : files)
     {
         auto path = FS::PathCombine(m_stagingPath, ".minecraft", file.path);
-        qDebug() << "Will download" << file.download << "to" << path;
+        qDebug() << "Will try to download" << file.downloads.front() << "to" << path;
-        auto dl = Net::Download::makeFile(file.download, path);
+        auto dl = Net::Download::makeFile(file.downloads.dequeue(), path);
         dl->addValidator(new Net::ChecksumValidator(file.hashAlgorithm, file.hash));
         m_filesNetJob->addNetAction(dl);
+
+        if (file.downloads.size() > 0) {
+            // FIXME: This really needs to be put into a ConcurrentTask of
+            // MultipleOptionsTask's , once those exist :)
+            connect(dl.get(), &NetAction::failed, [this, &file, path, dl]{
+                auto dl = Net::Download::makeFile(file.downloads.dequeue(), path);
+                dl->addValidator(new Net::ChecksumValidator(file.hashAlgorithm, file.hash));
+                m_filesNetJob->addNetAction(dl);
+                dl->succeeded();
+            });
+        }
     }
     connect(m_filesNetJob.get(), &NetJob::succeeded, this, [&]()
             {

launcher/modplatform/modrinth/ModrinthPackManifest.cpp

@@ -95,19 +95,6 @@ void loadIndexedVersions(Modpack& pack, QJsonDocument& doc)
     pack.versionsLoaded = true;
 }
 
-auto validateDownloadUrl(QUrl url) -> bool
-{
-    static QSet<QString> domainWhitelist{
-        "cdn.modrinth.com",
-        "github.com",
-        "raw.githubusercontent.com",
-        "gitlab.com"
-    };
-
-    auto domain = url.host();
-    return domainWhitelist.contains(domain);
-}
-
 auto loadIndexedVersion(QJsonObject &obj) -> ModpackVersion
 {
     ModpackVersion file;
@@ -137,9 +124,6 @@ auto loadIndexedVersion(QJsonObject &obj) -> ModpackVersion
 
         auto url = Json::requireString(parent, "url");
 
-        if(!validateDownloadUrl(url))
-            continue;
-
         file.download_url = url;
         if(is_primary)
             break;

launcher/modplatform/modrinth/ModrinthPackManifest.h

@@ -40,6 +40,7 @@
 
 #include <QByteArray>
 #include <QCryptographicHash>
+#include <QQueue>
 #include <QString>
 #include <QUrl>
 #include <QVector>
@@ -48,14 +49,12 @@ class MinecraftInstance;
 
 namespace Modrinth {
 
-struct File
+struct File {
-{
     QString path;
 
     QCryptographicHash::Algorithm hashAlgorithm;
     QByteArray hash;
-    // TODO: should this support multiple download URLs, like the JSON does?
+    QQueue<QUrl> downloads;
-    QUrl download;
 };
 
 struct ModpackExtra {