0xf8/pollymc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

S3 bucket listing support and network code refactors.

Browse Source 
* Adds support for listing all objects in an S3 bucket.
* Renames a bunch of network related classes (Download->Action)
* Net actions now have static constructors
This commit is contained in:
Petr Mrázek 2013-10-26 19:55:48 +02:00
parent c467ebf132
commit 9233477295
27 changed files with 405 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CMakeLists.txt
View File

@ -241,7 +241,7 @@ logic/InstanceLauncher.h
logic/InstanceLauncher.cpp
# network stuffs
logic/net/Download.h
logic/net/NetAction.h
logic/net/FileDownload.h
logic/net/FileDownload.cpp
logic/net/ByteArrayDownload.h
@ -250,12 +250,15 @@ logic/net/CacheDownload.h
logic/net/CacheDownload.cpp
logic/net/ForgeXzDownload.h
logic/net/ForgeXzDownload.cpp
logic/net/DownloadJob.h
logic/net/DownloadJob.cpp
logic/net/NetJob.h
logic/net/NetJob.cpp
logic/net/HttpMetaCache.h
logic/net/HttpMetaCache.cpp
logic/net/LoginTask.h
logic/net/LoginTask.cpp
logic/net/S3ListBucket.h
logic/net/S3ListBucket.cpp
# legacy instances
logic/LegacyInstance.h

4
gui/LegacyModEditDialog.cpp
View File

@ -218,8 +218,8 @@ void LegacyModEditDialog::on_addForgeBtn_clicked()
auto entry = MMC->metacache()->resolveEntry("minecraftforge", forge->filename);
if (entry->stale)
{
DownloadJob *fjob = new DownloadJob("Forge download");
fjob->addCacheDownload(forge->universal_url, entry);
NetJob *fjob = new NetJob("Forge download");
fjob->addNetAction(CacheDownload::make(forge->universal_url, entry));
ProgressDialog dlg(this);
dlg.exec(fjob);
if (dlg.result() == QDialog::Accepted)

4
gui/LegacyModEditDialog.h
View File

@ -17,7 +17,7 @@
#include <QDialog>
#include "logic/LegacyInstance.h"
#include <logic/net/DownloadJob.h>
#include <logic/net/NetJob.h>
namespace Ui
{
@ -74,5 +74,5 @@ private:
std::shared_ptr<ModList> m_jarmods;
std::shared_ptr<ModList> m_texturepacks;
LegacyInstance *m_inst;
DownloadJobPtr forgeJob;
NetJobPtr forgeJob;
};

4
gui/OneSixModEditDialog.cpp
View File

@ -166,8 +166,8 @@ void OneSixModEditDialog::on_forgeBtn_clicked()
auto entry = MMC->metacache()->resolveEntry("minecraftforge", forgeVersion->filename);
if (entry->stale)
{
DownloadJob *fjob = new DownloadJob("Forge download");
fjob->addCacheDownload(forgeVersion->installer_url, entry);
NetJob *fjob = new NetJob("Forge download");
fjob->addNetAction(CacheDownload::make(forgeVersion->installer_url, entry));
ProgressDialog dlg(this);
dlg.exec(fjob);
if (dlg.result() == QDialog::Accepted)

43
gui/mainwindow.cpp
View File

@ -176,7 +176,8 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWi
assets_downloader = new OneSixAssets();
connect(assets_downloader, SIGNAL(indexStarted()), SLOT(assetsIndexStarted()));
connect(assets_downloader, SIGNAL(filesStarted()), SLOT(assetsFilesStarted()));
connect(assets_downloader, SIGNAL(filesProgress(int, int, int)), SLOT(assetsFilesProgress(int, int, int)));
connect(assets_downloader, SIGNAL(filesProgress(int, int, int)),
SLOT(assetsFilesProgress(int, int, int)));
connect(assets_downloader, SIGNAL(failed()), SLOT(assetsFailed()));
connect(assets_downloader, SIGNAL(finished()), SLOT(assetsFinished()));
assets_downloader->start();
@ -465,8 +466,10 @@ void MainWindow::instanceActivated(QModelIndex index)
(BaseInstance *)index.data(InstanceList::InstancePointerRole).value<void *>();
bool autoLogin = MMC->settings()->get("AutoLogin").toBool();
if(autoLogin) doAutoLogin();
else doLogin();
if (autoLogin)
doAutoLogin();
else
doLogin();
}
void MainWindow::on_actionLaunchInstance_triggered()
@ -482,15 +485,15 @@ void MainWindow::doAutoLogin()
if (!m_selectedInstance)
return;
Keyring * k = Keyring::instance();
Keyring *k = Keyring::instance();
QStringList accounts = k->getStoredAccounts("minecraft");
if(!accounts.isEmpty())
if (!accounts.isEmpty())
{
QString username = accounts[0];
QString password = k->getPassword("minecraft", username);
if(!password.isEmpty())
if (!password.isEmpty())
{
QLOG_INFO() << "Automatically logging in with stored account: " << username;
m_activeInst = m_selectedInstance;
@ -498,7 +501,8 @@ void MainWindow::doAutoLogin()
}
else
{
QLOG_ERROR() << "Auto login set for account, but no password was found: " << username;
QLOG_ERROR() << "Auto login set for account, but no password was found: "
<< username;
doLogin(tr("Auto login attempted, but no password is stored."));
}
}
@ -515,10 +519,8 @@ void MainWindow::doLogin(QString username, QString password)
ProgressDialog *tDialog = new ProgressDialog(this);
LoginTask *loginTask = new LoginTask(uInfo, tDialog);
connect(loginTask, SIGNAL(succeeded()), SLOT(onLoginComplete()),
Qt::QueuedConnection);
connect(loginTask, SIGNAL(failed(QString)), SLOT(doLogin(QString)),
Qt::QueuedConnection);
connect(loginTask, SIGNAL(succeeded()), SLOT(onLoginComplete()), Qt::QueuedConnection);
connect(loginTask, SIGNAL(failed(QString)), SLOT(doLogin(QString)), Qt::QueuedConnection);
tDialog->exec(loginTask);
}
@ -573,10 +575,13 @@ void MainWindow::onLoginComplete()
delete updateTask;
}
auto job = new DownloadJob("Player skin: " + m_activeLogin.player_name);
auto job = new NetJob("Player skin: " + m_activeLogin.player_name);
auto meta = MMC->metacache()->resolveEntry("skins", m_activeLogin.player_name + ".png");
job->addCacheDownload(QUrl("http://skins.minecraft.net/MinecraftSkins/" + m_activeLogin.player_name + ".png"), meta);
auto action = CacheDownload::make(
QUrl("http://skins.minecraft.net/MinecraftSkins/" + m_activeLogin.player_name + ".png"),
meta);
job->addNetAction(action);
meta->stale = true;
job->start();
@ -586,7 +591,7 @@ void MainWindow::onLoginComplete()
// Add skin mapping
QByteArray data;
{
if(!listFile.open(QIODevice::ReadWrite))
if (!listFile.open(QIODevice::ReadWrite))
{
QLOG_ERROR() << "Failed to open/make skins list JSON";
return;
@ -601,7 +606,7 @@ void MainWindow::onLoginComplete()
QJsonObject mappings = root.value("mappings").toObject();
QJsonArray usernames = mappings.value(m_activeLogin.username).toArray();
if(!usernames.contains(m_activeLogin.player_name))
if (!usernames.contains(m_activeLogin.player_name))
{
usernames.prepend(m_activeLogin.player_name);
mappings[m_activeLogin.username] = usernames;
@ -642,12 +647,11 @@ void MainWindow::launchInstance(BaseInstance *instance, LoginResponse response)
this->hide();
}
console = new ConsoleWindow(proc);
connect(proc, SIGNAL(log(QString, MessageLevel::Enum)), console,
SLOT(write(QString, MessageLevel::Enum)));
connect(proc, SIGNAL(ended(BaseInstance*)), this, SLOT(instanceEnded(BaseInstance*)));
connect(proc, SIGNAL(ended(BaseInstance *)), this, SLOT(instanceEnded(BaseInstance *)));
if (instance->settings().get("ShowConsole").toBool())
{
@ -856,7 +860,7 @@ void MainWindow::checkSetDefaultJava()
JavaUtils ju;
java = ju.GetDefaultJava();
}
if(java)
if (java)
MMC->settings()->set("JavaPath", java->path);
else
MMC->settings()->set("JavaPath", QString("java"));
@ -876,7 +880,8 @@ void MainWindow::assetsFilesStarted()
void MainWindow::assetsFilesProgress(int succeeded, int failed, int total)
{
QString status = tr("Downloading assets: %1 / %2").arg(succeeded + failed).arg(total);
if(failed > 0) status += tr(" (%1 failed)").arg(failed);
if (failed > 0)
status += tr(" (%1 failed)").arg(failed);
status += tr("...");
m_statusRight->setText(status);
}

2
logic/BaseUpdate.h
View File

@ -19,7 +19,7 @@
#include <QList>
#include <QUrl>
#include "net/DownloadJob.h"
#include "net/NetJob.h"
#include "tasks/Task.h"

4
logic/LegacyUpdate.cpp
View File

@ -233,8 +233,8 @@ void LegacyUpdate::jarStart()
QString intended_version_id = inst->intendedVersionId();
urlstr += intended_version_id + "/" + intended_version_id + ".jar";
auto dljob = new DownloadJob("Minecraft.jar for version " + intended_version_id);
dljob->addFileDownload(QUrl(urlstr), inst->defaultBaseJar());
auto dljob = new NetJob("Minecraft.jar for version " + intended_version_id);
dljob->addNetAction(FileDownload::make(QUrl(urlstr), inst->defaultBaseJar()));
legacyDownloadJob.reset(dljob);
connect(dljob, SIGNAL(succeeded()), SLOT(jarFinished()));
connect(dljob, SIGNAL(failed()), SLOT(jarFailed()));

4
logic/LegacyUpdate.h
View File

@ -19,7 +19,7 @@
#include <QList>
#include <QUrl>
#include "net/DownloadJob.h"
#include "net/NetJob.h"
#include "tasks/Task.h"
#include "BaseUpdate.h"
@ -66,7 +66,7 @@ private:
QString lwjglTargetPath;
QString lwjglNativesPath;
private:
DownloadJobPtr legacyDownloadJob;
NetJobPtr legacyDownloadJob;
};

99
logic/OneSixAssets.cpp
View File

@ -2,19 +2,11 @@
#include <logger/QsLog.h>
#include <QtXml/QtXml>
#include "OneSixAssets.h"
#include "net/DownloadJob.h"
#include "net/NetJob.h"
#include "net/HttpMetaCache.h"
#include "net/S3ListBucket.h"
#include "MultiMC.h"
inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
{
QDomNodeList elementList = parent.elementsByTagName(tagname);
if (elementList.count())
return elementList.at(0).toElement();
else
return QDomElement();
}
class ThreadedDeleter : public QThread
{
Q_OBJECT
@ -22,18 +14,18 @@ public:
void run()
{
QLOG_INFO() << "Cleaning up assets folder...";
QDirIterator iter ( m_base, QDirIterator::Subdirectories );
QDirIterator iter(m_base, QDirIterator::Subdirectories);
int base_length = m_base.length();
while ( iter.hasNext() )
while (iter.hasNext())
{
QString filename = iter.next();
QFileInfo current ( filename );
QFileInfo current(filename);
// we keep the dirs... whatever
if ( current.isDir() )
if (current.isDir())
continue;
QString trimmedf = filename;
trimmedf.remove ( 0, base_length + 1 );
if ( m_whitelist.contains ( trimmedf ) )
trimmedf.remove(0, base_length + 1);
if (m_whitelist.contains(trimmedf))
{
QLOG_TRACE() << trimmedf << " gets to live";
}
@ -41,7 +33,7 @@ public:
{
// DO NOT TOLERATE JUNK
QLOG_TRACE() << trimmedf << " dies";
QFile f ( filename );
QFile f(filename);
f.remove();
}
}
@ -60,71 +52,41 @@ void OneSixAssets::downloadFinished()
deleter->start();
}
void OneSixAssets::fetchXMLFinished()
void OneSixAssets::S3BucketFinished()
{
QString prefix ( "http://s3.amazonaws.com/Minecraft.Resources/" );
QString fprefix ( "assets/" );
QString prefix("http://s3.amazonaws.com/Minecraft.Resources/");
nuke_whitelist.clear();
emit filesStarted();
auto firstJob = index_job->first();
QByteArray ba = std::dynamic_pointer_cast<ByteArrayDownload>(firstJob)->m_data;
auto objectList = std::dynamic_pointer_cast<S3ListBucket>(firstJob)->objects;
QString xmlErrorMsg;
QDomDocument doc;
if ( !doc.setContent ( ba, false, &xmlErrorMsg ) )
{
QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:" << xmlErrorMsg << ba;
emit failed();
return;
}
//QRegExp etag_match(".*([a-f0-9]{32}).*");
QDomNodeList contents = doc.elementsByTagName ( "Contents" );
NetJob *job = new NetJob("Assets");
DownloadJob *job = new DownloadJob("Assets");
connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) );
connect ( job, SIGNAL(failed()), SIGNAL(failed()) );
connect ( job, SIGNAL(filesProgress(int, int, int)), SIGNAL(filesProgress(int, int, int)) );
connect(job, SIGNAL(succeeded()), SLOT(downloadFinished()));
connect(job, SIGNAL(failed()), SIGNAL(failed()));
connect(job, SIGNAL(filesProgress(int, int, int)), SIGNAL(filesProgress(int, int, int)));
auto metacache = MMC->metacache();
for ( int i = 0; i < contents.length(); i++ )
for (auto object: objectList)
{
QDomElement element = contents.at ( i ).toElement();
if ( element.isNull() )
// Filter folder keys (zero size)
if (object.size == 0)
continue;
QDomElement keyElement = getDomElementByTagName ( element, "Key" );
QDomElement lastmodElement = getDomElementByTagName ( element, "LastModified" );
QDomElement etagElement = getDomElementByTagName ( element, "ETag" );
QDomElement sizeElement = getDomElementByTagName ( element, "Size" );
nuke_whitelist.append(object.Key);
if ( keyElement.isNull() || lastmodElement.isNull() || etagElement.isNull() || sizeElement.isNull() )
continue;
QString keyStr = keyElement.text();
QString lastModStr = lastmodElement.text();
QString etagStr = etagElement.text();
QString sizeStr = sizeElement.text();
//Filter folder keys
if ( sizeStr == "0" )
continue;
nuke_whitelist.append ( keyStr );
auto entry = metacache->resolveEntry("assets", keyStr, etagStr);
if(entry->stale)
auto entry = metacache->resolveEntry("assets", object.Key, object.ETag);
if (entry->stale)
{
job->addCacheDownload(QUrl(prefix + keyStr), entry);
job->addNetAction(CacheDownload::make(QUrl(prefix + object.Key), entry));
}
}
if(job->size())
if (job->size())
{
files_job.reset ( job );
files_job.reset(job);
files_job->start();
}
else
@ -136,11 +98,12 @@ void OneSixAssets::fetchXMLFinished()
void OneSixAssets::start()
{
auto job = new DownloadJob("Assets index");
job->addByteArrayDownload(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" ));
connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) );
auto job = new NetJob("Assets index");
job->addNetAction(
S3ListBucket::make(QUrl("http://s3.amazonaws.com/Minecraft.Resources/")));
connect(job, SIGNAL(succeeded()), SLOT(S3BucketFinished()));
emit indexStarted();
index_job.reset ( job );
index_job.reset(job);
job->start();
}

8
logic/OneSixAssets.h
View File

@ -1,5 +1,5 @@
#pragma once
#include "net/DownloadJob.h"
#include "net/NetJob.h"
class Private;
class ThreadedDeleter;
@ -15,13 +15,13 @@ signals:
void filesProgress(int, int, int);
public slots:
void fetchXMLFinished();
void S3BucketFinished();
void downloadFinished();
public:
void start();
private:
ThreadedDeleter * deleter;
QStringList nuke_whitelist;
DownloadJobPtr index_job;
DownloadJobPtr files_job;
NetJobPtr index_job;
NetJobPtr files_job;
};

14
logic/OneSixUpdate.cpp
View File

@ -72,8 +72,8 @@ void OneSixUpdate::versionFileStart()
QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/");
urlstr += targetVersion->descriptor() + "/" + targetVersion->descriptor() + ".json";
auto job = new DownloadJob("Version index");
job->addByteArrayDownload(QUrl(urlstr));
auto job = new NetJob("Version index");
job->addNetAction(ByteArrayDownload::make(QUrl(urlstr)));
specificVersionDownloadJob.reset(job);
connect(specificVersionDownloadJob.get(), SIGNAL(succeeded()), SLOT(versionFileFinished()));
connect(specificVersionDownloadJob.get(), SIGNAL(failed()), SLOT(versionFileFailed()));
@ -84,7 +84,7 @@ void OneSixUpdate::versionFileStart()
void OneSixUpdate::versionFileFinished()
{
DownloadPtr DlJob = specificVersionDownloadJob->first();
NetActionPtr DlJob = specificVersionDownloadJob->first();
OneSixInstance *inst = (OneSixInstance *)m_inst;
QString version_id = targetVersion->descriptor();
@ -154,8 +154,8 @@ void OneSixUpdate::jarlibStart()
QString targetstr("versions/");
targetstr += version->id + "/" + version->id + ".jar";
auto job = new DownloadJob("Libraries for instance " + inst->name());
job->addFileDownload(QUrl(urlstr), targetstr);
auto job = new NetJob("Libraries for instance " + inst->name());
job->addNetAction(FileDownload::make(QUrl(urlstr), targetstr));
jarlibDownloadJob.reset(job);
auto libs = version->getActiveNativeLibs();
@ -171,9 +171,9 @@ void OneSixUpdate::jarlibStart()
if (entry->stale)
{
if (lib->hint() == "forge-pack-xz")
jarlibDownloadJob->addForgeXzDownload(download_path, entry);
jarlibDownloadJob->addNetAction(ForgeXzDownload::make(download_path, entry));
else
jarlibDownloadJob->addCacheDownload(download_path, entry);
jarlibDownloadJob->addNetAction(CacheDownload::make(download_path, entry));
}
}
connect(jarlibDownloadJob.get(), SIGNAL(succeeded()), SLOT(jarlibFinished()));

6
logic/OneSixUpdate.h
View File

@ -18,7 +18,7 @@
#include <QObject>
#include <QList>
#include <QUrl>
#include "net/DownloadJob.h"
#include "net/NetJob.h"
#include "tasks/Task.h"
#include "BaseUpdate.h"
@ -43,8 +43,8 @@ private slots:
void jarlibFailed();
private:
DownloadJobPtr specificVersionDownloadJob;
DownloadJobPtr jarlibDownloadJob;
NetJobPtr specificVersionDownloadJob;
NetJobPtr jarlibDownloadJob;
// target version, determined during this task
std::shared_ptr<MinecraftVersion> targetVersion;

15
logic/lists/ForgeVersionList.cpp
View File

@ -14,7 +14,7 @@
*/
#include "ForgeVersionList.h"
#include <logic/net/DownloadJob.h>
#include <logic/net/NetJob.h>
#include "MultiMC.h"
#include <QtNetwork>
@ -159,14 +159,14 @@ ForgeListLoadTask::ForgeListLoadTask(ForgeVersionList *vlist) : Task()
void ForgeListLoadTask::executeTask()
{
auto job = new DownloadJob("Version index");
auto job = new NetJob("Version index");
// we do not care if the version is stale or not.
auto forgeListEntry = MMC->metacache()->resolveEntry("minecraftforge", "list.json");
// verify by poking the server.
forgeListEntry->stale = true;
job->addCacheDownload(QUrl(JSON_URL), forgeListEntry);
job->addNetAction(CacheDownload::make(QUrl(JSON_URL), forgeListEntry));
listJob.reset(job);
connect(listJob.get(), SIGNAL(succeeded()), SLOT(list_downloaded()));
connect(listJob.get(), SIGNAL(failed()), SLOT(list_failed()));
@ -178,7 +178,7 @@ void ForgeListLoadTask::list_failed()
{
auto DlJob = listJob->first();
auto reply = DlJob->m_reply;
if(reply)
if (reply)
{
QLOG_ERROR() << "Getting forge version list failed: " << reply->errorString();
}
@ -193,7 +193,7 @@ void ForgeListLoadTask::list_downloaded()
auto DlJob = listJob->first();
auto filename = std::dynamic_pointer_cast<CacheDownload>(DlJob)->m_target_path;
QFile listFile(filename);
if(!listFile.open(QIODevice::ReadOnly))
if (!listFile.open(QIODevice::ReadOnly))
return;
data = listFile.readAll();
DlJob.reset();
@ -202,7 +202,6 @@ void ForgeListLoadTask::list_downloaded()
QJsonParseError jsonError;
QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &jsonError);
if (jsonError.error != QJsonParseError::NoError)
{
emitFailed("Error parsing version list JSON:" + jsonError.errorString());

4
logic/lists/ForgeVersionList.h
View File

@ -23,7 +23,7 @@
#include <QNetworkReply>
#include "BaseVersionList.h"
#include "logic/tasks/Task.h"
#include "logic/net/DownloadJob.h"
#include "logic/net/NetJob.h"
class ForgeVersion;
typedef std::shared_ptr<ForgeVersion> ForgeVersionPtr;
@ -104,6 +104,6 @@ slots:
void list_failed();
protected:
DownloadJobPtr listJob;
NetJobPtr listJob;
ForgeVersionList *m_list;
};

2
logic/net/ByteArrayDownload.cpp
View File

@ -2,7 +2,7 @@
#include "MultiMC.h"
#include <logger/QsLog.h>
ByteArrayDownload::ByteArrayDownload(QUrl url) : Download()
ByteArrayDownload::ByteArrayDownload(QUrl url) : NetAction()
{
m_url = url;
m_status = Job_NotStarted;

23
logic/net/ByteArrayDownload.h
View File

@ -1,24 +1,29 @@
#pragma once
#include "Download.h"
#include "NetAction.h"
class ByteArrayDownload: public Download
typedef std::shared_ptr<class ByteArrayDownload> ByteArrayDownloadPtr;
class ByteArrayDownload : public NetAction
{
Q_OBJECT
public:
ByteArrayDownload(QUrl url);
static ByteArrayDownloadPtr make(QUrl url)
{
return ByteArrayDownloadPtr(new ByteArrayDownload(url));
}
public:
/// if not saving to file, downloaded data is placed here
QByteArray m_data;
public slots:
public
slots:
virtual void start();
protected slots:
protected
slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
void downloadError(QNetworkReply::NetworkError error);
void downloadFinished();
void downloadReadyRead();
};
typedef std::shared_ptr<ByteArrayDownload> ByteArrayDownloadPtr;

2
logic/net/CacheDownload.cpp
View File

@ -8,7 +8,7 @@
#include <logger/QsLog.h>
CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry)
: Download(), md5sum(QCryptographicHash::Md5)
: NetAction(), md5sum(QCryptographicHash::Md5)
{
m_url = url;
m_entry = entry;

22
logic/net/CacheDownload.h
View File

@ -1,11 +1,12 @@
#pragma once
#include "Download.h"
#include "NetAction.h"
#include "HttpMetaCache.h"
#include <QFile>
#include <qcryptographichash.h>
class CacheDownload : public Download
typedef std::shared_ptr<class CacheDownload> CacheDownloadPtr;
class CacheDownload : public NetAction
{
Q_OBJECT
public:
@ -18,17 +19,22 @@ public:
QFile m_output_file;
/// the hash-as-you-download
QCryptographicHash md5sum;
public:
explicit CacheDownload(QUrl url, MetaEntryPtr entry);
protected slots:
static CacheDownloadPtr make(QUrl url, MetaEntryPtr entry)
{
return CacheDownloadPtr(new CacheDownload(url, entry));
}
protected
slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
virtual void downloadError(QNetworkReply::NetworkError error);
virtual void downloadFinished();
virtual void downloadReadyRead();
public slots:
public
slots:
virtual void start();
};
typedef std::shared_ptr<CacheDownload> CacheDownloadPtr;

2
logic/net/FileDownload.cpp
View File

@ -6,7 +6,7 @@
FileDownload::FileDownload ( QUrl url, QString target_path )
:Download()
:NetAction()
{
m_url = url;
m_target_path = target_path;

25
logic/net/FileDownload.h
View File

@ -1,14 +1,16 @@
#pragma once
#include "Download.h"
#include "NetAction.h"
#include <QFile>
class FileDownload : public Download
typedef std::shared_ptr<class FileDownload> FileDownloadPtr;
class FileDownload : public NetAction
{
Q_OBJECT
public:
/// if true, check the md5sum against a provided md5sum
/// also, if a file exists, perform an md5sum first and don't download only if they don't match
/// also, if a file exists, perform an md5sum first and don't download only if they don't
/// match
bool m_check_md5;
/// the expected md5 checksum
QString m_expected_md5;
@ -18,18 +20,21 @@ public:
QString m_target_path;
/// this is the output file, if any
QFile m_output_file;
public:
explicit FileDownload(QUrl url, QString target_path);
protected slots:
static FileDownloadPtr make(QUrl url, QString target_path)
{
return FileDownloadPtr(new FileDownload(url, target_path));
}
protected
slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
virtual void downloadError(QNetworkReply::NetworkError error);
virtual void downloadFinished();
virtual void downloadReadyRead();
public slots:
public
slots:
virtual void start();
};
typedef std::shared_ptr<FileDownload> FileDownloadPtr;

2
logic/net/ForgeXzDownload.cpp
View File

@ -8,7 +8,7 @@
#include <logger/QsLog.h>
ForgeXzDownload::ForgeXzDownload(QUrl url, MetaEntryPtr entry)
: Download()
: NetAction()
{
QString urlstr = url.toString();
urlstr.append(".pack.xz");

22
logic/net/ForgeXzDownload.h
View File

@ -1,11 +1,12 @@
#pragma once
#include "Download.h"
#include "NetAction.h"
#include "HttpMetaCache.h"
#include <QFile>
#include <QTemporaryFile>
typedef std::shared_ptr<class ForgeXzDownload> ForgeXzDownloadPtr;
class ForgeXzDownload : public Download
class ForgeXzDownload : public NetAction
{
Q_OBJECT
public:
@ -19,17 +20,22 @@ public:
public:
explicit ForgeXzDownload(QUrl url, MetaEntryPtr entry);
protected slots:
static ForgeXzDownloadPtr make(QUrl url, MetaEntryPtr entry)
{
return ForgeXzDownloadPtr(new ForgeXzDownload(url, entry));
}
protected
slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
virtual void downloadError(QNetworkReply::NetworkError error);
virtual void downloadFinished();
virtual void downloadReadyRead();
public slots:
public
slots:
virtual void start();
private:
void decompressAndInstall();
};
typedef std::shared_ptr<ForgeXzDownload> ForgeXzDownloadPtr;

9
logic/net/Download.h → logic/net/NetAction.h
View File

@ -13,14 +13,15 @@ enum JobStatus
Job_Failed
};
class Download : public QObject
typedef std::shared_ptr<class NetAction> NetActionPtr;
class NetAction : public QObject
{
Q_OBJECT
protected:
explicit Download() : QObject(0) {};
explicit NetAction() : QObject(0) {};
public:
virtual ~Download() {};
virtual ~NetAction() {};
public:
/// the network reply
@ -50,5 +51,3 @@ protected slots:
public slots:
virtual void start() = 0;
};
typedef std::shared_ptr<Download> DownloadPtr;

52
logic/net/DownloadJob.cpp → logic/net/NetJob.cpp
View File

@ -1,4 +1,4 @@
#include "DownloadJob.h"
#include "NetJob.h"
#include "pathutils.h"
#include "MultiMC.h"
#include "FileDownload.h"
@ -7,47 +7,7 @@
#include <logger/QsLog.h>
ByteArrayDownloadPtr DownloadJob::addByteArrayDownload(QUrl url)
{
ByteArrayDownloadPtr ptr(new ByteArrayDownload(url));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
parts_progress.append(part_info());
total_progress++;
return ptr;
}
FileDownloadPtr DownloadJob::addFileDownload(QUrl url, QString rel_target_path)
{
FileDownloadPtr ptr(new FileDownload(url, rel_target_path));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
parts_progress.append(part_info());
total_progress++;
return ptr;
}
CacheDownloadPtr DownloadJob::addCacheDownload(QUrl url, MetaEntryPtr entry)
{
CacheDownloadPtr ptr(new CacheDownload(url, entry));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
parts_progress.append(part_info());
total_progress++;
return ptr;
}
ForgeXzDownloadPtr DownloadJob::addForgeXzDownload(QUrl url, MetaEntryPtr entry)
{
ForgeXzDownloadPtr ptr(new ForgeXzDownload(url, entry));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
parts_progress.append(part_info());
total_progress++;
return ptr;
}
void DownloadJob::partSucceeded(int index)
void NetJob::partSucceeded(int index)
{
// do progress. all slots are 1 in size at least
auto &slot = parts_progress[index];
@ -73,7 +33,7 @@ void DownloadJob::partSucceeded(int index)
}
}
void DownloadJob::partFailed(int index)
void NetJob::partFailed(int index)
{
auto &slot = parts_progress[index];
if (slot.failures == 3)
@ -97,7 +57,7 @@ void DownloadJob::partFailed(int index)
}
}
void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
{
auto &slot = parts_progress[index];
@ -111,7 +71,7 @@ void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTota
emit progress(current_progress, total_progress);
}
void DownloadJob::start()
void NetJob::start()
{
QLOG_INFO() << m_job_name.toLocal8Bit() << " started.";
for (auto iter : downloads)
@ -124,7 +84,7 @@ void DownloadJob::start()
}
}
QStringList DownloadJob::getFailedFiles()
QStringList NetJob::getFailedFiles()
{
QStringList failed;
for (auto download : downloads)

64
logic/net/DownloadJob.h → logic/net/NetJob.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <QtNetwork>
#include <QLabel>
#include "Download.h"
#include "NetAction.h"
#include "ByteArrayDownload.h"
#include "FileDownload.h"
#include "CacheDownload.h"
@ -9,51 +9,57 @@
#include "ForgeXzDownload.h"
#include "logic/tasks/ProgressProvider.h"
class DownloadJob;
typedef std::shared_ptr<DownloadJob> DownloadJobPtr;
class NetJob;
typedef std::shared_ptr<NetJob> NetJobPtr;
/**
* A single file for the downloader/cache to process.
*/
class DownloadJob : public ProgressProvider
class NetJob : public ProgressProvider
{
Q_OBJECT
public:
explicit DownloadJob(QString job_name)
:ProgressProvider(), m_job_name(job_name){};
ByteArrayDownloadPtr addByteArrayDownload(QUrl url);
FileDownloadPtr addFileDownload(QUrl url, QString rel_target_path);
CacheDownloadPtr addCacheDownload(QUrl url, MetaEntryPtr entry);
ForgeXzDownloadPtr addForgeXzDownload(QUrl url, MetaEntryPtr entry);
DownloadPtr operator[](int index)
explicit NetJob(QString job_name) : ProgressProvider(), m_job_name(job_name) {};
template <typename T>
bool addNetAction(T action)
{
NetActionPtr base = std::static_pointer_cast<NetAction>(action);
base->index_within_job = downloads.size();
downloads.append(action);
parts_progress.append(part_info());
total_progress++;
return true;
}
NetActionPtr operator[](int index)
{
return downloads[index];
};
DownloadPtr first()
}
;
NetActionPtr first()
{
if(downloads.size())
if (downloads.size())
return downloads[0];
return DownloadPtr();
return NetActionPtr();
}
int size() const
{
return downloads.size();
}
virtual void getProgress(qint64& current, qint64& total)
virtual void getProgress(qint64 &current, qint64 &total)
{
current = current_progress;
total = total_progress;
};
}
;
virtual QString getStatus() const
{
return m_job_name;
};
}
;
virtual bool isRunning() const
{
return m_running;
};
}
;
QStringList getFailedFiles();
signals:
void started();
@ -61,12 +67,15 @@ signals:
void filesProgress(int, int, int);
void succeeded();
void failed();
public slots:
public
slots:
virtual void start();
private slots:
private
slots:
void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);
void partSucceeded(int index);
void partFailed(int index);
private:
struct part_info
{
@ -75,7 +84,7 @@ private:
int failures = 0;
};
QString m_job_name;
QList<DownloadPtr> downloads;
QList<NetActionPtr> downloads;
QList<part_info> parts_progress;
qint64 current_progress = 0;
qint64 total_progress = 0;
@ -83,4 +92,3 @@ private:
int num_failed = 0;
bool m_running = false;
};

161
logic/net/S3ListBucket.cpp Normal file
View File

@ -0,0 +1,161 @@
#include "S3ListBucket.h"
#include "MultiMC.h"
#include <logger/QsLog.h>
#include <QUrlQuery>
#include <qxmlstream.h>
#include <QDomDocument>
inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
{
QDomNodeList elementList = parent.elementsByTagName(tagname);
if (elementList.count())
return elementList.at(0).toElement();
else
return QDomElement();
}
S3ListBucket::S3ListBucket(QUrl url) : NetAction()
{
m_url = url;
m_status = Job_NotStarted;
}
void S3ListBucket::start()
{
QUrl finalUrl = m_url;
if (current_marker.size())
{
QUrlQuery query;
query.addQueryItem("marker", current_marker);
finalUrl.setQuery(query);
}
QNetworkRequest request(finalUrl);
request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)");
auto worker = MMC->qnam();
QNetworkReply *rep = worker->get(request);
m_reply = std::shared_ptr<QNetworkReply>(rep);
connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
SLOT(downloadProgress(qint64, qint64)));
connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
SLOT(downloadError(QNetworkReply::NetworkError)));
connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
}
void S3ListBucket::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
{
emit progress(index_within_job, bytesSoFar + bytesReceived, bytesSoFar + bytesTotal);
}
void S3ListBucket::downloadError(QNetworkReply::NetworkError error)
{
// error happened during download.
QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit()
<< "Network error: " << error;
m_status = Job_Failed;
}
void S3ListBucket::processValidReply()
{
QLOG_TRACE() << "GOT: " << m_url.toString() << " marker:" << current_marker;
auto readContents = [&](QXmlStreamReader & xml)
{
QString Key, ETag, Size;
while (!(xml.tokenType() == QXmlStreamReader::EndElement && xml.name() == "Contents"))
{
if (xml.tokenType() == QXmlStreamReader::StartElement)
{
if (xml.name() == "Key")
{
Key = xml.readElementText();
}
if (xml.name() == "ETag")
{
ETag = xml.readElementText();
}
if (xml.name() == "Size")
{
Size = xml.readElementText();
}
}
xml.readNext();
}
if (xml.error() != QXmlStreamReader::NoError)
return;
objects.append({Key, ETag, Size.toLongLong()});
};
// nothing went wrong...
QString prefix("http://s3.amazonaws.com/Minecraft.Resources/");
QByteArray ba = m_reply->readAll();
QString xmlErrorMsg;
bool is_truncated = false;
QXmlStreamReader xml(ba);
while (!xml.atEnd() && !xml.hasError())
{
/* Read next element.*/
QXmlStreamReader::TokenType token = xml.readNext();
/* If token is just StartDocument, we'll go to next.*/
if (token == QXmlStreamReader::StartDocument)
{
continue;
}
if (token == QXmlStreamReader::StartElement)
{
/* If it's named person, we'll dig the information from there.*/
if (xml.name() == "Contents")
{
readContents(xml);
}
else if (xml.name() == "IsTruncated")
{
is_truncated = (xml.readElementText() == "true");
}
}
}
if (xml.hasError())
{
QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:"
<< xml.errorString() << ba;
emit failed(index_within_job);
return;
}
if(is_truncated)
{
current_marker = objects.last().Key;
bytesSoFar += m_reply->size();
m_reply.reset();
start();
}
else
{
m_status = Job_Finished;
m_reply.reset();
emit succeeded(index_within_job);
}
return;
}
void S3ListBucket::downloadFinished()
{
// if the download succeeded
if (m_status != Job_Failed)
{
processValidReply();
}
// else the download failed
else
{
m_reply.reset();
emit failed(index_within_job);
return;
}
}
void S3ListBucket::downloadReadyRead()
{
// ~_~
}

42
logic/net/S3ListBucket.h Normal file
View File

@ -0,0 +1,42 @@
#pragma once
#include "NetAction.h"
struct S3Object
{
QString Key;
QString ETag;
qlonglong size;
};
typedef std::shared_ptr<class S3ListBucket> S3ListBucketPtr;
class S3ListBucket : public NetAction
{
Q_OBJECT
public:
S3ListBucket(QUrl url);
static S3ListBucketPtr make(QUrl url)
{
return S3ListBucketPtr(new S3ListBucket(url));
}
public:
QList<S3Object> objects;
public
slots:
virtual void start() override;
protected
slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override;
virtual void downloadError(QNetworkReply::NetworkError error) override;
virtual void downloadFinished() override;
virtual void downloadReadyRead() override;
private:
void processValidReply();
private:
qint64 bytesSoFar = 0;
QString current_marker;
};