NOISSUE sanitize loading and downloading of metadata files

This commit is contained in:
Petr Mrázek 2017-03-18 02:22:36 +01:00
parent 0060b50625
commit e46aba9da5
20 changed files with 164 additions and 510 deletions

View File

@ -416,12 +416,8 @@ set(TOOLS_SOURCES
set(META_SOURCES
# Metadata sources
meta/tasks/RemoteLoadTask.cpp
meta/tasks/RemoteLoadTask.h
meta/tasks/LocalLoadTask.cpp
meta/tasks/LocalLoadTask.h
meta/format/Format.cpp
meta/format/Format.h
meta/JsonFormat.cpp
meta/JsonFormat.h
meta/BaseEntity.cpp
meta/BaseEntity.h
meta/VersionList.cpp
@ -430,8 +426,6 @@ set(META_SOURCES
meta/Version.h
meta/Index.cpp
meta/Index.h
meta/Util.cpp
meta/Util.h
meta/Reference.cpp
meta/Reference.h
)

View File

@ -16,26 +16,125 @@
#include "BaseEntity.h"
#include "Json.h"
#include "Util.h"
namespace Meta
#include "net/Download.h"
#include "net/HttpMetaCache.h"
#include "net/NetJob.h"
#include "Env.h"
#include "Json.h"
class ParsingValidator : public Net::Validator
{
BaseEntity::~BaseEntity()
public: /* con/des */
ParsingValidator(Meta::BaseEntity *entity) : m_entity(entity)
{
};
virtual ~ParsingValidator()
{
};
public: /* methods */
bool init(QNetworkRequest &) override
{
return true;
}
bool write(QByteArray & data) override
{
this->data.append(data);
return true;
}
bool abort() override
{
return true;
}
bool validate(QNetworkReply &) override
{
auto fname = m_entity->localFilename();
try
{
m_entity->parse(Json::requireObject(Json::requireDocument(data, fname), fname));
return true;
}
catch (Exception &e)
{
qWarning() << "Unable to parse response:" << e.cause();
return false;
}
}
private: /* data */
QByteArray data;
Meta::BaseEntity *m_entity;
};
Meta::BaseEntity::~BaseEntity()
{
}
QUrl BaseEntity::url() const
QUrl Meta::BaseEntity::url() const
{
return rootUrl().resolved(localFilename());
return QUrl("https://meta.multimc.org").resolved(localFilename());
}
void BaseEntity::notifyLocalLoadComplete()
bool Meta::BaseEntity::loadLocalFile()
{
m_localLoaded = true;
const QString fname = QDir("meta").absoluteFilePath(localFilename());
if (!QFile::exists(fname))
{
return false;
}
// TODO: check if the file has the expected checksum
try
{
parse(Json::requireObject(Json::requireDocument(fname, fname), fname));
return true;
}
catch (Exception &e)
{
qDebug() << QString("Unable to parse file %1: %2").arg(fname, e.cause());
// just make sure it's gone and we never consider it again.
QFile::remove(fname);
return false;
}
}
void BaseEntity::notifyRemoteLoadComplete()
void Meta::BaseEntity::load()
{
m_remoteLoaded = true;
}
if(!isLoaded())
{
loadLocalFile();
}
if(!shouldStartRemoteUpdate())
{
return;
}
NetJob *job = new NetJob(QObject::tr("Download of meta file %1").arg(localFilename()));
auto url = this->url();
auto entry = ENV.metacache()->resolveEntry("meta", localFilename());
entry->setStale(true);
auto dl = Net::Download::makeCached(url, entry);
/*
* The validator parses the file and loads it into the object.
* If that fails, the file is not written to storage.
*/
dl->addValidator(new ParsingValidator(this));
job->addNetAction(dl);
m_updateStatus = UpdateStatus::InProgress;
m_updateTask.reset(job);
QObject::connect(job, &NetJob::succeeded, [&]()
{
m_loadStatus = LoadStatus::Remote;
m_updateStatus = UpdateStatus::Succeeded;
m_updateTask.reset();
});
QObject::connect(job, &NetJob::failed, [&]()
{
m_updateStatus = UpdateStatus::Failed;
m_updateTask.reset();
});
m_updateTask->start();
}
#include "BaseEntity.moc"

View File

@ -15,9 +15,9 @@
#pragma once
#include <QObject>
#include <memory>
#include <QJsonObject>
#include <QObject>
#include "QObjectPtr.h"
#include "multimc_logic_export.h"
@ -26,29 +26,48 @@ namespace Meta
{
class MULTIMC_LOGIC_EXPORT BaseEntity
{
public: /* types */
using Ptr = std::shared_ptr<BaseEntity>;
enum class LoadStatus
{
NotLoaded,
Local,
Remote
};
enum class UpdateStatus
{
NotDone,
InProgress,
Failed,
Succeeded
};
public:
virtual ~BaseEntity();
using Ptr = std::shared_ptr<BaseEntity>;
virtual std::unique_ptr<Task> remoteUpdateTask() = 0;
virtual std::unique_ptr<Task> localUpdateTask() = 0;
virtual void merge(const std::shared_ptr<BaseEntity> &other) = 0;
virtual void parse(const QJsonObject &obj) = 0;
virtual QString localFilename() const = 0;
virtual QUrl url() const;
bool isComplete() const { return m_localLoaded || m_remoteLoaded; }
bool isLoaded() const
{
return m_loadStatus > LoadStatus::NotLoaded;
}
bool shouldStartRemoteUpdate() const
{
return m_updateStatus == UpdateStatus::NotDone;
}
bool isLocalLoaded() const { return m_localLoaded; }
bool isRemoteLoaded() const { return m_remoteLoaded; }
void load();
void notifyLocalLoadComplete();
void notifyRemoteLoadComplete();
protected: /* methods */
bool loadLocalFile();
private:
bool m_localLoaded = false;
bool m_remoteLoaded = false;
LoadStatus m_loadStatus = LoadStatus::NotLoaded;
UpdateStatus m_updateStatus = UpdateStatus::NotDone;
shared_qobject_ptr<Task> m_updateTask;
};
}

View File

@ -16,9 +16,7 @@
#include "Index.h"
#include "VersionList.h"
#include "tasks/LocalLoadTask.h"
#include "tasks/RemoteLoadTask.h"
#include "format/Format.h"
#include "JsonFormat.h"
namespace Meta
{
@ -78,15 +76,6 @@ QVariant Index::headerData(int section, Qt::Orientation orientation, int role) c
}
}
std::unique_ptr<Task> Index::remoteUpdateTask()
{
return std::unique_ptr<RemoteLoadTask>(new RemoteLoadTask(this));
}
std::unique_ptr<Task> Index::localUpdateTask()
{
return std::unique_ptr<LocalLoadTask>(new LocalLoadTask(this));
}
bool Index::hasUid(const QString &uid) const
{
return m_uids.contains(uid);

View File

@ -48,19 +48,12 @@ public:
int columnCount(const QModelIndex &parent) const override;
QVariant headerData(int section, Qt::Orientation orientation, int role) const override;
std::unique_ptr<Task> remoteUpdateTask() override;
std::unique_ptr<Task> localUpdateTask() override;
QString localFilename() const override { return "index.json"; }
// queries
VersionListPtr get(const QString &uid);
VersionPtr get(const QString &uid, const QString &version);
bool hasUid(const QString &uid) const;
/*
VersionListPtr getList(const QString &uid) const;
VersionListPtr getListGuaranteed(const QString &uid) const;
*/
QVector<VersionListPtr> lists() const { return m_lists; }

View File

@ -16,12 +16,6 @@ slots:
QCOMPARE(ENV.metadataIndex(), ENV.metadataIndex());
}
void test_providesTasks()
{
QVERIFY(ENV.metadataIndex()->localUpdateTask() != nullptr);
QVERIFY(ENV.metadataIndex()->remoteUpdateTask() != nullptr);
}
void test_hasUid_and_getList()
{
Meta::Index windex({std::make_shared<Meta::VersionList>("list1"), std::make_shared<Meta::VersionList>("list2"), std::make_shared<Meta::VersionList>("list3")});

View File

@ -13,15 +13,16 @@
* limitations under the License.
*/
#include "Format.h"
#include "minecraft/onesix/OneSixVersionFormat.h""
#include "meta/Index.h"
#include "meta/Version.h"
#include "meta/VersionList.h"
#include "JsonFormat.h"
// FIXME: remove this from here... somehow
#include "minecraft/onesix/OneSixVersionFormat.h"
#include "Json.h"
#include "Index.h"
#include "Version.h"
#include "VersionList.h"
using namespace Json;
namespace Meta

View File

@ -1,50 +0,0 @@
/* Copyright 2015-2017 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Util.h"
#include <QUrl>
#include <QDir>
#include "Env.h"
namespace Meta
{
QUrl rootUrl()
{
return QUrl("https://meta.multimc.org");
}
QUrl indexUrl()
{
return rootUrl().resolved(QStringLiteral("index.json"));
}
QUrl versionListUrl(const QString &uid)
{
return rootUrl().resolved(uid + "/index.json");
}
QUrl versionUrl(const QString &uid, const QString &version)
{
return rootUrl().resolved(uid + "/" + version + ".json");
}
QDir localDir()
{
return QDir("meta");
}
}

View File

@ -1,28 +0,0 @@
/* Copyright 2015-2017 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "multimc_logic_export.h"
class QUrl;
class QString;
class QDir;
namespace Meta
{
MULTIMC_LOGIC_EXPORT QUrl rootUrl();
MULTIMC_LOGIC_EXPORT QDir localDir();
}

View File

@ -17,9 +17,7 @@
#include <QDateTime>
#include "tasks/LocalLoadTask.h"
#include "tasks/RemoteLoadTask.h"
#include "format/Format.h"
#include "JsonFormat.h"
namespace Meta
{
@ -46,15 +44,6 @@ QDateTime Version::time() const
return QDateTime::fromMSecsSinceEpoch(m_time * 1000, Qt::UTC);
}
std::unique_ptr<Task> Version::remoteUpdateTask()
{
return std::unique_ptr<RemoteLoadTask>(new RemoteLoadTask(this));
}
std::unique_ptr<Task> Version::localUpdateTask()
{
return std::unique_ptr<LocalLoadTask>(new LocalLoadTask(this));
}
void Version::parse(const QJsonObject& obj)
{
parseVersion(obj, this);

View File

@ -56,8 +56,6 @@ public:
QVector<Reference> requires() const { return m_requires; }
VersionFilePtr data() const { return m_data; }
std::unique_ptr<Task> remoteUpdateTask() override;
std::unique_ptr<Task> localUpdateTask() override;
void merge(const std::shared_ptr<BaseEntity> &other) override;
void parse(const QJsonObject &obj) override;

View File

@ -18,64 +18,11 @@
#include <QDateTime>
#include "Version.h"
#include "tasks/RemoteLoadTask.h"
#include "tasks/LocalLoadTask.h"
#include "format/Format.h"
#include "JsonFormat.h"
#include "Reference.h"
namespace Meta
{
class WVLLoadTask : public Task
{
Q_OBJECT
public:
explicit WVLLoadTask(VersionList *list, QObject *parent = nullptr)
: Task(parent), m_list(list)
{
}
bool canAbort() const override
{
return !m_currentTask || m_currentTask->canAbort();
}
bool abort() override
{
return m_currentTask->abort();
}
private:
void executeTask() override
{
if (!m_list->isLocalLoaded())
{
m_currentTask = m_list->localUpdateTask();
connect(m_currentTask.get(), &Task::succeeded, this, &WVLLoadTask::next);
}
else
{
m_currentTask = m_list->remoteUpdateTask();
connect(m_currentTask.get(), &Task::succeeded, this, &WVLLoadTask::emitSucceeded);
}
connect(m_currentTask.get(), &Task::status, this, &WVLLoadTask::setStatus);
connect(m_currentTask.get(), &Task::progress, this, &WVLLoadTask::setProgress);
connect(m_currentTask.get(), &Task::failed, this, &WVLLoadTask::emitFailed);
m_currentTask->start();
}
void next()
{
m_currentTask = m_list->remoteUpdateTask();
connect(m_currentTask.get(), &Task::status, this, &WVLLoadTask::setStatus);
connect(m_currentTask.get(), &Task::progress, this, &WVLLoadTask::setProgress);
connect(m_currentTask.get(), &Task::succeeded, this, &WVLLoadTask::emitSucceeded);
m_currentTask->start();
}
VersionList *m_list;
std::unique_ptr<Task> m_currentTask;
};
VersionList::VersionList(const QString &uid, QObject *parent)
: BaseVersionList(parent), m_uid(uid)
{
@ -84,12 +31,13 @@ VersionList::VersionList(const QString &uid, QObject *parent)
Task *VersionList::getLoadTask()
{
return new WVLLoadTask(this);
// TODO: create a wrapper task that will chain from root to here.
return nullptr;
}
bool VersionList::isLoaded()
{
return isLocalLoaded() && isRemoteLoaded();
return isLoaded();
}
const BaseVersionPtr VersionList::at(int i) const
@ -167,15 +115,6 @@ QHash<int, QByteArray> VersionList::roleNames() const
return roles;
}
std::unique_ptr<Task> VersionList::remoteUpdateTask()
{
return std::unique_ptr<RemoteLoadTask>(new RemoteLoadTask(this));
}
std::unique_ptr<Task> VersionList::localUpdateTask()
{
return std::unique_ptr<LocalLoadTask>(new LocalLoadTask(this));
}
QString VersionList::localFilename() const
{
return m_uid + "/index.json";
@ -200,6 +139,7 @@ void VersionList::setName(const QString &name)
m_name = name;
emit nameChanged(name);
}
void VersionList::setVersions(const QVector<VersionPtr> &versions)
{
beginResetModel();

View File

@ -54,9 +54,6 @@ public:
RoleList providesRoles() const override;
QHash<int, QByteArray> roleNames() const override;
std::unique_ptr<Task> remoteUpdateTask() override;
std::unique_ptr<Task> localUpdateTask() override;
QString localFilename() const override;
QString uid() const { return m_uid; }

View File

@ -1,57 +0,0 @@
/* Copyright 2015-2017 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "LocalLoadTask.h"
#include <QFile>
#include "meta/format/Format.h"
#include "meta/Util.h"
#include "meta/Index.h"
#include "meta/Version.h"
#include "meta/VersionList.h"
#include "Env.h"
#include "Json.h"
namespace Meta
{
LocalLoadTask::LocalLoadTask(BaseEntity *entity, QObject *parent)
: Task(parent), m_entity(entity)
{
}
void LocalLoadTask::executeTask()
{
const QString fname = Meta::localDir().absoluteFilePath(m_entity->localFilename());
if (!QFile::exists(fname))
{
emitFailed(tr("File doesn't exist"));
return;
}
setStatus(tr("Reading %1...").arg(fname));
setProgress(0, 0);
try
{
m_entity->parse(Json::requireObject(Json::requireDocument(fname, fname), fname));
m_entity->notifyLocalLoadComplete();
emitSucceeded();
}
catch (Exception &e)
{
emitFailed(tr("Unable to parse file %1: %2").arg(fname, e.cause()));
}
}
}

View File

@ -1,39 +0,0 @@
/* Copyright 2015-2017 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "tasks/Task.h"
#include <memory>
namespace Meta
{
class BaseEntity;
class Index;
class VersionList;
class Version;
// FIXME: this is now just an odd function, get rid of it
class LocalLoadTask : public Task
{
Q_OBJECT
public:
explicit LocalLoadTask(BaseEntity *entity, QObject *parent = nullptr);
private:
void executeTask() override;
BaseEntity *m_entity;
};
}

View File

@ -1,103 +0,0 @@
/* Copyright 2015-2017 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "RemoteLoadTask.h"
#include "net/Download.h"
#include "net/HttpMetaCache.h"
#include "net/NetJob.h"
#include "meta/format/Format.h"
#include "meta/Util.h"
#include "meta/Index.h"
#include "meta/Version.h"
#include "meta/VersionList.h"
#include "Env.h"
#include "Json.h"
namespace Meta
{
RemoteLoadTask::RemoteLoadTask(BaseEntity *entity, QObject *parent)
: Task(parent), m_entity(entity)
{
}
class ParsingValidator : public Net::Validator
{
public: /* con/des */
ParsingValidator(BaseEntity *entity) : m_entity(entity)
{
};
virtual ~ParsingValidator()
{
};
public: /* methods */
bool init(QNetworkRequest &) override
{
return true;
}
bool write(QByteArray & data) override
{
this->data.append(data);
return true;
}
bool abort() override
{
return true;
}
bool validate(QNetworkReply &) override
{
auto fname = m_entity->localFilename();
try
{
m_entity->parse(Json::requireObject(Json::requireDocument(data, fname), fname));
m_entity->notifyRemoteLoadComplete();
return true;
}
catch (Exception &e)
{
qWarning() << "Unable to parse response:" << e.cause();
return false;
}
}
private: /* data */
QByteArray data;
BaseEntity *m_entity;
};
void RemoteLoadTask::executeTask()
{
// FIXME: leak here!!!
NetJob *job = new NetJob(tr("Download of meta file %1").arg(m_entity->localFilename()));
auto url = m_entity->url();
auto entry = ENV.metacache()->resolveEntry("meta", m_entity->localFilename());
entry->setStale(true);
m_dl = Net::Download::makeCached(url, entry);
/*
* The validator parses the file and loads it into the object.
* If that fails, the file is not written to storage.
*/
m_dl->addValidator(new ParsingValidator(m_entity));
job->addNetAction(m_dl);
connect(job, &NetJob::failed, this, &RemoteLoadTask::emitFailed);
connect(job, &NetJob::succeeded, this, &RemoteLoadTask::succeeded);
connect(job, &NetJob::status, this, &RemoteLoadTask::setStatus);
connect(job, &NetJob::progress, this, &RemoteLoadTask::setProgress);
job->start();
}
}

View File

@ -1,46 +0,0 @@
/* Copyright 2015-2017 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "tasks/Task.h"
#include <memory>
namespace Net
{
class Download;
}
namespace Meta
{
class BaseEntity;
class Index;
class VersionList;
class Version;
// FIXME: this is now just an oddly constructed NetJob, get rid of it.
class RemoteLoadTask : public Task
{
Q_OBJECT
public:
explicit RemoteLoadTask(BaseEntity *entity, QObject *parent = nullptr);
private:
void executeTask() override;
BaseEntity *m_entity;
std::shared_ptr<Net::Download> m_dl;
};
}

View File

@ -554,21 +554,8 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new MainWindow
job->start();
}
// run the things that load and download other things... FIXME: this is NOT the place
// FIXME: invisible actions in the background = NOPE.
// load the news
{
/*
if (!MMC->minecraftlist()->isLoaded())
{
m_versionLoadTask = MMC->minecraftlist()->getLoadTask();
startTask(m_versionLoadTask);
}
if (!MMC->lwjgllist()->isLoaded())
{
MMC->lwjgllist()->loadList();
}
*/
m_newsChecker->reloadNews();
updateNewsLabel();
}

View File

@ -97,7 +97,7 @@ QIcon PackagesPage::icon() const
void PackagesPage::on_refreshIndexBtn_clicked()
{
ProgressDialog(this).execWithTask(ENV.metadataIndex()->remoteUpdateTask());
ENV.metadataIndex()->load();
}
void PackagesPage::on_refreshFileBtn_clicked()
{
@ -106,7 +106,7 @@ void PackagesPage::on_refreshFileBtn_clicked()
{
return;
}
ProgressDialog(this).execWithTask(list->remoteUpdateTask());
list->load();
}
void PackagesPage::on_refreshVersionBtn_clicked()
{
@ -115,7 +115,7 @@ void PackagesPage::on_refreshVersionBtn_clicked()
{
return;
}
ProgressDialog(this).execWithTask(version->remoteUpdateTask());
version->load();
}
void PackagesPage::on_fileSearchEdit_textChanged(const QString &search)
@ -158,19 +158,7 @@ void PackagesPage::updateCurrentVersionList(const QModelIndex &index)
ui->fileName->setText(list->name());
m_versionProxy->setSourceModel(list.get());
ui->refreshFileBtn->setText(tr("Refresh %1").arg(list->humanReadable()));
if (!list->isLocalLoaded())
{
std::unique_ptr<Task> task = list->localUpdateTask();
connect(task.get(), &Task::finished, this, [this, list]()
{
if (list->count() == 0 && !list->isRemoteLoaded())
{
ProgressDialog(this).execWithTask(list->remoteUpdateTask());
}
});
ProgressDialog(this).execWithTask(task);
}
list->load();
}
else
{
@ -227,16 +215,5 @@ void PackagesPage::updateVersion()
void PackagesPage::opened()
{
if (!ENV.metadataIndex()->isLocalLoaded())
{
std::unique_ptr<Task> task = ENV.metadataIndex()->localUpdateTask();
connect(task.get(), &Task::finished, this, [this]()
{
if (!ENV.metadataIndex()->isRemoteLoaded())
{
ProgressDialog(this).execWithTask(ENV.metadataIndex()->remoteUpdateTask());
}
});
ProgressDialog(this).execWithTask(task);
}
ENV.metadataIndex()->load();
}