0xf8/pollymc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cache forge version list (it's huge)

Browse Source
This commit is contained in:
Petr Mrázek 2013-09-22 14:00:37 +02:00
parent ceca6959d2
commit 9d03a9c1e3
6 changed files with 118 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
logic/ForgeInstaller.cpp
View File

@ -1,10 +1,12 @@
#include "ForgeInstaller.h" #include "ForgeInstaller.h"
#include "OneSixVersion.h" #include "OneSixVersion.h"
#include "OneSixLibrary.h" #include "OneSixLibrary.h"
#include "net/HttpMetaCache.h"
#include <quazip.h> #include <quazip.h>
#include <quazipfile.h> #include <quazipfile.h>
#include <pathutils.h> #include <pathutils.h>
#include <QStringList> #include <QStringList>
#include "MultiMC.h"
ForgeInstaller::ForgeInstaller(QString filename, QString universal_url) ForgeInstaller::ForgeInstaller(QString filename, QString universal_url)
{ {
@ -53,6 +55,8 @@ ForgeInstaller::ForgeInstaller(QString filename, QString universal_url)
// where do we put the library? decode the mojang path // where do we put the library? decode the mojang path
OneSixLibrary lib(libraryName); OneSixLibrary lib(libraryName);
lib.finalize(); lib.finalize();
auto cacheentry = MMC->metacache()->resolveEntry("libraries", lib.storagePath());
finalPath = "libraries/" + lib.storagePath(); finalPath = "libraries/" + lib.storagePath();
if (!ensureFilePathExists(finalPath)) if (!ensureFilePathExists(finalPath))
return; return;
@ -71,6 +75,12 @@ ForgeInstaller::ForgeInstaller(QString filename, QString universal_url)
return; return;
if (!extraction.commit()) if (!extraction.commit())
return; return;
QCryptographicHash md5sum(QCryptographicHash::Md5);
md5sum.addData(data);
cacheentry->stale = false;
cacheentry->md5sum = md5sum.result().toHex().constData();
MMC->metacache()->updateEntry(cacheentry);
} }
file.close(); file.close();
@ -91,7 +101,7 @@ bool ForgeInstaller::apply(QSharedPointer<OneSixVersion> to)
{ {
QString libName = lib->name(); QString libName = lib->name();
// if this is the actual forge lib, set an absolute url for the download // if this is the actual forge lib, set an absolute url for the download
if(libName.contains("minecraftforge")) if (libName.contains("minecraftforge"))
{ {
lib->setAbsoluteUrl(m_universal_url); lib->setAbsoluteUrl(m_universal_url);
} }

1
logic/NostalgiaInstance.h
View File

@ -9,3 +9,4 @@ public:
explicit NostalgiaInstance(const QString &rootDir, SettingsObject * settings, QObject *parent = 0); explicit NostalgiaInstance(const QString &rootDir, SettingsObject * settings, QObject *parent = 0);
virtual QString getStatusbarDescription(); virtual QString getStatusbarDescription();
}; };

1
logic/OneSixInstance.cpp
View File

@ -320,3 +320,4 @@ QString OneSixInstance::instanceConfigFolder() const
{ {
return PathCombine(minecraftRoot(), "config"); return PathCombine(minecraftRoot(), "config");
} }

119
logic/lists/ForgeVersionList.cpp
View File

@ -3,7 +3,7 @@
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
* *
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
@ -25,10 +25,8 @@
#define JSON_URL "http://files.minecraftforge.net/minecraftforge/json" #define JSON_URL "http://files.minecraftforge.net/minecraftforge/json"
ForgeVersionList::ForgeVersionList(QObject *parent) : BaseVersionList(parent)
ForgeVersionList::ForgeVersionList(QObject* parent): BaseVersionList(parent)
{ {
} }
Task *ForgeVersionList::getLoadTask() Task *ForgeVersionList::getLoadTask()
@ -51,19 +49,19 @@ int ForgeVersionList::count() const
return m_vlist.count(); return m_vlist.count();
} }
int ForgeVersionList::columnCount(const QModelIndex& parent) const int ForgeVersionList::columnCount(const QModelIndex &parent) const
{ {
return 3; return 3;
} }
QVariant ForgeVersionList::data(const QModelIndex &index, int role) const QVariant ForgeVersionList::data(const QModelIndex &index, int role) const
{ {
if (!index.isValid()) if (!index.isValid())
return QVariant(); return QVariant();
if (index.row() > count()) if (index.row() > count())
return QVariant(); return QVariant();
auto version = m_vlist[index.row()].dynamicCast<ForgeVersion>(); auto version = m_vlist[index.row()].dynamicCast<ForgeVersion>();
switch (role) switch (role)
{ {
@ -72,22 +70,22 @@ QVariant ForgeVersionList::data(const QModelIndex &index, int role) const
{ {
case 0: case 0:
return version->name(); return version->name();
case 1: case 1:
return version->mcver; return version->mcver;
case 2: case 2:
return version->typeString(); return version->typeString();
default: default:
return QVariant(); return QVariant();
} }
case Qt::ToolTipRole: case Qt::ToolTipRole:
return version->descriptor(); return version->descriptor();
case VersionPointerRole: case VersionPointerRole:
return qVariantFromValue(m_vlist[index.row()]); return qVariantFromValue(m_vlist[index.row()]);
default: default:
return QVariant(); return QVariant();
} }
@ -102,33 +100,33 @@ QVariant ForgeVersionList::headerData(int section, Qt::Orientation orientation,
{ {
case 0: case 0:
return "Version"; return "Version";
case 1: case 1:
return "Minecraft"; return "Minecraft";
case 2: case 2:
return "Type"; return "Type";
default: default:
return QVariant(); return QVariant();
} }
case Qt::ToolTipRole: case Qt::ToolTipRole:
switch (section) switch (section)
{ {
case 0: case 0:
return "The name of the version."; return "The name of the version.";
case 1: case 1:
return "Minecraft version"; return "Minecraft version";
case 2: case 2:
return "The version's type."; return "The version's type.";
default: default:
return QVariant(); return QVariant();
} }
default: default:
return QVariant(); return QVariant();
} }
@ -139,7 +137,7 @@ BaseVersionPtr ForgeVersionList::getLatestStable() const
return BaseVersionPtr(); return BaseVersionPtr();
} }
void ForgeVersionList::updateListData(QList<BaseVersionPtr > versions) void ForgeVersionList::updateListData(QList<BaseVersionPtr> versions)
{ {
beginResetModel(); beginResetModel();
m_vlist = versions; m_vlist = versions;
@ -154,104 +152,112 @@ void ForgeVersionList::sort()
// NO-OP for now // NO-OP for now
} }
ForgeListLoadTask::ForgeListLoadTask(ForgeVersionList *vlist) : Task()
ForgeListLoadTask::ForgeListLoadTask(ForgeVersionList* vlist): Task()
{ {
m_list = vlist; m_list = vlist;
} }
void ForgeListLoadTask::executeTask() void ForgeListLoadTask::executeTask()
{ {
auto job = new DownloadJob("Version index"); auto job = new DownloadJob("Version index");
job->add(QUrl(JSON_URL)); // we do not care if the version is stale or not.
auto forgeListEntry = MMC->metacache()->resolveEntry("minecraftforge", "list.json");
job->add(QUrl(JSON_URL), forgeListEntry);
listJob.reset(job); listJob.reset(job);
connect(listJob.data(), SIGNAL(succeeded()), SLOT(list_downloaded())); connect(listJob.data(), SIGNAL(succeeded()), SLOT(list_downloaded()));
connect(listJob.data(), SIGNAL(failed()), SLOT(versionFileFailed())); connect(listJob.data(), SIGNAL(failed()), SLOT(versionFileFailed()));
connect(listJob.data(), SIGNAL(progress(qint64,qint64)), SIGNAL(progress(qint64,qint64))); connect(listJob.data(), SIGNAL(progress(qint64, qint64)), SIGNAL(progress(qint64, qint64)));
listJob->start(); listJob->start();
} }
void ForgeListLoadTask::list_downloaded() void ForgeListLoadTask::list_downloaded()
{ {
auto DlJob = listJob->first(); QByteArray data;
auto data = DlJob.dynamicCast<ByteArrayDownload>()->m_data; {
auto DlJob = listJob->first();
auto filename = DlJob.dynamicCast<CacheDownload>()->m_target_path;
QFile listFile(filename);
if(!listFile.open(QIODevice::ReadOnly))
return;
data = listFile.readAll();
DlJob.reset();
}
QJsonParseError jsonError; QJsonParseError jsonError;
QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &jsonError); QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &jsonError);
DlJob.reset();
if (jsonError.error != QJsonParseError::NoError) if (jsonError.error != QJsonParseError::NoError)
{ {
emitFailed("Error parsing version list JSON:" + jsonError.errorString()); emitFailed("Error parsing version list JSON:" + jsonError.errorString());
return; return;
} }
if(!jsonDoc.isObject()) if (!jsonDoc.isObject())
{ {
emitFailed("Error parsing version list JSON: jsonDoc is not an object"); emitFailed("Error parsing version list JSON: jsonDoc is not an object");
return; return;
} }
QJsonObject root = jsonDoc.object(); QJsonObject root = jsonDoc.object();
// Now, get the array of versions. // Now, get the array of versions.
if(!root.value("builds").isArray()) if (!root.value("builds").isArray())
{ {
emitFailed("Error parsing version list JSON: version list object is missing 'builds' array"); emitFailed(
"Error parsing version list JSON: version list object is missing 'builds' array");
return; return;
} }
QJsonArray builds = root.value("builds").toArray(); QJsonArray builds = root.value("builds").toArray();
QList<BaseVersionPtr > tempList; QList<BaseVersionPtr> tempList;
for (int i = 0; i < builds.count(); i++) for (int i = 0; i < builds.count(); i++)
{ {
// Load the version info. // Load the version info.
if(!builds[i].isObject()) if (!builds[i].isObject())
{ {
//FIXME: log this somewhere // FIXME: log this somewhere
continue; continue;
} }
QJsonObject obj = builds[i].toObject(); QJsonObject obj = builds[i].toObject();
int build_nr = obj.value("build").toDouble(0); int build_nr = obj.value("build").toDouble(0);
if(!build_nr) if (!build_nr)
continue; continue;
QJsonArray files = obj.value("files").toArray(); QJsonArray files = obj.value("files").toArray();
QString url, jobbuildver, mcver, buildtype, filename; QString url, jobbuildver, mcver, buildtype, filename;
QString changelog_url, installer_url; QString changelog_url, installer_url;
QString installer_filename; QString installer_filename;
bool valid = false; bool valid = false;
for(int j = 0; j < files.count(); j++) for (int j = 0; j < files.count(); j++)
{ {
if(!files[j].isObject()) if (!files[j].isObject())
continue; continue;
QJsonObject file = files[j].toObject(); QJsonObject file = files[j].toObject();
buildtype = file.value("buildtype").toString(); buildtype = file.value("buildtype").toString();
if((buildtype == "client" || buildtype == "universal") && !valid) if ((buildtype == "client" || buildtype == "universal") && !valid)
{ {
mcver = file.value("mcver").toString(); mcver = file.value("mcver").toString();
url = file.value("url").toString(); url = file.value("url").toString();
jobbuildver = file.value("jobbuildver").toString(); jobbuildver = file.value("jobbuildver").toString();
int lastSlash = url.lastIndexOf('/'); int lastSlash = url.lastIndexOf('/');
filename = url.mid(lastSlash+1); filename = url.mid(lastSlash + 1);
valid = true; valid = true;
} }
else if(buildtype == "changelog") else if (buildtype == "changelog")
{ {
QString ext = file.value("ext").toString(); QString ext = file.value("ext").toString();
if(ext.isEmpty()) if (ext.isEmpty())
continue; continue;
changelog_url = file.value("url").toString(); changelog_url = file.value("url").toString();
} }
else if(buildtype == "installer") else if (buildtype == "installer")
{ {
installer_url = file.value("url").toString(); installer_url = file.value("url").toString();
int lastSlash = installer_url.lastIndexOf('/'); int lastSlash = installer_url.lastIndexOf('/');
installer_filename = installer_url.mid(lastSlash+1); installer_filename = installer_url.mid(lastSlash + 1);
} }
} }
if(valid) if (valid)
{ {
// Now, we construct the version object and add it to the list. // Now, we construct the version object and add it to the list.
QSharedPointer<ForgeVersion> fVersion(new ForgeVersion()); QSharedPointer<ForgeVersion> fVersion(new ForgeVersion());
@ -260,7 +266,7 @@ void ForgeListLoadTask::list_downloaded()
fVersion->installer_url = installer_url; fVersion->installer_url = installer_url;
fVersion->jobbuildver = jobbuildver; fVersion->jobbuildver = jobbuildver;
fVersion->mcver = mcver; fVersion->mcver = mcver;
if(installer_filename.isEmpty()) if (installer_filename.isEmpty())
fVersion->filename = filename; fVersion->filename = filename;
else else
fVersion->filename = installer_filename; fVersion->filename = installer_filename;
@ -269,12 +275,7 @@ void ForgeListLoadTask::list_downloaded()
} }
} }
m_list->updateListData(tempList); m_list->updateListData(tempList);
emitSucceeded(); emitSucceeded();
return; return;
} }

63
logic/net/CacheDownload.cpp
View File

@ -7,8 +7,8 @@
#include <QDateTime> #include <QDateTime>
#include <QDebug> #include <QDebug>
CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry ) CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry)
:Download(), md5sum(QCryptographicHash::Md5) : Download(), md5sum(QCryptographicHash::Md5)
{ {
m_url = url; m_url = url;
m_entry = entry; m_entry = entry;
@ -19,38 +19,40 @@ CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry )
void CacheDownload::start() void CacheDownload::start()
{ {
if(!m_entry->stale) if (!m_entry->stale)
{ {
emit succeeded(index_within_job); emit succeeded(index_within_job);
return; return;
} }
m_output_file.setFileName(m_target_path); m_output_file.setFileName(m_target_path);
// if there already is a file and md5 checking is in effect and it can be opened // if there already is a file and md5 checking is in effect and it can be opened
if(!ensureFilePathExists(m_target_path)) if (!ensureFilePathExists(m_target_path))
{ {
emit failed(index_within_job); emit failed(index_within_job);
return; return;
} }
qDebug() << "Downloading " << m_url.toString(); qDebug() << "Downloading " << m_url.toString();
QNetworkRequest request ( m_url ); QNetworkRequest request(m_url);
request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1());
auto worker = MMC->qnam(); auto worker = MMC->qnam();
QNetworkReply * rep = worker->get ( request ); QNetworkReply *rep = worker->get(request);
m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater ); m_reply = QSharedPointer<QNetworkReply>(rep, &QObject::deleteLater);
connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); SLOT(downloadProgress(qint64, qint64)));
connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
SLOT(downloadError(QNetworkReply::NetworkError)));
connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
} }
void CacheDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) void CacheDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
{ {
emit progress (index_within_job, bytesReceived, bytesTotal ); emit progress(index_within_job, bytesReceived, bytesTotal);
} }
void CacheDownload::downloadError ( QNetworkReply::NetworkError error ) void CacheDownload::downloadError(QNetworkReply::NetworkError error)
{ {
// error happened during download. // error happened during download.
// TODO: log the reason why // TODO: log the reason why
@ -59,12 +61,12 @@ void CacheDownload::downloadError ( QNetworkReply::NetworkError error )
void CacheDownload::downloadFinished() void CacheDownload::downloadFinished()
{ {
// if the download succeeded // if the download succeeded
if ( m_status != Job_Failed ) if (m_status != Job_Failed)
{ {
// nothing went wrong... // nothing went wrong...
m_status = Job_Finished; m_status = Job_Finished;
if(m_opened_for_saving) if (m_opened_for_saving)
{ {
// save the data to the downloadable if we aren't saving to file // save the data to the downloadable if we aren't saving to file
m_output_file.close(); m_output_file.close();
@ -72,20 +74,23 @@ void CacheDownload::downloadFinished()
} }
else else
{ {
if ( m_output_file.open ( QIODevice::ReadOnly ) ) if (m_output_file.open(QIODevice::ReadOnly))
{ {
m_entry->md5sum = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); m_entry->md5sum =
QCryptographicHash::hash(m_output_file.readAll(), QCryptographicHash::Md5)
.toHex()
.constData();
m_output_file.close(); m_output_file.close();
} }
} }
QFileInfo output_file_info(m_target_path); QFileInfo output_file_info(m_target_path);
m_entry->etag = m_reply->rawHeader("ETag").constData(); m_entry->etag = m_reply->rawHeader("ETag").constData();
m_entry->last_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch(); m_entry->last_changed_timestamp =
output_file_info.lastModified().toUTC().toMSecsSinceEpoch();
m_entry->stale = false; m_entry->stale = false;
MMC->metacache()->updateEntry(m_entry); MMC->metacache()->updateEntry(m_entry);
m_reply.clear(); m_reply.clear();
emit succeeded(index_within_job); emit succeeded(index_within_job);
return; return;
@ -103,9 +108,9 @@ void CacheDownload::downloadFinished()
void CacheDownload::downloadReadyRead() void CacheDownload::downloadReadyRead()
{ {
if(!m_opened_for_saving) if (!m_opened_for_saving)
{ {
if ( !m_output_file.open ( QIODevice::WriteOnly ) ) if (!m_output_file.open(QIODevice::WriteOnly))
{ {
/* /*
* Can't open the file... the job failed * Can't open the file... the job failed
@ -118,5 +123,5 @@ void CacheDownload::downloadReadyRead()
} }
QByteArray ba = m_reply->readAll(); QByteArray ba = m_reply->readAll();
md5sum.addData(ba); md5sum.addData(ba);
m_output_file.write ( ba ); m_output_file.write(ba);
} }

19
logic/net/HttpMetaCache.h
View File

@ -24,25 +24,28 @@ public:
// supply path to the cache index file // supply path to the cache index file
HttpMetaCache(QString path); HttpMetaCache(QString path);
~HttpMetaCache(); ~HttpMetaCache();
// get the entry solely from the cache // get the entry solely from the cache
// you probably don't want this, unless you have some specific caching needs. // you probably don't want this, unless you have some specific caching needs.
MetaEntryPtr getEntry(QString base, QString resource_path); MetaEntryPtr getEntry(QString base, QString resource_path);
// get the entry from cache and verify that it isn't stale (within reason) // get the entry from cache and verify that it isn't stale (within reason)
MetaEntryPtr resolveEntry(QString base, QString resource_path, QString expected_etag = QString()); MetaEntryPtr resolveEntry(QString base, QString resource_path,
QString expected_etag = QString());
// add a previously resolved stale entry // add a previously resolved stale entry
bool updateEntry(MetaEntryPtr stale_entry); bool updateEntry(MetaEntryPtr stale_entry);
void addBase(QString base, QString base_root); void addBase(QString base, QString base_root);
// (re)start a timer that calls SaveNow later. // (re)start a timer that calls SaveNow later.
void SaveEventually(); void SaveEventually();
void Load(); void Load();
QString getBasePath ( QString base ); QString getBasePath(QString base);
public slots: public
slots:
void SaveNow(); void SaveNow();
private: private:
// create a new stale entry, given the parameters // create a new stale entry, given the parameters
MetaEntryPtr staleEntry(QString base, QString resource_path); MetaEntryPtr staleEntry(QString base, QString resource_path);