Use HttpMetaCache to minimize network use.
This commit is contained in:
		@@ -184,6 +184,10 @@ logic/ModList.h
 | 
				
			|||||||
logic/InstanceLauncher.h
 | 
					logic/InstanceLauncher.h
 | 
				
			||||||
 | 
					
 | 
				
			||||||
# network stuffs
 | 
					# network stuffs
 | 
				
			||||||
 | 
					logic/net/Download.h
 | 
				
			||||||
 | 
					logic/net/FileDownload.h
 | 
				
			||||||
 | 
					logic/net/ByteArrayDownload.h
 | 
				
			||||||
 | 
					logic/net/CacheDownload.h
 | 
				
			||||||
logic/net/DownloadJob.h
 | 
					logic/net/DownloadJob.h
 | 
				
			||||||
logic/net/HttpMetaCache.h
 | 
					logic/net/HttpMetaCache.h
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@@ -252,6 +256,9 @@ logic/ModList.cpp
 | 
				
			|||||||
logic/InstanceLauncher.cpp
 | 
					logic/InstanceLauncher.cpp
 | 
				
			||||||
 | 
					
 | 
				
			||||||
# network stuffs - to be moved into a depend lib ~_~
 | 
					# network stuffs - to be moved into a depend lib ~_~
 | 
				
			||||||
 | 
					logic/net/FileDownload.cpp
 | 
				
			||||||
 | 
					logic/net/ByteArrayDownload.cpp
 | 
				
			||||||
 | 
					logic/net/CacheDownload.cpp
 | 
				
			||||||
logic/net/DownloadJob.cpp
 | 
					logic/net/DownloadJob.cpp
 | 
				
			||||||
logic/net/HttpMetaCache.cpp
 | 
					logic/net/HttpMetaCache.cpp
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 
 | 
				
			|||||||
							
								
								
									
										21
									
								
								MultiMC.cpp
									
									
									
									
									
								
							
							
						
						
									
										21
									
								
								MultiMC.cpp
									
									
									
									
									
								
							@@ -8,6 +8,7 @@
 | 
				
			|||||||
#include "logic/lists/InstanceList.h"
 | 
					#include "logic/lists/InstanceList.h"
 | 
				
			||||||
#include "logic/lists/IconList.h"
 | 
					#include "logic/lists/IconList.h"
 | 
				
			||||||
#include "logic/InstanceLauncher.h"
 | 
					#include "logic/InstanceLauncher.h"
 | 
				
			||||||
 | 
					#include "logic/net/HttpMetaCache.h"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
#include "pathutils.h"
 | 
					#include "pathutils.h"
 | 
				
			||||||
@@ -112,11 +113,16 @@ MultiMC::MultiMC ( int& argc, char** argv )
 | 
				
			|||||||
	
 | 
						
 | 
				
			||||||
	// load settings
 | 
						// load settings
 | 
				
			||||||
	initGlobalSettings();
 | 
						initGlobalSettings();
 | 
				
			||||||
 | 
						
 | 
				
			||||||
	// and instances
 | 
						// and instances
 | 
				
			||||||
	m_instances = new InstanceList(m_settings->get("InstanceDir").toString(),this);
 | 
						m_instances = new InstanceList(m_settings->get("InstanceDir").toString(),this);
 | 
				
			||||||
	std::cout << "Loading Instances..." << std::endl;
 | 
						std::cout << "Loading Instances..." << std::endl;
 | 
				
			||||||
	m_instances->loadList();
 | 
						m_instances->loadList();
 | 
				
			||||||
	// network manager
 | 
						
 | 
				
			||||||
 | 
						// init the http meta cache
 | 
				
			||||||
 | 
						initHttpMetaCache();
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						// create the global network manager
 | 
				
			||||||
	m_qnam = new QNetworkAccessManager(this);
 | 
						m_qnam = new QNetworkAccessManager(this);
 | 
				
			||||||
	
 | 
						
 | 
				
			||||||
	// Register meta types.
 | 
						// Register meta types.
 | 
				
			||||||
@@ -137,11 +143,12 @@ MultiMC::MultiMC ( int& argc, char** argv )
 | 
				
			|||||||
MultiMC::~MultiMC()
 | 
					MultiMC::~MultiMC()
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	delete m_settings;
 | 
						delete m_settings;
 | 
				
			||||||
 | 
						delete m_metacache;
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
void MultiMC::initGlobalSettings()
 | 
					void MultiMC::initGlobalSettings()
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	m_settings = new INISettingsObject(applicationDirPath() + "/multimc.cfg", this);
 | 
						m_settings = new INISettingsObject("multimc.cfg", this);
 | 
				
			||||||
		// Updates
 | 
							// Updates
 | 
				
			||||||
	m_settings->registerSetting(new Setting("UseDevBuilds", false));
 | 
						m_settings->registerSetting(new Setting("UseDevBuilds", false));
 | 
				
			||||||
	m_settings->registerSetting(new Setting("AutoUpdate", true));
 | 
						m_settings->registerSetting(new Setting("AutoUpdate", true));
 | 
				
			||||||
@@ -189,6 +196,16 @@ void MultiMC::initGlobalSettings()
 | 
				
			|||||||
	m_settings->registerSetting(new Setting("TheCat", false));
 | 
						m_settings->registerSetting(new Setting("TheCat", false));
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void MultiMC::initHttpMetaCache()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						m_metacache = new HttpMetaCache("metacache");
 | 
				
			||||||
 | 
						m_metacache->addBase("assets", QDir("assets").absolutePath());
 | 
				
			||||||
 | 
						m_metacache->addBase("versions", QDir("versions").absolutePath());
 | 
				
			||||||
 | 
						m_metacache->addBase("libraries", QDir("libraries").absolutePath());
 | 
				
			||||||
 | 
						m_metacache->Load();
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
IconList* MultiMC::icons()
 | 
					IconList* MultiMC::icons()
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	if ( !m_icons )
 | 
						if ( !m_icons )
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -4,6 +4,7 @@
 | 
				
			|||||||
#include "MultiMCVersion.h"
 | 
					#include "MultiMCVersion.h"
 | 
				
			||||||
#include "config.h"
 | 
					#include "config.h"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					class HttpMetaCache;
 | 
				
			||||||
class SettingsObject;
 | 
					class SettingsObject;
 | 
				
			||||||
class InstanceList;
 | 
					class InstanceList;
 | 
				
			||||||
class IconList;
 | 
					class IconList;
 | 
				
			||||||
@@ -55,14 +56,21 @@ public:
 | 
				
			|||||||
	{
 | 
						{
 | 
				
			||||||
		return m_qnam;
 | 
							return m_qnam;
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						HttpMetaCache * metacache()
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							return m_metacache;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
private:
 | 
					private:
 | 
				
			||||||
	void initGlobalSettings();
 | 
						void initGlobalSettings();
 | 
				
			||||||
	
 | 
						
 | 
				
			||||||
 | 
						void initHttpMetaCache();
 | 
				
			||||||
private:
 | 
					private:
 | 
				
			||||||
	SettingsObject * m_settings = nullptr;
 | 
						SettingsObject * m_settings = nullptr;
 | 
				
			||||||
	InstanceList * m_instances = nullptr;
 | 
						InstanceList * m_instances = nullptr;
 | 
				
			||||||
	IconList * m_icons = nullptr;
 | 
						IconList * m_icons = nullptr;
 | 
				
			||||||
	QNetworkAccessManager * m_qnam = nullptr;
 | 
						QNetworkAccessManager * m_qnam = nullptr;
 | 
				
			||||||
 | 
						HttpMetaCache * m_metacache = nullptr;
 | 
				
			||||||
	Status m_status = MultiMC::Failed;
 | 
						Status m_status = MultiMC::Failed;
 | 
				
			||||||
	MultiMCVersion m_version = {VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION, VERSION_BUILD};
 | 
						MultiMCVersion m_version = {VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION, VERSION_BUILD};
 | 
				
			||||||
};
 | 
					};
 | 
				
			||||||
@@ -227,7 +227,9 @@ void LegacyUpdate::jarStart()
 | 
				
			|||||||
	QString intended_version_id = inst->intendedVersionId();
 | 
						QString intended_version_id = inst->intendedVersionId();
 | 
				
			||||||
	urlstr += intended_version_id + "/" + intended_version_id + ".jar";
 | 
						urlstr += intended_version_id + "/" + intended_version_id + ".jar";
 | 
				
			||||||
	
 | 
						
 | 
				
			||||||
	legacyDownloadJob.reset(new DownloadJob(QUrl(urlstr), inst->defaultBaseJar()));
 | 
						auto dljob = new DownloadJob("Minecraft.jar for version " + intended_version_id);
 | 
				
			||||||
 | 
						dljob->add(QUrl(urlstr), inst->defaultBaseJar());
 | 
				
			||||||
 | 
						legacyDownloadJob.reset();
 | 
				
			||||||
	connect(legacyDownloadJob.data(), SIGNAL(finished()), SLOT(jarFinished()));
 | 
						connect(legacyDownloadJob.data(), SIGNAL(finished()), SLOT(jarFinished()));
 | 
				
			||||||
	connect(legacyDownloadJob.data(), SIGNAL(failed()), SLOT(jarFailed()));
 | 
						connect(legacyDownloadJob.data(), SIGNAL(failed()), SLOT(jarFailed()));
 | 
				
			||||||
	connect(legacyDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
 | 
						connect(legacyDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -3,6 +3,8 @@
 | 
				
			|||||||
#include <QtXml/QtXml>
 | 
					#include <QtXml/QtXml>
 | 
				
			||||||
#include "OneSixAssets.h"
 | 
					#include "OneSixAssets.h"
 | 
				
			||||||
#include "net/DownloadJob.h"
 | 
					#include "net/DownloadJob.h"
 | 
				
			||||||
 | 
					#include "net/HttpMetaCache.h"
 | 
				
			||||||
 | 
					#include "MultiMC.h"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
 | 
					inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
@@ -65,7 +67,7 @@ void OneSixAssets::fetchXMLFinished()
 | 
				
			|||||||
	nuke_whitelist.clear();
 | 
						nuke_whitelist.clear();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	auto firstJob = index_job->first();
 | 
						auto firstJob = index_job->first();
 | 
				
			||||||
	QByteArray ba = firstJob->m_data;
 | 
						QByteArray ba  = firstJob.dynamicCast<ByteArrayDownload>()->m_data;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	QString xmlErrorMsg;
 | 
						QString xmlErrorMsg;
 | 
				
			||||||
	QDomDocument doc;
 | 
						QDomDocument doc;
 | 
				
			||||||
@@ -76,10 +78,12 @@ void OneSixAssets::fetchXMLFinished()
 | 
				
			|||||||
	//QRegExp etag_match(".*([a-f0-9]{32}).*");
 | 
						//QRegExp etag_match(".*([a-f0-9]{32}).*");
 | 
				
			||||||
	QDomNodeList contents = doc.elementsByTagName ( "Contents" );
 | 
						QDomNodeList contents = doc.elementsByTagName ( "Contents" );
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	DownloadJob *job = new DownloadJob();
 | 
						DownloadJob *job = new DownloadJob("Assets");
 | 
				
			||||||
	connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) );
 | 
						connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) );
 | 
				
			||||||
	connect ( job, SIGNAL(failed()), SIGNAL(failed()) );
 | 
						connect ( job, SIGNAL(failed()), SIGNAL(failed()) );
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						auto metacache = MMC->metacache();
 | 
				
			||||||
 | 
						
 | 
				
			||||||
	for ( int i = 0; i < contents.length(); i++ )
 | 
						for ( int i = 0; i < contents.length(); i++ )
 | 
				
			||||||
	{
 | 
						{
 | 
				
			||||||
		QDomElement element = contents.at ( i ).toElement();
 | 
							QDomElement element = contents.at ( i ).toElement();
 | 
				
			||||||
@@ -104,22 +108,12 @@ void OneSixAssets::fetchXMLFinished()
 | 
				
			|||||||
		if ( sizeStr == "0" )
 | 
							if ( sizeStr == "0" )
 | 
				
			||||||
			continue;
 | 
								continue;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
		QString filename = fprefix + keyStr;
 | 
					 | 
				
			||||||
		QFile check_file ( filename );
 | 
					 | 
				
			||||||
		QString client_etag = "nonsense";
 | 
					 | 
				
			||||||
		// if there already is a file and md5 checking is in effect and it can be opened
 | 
					 | 
				
			||||||
		if ( check_file.exists() && check_file.open ( QIODevice::ReadOnly ) )
 | 
					 | 
				
			||||||
		{
 | 
					 | 
				
			||||||
			// check the md5 against the expected one
 | 
					 | 
				
			||||||
			client_etag = QCryptographicHash::hash ( check_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
 | 
					 | 
				
			||||||
			check_file.close();
 | 
					 | 
				
			||||||
		}
 | 
					 | 
				
			||||||
		
 | 
					 | 
				
			||||||
		QString trimmedEtag = etagStr.remove ( '"' );
 | 
					 | 
				
			||||||
		nuke_whitelist.append ( keyStr );
 | 
							nuke_whitelist.append ( keyStr );
 | 
				
			||||||
		if(trimmedEtag != client_etag)
 | 
							
 | 
				
			||||||
 | 
							auto entry = metacache->resolveEntry("assets", keyStr, etagStr);
 | 
				
			||||||
 | 
							if(entry->stale)
 | 
				
			||||||
		{
 | 
							{
 | 
				
			||||||
			job->add ( QUrl ( prefix + keyStr ), filename );
 | 
								job->add(QUrl(prefix + keyStr), entry);
 | 
				
			||||||
		}
 | 
							}
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
	if(job->size())
 | 
						if(job->size())
 | 
				
			||||||
@@ -135,7 +129,8 @@ void OneSixAssets::fetchXMLFinished()
 | 
				
			|||||||
}
 | 
					}
 | 
				
			||||||
void OneSixAssets::start()
 | 
					void OneSixAssets::start()
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	DownloadJob * job = new DownloadJob(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" ));
 | 
						auto job = new DownloadJob("Assets index");
 | 
				
			||||||
 | 
						job->add(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" ));
 | 
				
			||||||
	connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) );
 | 
						connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) );
 | 
				
			||||||
	index_job.reset ( job );
 | 
						index_job.reset ( job );
 | 
				
			||||||
	job->start();
 | 
						job->start();
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -12,7 +12,7 @@
 | 
				
			|||||||
 * See the License for the specific language governing permissions and
 | 
					 * See the License for the specific language governing permissions and
 | 
				
			||||||
 * limitations under the License.
 | 
					 * limitations under the License.
 | 
				
			||||||
 */
 | 
					 */
 | 
				
			||||||
 | 
					#include "MultiMC.h"
 | 
				
			||||||
#include "OneSixUpdate.h"
 | 
					#include "OneSixUpdate.h"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
#include <QtNetwork>
 | 
					#include <QtNetwork>
 | 
				
			||||||
@@ -72,7 +72,9 @@ void OneSixUpdate::versionFileStart()
 | 
				
			|||||||
	
 | 
						
 | 
				
			||||||
	QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/");
 | 
						QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/");
 | 
				
			||||||
	urlstr += targetVersion->descriptor + "/" + targetVersion->descriptor + ".json";
 | 
						urlstr += targetVersion->descriptor + "/" + targetVersion->descriptor + ".json";
 | 
				
			||||||
	specificVersionDownloadJob.reset(new DownloadJob(QUrl(urlstr)));
 | 
						auto job = new DownloadJob("Version index");
 | 
				
			||||||
 | 
						job->add(QUrl(urlstr));
 | 
				
			||||||
 | 
						specificVersionDownloadJob.reset(job);
 | 
				
			||||||
	connect(specificVersionDownloadJob.data(), SIGNAL(succeeded()), SLOT(versionFileFinished()));
 | 
						connect(specificVersionDownloadJob.data(), SIGNAL(succeeded()), SLOT(versionFileFinished()));
 | 
				
			||||||
	connect(specificVersionDownloadJob.data(), SIGNAL(failed()), SLOT(versionFileFailed()));
 | 
						connect(specificVersionDownloadJob.data(), SIGNAL(failed()), SLOT(versionFileFailed()));
 | 
				
			||||||
	connect(specificVersionDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
 | 
						connect(specificVersionDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64)));
 | 
				
			||||||
@@ -92,7 +94,7 @@ void OneSixUpdate::versionFileFinished()
 | 
				
			|||||||
		// FIXME: detect errors here, download to a temp file, swap
 | 
							// FIXME: detect errors here, download to a temp file, swap
 | 
				
			||||||
		QFile  vfile1 (version1);
 | 
							QFile  vfile1 (version1);
 | 
				
			||||||
		vfile1.open(QIODevice::Truncate | QIODevice::WriteOnly );
 | 
							vfile1.open(QIODevice::Truncate | QIODevice::WriteOnly );
 | 
				
			||||||
		vfile1.write(DlJob->m_data);
 | 
							vfile1.write(DlJob.dynamicCast<ByteArrayDownload>()->m_data);
 | 
				
			||||||
		vfile1.close();
 | 
							vfile1.close();
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
	
 | 
						
 | 
				
			||||||
@@ -134,16 +136,22 @@ void OneSixUpdate::jarlibStart()
 | 
				
			|||||||
	QString targetstr ("versions/");
 | 
						QString targetstr ("versions/");
 | 
				
			||||||
	targetstr += version->id + "/" + version->id + ".jar";
 | 
						targetstr += version->id + "/" + version->id + ".jar";
 | 
				
			||||||
	
 | 
						
 | 
				
			||||||
	jarlibDownloadJob.reset(new DownloadJob(QUrl(urlstr), targetstr));
 | 
						auto job = new DownloadJob("Libraries for instance " + inst->name());
 | 
				
			||||||
 | 
						job->add(QUrl(urlstr), targetstr);
 | 
				
			||||||
 | 
						jarlibDownloadJob.reset(job);
 | 
				
			||||||
	
 | 
						
 | 
				
			||||||
	auto libs = version->getActiveNativeLibs();
 | 
						auto libs = version->getActiveNativeLibs();
 | 
				
			||||||
	libs.append(version->getActiveNormalLibs());
 | 
						libs.append(version->getActiveNormalLibs());
 | 
				
			||||||
	
 | 
						
 | 
				
			||||||
 | 
						auto metacache = MMC->metacache();
 | 
				
			||||||
	for(auto lib: libs)
 | 
						for(auto lib: libs)
 | 
				
			||||||
	{
 | 
						{
 | 
				
			||||||
		QString download_path = lib->downloadPath();
 | 
							QString download_path = lib->downloadPath();
 | 
				
			||||||
		QString storage_path = "libraries/" + lib->storagePath();
 | 
							auto entry = metacache->resolveEntry("libraries", lib->storagePath());
 | 
				
			||||||
		jarlibDownloadJob->add(download_path, storage_path);
 | 
							if(entry->stale)
 | 
				
			||||||
 | 
							{
 | 
				
			||||||
 | 
								jarlibDownloadJob->add(download_path, entry);
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
	connect(jarlibDownloadJob.data(), SIGNAL(succeeded()), SLOT(jarlibFinished()));
 | 
						connect(jarlibDownloadJob.data(), SIGNAL(succeeded()), SLOT(jarlibFinished()));
 | 
				
			||||||
	connect(jarlibDownloadJob.data(), SIGNAL(failed()), SLOT(jarlibFailed()));
 | 
						connect(jarlibDownloadJob.data(), SIGNAL(failed()), SLOT(jarlibFailed()));
 | 
				
			||||||
 
 | 
				
			|||||||
							
								
								
									
										62
									
								
								logic/net/ByteArrayDownload.cpp
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										62
									
								
								logic/net/ByteArrayDownload.cpp
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,62 @@
 | 
				
			|||||||
 | 
					#include "ByteArrayDownload.h"
 | 
				
			||||||
 | 
					#include "MultiMC.h"
 | 
				
			||||||
 | 
					#include <QDebug>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					ByteArrayDownload::ByteArrayDownload ( QUrl url )
 | 
				
			||||||
 | 
						: Download()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						m_url = url;
 | 
				
			||||||
 | 
						m_status = Job_NotStarted;
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void ByteArrayDownload::start()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						qDebug() << "Downloading " << m_url.toString();
 | 
				
			||||||
 | 
						QNetworkRequest request ( m_url );
 | 
				
			||||||
 | 
						auto worker = MMC->qnam();
 | 
				
			||||||
 | 
						QNetworkReply * rep = worker->get ( request );
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void ByteArrayDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						emit progress (index_within_job, bytesReceived, bytesTotal );
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void ByteArrayDownload::downloadError ( QNetworkReply::NetworkError error )
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						// error happened during download.
 | 
				
			||||||
 | 
						// TODO: log the reason why
 | 
				
			||||||
 | 
						m_status = Job_Failed;
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void ByteArrayDownload::downloadFinished()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						// if the download succeeded
 | 
				
			||||||
 | 
						if ( m_status != Job_Failed )
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							// nothing went wrong...
 | 
				
			||||||
 | 
							m_status = Job_Finished;
 | 
				
			||||||
 | 
							m_data = m_reply->readAll();
 | 
				
			||||||
 | 
							m_reply.clear();
 | 
				
			||||||
 | 
							emit succeeded(index_within_job);
 | 
				
			||||||
 | 
							return;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						// else the download failed
 | 
				
			||||||
 | 
						else
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							m_reply.clear();
 | 
				
			||||||
 | 
							emit failed(index_within_job);
 | 
				
			||||||
 | 
							return;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void ByteArrayDownload::downloadReadyRead()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						// ~_~
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
							
								
								
									
										24
									
								
								logic/net/ByteArrayDownload.h
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										24
									
								
								logic/net/ByteArrayDownload.h
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,24 @@
 | 
				
			|||||||
 | 
					#pragma once
 | 
				
			||||||
 | 
					#include "Download.h"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					class ByteArrayDownload: public Download
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						Q_OBJECT
 | 
				
			||||||
 | 
					public:
 | 
				
			||||||
 | 
						ByteArrayDownload(QUrl url);
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
					public:
 | 
				
			||||||
 | 
						/// if not saving to file, downloaded data is placed here
 | 
				
			||||||
 | 
						QByteArray m_data;
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
					public slots:
 | 
				
			||||||
 | 
						virtual void start();
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
					protected slots:
 | 
				
			||||||
 | 
						void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 | 
				
			||||||
 | 
						void downloadError(QNetworkReply::NetworkError error);
 | 
				
			||||||
 | 
						void downloadFinished();
 | 
				
			||||||
 | 
						void downloadReadyRead();
 | 
				
			||||||
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					typedef QSharedPointer<ByteArrayDownload> ByteArrayDownloadPtr;
 | 
				
			||||||
							
								
								
									
										122
									
								
								logic/net/CacheDownload.cpp
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										122
									
								
								logic/net/CacheDownload.cpp
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,122 @@
 | 
				
			|||||||
 | 
					#include "MultiMC.h"
 | 
				
			||||||
 | 
					#include "CacheDownload.h"
 | 
				
			||||||
 | 
					#include <pathutils.h>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					#include <QCryptographicHash>
 | 
				
			||||||
 | 
					#include <QFileInfo>
 | 
				
			||||||
 | 
					#include <QDateTime>
 | 
				
			||||||
 | 
					#include <QDebug>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry )
 | 
				
			||||||
 | 
						:Download(), md5sum(QCryptographicHash::Md5)
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						m_url = url;
 | 
				
			||||||
 | 
						m_entry = entry;
 | 
				
			||||||
 | 
						m_target_path = entry->getFullPath();
 | 
				
			||||||
 | 
						m_status = Job_NotStarted;
 | 
				
			||||||
 | 
						m_opened_for_saving = false;
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void CacheDownload::start()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						if(!m_entry->stale)
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							emit succeeded(index_within_job);
 | 
				
			||||||
 | 
							return;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						m_output_file.setFileName(m_target_path);
 | 
				
			||||||
 | 
						// if there already is a file and md5 checking is in effect and it can be opened
 | 
				
			||||||
 | 
						if(!ensureFilePathExists(m_target_path))
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							emit failed(index_within_job);
 | 
				
			||||||
 | 
							return;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						qDebug() << "Downloading " << m_url.toString();
 | 
				
			||||||
 | 
						QNetworkRequest request ( m_url );
 | 
				
			||||||
 | 
						request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); 
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						auto worker = MMC->qnam();
 | 
				
			||||||
 | 
						QNetworkReply * rep = worker->get ( request );
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void CacheDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						emit progress (index_within_job, bytesReceived, bytesTotal );
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void CacheDownload::downloadError ( QNetworkReply::NetworkError error )
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						// error happened during download.
 | 
				
			||||||
 | 
						// TODO: log the reason why
 | 
				
			||||||
 | 
						m_status = Job_Failed;
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					void CacheDownload::downloadFinished()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						// if the download succeeded
 | 
				
			||||||
 | 
						if ( m_status != Job_Failed )
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							
 | 
				
			||||||
 | 
							// nothing went wrong...
 | 
				
			||||||
 | 
							m_status = Job_Finished;
 | 
				
			||||||
 | 
							if(m_opened_for_saving)
 | 
				
			||||||
 | 
							{
 | 
				
			||||||
 | 
								// save the data to the downloadable if we aren't saving to file
 | 
				
			||||||
 | 
								m_output_file.close();
 | 
				
			||||||
 | 
								m_entry->md5sum = md5sum.result().toHex().constData();
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
 | 
							else
 | 
				
			||||||
 | 
							{
 | 
				
			||||||
 | 
								if ( m_output_file.open ( QIODevice::ReadOnly ) )
 | 
				
			||||||
 | 
								{
 | 
				
			||||||
 | 
									m_entry->md5sum = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
 | 
				
			||||||
 | 
									m_output_file.close();
 | 
				
			||||||
 | 
								}
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
 | 
							QFileInfo output_file_info(m_target_path);
 | 
				
			||||||
 | 
							
 | 
				
			||||||
 | 
							
 | 
				
			||||||
 | 
							m_entry->etag = m_reply->rawHeader("ETag").constData();
 | 
				
			||||||
 | 
							m_entry->last_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch();
 | 
				
			||||||
 | 
							m_entry->stale = false;
 | 
				
			||||||
 | 
							MMC->metacache()->updateEntry(m_entry);
 | 
				
			||||||
 | 
							
 | 
				
			||||||
 | 
							m_reply.clear();
 | 
				
			||||||
 | 
							emit succeeded(index_within_job);
 | 
				
			||||||
 | 
							return;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						// else the download failed
 | 
				
			||||||
 | 
						else
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							m_output_file.close();
 | 
				
			||||||
 | 
							m_output_file.remove();
 | 
				
			||||||
 | 
							m_reply.clear();
 | 
				
			||||||
 | 
							emit failed(index_within_job);
 | 
				
			||||||
 | 
							return;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void CacheDownload::downloadReadyRead()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						if(!m_opened_for_saving)
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							if ( !m_output_file.open ( QIODevice::WriteOnly ) )
 | 
				
			||||||
 | 
							{
 | 
				
			||||||
 | 
								/*
 | 
				
			||||||
 | 
								* Can't open the file... the job failed
 | 
				
			||||||
 | 
								*/
 | 
				
			||||||
 | 
								m_reply->abort();
 | 
				
			||||||
 | 
								emit failed(index_within_job);
 | 
				
			||||||
 | 
								return;
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
 | 
							m_opened_for_saving = true;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						QByteArray ba = m_reply->readAll();
 | 
				
			||||||
 | 
						md5sum.addData(ba);
 | 
				
			||||||
 | 
						m_output_file.write ( ba );
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
							
								
								
									
										34
									
								
								logic/net/CacheDownload.h
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										34
									
								
								logic/net/CacheDownload.h
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,34 @@
 | 
				
			|||||||
 | 
					#pragma once
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					#include "Download.h"
 | 
				
			||||||
 | 
					#include "HttpMetaCache.h"
 | 
				
			||||||
 | 
					#include <QFile>
 | 
				
			||||||
 | 
					#include <qcryptographichash.h>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					class CacheDownload : public Download
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						Q_OBJECT
 | 
				
			||||||
 | 
					public:
 | 
				
			||||||
 | 
						MetaEntryPtr m_entry;
 | 
				
			||||||
 | 
						/// is the saving file already open?
 | 
				
			||||||
 | 
						bool m_opened_for_saving;
 | 
				
			||||||
 | 
						/// if saving to file, use the one specified in this string
 | 
				
			||||||
 | 
						QString m_target_path;
 | 
				
			||||||
 | 
						/// this is the output file, if any
 | 
				
			||||||
 | 
						QFile m_output_file;
 | 
				
			||||||
 | 
						/// the hash-as-you-download
 | 
				
			||||||
 | 
						QCryptographicHash md5sum;
 | 
				
			||||||
 | 
					public:
 | 
				
			||||||
 | 
						explicit CacheDownload(QUrl url, MetaEntryPtr entry);
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
					protected slots:
 | 
				
			||||||
 | 
						virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 | 
				
			||||||
 | 
						virtual void downloadError(QNetworkReply::NetworkError error);
 | 
				
			||||||
 | 
						virtual void downloadFinished();
 | 
				
			||||||
 | 
						virtual void downloadReadyRead();
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
					public slots:
 | 
				
			||||||
 | 
						virtual void start();
 | 
				
			||||||
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					typedef QSharedPointer<CacheDownload> CacheDownloadPtr;
 | 
				
			||||||
							
								
								
									
										54
									
								
								logic/net/Download.h
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										54
									
								
								logic/net/Download.h
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,54 @@
 | 
				
			|||||||
 | 
					#pragma once
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					#include <QObject>
 | 
				
			||||||
 | 
					#include <QUrl>
 | 
				
			||||||
 | 
					#include <QSharedPointer>
 | 
				
			||||||
 | 
					#include <QNetworkReply>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					enum JobStatus
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						Job_NotStarted,
 | 
				
			||||||
 | 
						Job_InProgress,
 | 
				
			||||||
 | 
						Job_Finished,
 | 
				
			||||||
 | 
						Job_Failed
 | 
				
			||||||
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					class Download : public QObject
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						Q_OBJECT
 | 
				
			||||||
 | 
					protected:
 | 
				
			||||||
 | 
						explicit Download(): QObject(0){};
 | 
				
			||||||
 | 
					public:
 | 
				
			||||||
 | 
						virtual ~Download() {};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					public:
 | 
				
			||||||
 | 
						/// the network reply
 | 
				
			||||||
 | 
						QSharedPointer<QNetworkReply> m_reply;
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						/// source URL
 | 
				
			||||||
 | 
						QUrl m_url;
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						/// The file's status
 | 
				
			||||||
 | 
						JobStatus m_status;
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						/// index within the parent job
 | 
				
			||||||
 | 
						int index_within_job = 0;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					signals:
 | 
				
			||||||
 | 
						void started(int index);
 | 
				
			||||||
 | 
						void progress(int index, qint64 current, qint64 total);
 | 
				
			||||||
 | 
						void succeeded(int index);
 | 
				
			||||||
 | 
						void failed(int index);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					protected slots:
 | 
				
			||||||
 | 
						virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0;
 | 
				
			||||||
 | 
						virtual void downloadError(QNetworkReply::NetworkError error) = 0;
 | 
				
			||||||
 | 
						virtual void downloadFinished() = 0;
 | 
				
			||||||
 | 
						virtual void downloadReadyRead() = 0;
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
					public slots:
 | 
				
			||||||
 | 
						virtual void start() = 0;
 | 
				
			||||||
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					typedef QSharedPointer<Download> DownloadPtr;
 | 
				
			||||||
@@ -1,136 +1,29 @@
 | 
				
			|||||||
#include "DownloadJob.h"
 | 
					#include "DownloadJob.h"
 | 
				
			||||||
#include "pathutils.h"
 | 
					#include "pathutils.h"
 | 
				
			||||||
#include "MultiMC.h"
 | 
					#include "MultiMC.h"
 | 
				
			||||||
 | 
					#include "FileDownload.h"
 | 
				
			||||||
 | 
					#include "ByteArrayDownload.h"
 | 
				
			||||||
 | 
					#include "CacheDownload.h"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
Download::Download (QUrl url, QString target_path, QString expected_md5 )
 | 
					ByteArrayDownloadPtr DownloadJob::add ( QUrl url )
 | 
				
			||||||
	:Job()
 | 
					 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	m_url = url;
 | 
						ByteArrayDownloadPtr ptr (new ByteArrayDownload(url));
 | 
				
			||||||
	m_target_path = target_path;
 | 
						ptr->index_within_job = downloads.size();
 | 
				
			||||||
	m_expected_md5 = expected_md5;
 | 
						downloads.append(ptr);
 | 
				
			||||||
 | 
						return ptr;
 | 
				
			||||||
	m_check_md5 = m_expected_md5.size();
 | 
					 | 
				
			||||||
	m_save_to_file = m_target_path.size();
 | 
					 | 
				
			||||||
	m_status = Job_NotStarted;
 | 
					 | 
				
			||||||
	m_opened_for_saving = false;
 | 
					 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
void Download::start()
 | 
					FileDownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path)
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	if ( m_save_to_file )
 | 
						FileDownloadPtr ptr (new FileDownload(url, rel_target_path));
 | 
				
			||||||
	{
 | 
						ptr->index_within_job = downloads.size();
 | 
				
			||||||
		QString filename = m_target_path;
 | 
						downloads.append(ptr);
 | 
				
			||||||
		m_output_file.setFileName ( filename );
 | 
						return ptr;
 | 
				
			||||||
		// if there already is a file and md5 checking is in effect and it can be opened
 | 
					 | 
				
			||||||
		if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) )
 | 
					 | 
				
			||||||
		{
 | 
					 | 
				
			||||||
			// check the md5 against the expected one
 | 
					 | 
				
			||||||
			QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
 | 
					 | 
				
			||||||
			m_output_file.close();
 | 
					 | 
				
			||||||
			// skip this file if they match
 | 
					 | 
				
			||||||
			if ( m_check_md5 && hash == m_expected_md5 )
 | 
					 | 
				
			||||||
			{
 | 
					 | 
				
			||||||
				qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
 | 
					 | 
				
			||||||
				emit succeeded(index_within_job);
 | 
					 | 
				
			||||||
				return;
 | 
					 | 
				
			||||||
			}
 | 
					 | 
				
			||||||
			else
 | 
					 | 
				
			||||||
			{
 | 
					 | 
				
			||||||
				m_expected_md5 = hash;
 | 
					 | 
				
			||||||
			}
 | 
					 | 
				
			||||||
		}
 | 
					 | 
				
			||||||
		if(!ensureFilePathExists(filename))
 | 
					 | 
				
			||||||
		{
 | 
					 | 
				
			||||||
			emit failed(index_within_job);
 | 
					 | 
				
			||||||
			return;
 | 
					 | 
				
			||||||
		}
 | 
					 | 
				
			||||||
	}
 | 
					 | 
				
			||||||
	qDebug() << "Downloading " << m_url.toString();
 | 
					 | 
				
			||||||
	QNetworkRequest request ( m_url );
 | 
					 | 
				
			||||||
	request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); 
 | 
					 | 
				
			||||||
	
 | 
					 | 
				
			||||||
	auto worker = MMC->qnam();
 | 
					 | 
				
			||||||
	QNetworkReply * rep = worker->get ( request );
 | 
					 | 
				
			||||||
	
 | 
					 | 
				
			||||||
	m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
 | 
					 | 
				
			||||||
	connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
 | 
					 | 
				
			||||||
	connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
 | 
					 | 
				
			||||||
	connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
 | 
					 | 
				
			||||||
	connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
 | 
					 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
void Download::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
 | 
					CacheDownloadPtr DownloadJob::add ( QUrl url, MetaEntryPtr entry)
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	emit progress (index_within_job, bytesReceived, bytesTotal );
 | 
						CacheDownloadPtr ptr (new CacheDownload(url, entry));
 | 
				
			||||||
}
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
void Download::downloadError ( QNetworkReply::NetworkError error )
 | 
					 | 
				
			||||||
{
 | 
					 | 
				
			||||||
	// error happened during download.
 | 
					 | 
				
			||||||
	// TODO: log the reason why
 | 
					 | 
				
			||||||
	m_status = Job_Failed;
 | 
					 | 
				
			||||||
}
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
void Download::downloadFinished()
 | 
					 | 
				
			||||||
{
 | 
					 | 
				
			||||||
	// if the download succeeded
 | 
					 | 
				
			||||||
	if ( m_status != Job_Failed )
 | 
					 | 
				
			||||||
	{
 | 
					 | 
				
			||||||
		// nothing went wrong...
 | 
					 | 
				
			||||||
		m_status = Job_Finished;
 | 
					 | 
				
			||||||
		// save the data to the downloadable if we aren't saving to file
 | 
					 | 
				
			||||||
		if ( !m_save_to_file )
 | 
					 | 
				
			||||||
		{
 | 
					 | 
				
			||||||
			m_data = m_reply->readAll();
 | 
					 | 
				
			||||||
		}
 | 
					 | 
				
			||||||
		else
 | 
					 | 
				
			||||||
		{
 | 
					 | 
				
			||||||
			m_output_file.close();
 | 
					 | 
				
			||||||
		}
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
		//TODO: check md5 here!
 | 
					 | 
				
			||||||
		m_reply.clear();
 | 
					 | 
				
			||||||
		emit succeeded(index_within_job);
 | 
					 | 
				
			||||||
		return;
 | 
					 | 
				
			||||||
	}
 | 
					 | 
				
			||||||
	// else the download failed
 | 
					 | 
				
			||||||
	else
 | 
					 | 
				
			||||||
	{
 | 
					 | 
				
			||||||
		if ( m_save_to_file )
 | 
					 | 
				
			||||||
		{
 | 
					 | 
				
			||||||
			m_output_file.close();
 | 
					 | 
				
			||||||
			m_output_file.remove();
 | 
					 | 
				
			||||||
		}
 | 
					 | 
				
			||||||
		m_reply.clear();
 | 
					 | 
				
			||||||
		emit failed(index_within_job);
 | 
					 | 
				
			||||||
		return;
 | 
					 | 
				
			||||||
	}
 | 
					 | 
				
			||||||
}
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
void Download::downloadReadyRead()
 | 
					 | 
				
			||||||
{
 | 
					 | 
				
			||||||
	if( m_save_to_file )
 | 
					 | 
				
			||||||
	{
 | 
					 | 
				
			||||||
		if(!m_opened_for_saving)
 | 
					 | 
				
			||||||
		{
 | 
					 | 
				
			||||||
			if ( !m_output_file.open ( QIODevice::WriteOnly ) )
 | 
					 | 
				
			||||||
			{
 | 
					 | 
				
			||||||
				/*
 | 
					 | 
				
			||||||
				* Can't open the file... the job failed
 | 
					 | 
				
			||||||
				*/
 | 
					 | 
				
			||||||
				m_reply->abort();
 | 
					 | 
				
			||||||
				emit failed(index_within_job);
 | 
					 | 
				
			||||||
				return;
 | 
					 | 
				
			||||||
			}
 | 
					 | 
				
			||||||
			m_opened_for_saving = true;
 | 
					 | 
				
			||||||
		}
 | 
					 | 
				
			||||||
		m_output_file.write ( m_reply->readAll() );
 | 
					 | 
				
			||||||
	}
 | 
					 | 
				
			||||||
}
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expected_md5 )
 | 
					 | 
				
			||||||
{
 | 
					 | 
				
			||||||
	DownloadPtr ptr (new Download(url, rel_target_path, expected_md5));
 | 
					 | 
				
			||||||
	ptr->index_within_job = downloads.size();
 | 
						ptr->index_within_job = downloads.size();
 | 
				
			||||||
	downloads.append(ptr);
 | 
						downloads.append(ptr);
 | 
				
			||||||
	return ptr;
 | 
						return ptr;
 | 
				
			||||||
@@ -139,16 +32,17 @@ DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expect
 | 
				
			|||||||
void DownloadJob::partSucceeded ( int index )
 | 
					void DownloadJob::partSucceeded ( int index )
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	num_succeeded++;
 | 
						num_succeeded++;
 | 
				
			||||||
 | 
						qDebug() << m_job_name.toLocal8Bit() << " progress: " << num_succeeded << "/" << downloads.size();
 | 
				
			||||||
	if(num_failed + num_succeeded == downloads.size())
 | 
						if(num_failed + num_succeeded == downloads.size())
 | 
				
			||||||
	{
 | 
						{
 | 
				
			||||||
		if(num_failed)
 | 
							if(num_failed)
 | 
				
			||||||
		{
 | 
							{
 | 
				
			||||||
			qDebug() << "Download JOB failed: " << this;
 | 
								qDebug() << m_job_name.toLocal8Bit() << " failed.";
 | 
				
			||||||
			emit failed();
 | 
								emit failed();
 | 
				
			||||||
		}
 | 
							}
 | 
				
			||||||
		else
 | 
							else
 | 
				
			||||||
		{
 | 
							{
 | 
				
			||||||
			qDebug() << "Download JOB succeeded: " << this;
 | 
								qDebug() << m_job_name.toLocal8Bit() << " succeeded.";
 | 
				
			||||||
			emit succeeded();
 | 
								emit succeeded();
 | 
				
			||||||
		}
 | 
							}
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
@@ -159,7 +53,7 @@ void DownloadJob::partFailed ( int index )
 | 
				
			|||||||
	num_failed++;
 | 
						num_failed++;
 | 
				
			||||||
	if(num_failed + num_succeeded == downloads.size())
 | 
						if(num_failed + num_succeeded == downloads.size())
 | 
				
			||||||
	{
 | 
						{
 | 
				
			||||||
		qDebug() << "Download JOB failed: " << this;
 | 
							qDebug() << m_job_name.toLocal8Bit() << " failed.";
 | 
				
			||||||
		emit failed();
 | 
							emit failed();
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
@@ -172,7 +66,7 @@ void DownloadJob::partProgress ( int index, qint64 bytesReceived, qint64 bytesTo
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
void DownloadJob::start()
 | 
					void DownloadJob::start()
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	qDebug() << "Download JOB started: " << this;
 | 
						qDebug() << m_job_name.toLocal8Bit() << " started.";
 | 
				
			||||||
	for(auto iter: downloads)
 | 
						for(auto iter: downloads)
 | 
				
			||||||
	{
 | 
						{
 | 
				
			||||||
		connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
 | 
							connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -1,98 +1,28 @@
 | 
				
			|||||||
#pragma once
 | 
					#pragma once
 | 
				
			||||||
#include <QtNetwork>
 | 
					#include <QtNetwork>
 | 
				
			||||||
 | 
					#include "Download.h"
 | 
				
			||||||
 | 
					#include "ByteArrayDownload.h"
 | 
				
			||||||
 | 
					#include "FileDownload.h"
 | 
				
			||||||
 | 
					#include "CacheDownload.h"
 | 
				
			||||||
 | 
					#include "HttpMetaCache.h"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class DownloadJob;
 | 
					class DownloadJob;
 | 
				
			||||||
class Download;
 | 
					 | 
				
			||||||
typedef QSharedPointer<DownloadJob> DownloadJobPtr;
 | 
					typedef QSharedPointer<DownloadJob> DownloadJobPtr;
 | 
				
			||||||
typedef QSharedPointer<Download> DownloadPtr;
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
enum JobStatus
 | 
					 | 
				
			||||||
{
 | 
					 | 
				
			||||||
	Job_NotStarted,
 | 
					 | 
				
			||||||
	Job_InProgress,
 | 
					 | 
				
			||||||
	Job_Finished,
 | 
					 | 
				
			||||||
	Job_Failed
 | 
					 | 
				
			||||||
};
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
class Job : public QObject
 | 
					 | 
				
			||||||
{
 | 
					 | 
				
			||||||
	Q_OBJECT
 | 
					 | 
				
			||||||
protected:
 | 
					 | 
				
			||||||
	explicit Job(): QObject(0){};
 | 
					 | 
				
			||||||
public:
 | 
					 | 
				
			||||||
	virtual ~Job() {};
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
public slots:
 | 
					 | 
				
			||||||
	virtual void start() = 0;
 | 
					 | 
				
			||||||
};
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
class Download: public Job
 | 
					 | 
				
			||||||
{
 | 
					 | 
				
			||||||
	friend class DownloadJob;
 | 
					 | 
				
			||||||
	Q_OBJECT
 | 
					 | 
				
			||||||
protected:
 | 
					 | 
				
			||||||
	Download(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
 | 
					 | 
				
			||||||
public:
 | 
					 | 
				
			||||||
	/// the network reply
 | 
					 | 
				
			||||||
	QSharedPointer<QNetworkReply> m_reply;
 | 
					 | 
				
			||||||
	/// source URL
 | 
					 | 
				
			||||||
	QUrl m_url;
 | 
					 | 
				
			||||||
	
 | 
					 | 
				
			||||||
	/// if true, check the md5sum against a provided md5sum
 | 
					 | 
				
			||||||
	/// also, if a file exists, perform an md5sum first and don't download only if they don't match
 | 
					 | 
				
			||||||
	bool m_check_md5;
 | 
					 | 
				
			||||||
	/// the expected md5 checksum
 | 
					 | 
				
			||||||
	QString m_expected_md5;
 | 
					 | 
				
			||||||
	
 | 
					 | 
				
			||||||
	/// save to file?
 | 
					 | 
				
			||||||
	bool m_save_to_file;
 | 
					 | 
				
			||||||
	/// is the saving file already open?
 | 
					 | 
				
			||||||
	bool m_opened_for_saving;
 | 
					 | 
				
			||||||
	/// if saving to file, use the one specified in this string
 | 
					 | 
				
			||||||
	QString m_target_path;
 | 
					 | 
				
			||||||
	/// this is the output file, if any
 | 
					 | 
				
			||||||
	QFile m_output_file;
 | 
					 | 
				
			||||||
	/// if not saving to file, downloaded data is placed here
 | 
					 | 
				
			||||||
	QByteArray m_data;
 | 
					 | 
				
			||||||
	
 | 
					 | 
				
			||||||
	int currentProgress = 0;
 | 
					 | 
				
			||||||
	int totalProgress = 0;
 | 
					 | 
				
			||||||
	
 | 
					 | 
				
			||||||
	/// The file's status
 | 
					 | 
				
			||||||
	JobStatus m_status;
 | 
					 | 
				
			||||||
	
 | 
					 | 
				
			||||||
	int index_within_job = 0;
 | 
					 | 
				
			||||||
signals:
 | 
					 | 
				
			||||||
	void started(int index);
 | 
					 | 
				
			||||||
	void progress(int index, qint64 current, qint64 total);
 | 
					 | 
				
			||||||
	void succeeded(int index);
 | 
					 | 
				
			||||||
	void failed(int index);
 | 
					 | 
				
			||||||
public slots:
 | 
					 | 
				
			||||||
	virtual void start();
 | 
					 | 
				
			||||||
	
 | 
					 | 
				
			||||||
private slots:
 | 
					 | 
				
			||||||
	void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);;
 | 
					 | 
				
			||||||
	void downloadError(QNetworkReply::NetworkError error);
 | 
					 | 
				
			||||||
	void downloadFinished();
 | 
					 | 
				
			||||||
	void downloadReadyRead();
 | 
					 | 
				
			||||||
};
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
/**
 | 
					/**
 | 
				
			||||||
 * A single file for the downloader/cache to process.
 | 
					 * A single file for the downloader/cache to process.
 | 
				
			||||||
 */
 | 
					 */
 | 
				
			||||||
class DownloadJob : public Job
 | 
					class DownloadJob : public QObject
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	Q_OBJECT
 | 
						Q_OBJECT
 | 
				
			||||||
public:
 | 
					public:
 | 
				
			||||||
	explicit DownloadJob()
 | 
						explicit DownloadJob(QString job_name)
 | 
				
			||||||
		:Job(){};
 | 
							:QObject(), m_job_name(job_name){};
 | 
				
			||||||
	explicit DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString())
 | 
						
 | 
				
			||||||
		:Job()
 | 
						ByteArrayDownloadPtr add(QUrl url);
 | 
				
			||||||
	{
 | 
						FileDownloadPtr      add(QUrl url, QString rel_target_path);
 | 
				
			||||||
		add(url, rel_target_path, expected_md5);
 | 
						CacheDownloadPtr     add(QUrl url, MetaEntryPtr entry);
 | 
				
			||||||
	};
 | 
					 | 
				
			||||||
	
 | 
						
 | 
				
			||||||
	DownloadPtr add(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
 | 
					 | 
				
			||||||
	DownloadPtr operator[](int index)
 | 
						DownloadPtr operator[](int index)
 | 
				
			||||||
	{
 | 
						{
 | 
				
			||||||
		return downloads[index];
 | 
							return downloads[index];
 | 
				
			||||||
@@ -119,6 +49,7 @@ private slots:
 | 
				
			|||||||
	void partSucceeded(int index);
 | 
						void partSucceeded(int index);
 | 
				
			||||||
	void partFailed(int index);
 | 
						void partFailed(int index);
 | 
				
			||||||
private:
 | 
					private:
 | 
				
			||||||
 | 
						QString m_job_name;
 | 
				
			||||||
	QList<DownloadPtr> downloads;
 | 
						QList<DownloadPtr> downloads;
 | 
				
			||||||
	int num_succeeded = 0;
 | 
						int num_succeeded = 0;
 | 
				
			||||||
	int num_failed = 0;
 | 
						int num_failed = 0;
 | 
				
			||||||
 
 | 
				
			|||||||
							
								
								
									
										111
									
								
								logic/net/FileDownload.cpp
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										111
									
								
								logic/net/FileDownload.cpp
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,111 @@
 | 
				
			|||||||
 | 
					#include "MultiMC.h"
 | 
				
			||||||
 | 
					#include "FileDownload.h"
 | 
				
			||||||
 | 
					#include <pathutils.h>
 | 
				
			||||||
 | 
					#include <QCryptographicHash>
 | 
				
			||||||
 | 
					#include <QDebug>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					FileDownload::FileDownload ( QUrl url, QString target_path )
 | 
				
			||||||
 | 
						:Download()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						m_url = url;
 | 
				
			||||||
 | 
						m_target_path = target_path;
 | 
				
			||||||
 | 
						m_check_md5 = false;
 | 
				
			||||||
 | 
						m_status = Job_NotStarted;
 | 
				
			||||||
 | 
						m_opened_for_saving = false;
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void FileDownload::start()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						QString filename = m_target_path;
 | 
				
			||||||
 | 
						m_output_file.setFileName ( filename );
 | 
				
			||||||
 | 
						// if there already is a file and md5 checking is in effect and it can be opened
 | 
				
			||||||
 | 
						if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) )
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							// check the md5 against the expected one
 | 
				
			||||||
 | 
							QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData();
 | 
				
			||||||
 | 
							m_output_file.close();
 | 
				
			||||||
 | 
							// skip this file if they match
 | 
				
			||||||
 | 
							if ( m_check_md5 && hash == m_expected_md5 )
 | 
				
			||||||
 | 
							{
 | 
				
			||||||
 | 
								qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
 | 
				
			||||||
 | 
								emit succeeded(index_within_job);
 | 
				
			||||||
 | 
								return;
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
 | 
							else
 | 
				
			||||||
 | 
							{
 | 
				
			||||||
 | 
								m_expected_md5 = hash;
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						if(!ensureFilePathExists(filename))
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							emit failed(index_within_job);
 | 
				
			||||||
 | 
							return;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						qDebug() << "Downloading " << m_url.toString();
 | 
				
			||||||
 | 
						QNetworkRequest request ( m_url );
 | 
				
			||||||
 | 
						request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); 
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						auto worker = MMC->qnam();
 | 
				
			||||||
 | 
						QNetworkReply * rep = worker->get ( request );
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) );
 | 
				
			||||||
 | 
						connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void FileDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						emit progress (index_within_job, bytesReceived, bytesTotal );
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void FileDownload::downloadError ( QNetworkReply::NetworkError error )
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						// error happened during download.
 | 
				
			||||||
 | 
						// TODO: log the reason why
 | 
				
			||||||
 | 
						m_status = Job_Failed;
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void FileDownload::downloadFinished()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						// if the download succeeded
 | 
				
			||||||
 | 
						if ( m_status != Job_Failed )
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							// nothing went wrong...
 | 
				
			||||||
 | 
							m_status = Job_Finished;
 | 
				
			||||||
 | 
							m_output_file.close();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
							m_reply.clear();
 | 
				
			||||||
 | 
							emit succeeded(index_within_job);
 | 
				
			||||||
 | 
							return;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						// else the download failed
 | 
				
			||||||
 | 
						else
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							m_output_file.close();
 | 
				
			||||||
 | 
							m_reply.clear();
 | 
				
			||||||
 | 
							emit failed(index_within_job);
 | 
				
			||||||
 | 
							return;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void FileDownload::downloadReadyRead()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						if(!m_opened_for_saving)
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							if ( !m_output_file.open ( QIODevice::WriteOnly ) )
 | 
				
			||||||
 | 
							{
 | 
				
			||||||
 | 
								/*
 | 
				
			||||||
 | 
								* Can't open the file... the job failed
 | 
				
			||||||
 | 
								*/
 | 
				
			||||||
 | 
								m_reply->abort();
 | 
				
			||||||
 | 
								emit failed(index_within_job);
 | 
				
			||||||
 | 
								return;
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
 | 
							m_opened_for_saving = true;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						m_output_file.write ( m_reply->readAll() );
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
							
								
								
									
										35
									
								
								logic/net/FileDownload.h
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										35
									
								
								logic/net/FileDownload.h
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,35 @@
 | 
				
			|||||||
 | 
					#pragma once
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					#include "Download.h"
 | 
				
			||||||
 | 
					#include <QFile>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					class FileDownload : public Download
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						Q_OBJECT
 | 
				
			||||||
 | 
					public:
 | 
				
			||||||
 | 
						/// if true, check the md5sum against a provided md5sum
 | 
				
			||||||
 | 
						/// also, if a file exists, perform an md5sum first and don't download only if they don't match
 | 
				
			||||||
 | 
						bool m_check_md5;
 | 
				
			||||||
 | 
						/// the expected md5 checksum
 | 
				
			||||||
 | 
						QString m_expected_md5;
 | 
				
			||||||
 | 
						/// is the saving file already open?
 | 
				
			||||||
 | 
						bool m_opened_for_saving;
 | 
				
			||||||
 | 
						/// if saving to file, use the one specified in this string
 | 
				
			||||||
 | 
						QString m_target_path;
 | 
				
			||||||
 | 
						/// this is the output file, if any
 | 
				
			||||||
 | 
						QFile m_output_file;
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
					public:
 | 
				
			||||||
 | 
						explicit FileDownload(QUrl url, QString target_path);
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
					protected slots:
 | 
				
			||||||
 | 
						virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 | 
				
			||||||
 | 
						virtual void downloadError(QNetworkReply::NetworkError error);
 | 
				
			||||||
 | 
						virtual void downloadFinished();
 | 
				
			||||||
 | 
						virtual void downloadReadyRead();
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
					public slots:
 | 
				
			||||||
 | 
						virtual void start();
 | 
				
			||||||
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					typedef QSharedPointer<FileDownload> FileDownloadPtr;
 | 
				
			||||||
@@ -1,38 +1,136 @@
 | 
				
			|||||||
 | 
					#include "MultiMC.h"
 | 
				
			||||||
#include "HttpMetaCache.h"
 | 
					#include "HttpMetaCache.h"
 | 
				
			||||||
#include <pathutils.h>
 | 
					#include <pathutils.h>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					#include <QFileInfo>
 | 
				
			||||||
#include <QFile>
 | 
					#include <QFile>
 | 
				
			||||||
#include <qjsondocument.h>
 | 
					#include <QTemporaryFile>
 | 
				
			||||||
#include <qjsonarray.h>
 | 
					#include <QSaveFile>
 | 
				
			||||||
#include <qjsonobject.h>
 | 
					#include <QDateTime>
 | 
				
			||||||
#include <qfileinfo.h>
 | 
					#include <QCryptographicHash>
 | 
				
			||||||
#include <qtemporaryfile.h>
 | 
					
 | 
				
			||||||
#include <qsavefile.h>
 | 
					#include <QDebug>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					#include <QJsonDocument>
 | 
				
			||||||
 | 
					#include <QJsonArray>
 | 
				
			||||||
 | 
					#include <QJsonObject>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					QString MetaEntry::getFullPath()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						return PathCombine(MMC->metacache()->getBasePath(base), path);
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
HttpMetaCache::HttpMetaCache(QString path)
 | 
					HttpMetaCache::HttpMetaCache(QString path)
 | 
				
			||||||
 | 
						:QObject()
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	m_index_file = path;
 | 
						m_index_file = path;
 | 
				
			||||||
}
 | 
						saveBatchingTimer.setSingleShot(true);
 | 
				
			||||||
HttpMetaCache::~HttpMetaCache()
 | 
						saveBatchingTimer.setTimerType(Qt::VeryCoarseTimer);
 | 
				
			||||||
{
 | 
						connect(&saveBatchingTimer,SIGNAL(timeout()),SLOT(SaveNow()));
 | 
				
			||||||
	Save();
 | 
					 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
void HttpMetaCache::addEntry ( QString base, QString resource_path, QString etag )
 | 
					HttpMetaCache::~HttpMetaCache()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						saveBatchingTimer.stop();
 | 
				
			||||||
 | 
						SaveNow();
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					void HttpMetaCache::SaveEventually()
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						saveBatchingTimer.stop();
 | 
				
			||||||
 | 
						saveBatchingTimer.start(30000);
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					MetaEntryPtr HttpMetaCache::getEntry ( QString base, QString resource_path )
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	// no base. no base path. can't store
 | 
						// no base. no base path. can't store
 | 
				
			||||||
	if(!m_entries.contains(base))
 | 
						if(!m_entries.contains(base))
 | 
				
			||||||
		return;
 | 
					 | 
				
			||||||
	QString real_path = PathCombine(m_entries[base].base_path, resource_path);
 | 
					 | 
				
			||||||
	QFileInfo finfo(real_path);
 | 
					 | 
				
			||||||
	
 | 
					 | 
				
			||||||
	// just ignore it, it's garbage if it's not a proper file
 | 
					 | 
				
			||||||
	if(!finfo.isFile() || !finfo.isReadable())
 | 
					 | 
				
			||||||
	{
 | 
						{
 | 
				
			||||||
		// TODO: log problem
 | 
							// TODO: log problem
 | 
				
			||||||
		return;
 | 
							return MetaEntryPtr();
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						EntryMap & map = m_entries[base];
 | 
				
			||||||
 | 
						if(map.entry_list.contains(resource_path))
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							return map.entry_list[resource_path];
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						return MetaEntryPtr();
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					MetaEntryPtr HttpMetaCache::resolveEntry ( QString base, QString resource_path, QString expected_etag )
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						auto entry = getEntry(base, resource_path);
 | 
				
			||||||
 | 
						// it's not present? generate a default stale entry
 | 
				
			||||||
 | 
						if(!entry)
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							return staleEntry(base, resource_path);
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
	
 | 
						
 | 
				
			||||||
	Save();
 | 
						auto & selected_base = m_entries[base];
 | 
				
			||||||
 | 
						QString real_path = PathCombine(selected_base.base_path, resource_path);
 | 
				
			||||||
 | 
						QFileInfo finfo(real_path);
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						// is the file really there? if not -> stale
 | 
				
			||||||
 | 
						if(!finfo.isFile() || !finfo.isReadable())
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							// if the file doesn't exist, we disown the entry
 | 
				
			||||||
 | 
							selected_base.entry_list.remove(resource_path);
 | 
				
			||||||
 | 
							return staleEntry(base, resource_path);
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						if(!expected_etag.isEmpty() && expected_etag != entry->etag)
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							// if the etag doesn't match expected, we disown the entry
 | 
				
			||||||
 | 
							selected_base.entry_list.remove(resource_path);
 | 
				
			||||||
 | 
							return staleEntry(base, resource_path);
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						// if the file changed, check md5sum
 | 
				
			||||||
 | 
						qint64 file_last_changed = finfo.lastModified().toUTC().toMSecsSinceEpoch();
 | 
				
			||||||
 | 
						if(file_last_changed != entry->last_changed_timestamp)
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							QFile input(real_path);
 | 
				
			||||||
 | 
							input.open(QIODevice::ReadOnly);
 | 
				
			||||||
 | 
							QString md5sum = QCryptographicHash::hash(input.readAll(), QCryptographicHash::Md5).toHex().constData();
 | 
				
			||||||
 | 
							if(entry->md5sum != md5sum)
 | 
				
			||||||
 | 
							{
 | 
				
			||||||
 | 
								selected_base.entry_list.remove(resource_path);
 | 
				
			||||||
 | 
								return staleEntry(base, resource_path);
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
 | 
							// md5sums matched... keep entry and save the new state to file
 | 
				
			||||||
 | 
							entry->last_changed_timestamp = file_last_changed;
 | 
				
			||||||
 | 
							SaveEventually();
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
						// entry passed all the checks we cared about.
 | 
				
			||||||
 | 
						return entry;
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					bool HttpMetaCache::updateEntry ( MetaEntryPtr stale_entry )
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						if(!m_entries.contains(stale_entry->base))
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							qDebug() << "Cannot add entry with unknown base: " << stale_entry->base.toLocal8Bit();
 | 
				
			||||||
 | 
							return false;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						if(stale_entry->stale)
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							qDebug() << "Cannot add stale entry: " << stale_entry->getFullPath().toLocal8Bit();
 | 
				
			||||||
 | 
							return false;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						m_entries[stale_entry->base].entry_list[stale_entry->path] = stale_entry;
 | 
				
			||||||
 | 
						SaveEventually();
 | 
				
			||||||
 | 
						return true;
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					MetaEntryPtr HttpMetaCache::staleEntry(QString base, QString resource_path)
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						auto foo = new MetaEntry;
 | 
				
			||||||
 | 
						foo->base = base;
 | 
				
			||||||
 | 
						foo->path = resource_path;
 | 
				
			||||||
 | 
						foo->stale = true;
 | 
				
			||||||
 | 
						return MetaEntryPtr(foo);
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
void HttpMetaCache::addBase ( QString base, QString base_root )
 | 
					void HttpMetaCache::addBase ( QString base, QString base_root )
 | 
				
			||||||
@@ -46,6 +144,16 @@ void HttpMetaCache::addBase ( QString base, QString base_root )
 | 
				
			|||||||
	m_entries[base] = foo;
 | 
						m_entries[base] = foo;
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					QString HttpMetaCache::getBasePath ( QString base )
 | 
				
			||||||
 | 
					{
 | 
				
			||||||
 | 
						if(m_entries.contains(base))
 | 
				
			||||||
 | 
						{
 | 
				
			||||||
 | 
							return m_entries[base].base_path;
 | 
				
			||||||
 | 
						}
 | 
				
			||||||
 | 
						return QString();
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
void HttpMetaCache::Load()
 | 
					void HttpMetaCache::Load()
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	QFile index(m_index_file);
 | 
						QFile index(m_index_file);
 | 
				
			||||||
@@ -65,12 +173,12 @@ void HttpMetaCache::Load()
 | 
				
			|||||||
	
 | 
						
 | 
				
			||||||
	// read the entry array
 | 
						// read the entry array
 | 
				
			||||||
	auto entries_val =root.value("entries");
 | 
						auto entries_val =root.value("entries");
 | 
				
			||||||
	if(!version_val.isArray())
 | 
						if(!entries_val.isArray())
 | 
				
			||||||
		return;
 | 
							return;
 | 
				
			||||||
	QJsonArray array = json.array();
 | 
						QJsonArray array = entries_val.toArray();
 | 
				
			||||||
	for(auto element: array)
 | 
						for(auto element: array)
 | 
				
			||||||
	{
 | 
						{
 | 
				
			||||||
		if(!element.isObject());
 | 
							if(!element.isObject())
 | 
				
			||||||
			return;
 | 
								return;
 | 
				
			||||||
		auto element_obj = element.toObject();
 | 
							auto element_obj = element.toObject();
 | 
				
			||||||
		QString base = element_obj.value("base").toString();
 | 
							QString base = element_obj.value("base").toString();
 | 
				
			||||||
@@ -83,11 +191,13 @@ void HttpMetaCache::Load()
 | 
				
			|||||||
		foo->md5sum = element_obj.value("md5sum").toString();
 | 
							foo->md5sum = element_obj.value("md5sum").toString();
 | 
				
			||||||
		foo->etag = element_obj.value("etag").toString();
 | 
							foo->etag = element_obj.value("etag").toString();
 | 
				
			||||||
		foo->last_changed_timestamp = element_obj.value("last_changed_timestamp").toDouble();
 | 
							foo->last_changed_timestamp = element_obj.value("last_changed_timestamp").toDouble();
 | 
				
			||||||
 | 
							// presumed innocent until closer examination
 | 
				
			||||||
 | 
							foo->stale = false;
 | 
				
			||||||
		entrymap.entry_list[path] = MetaEntryPtr( foo );
 | 
							entrymap.entry_list[path] = MetaEntryPtr( foo );
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
void HttpMetaCache::Save()
 | 
					void HttpMetaCache::SaveNow()
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
	QSaveFile tfile(m_index_file);
 | 
						QSaveFile tfile(m_index_file);
 | 
				
			||||||
	if(!tfile.open(QIODevice::WriteOnly | QIODevice::Truncate))
 | 
						if(!tfile.open(QIODevice::WriteOnly | QIODevice::Truncate))
 | 
				
			||||||
@@ -118,14 +228,3 @@ void HttpMetaCache::Save()
 | 
				
			|||||||
		return;
 | 
							return;
 | 
				
			||||||
	tfile.commit();
 | 
						tfile.commit();
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
MetaEntryPtr HttpMetaCache::getEntryForResource ( QString base, QString resource_path )
 | 
					 | 
				
			||||||
{
 | 
					 | 
				
			||||||
	if(!m_entries.contains(base))
 | 
					 | 
				
			||||||
		return MetaEntryPtr();
 | 
					 | 
				
			||||||
	auto & entrymap = m_entries[base];
 | 
					 | 
				
			||||||
	if(!entrymap.entry_list.contains(resource_path))
 | 
					 | 
				
			||||||
		return MetaEntryPtr();
 | 
					 | 
				
			||||||
	return entrymap.entry_list[resource_path];
 | 
					 | 
				
			||||||
}
 | 
					 | 
				
			||||||
 
 | 
				
			|||||||
@@ -2,6 +2,7 @@
 | 
				
			|||||||
#include <QString>
 | 
					#include <QString>
 | 
				
			||||||
#include <QSharedPointer>
 | 
					#include <QSharedPointer>
 | 
				
			||||||
#include <QMap>
 | 
					#include <QMap>
 | 
				
			||||||
 | 
					#include <qtimer.h>
 | 
				
			||||||
 | 
					
 | 
				
			||||||
struct MetaEntry
 | 
					struct MetaEntry
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
@@ -9,23 +10,42 @@ struct MetaEntry
 | 
				
			|||||||
	QString path;
 | 
						QString path;
 | 
				
			||||||
	QString md5sum;
 | 
						QString md5sum;
 | 
				
			||||||
	QString etag;
 | 
						QString etag;
 | 
				
			||||||
	quint64 last_changed_timestamp = 0;
 | 
						qint64 last_changed_timestamp = 0;
 | 
				
			||||||
 | 
						bool stale = true;
 | 
				
			||||||
 | 
						QString getFullPath();
 | 
				
			||||||
};
 | 
					};
 | 
				
			||||||
 | 
					
 | 
				
			||||||
typedef QSharedPointer<MetaEntry> MetaEntryPtr;
 | 
					typedef QSharedPointer<MetaEntry> MetaEntryPtr;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
class HttpMetaCache
 | 
					class HttpMetaCache : public QObject
 | 
				
			||||||
{
 | 
					{
 | 
				
			||||||
 | 
						Q_OBJECT
 | 
				
			||||||
public:
 | 
					public:
 | 
				
			||||||
	// supply path to the cache index file
 | 
						// supply path to the cache index file
 | 
				
			||||||
	HttpMetaCache(QString path);
 | 
						HttpMetaCache(QString path);
 | 
				
			||||||
	~HttpMetaCache();
 | 
						~HttpMetaCache();
 | 
				
			||||||
	MetaEntryPtr getEntryForResource(QString base, QString resource_path);
 | 
						
 | 
				
			||||||
	void addEntry(QString base, QString resource_path, QString etag);
 | 
						// get the entry solely from the cache
 | 
				
			||||||
 | 
						// you probably don't want this, unless you have some specific caching needs.
 | 
				
			||||||
 | 
						MetaEntryPtr getEntry(QString base, QString resource_path);
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						// get the entry from cache and verify that it isn't stale (within reason)
 | 
				
			||||||
 | 
						MetaEntryPtr resolveEntry(QString base, QString resource_path, QString expected_etag = QString());
 | 
				
			||||||
 | 
						
 | 
				
			||||||
 | 
						// add a previously resolved stale entry
 | 
				
			||||||
 | 
						bool updateEntry(MetaEntryPtr stale_entry);
 | 
				
			||||||
 | 
						
 | 
				
			||||||
	void addBase(QString base, QString base_root);
 | 
						void addBase(QString base, QString base_root);
 | 
				
			||||||
private:
 | 
						
 | 
				
			||||||
	void Save();
 | 
						// (re)start a timer that calls SaveNow later.
 | 
				
			||||||
 | 
						void SaveEventually();
 | 
				
			||||||
	void Load();
 | 
						void Load();
 | 
				
			||||||
 | 
						QString getBasePath ( QString base );
 | 
				
			||||||
 | 
					public slots:
 | 
				
			||||||
 | 
						void SaveNow();
 | 
				
			||||||
 | 
					private:
 | 
				
			||||||
 | 
						// create a new stale entry, given the parameters
 | 
				
			||||||
 | 
						MetaEntryPtr staleEntry(QString base, QString resource_path);
 | 
				
			||||||
	struct EntryMap
 | 
						struct EntryMap
 | 
				
			||||||
	{
 | 
						{
 | 
				
			||||||
		QString base_path;
 | 
							QString base_path;
 | 
				
			||||||
@@ -33,4 +53,5 @@ private:
 | 
				
			|||||||
	};
 | 
						};
 | 
				
			||||||
	QMap<QString, EntryMap> m_entries;
 | 
						QMap<QString, EntryMap> m_entries;
 | 
				
			||||||
	QString m_index_file;
 | 
						QString m_index_file;
 | 
				
			||||||
 | 
						QTimer saveBatchingTimer;
 | 
				
			||||||
};
 | 
					};
 | 
				
			||||||
		Reference in New Issue
	
	Block a user