From 2837236d81b882f041a1cefadc86ca9d5f09ceeb Mon Sep 17 00:00:00 2001 From: Rachel Powers <508861+Ryex@users.noreply.github.com> Date: Thu, 9 Feb 2023 19:48:40 -0700 Subject: [PATCH] fix: intelegent recursive links & symlink follow on export Signed-off-by: Rachel Powers <508861+Ryex@users.noreply.github.com> --- launcher/FileSystem.cpp | 66 ++++++++- launcher/FileSystem.h | 28 +++- launcher/InstanceCopyPrefs.cpp | 9 ++ launcher/InstanceCopyPrefs.h | 1 + launcher/InstanceCopyTask.cpp | 16 ++- launcher/MMCZip.cpp | 11 +- launcher/ui/dialogs/CopyInstanceDialog.cpp | 21 ++- tests/FileSystem_test.cpp | 148 +++++++++++++++++++++ 8 files changed, 278 insertions(+), 22 deletions(-) diff --git a/launcher/FileSystem.cpp b/launcher/FileSystem.cpp index c363f6ea..4ee3899b 100644 --- a/launcher/FileSystem.cpp +++ b/launcher/FileSystem.cpp @@ -242,6 +242,14 @@ bool copy::operator()(const QString& offset, bool dryRun) return err.value() == 0; } +/// qDebug print support for the LinkPair struct +QDebug operator<<(QDebug debug, const LinkPair& lp) +{ + QDebugStateSaver saver(debug); + + debug.nospace() << "LinkPair{ src: " << lp.src << " , dst: " << lp.dst << " }"; + return debug; +} bool create_link::operator()(const QString& offset, bool dryRun) { @@ -265,7 +273,7 @@ bool create_link::operator()(const QString& offset, bool dryRun) * @param offset subdirectory form src to link to dest * @return if there was an error during the attempt to link */ -void create_link::make_link_list( const QString& offset) +void create_link::make_link_list(const QString& offset) { for (auto pair : m_path_pairs) { const QString& srcPath = pair.src; @@ -297,14 +305,26 @@ void create_link::make_link_list( const QString& offset) link_file(src, ""); } else { if (m_debug) - qDebug() << "linking recursivly:" << src << "to" << dst; + qDebug() << "linking recursivly:" << src << "to" << dst << "max_depth:" << m_max_depth; QDir src_dir(src); QDirIterator source_it(src, QDir::Filter::Files | QDir::Filter::Hidden, QDirIterator::Subdirectories); + QStringList linkedPaths; + while (source_it.hasNext()) { auto src_path = source_it.next(); auto relative_path = src_dir.relativeFilePath(src_path); + if (m_max_depth >= 0 && PathDepth(relative_path) > m_max_depth){ + relative_path = PathTruncate(relative_path, m_max_depth); + src_path = src_dir.filePath(relative_path); + if (linkedPaths.contains(src_path)) { + continue; + } + } + + linkedPaths.append(src_path); + link_file(src_path, relative_path); } } @@ -312,7 +332,7 @@ void create_link::make_link_list( const QString& offset) } bool create_link::make_links() -{ +{ for (auto link : m_links_to_make) { QString src_path = link.src; @@ -556,11 +576,49 @@ QString PathCombine(const QString& path1, const QString& path2, const QString& p return PathCombine(PathCombine(path1, path2, path3), path4); } -QString AbsolutePath(QString path) +QString AbsolutePath(const QString& path) { return QFileInfo(path).absolutePath(); } +int PathDepth(const QString& path) +{ + if (path.isEmpty()) return 0; + + QFileInfo info(path); + + auto parts = QDir::toNativeSeparators(info.path()).split(QDir::separator(), Qt::SkipEmptyParts); + + int numParts = QDir::toNativeSeparators(info.path()).split(QDir::separator(), Qt::SkipEmptyParts).length(); + numParts -= parts.count("."); + numParts -= parts.count("..") * 2; + + return numParts; +} + +QString PathTruncate(const QString& path, int depth) +{ + if (path.isEmpty() || (depth < 0) ) return ""; + + QString trunc = QFileInfo(path).path(); + + if (PathDepth(trunc) > depth ) { + return PathTruncate(trunc, depth); + } + + auto parts = QDir::toNativeSeparators(trunc).split(QDir::separator(), Qt::SkipEmptyParts); + if (parts.startsWith(".") && !path.startsWith(".")) { + parts.removeFirst(); + } + if (path.startsWith(QDir::separator())) { + parts.prepend(""); + } + + trunc = parts.join(QDir::separator()); + + return trunc; +} + QString ResolveExecutable(QString path) { if (path.isEmpty()) { diff --git a/launcher/FileSystem.h b/launcher/FileSystem.h index 83ff99a4..7485206a 100644 --- a/launcher/FileSystem.h +++ b/launcher/FileSystem.h @@ -200,6 +200,11 @@ class create_link : public QObject { m_recursive = recursive; return *this; } + create_link& setMaxDepth(int depth) + { + m_max_depth = depth; + return *this; + } create_link& debug(bool d) { m_debug = d; @@ -239,6 +244,9 @@ class create_link : public QObject { bool m_whitelist = false; bool m_recursive = true; + /// @brief >= -1 = infinite, 0 = link files at src/* to dest/*, 1 = link files at src/*/* to dest/*/*, etc. + int m_max_depth = -1; + QList m_path_pairs; QList m_path_results; QList m_links_to_make; @@ -272,7 +280,25 @@ QString PathCombine(const QString& path1, const QString& path2); QString PathCombine(const QString& path1, const QString& path2, const QString& path3); QString PathCombine(const QString& path1, const QString& path2, const QString& path3, const QString& path4); -QString AbsolutePath(QString path); +QString AbsolutePath(const QString& path); + +/** + * @brief depth of path. "foo.txt" -> 0 , "bar/foo.txt" -> 1, /baz/bar/foo.txt -> 2, etc. + * + * @param path path to measure + * @return int number of componants before base path + */ +int PathDepth(const QString& path); + + +/** + * @brief cut off segments of path untill it is a max of length depth + * + * @param path path to truncate + * @param depth max depth of new path + * @return QString truncated path + */ +QString PathTruncate(const QString& path, int depth); /** * Resolve an executable diff --git a/launcher/InstanceCopyPrefs.cpp b/launcher/InstanceCopyPrefs.cpp index c03d0aae..0650002b 100644 --- a/launcher/InstanceCopyPrefs.cpp +++ b/launcher/InstanceCopyPrefs.cpp @@ -16,8 +16,13 @@ bool InstanceCopyPrefs::allTrue() const copyScreenshots; } + // Returns a single RegEx string of the selected folders/files to filter out (ex: ".minecraft/saves|.minecraft/server.dat") QString InstanceCopyPrefs::getSelectedFiltersAsRegex() const +{ + return getSelectedFiltersAsRegex({}); +} +QString InstanceCopyPrefs::getSelectedFiltersAsRegex(const QStringList& additionalFilters) const { QStringList filters; @@ -42,6 +47,10 @@ QString InstanceCopyPrefs::getSelectedFiltersAsRegex() const if(!copyScreenshots) filters << "screenshots"; + for (auto filter : additionalFilters) { + filters << filter; + } + // If we have any filters to add, join them as a single regex string to return: if (!filters.isEmpty()) { const QString MC_ROOT = "[.]?minecraft/"; diff --git a/launcher/InstanceCopyPrefs.h b/launcher/InstanceCopyPrefs.h index 14ae9551..c7bde068 100644 --- a/launcher/InstanceCopyPrefs.h +++ b/launcher/InstanceCopyPrefs.h @@ -10,6 +10,7 @@ struct InstanceCopyPrefs { public: [[nodiscard]] bool allTrue() const; [[nodiscard]] QString getSelectedFiltersAsRegex() const; + [[nodiscard]] QString getSelectedFiltersAsRegex(const QStringList& additionalFilters) const; // Getters [[nodiscard]] bool isCopySavesEnabled() const; [[nodiscard]] bool isKeepPlaytimeEnabled() const; diff --git a/launcher/InstanceCopyTask.cpp b/launcher/InstanceCopyTask.cpp index ba0052fa..40babd0f 100644 --- a/launcher/InstanceCopyTask.cpp +++ b/launcher/InstanceCopyTask.cpp @@ -4,13 +4,14 @@ #include "NullInstance.h" #include "pathmatcher/RegexpMatcher.h" #include +#include InstanceCopyTask::InstanceCopyTask(InstancePtr origInstance, const InstanceCopyPrefs& prefs) { m_origInstance = origInstance; m_keepPlaytime = prefs.isKeepPlaytimeEnabled(); - QString filters = prefs.getSelectedFiltersAsRegex(); + m_useLinks = prefs.isUseSymLinksEnabled(); @@ -18,6 +19,14 @@ InstanceCopyTask::InstanceCopyTask(InstancePtr origInstance, const InstanceCopyP m_useHardLinks = prefs.isLinkRecursivelyEnabled() && prefs.isUseHardLinksEnabled(); m_copySaves = prefs.isLinkRecursivelyEnabled() && prefs.isDontLinkSavesEnabled() && prefs.isCopySavesEnabled(); m_useClone = prefs.isUseCloneEnabled(); + + QString filters = prefs.getSelectedFiltersAsRegex(); + if (m_useLinks || m_useHardLinks) { + if (!filters.isEmpty()) filters += "|"; + filters += "instance.cfg"; + } + + qDebug() << "CopyFilters:" << filters; if (!filters.isEmpty()) { @@ -46,9 +55,10 @@ void InstanceCopyTask::executeTask() folderClone.matcher(m_matcher.get()); return folderClone(); - } else if (m_useLinks) { + } else if (m_useLinks || m_useHardLinks) { FS::create_link folderLink(m_origInstance->instanceRoot(), m_stagingPath); - folderLink.linkRecursively(m_linkRecursively).useHardLinks(m_useHardLinks).matcher(m_matcher.get()); + int depth = m_linkRecursively ? -1 : 0; // we need to at least link the top level instead of the instance folder + folderLink.linkRecursively(true).setMaxDepth(depth).useHardLinks(m_useHardLinks).matcher(m_matcher.get()); bool there_were_errors = false; diff --git a/launcher/MMCZip.cpp b/launcher/MMCZip.cpp index b4b663c1..1a336375 100644 --- a/launcher/MMCZip.cpp +++ b/launcher/MMCZip.cpp @@ -102,8 +102,13 @@ bool MMCZip::compressDirFiles(QuaZip *zip, QString dir, QFileInfoList files, boo for (auto e : files) { auto filePath = directory.relativeFilePath(e.absoluteFilePath()); auto srcPath = e.absoluteFilePath(); - if (followSymlinks) - srcPath = e.canonicalFilePath(); + if (followSymlinks) { + if (e.isSymLink()) { + srcPath = e.symLinkTarget(); + } else { + srcPath = e.canonicalFilePath(); + } + } if( !JlCompress::compressFile(zip, srcPath, filePath)) return false; } @@ -119,7 +124,7 @@ bool MMCZip::compressDirFiles(QString fileCompressed, QString dir, QFileInfoList return false; } - auto result = compressDirFiles(&zip, dir, files); + auto result = compressDirFiles(&zip, dir, files, followSymlinks); zip.close(); if(zip.getZipError()!=0) { diff --git a/launcher/ui/dialogs/CopyInstanceDialog.cpp b/launcher/ui/dialogs/CopyInstanceDialog.cpp index c6cbefcf..9fe129f1 100644 --- a/launcher/ui/dialogs/CopyInstanceDialog.cpp +++ b/launcher/ui/dialogs/CopyInstanceDialog.cpp @@ -171,17 +171,18 @@ void CopyInstanceDialog::updateSelectAllCheckbox() void CopyInstanceDialog::updateUseCloneCheckbox() { - ui->useCloneCheckbox->setEnabled(m_cloneSupported && !ui->linkFilesGroup->isChecked()); - ui->useCloneCheckbox->setChecked(m_cloneSupported && m_selectedOptions.isUseCloneEnabled()); + ui->useCloneCheckbox->setEnabled(m_cloneSupported && !ui->symbolicLinksCheckbox->isChecked() && !ui->hardLinksCheckbox->isChecked()); + ui->useCloneCheckbox->setChecked(m_cloneSupported && m_selectedOptions.isUseCloneEnabled() && !ui->symbolicLinksCheckbox->isChecked() && + !ui->hardLinksCheckbox->isChecked()); } void CopyInstanceDialog::updateLinkOptions() { - ui->symbolicLinksCheckbox->setEnabled(m_linkSupported && !ui->hardLinksCheckbox->isChecked()); - ui->hardLinksCheckbox->setEnabled(m_linkSupported && !ui->symbolicLinksCheckbox->isChecked()); + ui->symbolicLinksCheckbox->setEnabled(m_linkSupported && !ui->hardLinksCheckbox->isChecked() && !ui->useCloneCheckbox->isChecked()); + ui->hardLinksCheckbox->setEnabled(m_linkSupported && !ui->symbolicLinksCheckbox->isChecked() && !ui->useCloneCheckbox->isChecked()); - ui->symbolicLinksCheckbox->setChecked(m_linkSupported && m_selectedOptions.isUseSymLinksEnabled()); - ui->hardLinksCheckbox->setChecked(m_linkSupported && m_selectedOptions.isUseHardLinksEnabled()); + ui->symbolicLinksCheckbox->setChecked(m_linkSupported && m_selectedOptions.isUseSymLinksEnabled() && !ui->useCloneCheckbox->isChecked()); + ui->hardLinksCheckbox->setChecked(m_linkSupported && m_selectedOptions.isUseHardLinksEnabled() && !ui->useCloneCheckbox->isChecked()); bool linksInUse = (ui->symbolicLinksCheckbox->isChecked() || ui->hardLinksCheckbox->isChecked()); ui->recursiveLinkCheckbox->setEnabled(m_linkSupported && linksInUse && !ui->hardLinksCheckbox->isChecked()); @@ -278,16 +279,14 @@ void CopyInstanceDialog::on_hardLinksCheckbox_stateChanged(int state) if (state == Qt::Checked && !ui->recursiveLinkCheckbox->isChecked()) { ui->recursiveLinkCheckbox->setChecked(true); } + updateUseCloneCheckbox(); updateLinkOptions(); } void CopyInstanceDialog::on_recursiveLinkCheckbox_stateChanged(int state) { m_selectedOptions.enableLinkRecursively(state == Qt::Checked); - if (state != Qt::Checked) { - ui->hardLinksCheckbox->setChecked(false); - ui->dontLinkSavesCheckbox->setChecked(false); - } + updateLinkOptions(); } @@ -299,6 +298,6 @@ void CopyInstanceDialog::on_dontLinkSavesCheckbox_stateChanged(int state) void CopyInstanceDialog::on_useCloneCheckbox_stateChanged(int state) { m_selectedOptions.enableUseClone(m_cloneSupported && (state == Qt::Checked)); - ui->linkFilesGroup->setEnabled(!m_selectedOptions.isUseCloneEnabled()); updateUseCloneCheckbox(); + updateLinkOptions(); } \ No newline at end of file diff --git a/tests/FileSystem_test.cpp b/tests/FileSystem_test.cpp index 4ccc4003..4418dd62 100644 --- a/tests/FileSystem_test.cpp +++ b/tests/FileSystem_test.cpp @@ -57,6 +57,11 @@ class LinkTask : public Task { m_lnk->whitelist(b); } + void setMaxDepth(int depth) + { + m_lnk->setMaxDepth(depth); + } + private: void executeTask() override { @@ -630,6 +635,149 @@ slots: QVERIFY(target_dir.entryList(filter).contains("pack.mcmeta")); } } + + void test_link_with_max_depth() + { + QString folder = QFINDTESTDATA("testdata/FileSystem/test_folder"); + auto f = [&folder, this]() + { + QTemporaryDir tempDir; + tempDir.setAutoRemove(true); + qDebug() << "From:" << folder << "To:" << tempDir.path(); + + QDir target_dir(FS::PathCombine(tempDir.path(), "test_folder")); + qDebug() << tempDir.path(); + qDebug() << target_dir.path(); + + LinkTask lnk_tsk(folder, target_dir.path()); + lnk_tsk.linkRecursively(true); + lnk_tsk.setMaxDepth(0); + QObject::connect(&lnk_tsk, &Task::finished, [&]{ + QVERIFY2(lnk_tsk.wasSuccessful(), "Task finished but was not successful when it should have been."); + }); + lnk_tsk.start(); + + QVERIFY2(QTest::qWaitFor([&]() { + return lnk_tsk.isFinished(); + }, 100000), "Task didn't finish as it should."); + + QVERIFY(!QFileInfo(target_dir.path()).isSymLink()); + + auto filter = QDir::Filter::Files | QDir::Filter::Dirs | QDir::Filter::Hidden; + for(auto entry: target_dir.entryList(filter)) + { + qDebug() << entry; + if (entry == "." || entry == "..") continue; + QFileInfo entry_lnk_info(target_dir.filePath(entry)); + QVERIFY(entry_lnk_info.isSymLink()); + } + + QFileInfo lnk_info(target_dir.path()); + QVERIFY(lnk_info.exists()); + QVERIFY(!lnk_info.isSymLink()); + + QVERIFY(target_dir.entryList().contains("pack.mcmeta")); + QVERIFY(target_dir.entryList().contains("assets")); + + + }; + + // first try variant without trailing / + QVERIFY(!folder.endsWith('/')); + f(); + + // then variant with trailing / + folder.append('/'); + QVERIFY(folder.endsWith('/')); + f(); + } + + void test_link_with_no_max_depth() + { + QString folder = QFINDTESTDATA("testdata/FileSystem/test_folder"); + auto f = [&folder]() + { + QTemporaryDir tempDir; + tempDir.setAutoRemove(true); + qDebug() << "From:" << folder << "To:" << tempDir.path(); + + QDir target_dir(FS::PathCombine(tempDir.path(), "test_folder")); + qDebug() << tempDir.path(); + qDebug() << target_dir.path(); + + LinkTask lnk_tsk(folder, target_dir.path()); + lnk_tsk.linkRecursively(true); + lnk_tsk.setMaxDepth(-1); + QObject::connect(&lnk_tsk, &Task::finished, [&]{ + QVERIFY2(lnk_tsk.wasSuccessful(), "Task finished but was not successful when it should have been."); + }); + lnk_tsk.start(); + + QVERIFY2(QTest::qWaitFor([&]() { + return lnk_tsk.isFinished(); + }, 100000), "Task didn't finish as it should."); + + + std::function verify_check = [&](QString check_path) { + QDir check_dir(check_path); + auto filter = QDir::Filter::Files | QDir::Filter::Dirs | QDir::Filter::Hidden; + for(auto entry: check_dir.entryList(filter)) + { + QFileInfo entry_lnk_info(check_dir.filePath(entry)); + qDebug() << entry << check_dir.filePath(entry); + if (!entry_lnk_info.isDir()){ + QVERIFY(entry_lnk_info.isSymLink()); + } else if (entry != "." && entry != "..") { + qDebug() << "Decending tree to verify symlinks:" << check_dir.filePath(entry); + verify_check(entry_lnk_info.filePath()); + } + } + }; + + verify_check(target_dir.path()); + + + QFileInfo lnk_info(target_dir.path()); + QVERIFY(lnk_info.exists()); + + QVERIFY(target_dir.entryList().contains("pack.mcmeta")); + QVERIFY(target_dir.entryList().contains("assets")); + }; + + // first try variant without trailing / + QVERIFY(!folder.endsWith('/')); + f(); + + // then variant with trailing / + folder.append('/'); + QVERIFY(folder.endsWith('/')); + f(); + } + + void test_path_depth() { + QCOMPARE_EQ(FS::PathDepth(""), 0); + QCOMPARE_EQ(FS::PathDepth("."), 0); + QCOMPARE_EQ(FS::PathDepth("foo.txt"), 0); + QCOMPARE_EQ(FS::PathDepth("./foo.txt"), 0); + QCOMPARE_EQ(FS::PathDepth("./bar/foo.txt"), 1); + QCOMPARE_EQ(FS::PathDepth("../bar/foo.txt"), 0); + QCOMPARE_EQ(FS::PathDepth("/bar/foo.txt"), 1); + QCOMPARE_EQ(FS::PathDepth("baz/bar/foo.txt"), 2); + QCOMPARE_EQ(FS::PathDepth("/baz/bar/foo.txt"), 2); + QCOMPARE_EQ(FS::PathDepth("./baz/bar/foo.txt"), 2); + QCOMPARE_EQ(FS::PathDepth("/baz/../bar/foo.txt"), 1); + } + + void test_path_trunc() { + QCOMPARE_EQ(FS::PathTruncate("", 0), ""); + QCOMPARE_EQ(FS::PathTruncate("foo.txt", 0), ""); + QCOMPARE_EQ(FS::PathTruncate("foo.txt", 1), ""); + QCOMPARE_EQ(FS::PathTruncate("./bar/foo.txt", 0), "./bar"); + QCOMPARE_EQ(FS::PathTruncate("./bar/foo.txt", 1), "./bar"); + QCOMPARE_EQ(FS::PathTruncate("/bar/foo.txt", 1), "/bar"); + QCOMPARE_EQ(FS::PathTruncate("bar/foo.txt", 1), "bar"); + QCOMPARE_EQ(FS::PathTruncate("baz/bar/foo.txt", 2), "baz/bar"); + } }; QTEST_GUILESS_MAIN(FileSystemTest)