diff --git a/src/tl/tl/tlFileUtils.cc b/src/tl/tl/tlFileUtils.cc index e575dbb634..747d21c6fe 100644 --- a/src/tl/tl/tlFileUtils.cc +++ b/src/tl/tl/tlFileUtils.cc @@ -584,6 +584,48 @@ cp_dir_recursive (const std::string &source, const std::string &target) return true; } +bool +mv_dir_recursive (const std::string &source, const std::string &target) +{ + std::vector entries; + std::string path = tl::absolute_file_path (source); + std::string path_to = tl::absolute_file_path (target); + + bool error = false; + + entries = dir_entries (path, false /*without_files*/, true /*with_dirs*/); + for (std::vector::const_iterator e = entries.begin (); e != entries.end (); ++e) { + std::string tc = tl::combine_path (path_to, *e); + if (! mkpath (tc)) { +#if defined(FILE_UTILS_VERBOSE) + tl::error << tr ("Unable to create target directory: ") << tc; +#endif + error = true; + } else if (! mv_dir_recursive (tl::combine_path (path, *e), tc)) { + error = true; + } + } + + entries = dir_entries (path, true /*with_files*/, false /*without_dirs*/); + for (std::vector::const_iterator e = entries.begin (); e != entries.end (); ++e) { + if (! tl::rename_file (tl::combine_path (path, *e), tl::combine_path (path_to, *e))) { +#if defined(FILE_UTILS_VERBOSE) + tl::error << tr ("Unable to move file from ") << tl::combine_path (path, *e) << tr (" to ") << tl::combine_path (path_to, *e); +#endif + error = true; + } + } + + if (! tl::rm_dir (path)) { +#if defined(FILE_UTILS_VERBOSE) + tl::error << tr ("Unable to remove folder ") << path; +#endif + error = true; + } + + return ! error; +} + std::string absolute_path (const std::string &s) { std::vector parts = split_path (absolute_file_path (s)); diff --git a/src/tl/tl/tlFileUtils.h b/src/tl/tl/tlFileUtils.h index f192599ff5..0649b4078c 100644 --- a/src/tl/tl/tlFileUtils.h +++ b/src/tl/tl/tlFileUtils.h @@ -52,11 +52,18 @@ bool TL_PUBLIC rm_dir_recursive (const std::string &path); bool TL_PUBLIC mkpath (const std::string &path); /** - * @brief Recursively remove the given directory, the files from that directory and all sub-directories (version with std::string) + * @brief Recursively copy the given directory * @return True, if successful. false otherwise. */ bool TL_PUBLIC cp_dir_recursive (const std::string &source, const std::string &target); +/** + * @brief Recursively move the contents of the given directory + * @return True, if successful. false otherwise. + * After this operation, the source directory is deleted. + */ +bool TL_PUBLIC mv_dir_recursive (const std::string &source, const std::string &target); + /** * @brief Gets the absolute path for a given file path * This will deliver the directory of the file as absolute path. diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc index 59e619234f..bb01826407 100644 --- a/src/tl/tl/tlGit.cc +++ b/src/tl/tl/tlGit.cc @@ -122,8 +122,28 @@ checkout_progress(const char * /*path*/, size_t cur, size_t tot, void *payload) void -GitObject::read (const std::string &url, const std::string &filter, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +GitObject::read (const std::string &org_url, const std::string &org_filter, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { + std::string url = org_url; + std::string filter = org_filter; + + std::string subdir; + + std::string url_terminator (".git/"); + size_t url_end = url.find (url_terminator); + if (url_end != std::string::npos) { + + subdir = std::string (url, url_end + url_terminator.size ()); + + url = std::string (url, 0, url_end + url_terminator.size () - 1); + if (filter.empty ()) { + filter = subdir + "/**"; + } else { + filter = subdir + "/" + filter; + } + + } + // @@@ use callback, timeout? tl::RelativeProgress progress (tl::to_string (tr ("Download progress")), 10000, 1 /*yield always*/); @@ -136,8 +156,10 @@ GitObject::read (const std::string &url, const std::string &filter, const std::s checkout_opts.paths.strings = (char **) &paths_cstr; } + /* checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; + @@@*/ checkout_opts.progress_cb = &checkout_progress; checkout_opts.progress_payload = (void *) &progress; @@ -161,13 +183,44 @@ GitObject::read (const std::string &url, const std::string &filter, const std::s int error = git_clone (&cloned_repo, url.c_str (), m_local_path.c_str (), &clone_opts); if (error != 0) { const git_error *err = git_error_last (); - throw tl::Exception (tl::to_string (tr ("Error cloning Git repo (%d): %s")), err->klass, (const char *) err->message); + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo: %s")), (const char *) err->message); + } + + if (! cloned_repo) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - no data available"))); } - if (cloned_repo) { - git_repository_free (cloned_repo); - // remove the worktree we don't need - tl::rm_dir_recursive (tl::combine_path (m_local_path, ".git")); + git_repository_free (cloned_repo); + + // remove the worktree as we don't need it + tl::rm_dir_recursive (tl::combine_path (m_local_path, ".git")); + + // pull subfolder files to target path level + if (! subdir.empty ()) { + + std::string pp = tl::combine_path (m_local_path, subdir); + if (! tl::is_dir (pp)) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to fetch subdirectory: ")) + pp); + } + + // rename the source to a temporary folder so we don't overwrite the source folder with something from within + std::string tmp_dir = "__temp__"; + for (unsigned int i = 0; ; ++i) { + if (! tl::file_exists (tl::combine_path (m_local_path, tmp_dir + tl::to_string (i)))) { + tmp_dir += tl::to_string (i); + break; + } + } + auto pc = tl::split (subdir, "/"); + if (! tl::rename_file (tl::combine_path (m_local_path, pc.front ()), tmp_dir)) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to rename temp folder"))); + } + pc.front () = tmp_dir; + + if (! tl::mv_dir_recursive (tl::combine_path (m_local_path, tl::join (pc, "/")), m_local_path)) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to move subdir components"))); + } + } } diff --git a/src/tl/unit_tests/tlFileUtilsTests.cc b/src/tl/unit_tests/tlFileUtilsTests.cc index f328b28b17..4814901c2e 100644 --- a/src/tl/unit_tests/tlFileUtilsTests.cc +++ b/src/tl/unit_tests/tlFileUtilsTests.cc @@ -285,6 +285,58 @@ TEST (3_NOQT) } } +TEST (4_mv_dir) +{ + std::string tmp_dir = tl::absolute_file_path (tmp_file ()); + std::string adir = tl::combine_path (tmp_dir, "a"); + + std::string b1dir = tl::combine_path (adir, "b1"); + tl::mkpath (b1dir); + EXPECT_EQ (tl::file_exists (b1dir), true); + + std::string b2dir = tl::combine_path (adir, "b2"); + tl::mkpath (b2dir); + EXPECT_EQ (tl::file_exists (b1dir), true); + + { + tl::OutputStream os (tl::absolute_file_path (tl::combine_path (b2dir, "x"))); + os << "hello, world!\n"; + } + + { + tl::OutputStream os (tl::absolute_file_path (tl::combine_path (b2dir, "y"))); + os << "hello, world II!\n"; + } + + std::string acopydir = tl::combine_path (tmp_dir, "acopy"); + tl::rm_dir_recursive (acopydir); + tl::mkpath (acopydir); + + tl::mv_dir_recursive (adir, acopydir); + + EXPECT_EQ (tl::file_exists (acopydir), true); + EXPECT_EQ (tl::file_exists (adir), false); + + std::string b1copydir = tl::combine_path (acopydir, "b1"); + EXPECT_EQ (tl::file_exists (b1copydir), true); + std::string b2copydir = tl::combine_path (acopydir, "b2"); + EXPECT_EQ (tl::file_exists (b2copydir), true); + + { + std::string xfile = tl::combine_path (b2copydir, "x"); + EXPECT_EQ (tl::file_exists (xfile), true); + tl::InputStream is (xfile); + EXPECT_EQ (is.read_all (), "hello, world!\n"); + } + + { + std::string yfile = tl::combine_path (b2copydir, "y"); + EXPECT_EQ (tl::file_exists (yfile), true); + tl::InputStream is (yfile); + EXPECT_EQ (is.read_all (), "hello, world II!\n"); + } +} + // Secret mode switchers for testing namespace tl { @@ -797,6 +849,13 @@ TEST (18) tl::InputStream is (zfile); EXPECT_EQ (is.read_all (), "hello, world!\n"); } + + // rename directory + tl::rename_file (tl::combine_path (tp, "dir"), "dirx"); + + EXPECT_EQ (tl::file_exists (tl::combine_path (tp, "dir")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (tp, "dirx")), true); + EXPECT_EQ (tl::is_dir (tl::combine_path (tp, "dirx")), true); } // get_home_path diff --git a/src/tl/unit_tests/tlGitTests.cc b/src/tl/unit_tests/tlGitTests.cc index e7e3c1544c..86839d7c19 100644 --- a/src/tl/unit_tests/tlGitTests.cc +++ b/src/tl/unit_tests/tlGitTests.cc @@ -24,35 +24,113 @@ #include "tlGit.h" #include "tlUnitTest.h" +#include "tlFileUtils.h" -// @@@ -static std::string test_url ("https://github.com/klayoutmatthias/xsection.git"); +static std::string test_url ("https://github.com/klayout/klayout_git_test.git"); -TEST(1) +TEST(1_plain) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); repo.read (test_url, std::string (), std::string ()); - printf("@@@ done: %s\n", path.c_str ()); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/macros/xsection.lym")), true); } -TEST(2) +TEST(2_pathspecs) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("src/**"), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/macros/xsection.lym")), true); +} + +TEST(3_subdir) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url + "/src", std::string (), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros/xsection.lym")), true); +} + +TEST(4_single_file) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); repo.read (test_url, std::string ("LICENSE"), std::string ()); - printf("@@@ done: %s\n", path.c_str ()); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src")), false); +} + +TEST(5_single_file_from_subdir) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url + "/src", std::string ("grain.xml"), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.7") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); } -TEST(3) +TEST(6_branch) { std::string path = tl::TestBase::tmp_file ("repo"); tl::GitObject repo (path); - repo.read (test_url, std::string (), std::string ("brxxx")); + repo.read (test_url + "/src", std::string ("grain.xml"), std::string ("wip")); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.4") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} - printf("@@@ done: %s\n", path.c_str ()); +TEST(7_invalid_branch) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + try { + repo.read (test_url, std::string (), std::string ("brxxx")); + EXPECT_EQ (true, false); + } catch (tl::Exception &ex) { + EXPECT_EQ (ex.msg (), "Error cloning Git repo: reference 'refs/remotes/origin/brxxx' not found"); + } } #endif