diff --git a/build.sh b/build.sh index 8d647deedb..e6d0d096f6 100755 --- a/build.sh +++ b/build.sh @@ -39,6 +39,7 @@ HAVE_QT=1 HAVE_PNG=0 HAVE_CURL=0 HAVE_EXPAT=0 +HAVE_GIT2=1 RUBYINCLUDE="" RUBYINCLUDE2="" @@ -209,6 +210,9 @@ while [ "$*" != "" ]; do -libexpat) HAVE_EXPAT=1 ;; + -nolibgit2) + HAVE_GIT2=0 + ;; -qt5) echo "*** WARNING: -qt5 option is ignored - Qt version is auto-detected now." ;; @@ -265,6 +269,7 @@ while [ "$*" != "" ]; do echo " -libcurl Use libcurl instead of QtNetwork (for Qt<4.7)" echo " -libexpat Use libexpat instead of QtXml" echo " -libpng Use libpng instead of Qt for PNG generation" + echo " -nolibgit2 Do not include libgit2 for Git package support" echo "" echo "Environment Variables:" echo "" @@ -495,6 +500,9 @@ fi if [ $HAVE_PNG != 0 ]; then echo " Uses libpng for PNG generation" fi +if [ $HAVE_GIT2 != 0 ]; then + echo " Uses libgit2 for Git access" +fi if [ "$RPATH" = "" ]; then RPATH="$BIN" fi @@ -578,6 +586,7 @@ echo " HAVE_64BIT_COORD=$HAVE_64BIT_COORD" echo " HAVE_CURL=$HAVE_CURL" echo " HAVE_PNG=$HAVE_PNG" echo " HAVE_EXPAT=$HAVE_EXPAT" +echo " HAVE_GIT2=$HAVE_GIT2" echo " RPATH=$RPATH" mkdir -p $BUILD @@ -650,6 +659,7 @@ qmake_options=( HAVE_CURL="$HAVE_CURL" HAVE_EXPAT="$HAVE_EXPAT" HAVE_PNG="$HAVE_PNG" + HAVE_GIT2="$HAVE_GIT2" PREFIX="$BIN" RPATH="$RPATH" KLAYOUT_VERSION="$KLAYOUT_VERSION" diff --git a/scripts/makedeb.sh b/scripts/makedeb.sh index 81fc8291d3..204266666f 100755 --- a/scripts/makedeb.sh +++ b/scripts/makedeb.sh @@ -19,16 +19,16 @@ fi # TODO: derive this list automatically? case $target in ubuntu16) - depends="libqt4-designer (>= 4.8.6), libqt4-xml (>= 4.8.6), libqt4-sql (>= 4.8.6), libqt4-network (>= 4.8.6), libqtcore4 (>= 4.8.6), libqtgui4 (>= 4.8.6), zlib1g (>= 1.2.8), libruby2.3 (>= 2.3.1), python3 (>= 3.5.1), libpython3.5 (>= 3.5.1), libstdc++6 (>= 4.6.3), libc6 (>= 2.15)" + depends="libqt4-designer (>= 4.8.6), libqt4-xml (>= 4.8.6), libqt4-sql (>= 4.8.6), libqt4-network (>= 4.8.6), libqtcore4 (>= 4.8.6), libqtgui4 (>= 4.8.6), zlib1g (>= 1.2.8), libgit2-24 (>= 0.24.0), libruby2.3 (>= 2.3.1), python3 (>= 3.5.1), libpython3.5 (>= 3.5.1), libstdc++6 (>= 4.6.3), libc6 (>= 2.15)" ;; ubuntu18) - depends="libqt4-designer (>= 4.8.7), libqt4-xml (>= 4.8.7), libqt4-sql (>= 4.8.7), libqt4-network (>= 4.8.7), libqtcore4 (>= 4.8.7), libqtgui4 (>= 4.8.7), zlib1g (>= 1.2.11), libruby2.5 (>= 2.5.1), python3 (>= 3.6.5), libpython3.6 (>= 3.6.5), libstdc++6 (>= 8), libc6 (>= 2.27)" + depends="libqt4-designer (>= 4.8.7), libqt4-xml (>= 4.8.7), libqt4-sql (>= 4.8.7), libqt4-network (>= 4.8.7), libqtcore4 (>= 4.8.7), libqtgui4 (>= 4.8.7), zlib1g (>= 1.2.11), libgit2-26 (>= 0.26.0), libruby2.5 (>= 2.5.1), python3 (>= 3.6.5), libpython3.6 (>= 3.6.5), libstdc++6 (>= 8), libc6 (>= 2.27)" ;; ubuntu20) - depends="libqt5core5a (>= 5.12.8), libqt5designer5 (>= 5.12.8), libqt5gui5 (>= 5.12.8), libqt5multimedia5 (>= 5.12.8), libqt5multimediawidgets5 (>= 5.12.8), libqt5network5 (>= 5.12.8), libqt5opengl5 (>= 5.12.8), libqt5printsupport5 (>= 5.12.8), libqt5sql5 (>= 5.12.8), libqt5svg5 (>= 5.12.8), libqt5widgets5 (>= 5.12.8), libqt5xml5 (>= 5.12.8), libqt5xmlpatterns5 (>= 5.12.8), zlib1g (>= 1.2.11), libruby2.7 (>= 2.7.0), python3 (>= 3.8.2), libpython3.8 (>= 3.8.2), libstdc++6 (>=10), libc6 (>= 2.31)" + depends="libqt5core5a (>= 5.12.8), libqt5designer5 (>= 5.12.8), libqt5gui5 (>= 5.12.8), libqt5multimedia5 (>= 5.12.8), libqt5multimediawidgets5 (>= 5.12.8), libqt5network5 (>= 5.12.8), libqt5opengl5 (>= 5.12.8), libqt5printsupport5 (>= 5.12.8), libqt5sql5 (>= 5.12.8), libqt5svg5 (>= 5.12.8), libqt5widgets5 (>= 5.12.8), libqt5xml5 (>= 5.12.8), libqt5xmlpatterns5 (>= 5.12.8), zlib1g (>= 1.2.11), libgit2-28 (>= 0.28.4), libruby2.7 (>= 2.7.0), python3 (>= 3.8.2), libpython3.8 (>= 3.8.2), libstdc++6 (>=10), libc6 (>= 2.31)" ;; ubuntu22) - depends="libqt5core5a (>= 5.15.3), libqt5designer5 (>= 5.15.3), libqt5gui5 (>= 5.15.3), libqt5multimedia5 (>= 5.15.3), libqt5multimediawidgets5 (>= 5.15.3), libqt5network5 (>= 5.15.3), libqt5opengl5 (>= 5.15.3), libqt5printsupport5 (>= 5.15.3), libqt5sql5 (>= 5.15.3), libqt5svg5 (>= 5.15.3), libqt5widgets5 (>= 5.15.3), libqt5xml5 (>= 5.15.3), libqt5xmlpatterns5 (>= 5.15.3), zlib1g (>= 1.2.11), libruby3.0 (>= 3.0.2), python3 (>= 3.10.4), libpython3.10 (>= 3.10.4), libstdc++6 (>=12), libc6 (>= 2.35)" + depends="libqt5core5a (>= 5.15.3), libqt5designer5 (>= 5.15.3), libqt5gui5 (>= 5.15.3), libqt5multimedia5 (>= 5.15.3), libqt5multimediawidgets5 (>= 5.15.3), libqt5network5 (>= 5.15.3), libqt5opengl5 (>= 5.15.3), libqt5printsupport5 (>= 5.15.3), libqt5sql5 (>= 5.15.3), libqt5svg5 (>= 5.15.3), libqt5widgets5 (>= 5.15.3), libqt5xml5 (>= 5.15.3), libqt5xmlpatterns5 (>= 5.15.3), zlib1g (>= 1.2.11), libgit2-1.1 (>= 1.1.0), libruby3.0 (>= 3.0.2), python3 (>= 3.10.4), libpython3.10 (>= 3.10.4), libstdc++6 (>=12), libc6 (>= 2.35)" ;; *) echo "Unknown target '$target' (given as first argument)" diff --git a/scripts/rpm-data/klayout.spec b/scripts/rpm-data/klayout.spec index 1ad64705f7..5a9bad6f24 100644 --- a/scripts/rpm-data/klayout.spec +++ b/scripts/rpm-data/klayout.spec @@ -53,6 +53,7 @@ Requires: qt5-qttools-devel >= 5.11.1 Requires: ruby >= 2.0.0 Requires: python3 >= 3.6.0 Requires: qt-x11 >= 4.8.5 +Requires: libgit2 >= 0.26.8 %define buildopt -j2 %endif @@ -62,7 +63,7 @@ Requires: libcurl >= 7.19.7 Requires: ruby >= 1.8.7 Requires: python >= 2.6.6 Requires: qt-x11 >= 4.6.2 -%define buildopt -libcurl -j2 +%define buildopt -libcurl -j2 -nolibgit2 %endif %if "%{target_system}" == "opensuse42_2" @@ -70,7 +71,7 @@ Requires: qt-x11 >= 4.6.2 Requires: ruby2.3 >= 2.3.1 Requires: python3 >= 3.4.6 Requires: libqt4-x11 >= 4.8.6 -%define buildopt -j2 +%define buildopt -j2 -nolibgit2 %endif %if "%{target_system}" == "opensuse42_3" @@ -78,13 +79,14 @@ Requires: libqt4-x11 >= 4.8.6 Requires: ruby2.3 >= 2.3.1 Requires: python3 >= 3.4.6 Requires: libqt4-x11 >= 4.8.6 -%define buildopt -j2 +%define buildopt -j2 -nolibgit2 %endif %if "%{target_system}" == "opensuse15" # OpenSuSE Leap 15 requirements Requires: ruby >= 2.5 Requires: python3 >= 3.6 +Requires: libgit2 >= 1.3.0 Requires: libqt5-qtbase >= 5.15.2 Requires: libQt5PrintSupport5 >= 5.15.2 Requires: libQt5Designer5 >= 5.15.2 diff --git a/setup.py b/setup.py index 744b465477..cc195adb58 100644 --- a/setup.py +++ b/setup.py @@ -422,15 +422,12 @@ def macros(self): macros = [ ("HAVE_CURL", 1), ("HAVE_EXPAT", 1), + ("HAVE_PNG", 1), ("KLAYOUT_MAJOR_VERSION", self.major_version()), ("KLAYOUT_MINOR_VERSION", self.minor_version()), ("GSI_ALIAS_INSPECT", 1), ] - if platform.system() == "Darwin" and check_libpng(): - macros += [("HAVE_PNG", 1)] - else: - macros += [("HAVE_PNG", 1)] return macros def minor_version(self): diff --git a/src/db/db/db.pro b/src/db/db/db.pro index 6914e12fa4..513cca3a01 100644 --- a/src/db/db/db.pro +++ b/src/db/db/db.pro @@ -404,7 +404,7 @@ HEADERS = \ dbShapeCollection.h \ dbShapeCollectionUtils.h -!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") { +!equals(HAVE_QT, "0") { RESOURCES = \ dbResources.qrc \ diff --git a/src/drc/drc/drc.pro b/src/drc/drc/drc.pro index c43e875d03..2304dc1327 100644 --- a/src/drc/drc/drc.pro +++ b/src/drc/drc/drc.pro @@ -13,7 +13,7 @@ HEADERS = \ drcCommon.h \ drcForceLink.h \ -!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") { +!equals(HAVE_QT, "0") { RESOURCES = \ drcResources.qrc } diff --git a/src/klayout.pri b/src/klayout.pri index 0cb25c0ab2..27a7051a05 100644 --- a/src/klayout.pri +++ b/src/klayout.pri @@ -103,6 +103,15 @@ equals(HAVE_PTHREADS, "1") { DEFINES += HAVE_PTHREADS } +equals(HAVE_GIT2, "1") { + !isEmpty(BITS_PATH) { + include($$BITS_PATH/git2/git2.pri) + } else { + LIBS += -lgit2 + } + DEFINES += HAVE_GIT2 +} + equals(HAVE_RUBY, "1") { !isEmpty(BITS_PATH) { include($$BITS_PATH/ruby/ruby.pri) diff --git a/src/lay/lay/lay.pro b/src/lay/lay/lay.pro index 3500502b89..23295c1f8f 100644 --- a/src/lay/lay/lay.pro +++ b/src/lay/lay/lay.pro @@ -33,6 +33,7 @@ HEADERS = \ layResourceHelpProvider.h \ layRuntimeErrorForm.h \ layReaderErrorForm.h \ + laySaltParsedURL.h \ laySearchReplaceConfigPage.h \ laySearchReplaceDialog.h \ laySearchReplacePropertiesWidgets.h \ @@ -144,6 +145,7 @@ SOURCES = \ layResourceHelpProvider.cc \ layRuntimeErrorForm.cc \ layReaderErrorForm.cc \ + laySaltParsedURL.cc \ laySearchReplaceConfigPage.cc \ laySearchReplaceDialog.cc \ laySearchReplacePlugin.cc \ diff --git a/src/lay/lay/layApplication.cc b/src/lay/lay/layApplication.cc index e7c54f7687..1b008fb7a6 100644 --- a/src/lay/lay/layApplication.cc +++ b/src/lay/lay/layApplication.cc @@ -770,7 +770,7 @@ ApplicationBase::init_app () size_t local_folders = (lay::get_appdata_path ().empty () ? 0 : 1); for (std::vector ::const_iterator p = m_klayout_path.begin (); p != m_klayout_path.end (); ++p) { - if (p - m_klayout_path.begin () < local_folders) { + if (size_t (p - m_klayout_path.begin ()) < local_folders) { mc->add_path (*p, tl::to_string (QObject::tr ("Local")), std::string (), false); } else if (m_klayout_path.size () == 1 + local_folders) { mc->add_path (*p, tl::to_string (QObject::tr ("Global")), std::string (), true); diff --git a/src/lay/lay/layInit.cc b/src/lay/lay/layInit.cc index 18e423c3ae..ec108ab79f 100644 --- a/src/lay/lay/layInit.cc +++ b/src/lay/lay/layInit.cc @@ -152,7 +152,7 @@ void init (const std::vector &_paths) try { s_plugins.push_back (do_load_plugin (imp)); modules.insert (*im); - } catch (tl::Exception (&ex)) { + } catch (tl::Exception &ex) { tl::error << ex.msg (); } } diff --git a/src/lay/lay/laySalt.cc b/src/lay/lay/laySalt.cc index d6fcd501f7..efb77844e1 100644 --- a/src/lay/lay/laySalt.cc +++ b/src/lay/lay/laySalt.cc @@ -21,11 +21,18 @@ */ #include "laySalt.h" +#include "laySaltParsedURL.h" + #include "tlString.h" #include "tlFileUtils.h" #include "tlLog.h" #include "tlInternational.h" #include "tlWebDAV.h" +#include "tlEnv.h" +#if defined(HAVE_GIT2) +# include "tlGit.h" +#endif + #include "lymMacro.h" #include @@ -63,6 +70,13 @@ Salt::root () return m_root; } +bool +Salt::download_package_information () const +{ + // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO + return tl::app_flag ("always-download-package-info") || m_root.sparse (); +} + Salt::flat_iterator Salt::begin_flat () { @@ -483,11 +497,27 @@ Salt::create_grain (const SaltGrain &templ, SaltGrain &target, double timeout, t } else if (! templ.url ().empty ()) { - if (templ.url ().find ("http:") == 0 || templ.url ().find ("https:") == 0) { + lay::SaltParsedURL purl (templ.url ()); + + if (purl.url ().find ("http:") == 0 || purl.url ().find ("https:") == 0) { - // otherwise download from the URL - tl::info << QObject::tr ("Downloading package from '%1' to '%2' ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); - res = tl::WebDAVObject::download (templ.url (), target.path (), timeout, callback); + // otherwise download from the URL using Git or SVN + + if (purl.protocol () == Git) { + +#if defined(HAVE_GIT2) + tl::info << QObject::tr ("Downloading package from '%1' to '%2' using Git protocol (ref='%3', subdir='%4') ..").arg (tl::to_qstring (purl.url ())).arg (tl::to_qstring (target.path ())).arg (tl::to_qstring (purl.branch ())).arg (tl::to_qstring (purl.subfolder ())); + res = tl::GitObject::download (purl.url (), target.path (), purl.subfolder (), purl.branch (), timeout, callback); +#else + throw tl::Exception (tl::to_string (QObject::tr ("Unable to install package '%1' - git protocol not compiled in").arg (tl::to_qstring (target.name ())))); +#endif + + } else if (purl.protocol () == WebDAV || purl.protocol () == DefaultProtocol) { + + tl::info << QObject::tr ("Downloading package from '%1' to '%2' using SVN/WebDAV protocol ..").arg (tl::to_qstring (purl.url ())).arg (tl::to_qstring (target.path ())); + res = tl::WebDAVObject::download (purl.url (), target.path (), timeout, callback); + + } } else { diff --git a/src/lay/lay/laySalt.h b/src/lay/lay/laySalt.h index 7970862a32..71c5a69d00 100644 --- a/src/lay/lay/laySalt.h +++ b/src/lay/lay/laySalt.h @@ -199,6 +199,11 @@ Q_OBJECT */ SaltGrains &root (); + /** + * @brief Gets a value indicating whether the collection wants package information to be downloaded always + */ + bool download_package_information () const; + signals: /** * @brief A signal triggered before one of the collections changed diff --git a/src/lay/lay/laySaltController.cc b/src/lay/lay/laySaltController.cc index 7c99e30db6..51b53ae663 100644 --- a/src/lay/lay/laySaltController.cc +++ b/src/lay/lay/laySaltController.cc @@ -23,6 +23,7 @@ #include "laySaltController.h" #include "laySaltManagerDialog.h" #include "laySaltDownloadManager.h" +#include "laySaltParsedURL.h" #include "layConfig.h" #include "layMainWindow.h" #include "layQtTools.h" @@ -142,7 +143,7 @@ SaltController::show_editor () { // while running the dialog, don't watch file events - that would interfere with // the changes applied by the dialog itself. - lay::BusySection busy_section; // disable file watcher + tl::FileSystemWatcherDisabled disable_file_watcher; // disable file watcher mp_salt_dialog->exec (); } @@ -156,7 +157,7 @@ SaltController::show_editor () void SaltController::sync_file_watcher () { - lay::BusySection busy_section; // disable file watcher + tl::FileSystemWatcherDisabled disable_file_watcher; // disable file watcher if (m_file_watcher) { m_file_watcher->clear (); @@ -202,12 +203,19 @@ SaltController::install_packages (const std::vector &packages, bool } } - if (n.find ("http:") == 0 || n.find ("https:") == 0 || n.find ("file:") == 0 || n[0] == '/' || n[0] == '\\') { + lay::SaltParsedURL purl (n); + const std::string &url = purl.url (); + + if (url.find ("http:") == 0 || url.find ("https:") == 0 || url.find ("file:") == 0 || url[0] == '/' || url[0] == '\\') { + // its a URL manager.register_download (std::string (), std::string (), n, v); + } else { + // its a plain name manager.register_download (n, std::string (), std::string (), v); + } } @@ -223,7 +231,7 @@ SaltController::install_packages (const std::vector &packages, bool { // while running the dialog, don't watch file events - that would interfere with // the changes applied by the dialog itself. - lay::BusySection busy_section; // disable file watcher + tl::FileSystemWatcherDisabled disable_file_watcher; // disable file watcher result = manager.execute (0, m_salt); } diff --git a/src/lay/lay/laySaltDownloadManager.cc b/src/lay/lay/laySaltDownloadManager.cc index e104e3531b..559b593336 100644 --- a/src/lay/lay/laySaltDownloadManager.cc +++ b/src/lay/lay/laySaltDownloadManager.cc @@ -27,6 +27,7 @@ #include "tlFileUtils.h" #include "tlWebDAV.h" #include "tlLog.h" +#include "tlEnv.h" #include #include @@ -310,42 +311,81 @@ SaltDownloadManager::fetch_missing (const lay::Salt &salt, const lay::Salt &salt ++progress; // Add URL and token from the package index + // + // In order to do so, we try to use the information from that package index as far as possible. + // Downloading a package definition from the original package URL may be expensive in case of + // large GIT repositories. + // + // Downloading is required if: + // - A package download is requested without a name (package can't be looked up in the package index) + // - Or a name is given, but not found in the package index + // + // Downloading can be bypassed if the package index (salt mine) specifies "sparse=false". + // In that case, the package index will have all information about the package. + if (! p->name.empty ()) { const lay::SaltGrain *g = salt_mine.grain_by_name (p->name); if (! g) { if (p->url.empty ()) { - throw tl::Exception (tl::to_string (QObject::tr ("Package '%1' not found in index - cannot resolve download URL").arg (tl::to_qstring (p->name)))); + throw tl::Exception (tl::to_string (tr ("Package '%s' not found in index - cannot resolve download URL")), p->name); } } else { if (p->url.empty ()) { if (tl::verbosity() >= 20) { - tl::log << "Resolved package URL for package " << p->name << ": " << g->url (); + tl::log << tr ("Resolved package URL for package") << " '" << p->name << "': " << g->url (); } p->url = g->url (); } p->token = g->token (); + p->grain = *g; + p->downloaded = true; } } - try { - p->grain = SaltGrain::from_url (p->url); - } catch (tl::Exception &ex) { - throw tl::Exception (tl::to_string (QObject::tr ("Error fetching spec file for package '%1': %2").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (ex.msg ())))); - } + if (! p->downloaded && salt_mine.download_package_information ()) { + + // If requested, download package information to complete information from index or dependencies + if (tl::verbosity() >= 10) { + tl::log << tl::sprintf (tl::to_string (tr ("Reading package description for package '%s' from: '%s")), p->name, p->url); + } + try { + p->grain = SaltGrain::from_url (p->url); + p->downloaded = true; + } catch (tl::Exception &ex) { + throw tl::Exception (tl::to_string (tr ("Error fetching spec file for package from '%s': %s")), p->url, ex.msg ()); + } - if (p->version.empty ()) { - p->version = p->grain.version (); } - p->name = p->grain.name (); - p->downloaded = true; + if (! p->downloaded) { + + if (p->name.empty ()) { + throw tl::Exception (tl::to_string (tr ("No name given package from '%s' (from dependencies or command line installation request)")), p->url); + } + + if (tl::verbosity() >= 10) { + tl::warn << tl::sprintf (tl::to_string (tr ("Package '%s' not downloaded from: %s. Dependencies may not be resolved.")), p->name, p->url); + } + + } else { + + if (p->version.empty ()) { + p->version = p->grain.version (); + } + if (p->name.empty ()) { + p->name = p->grain.name (); + } + + if (SaltGrain::compare_versions (p->grain.version (), p->version) < 0) { + throw tl::Exception (tl::to_string (tr ("Package '%s': package in repository is too old (%s) to satisfy requirements (%s)")), p->name, p->grain.version (), p->version); + } - if (SaltGrain::compare_versions (p->grain.version (), p->version) < 0) { - throw tl::Exception (tl::to_string (QObject::tr ("Package '%1': package in repository is too old (%2) to satisfy requirements (%3)").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (p->grain.version ())).arg (tl::to_qstring (p->version)))); } + p->downloaded = true; + } } diff --git a/src/lay/lay/laySaltGrain.cc b/src/lay/lay/laySaltGrain.cc index 932ceb3100..cd54e841f9 100644 --- a/src/lay/lay/laySaltGrain.cc +++ b/src/lay/lay/laySaltGrain.cc @@ -22,11 +22,15 @@ #include "laySaltGrain.h" #include "laySaltController.h" +#include "laySaltParsedURL.h" #include "tlString.h" #include "tlXMLParser.h" #include "tlHttpStream.h" -#include "tlWebDAV.h" #include "tlFileUtils.h" +#include "tlWebDAV.h" +#if defined(HAVE_GIT2) +# include "tlGit.h" +#endif #include #include @@ -65,7 +69,8 @@ SaltGrain::operator== (const SaltGrain &other) const m_license == other.m_license && m_hidden == other.m_hidden && m_authored_time == other.m_authored_time && - m_installed_time == other.m_installed_time; + m_installed_time == other.m_installed_time + ; } void @@ -253,18 +258,10 @@ SaltGrain::compare_versions (const std::string &v1, const std::string &v2) } } -std::string -SaltGrain::spec_url (const std::string &url) +const std::string & +SaltGrain::spec_file () { - std::string res = url; - if (! res.empty()) { - // TODO: use system path separator unless this is a URL - if (res [res.size () - 1] != '/') { - res += "/"; - } - res += grain_filename; - } - return res; + return grain_filename; } bool @@ -512,14 +509,21 @@ SaltGrain::from_path (const std::string &path) } tl::InputStream * -SaltGrain::stream_from_url (std::string &url, double timeout, tl::InputHttpStreamCallback *callback) +SaltGrain::stream_from_url (std::string &generic_url, double timeout, tl::InputHttpStreamCallback *callback) { - if (url.empty ()) { + if (generic_url.empty ()) { throw tl::Exception (tl::to_string (QObject::tr ("No download link available"))); } + if (tl::verbosity () >= 20) { + tl::info << tr ("Downloading package info from ") << generic_url; + } + + lay::SaltParsedURL purl (generic_url); + const std::string &url = purl.url (); + // base relative URL's on the salt mine URL - if (url.find ("http:") != 0 && url.find ("https:") != 0 && url.find ("file:") != 0 && !url.empty() && url[0] != '/' && url[0] != '\\' && lay::SaltController::instance ()) { + if (purl.protocol () == lay::DefaultProtocol && url.find ("http:") != 0 && url.find ("https:") != 0 && url.find ("file:") != 0 && !url.empty() && url[0] != '/' && url[0] != '\\' && lay::SaltController::instance ()) { // replace the last component ("repository.xml") by the given path QUrl sami_url (tl::to_qstring (lay::SaltController::instance ()->salt_mine_url ())); @@ -529,15 +533,27 @@ SaltGrain::stream_from_url (std::string &url, double timeout, tl::InputHttpStrea } sami_url.setPath (path_comp.join (QString::fromUtf8 ("/"))); - url = tl::to_string (sami_url.toString ()); + // return the full path as a file path, not an URL + generic_url = tl::to_string (sami_url.toString ()); } - std::string spec_url = SaltGrain::spec_url (url); - if (spec_url.find ("http:") == 0 || spec_url.find ("https:") == 0) { - return tl::WebDAVObject::download_item (spec_url, timeout, callback); + if (url.find ("http:") == 0 || url.find ("https:") == 0) { + + if (purl.protocol () == lay::Git) { +#if defined(HAVE_GIT2) + return tl::GitObject::download_item (url, SaltGrain::spec_file (), purl.subfolder (), purl.branch (), timeout, callback); +#else + throw tl::Exception (tl::to_string (QObject::tr ("Cannot download from Git - Git support not compiled in"))); +#endif + } else { + return tl::WebDAVObject::download_item (url + "/" + SaltGrain::spec_file (), timeout, callback); + } + } else { - return new tl::InputStream (spec_url); + + return new tl::InputStream (url); + } } diff --git a/src/lay/lay/laySaltGrain.h b/src/lay/lay/laySaltGrain.h index 92ef33e192..bf0f2ef90f 100644 --- a/src/lay/lay/laySaltGrain.h +++ b/src/lay/lay/laySaltGrain.h @@ -49,6 +49,9 @@ namespace lay */ struct SaltGrainDependency { + SaltGrainDependency () + { } + std::string name; std::string url; std::string version; @@ -466,6 +469,8 @@ class LAY_PUBLIC SaltGrain * This method will return a grain constructed from the downloaded data. * The data is read from "URL/grain.xml". This method will throw an * exception if an error occurs during reading. + * + * CAUTION: with GIT protocol and large repositories, this function may be very expensive. */ static SaltGrain from_url (const std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); @@ -474,13 +479,15 @@ class LAY_PUBLIC SaltGrain * The stream is a new'd object and needs to be deleted by the caller. * "url" is the download URL on input and gets modified to match the * actual URL if it is a relative one. + * + * CAUTION: with GIT protocol and large repositories, this function may be very expensive. */ static tl::InputStream *stream_from_url (std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); /** - * @brief Forms the spec file download URL from a given download URL + * @brief Gets the name of the spec file ("grain.xml") */ - static std::string spec_url (const std::string &url); + static const std::string &spec_file (); /** * @brief Returns a value indicating whether the given path represents is a grain diff --git a/src/lay/lay/laySaltGrainPropertiesDialog.cc b/src/lay/lay/laySaltGrainPropertiesDialog.cc index 7a3f098b9d..061eee1ca2 100644 --- a/src/lay/lay/laySaltGrainPropertiesDialog.cc +++ b/src/lay/lay/laySaltGrainPropertiesDialog.cc @@ -25,6 +25,7 @@ #include "tlString.h" #include "tlExceptions.h" #include "tlHttpStream.h" +#include "tlEnv.h" #include #include @@ -43,6 +44,14 @@ namespace lay // ---------------------------------------------------------------------------------------------------- +static bool download_package_information () +{ + // $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO + return tl::app_flag ("always-download-package-info"); +} + +// ---------------------------------------------------------------------------------------------------- + /** * @brief A delegate for editing a field of the dependency list */ @@ -583,7 +592,7 @@ SaltGrainPropertiesDialog::accept () << tr ("If the dependency package has a version itself, the version is automatically set to its current version."); } - if (!d->url.empty ()) { + if (!d->url.empty () && download_package_information ()) { SaltGrain gdep; try { gdep = SaltGrain::from_url (d->url); diff --git a/src/lay/lay/laySaltGrains.cc b/src/lay/lay/laySaltGrains.cc index bacb19d5fa..846d8052c1 100644 --- a/src/lay/lay/laySaltGrains.cc +++ b/src/lay/lay/laySaltGrains.cc @@ -34,6 +34,7 @@ namespace lay { SaltGrains::SaltGrains () + : m_sparse (true) { // .. nothing yet .. } @@ -54,6 +55,12 @@ SaltGrains::set_name (const std::string &n) m_name = n; } +void +SaltGrains::set_sparse (const bool &f) +{ + m_sparse = f; +} + void SaltGrains::set_title (const std::string &t) { @@ -302,6 +309,7 @@ SaltGrains::consolidate () static tl::XMLElementList s_group_struct = tl::make_member (&SaltGrains::name, &SaltGrains::set_name, "name") + + tl::make_member (&SaltGrains::sparse, &SaltGrains::set_sparse, "sparse") + tl::make_member (&SaltGrains::include, "include") + tl::make_element (&SaltGrains::begin_collections, &SaltGrains::end_collections, &SaltGrains::add_collection, "group", &s_group_struct) + tl::make_element (&SaltGrains::begin_grains, &SaltGrains::end_grains, &SaltGrains::add_grain, "salt-grain", SaltGrain::xml_elements ()); @@ -354,6 +362,9 @@ SaltGrains::include (const std::string &src_in) lay::SaltGrains g; g.load (src); + if (g.sparse ()) { + m_sparse = true; + } m_collections.splice (m_collections.end (), g.m_collections); m_grains.splice (m_grains.end (), g.m_grains); diff --git a/src/lay/lay/laySaltGrains.h b/src/lay/lay/laySaltGrains.h index a277e78638..c43e5fbafa 100644 --- a/src/lay/lay/laySaltGrains.h +++ b/src/lay/lay/laySaltGrains.h @@ -78,6 +78,24 @@ class LAY_PUBLIC SaltGrains */ void set_name (const std::string &p); + /** + * @brief Gets a value indicating that the information in the grain collection is sparse + * + * If this flag is set to true (the default), the information in the collection needs + * to be completed by pulling the original definition of the grain for the grain's URL. + * If the flag is false, the information is complete and reflects the grain's original + * definition. + */ + const bool &sparse () const + { + return m_sparse; + } + + /** + * @brief Sets a value indicating that the information in the grain collection is sparse + */ + void set_sparse (const bool &f); + /** * @brief Gets the title of the grain collection * @@ -225,6 +243,7 @@ class LAY_PUBLIC SaltGrains collections_type m_collections; grains_type m_grains; std::string m_url; + bool m_sparse; }; } diff --git a/src/lay/lay/laySaltManagerDialog.cc b/src/lay/lay/laySaltManagerDialog.cc index f439be7b80..4e86323bde 100644 --- a/src/lay/lay/laySaltManagerDialog.cc +++ b/src/lay/lay/laySaltManagerDialog.cc @@ -30,11 +30,13 @@ #include "ui_SaltGrainTemplateSelectionDialog.h" #include "tlString.h" #include "tlExceptions.h" +#include "tlEnv.h" #include "rba.h" #include "pya.h" #include +#include #include #include #include @@ -289,7 +291,9 @@ SaltManagerDialog::SaltManagerDialog (QWidget *parent, lay::Salt *salt, const st : QDialog (parent), m_salt_mine_url (salt_mine_url), dm_update_models (this, &SaltManagerDialog::update_models), m_current_tab (-1), - mp_downloaded_target (0) + mp_downloaded_target (0), + dm_mine_update_selected_changed (this, &SaltManagerDialog::do_mine_update_selected_changed), + dm_mine_new_selected_changed (this, &SaltManagerDialog::do_mine_new_selected_changed) { Ui::SaltManagerDialog::setupUi (this); mp_properties_dialog = new lay::SaltGrainPropertiesDialog (this); @@ -845,6 +849,8 @@ SaltManagerDialog::salt_mine_about_to_change () void SaltManagerDialog::refresh () { + m_salt_grain_cache.clear (); + if (! m_salt_mine_url.empty ()) { tl::log << tl::to_string (tr ("Downloading package repository from %1").arg (tl::to_qstring (m_salt_mine_url))); @@ -1094,8 +1100,12 @@ SaltManagerDialog::current_grains () void SaltManagerDialog::mine_update_selected_changed () { -BEGIN_PROTECTED + dm_mine_update_selected_changed (); +} +void +SaltManagerDialog::do_mine_update_selected_changed () +{ SaltModel *model = dynamic_cast (salt_mine_view_update->model ()); tl_assert (model != 0); @@ -1108,15 +1118,17 @@ BEGIN_PROTECTED details_update_frame->setEnabled (g != 0); get_remote_grain_info (g, details_update_text); - -END_PROTECTED } void SaltManagerDialog::mine_new_selected_changed () { -BEGIN_PROTECTED + dm_mine_new_selected_changed (); +} +void +SaltManagerDialog::do_mine_new_selected_changed () +{ SaltModel *model = dynamic_cast (salt_mine_view_new->model ()); tl_assert (model != 0); @@ -1129,8 +1141,73 @@ BEGIN_PROTECTED details_new_frame->setEnabled (g != 0); get_remote_grain_info (g, details_new_text); +} + +namespace +{ + +/** + * @brief A callback to keep the UI alive (mainly used for Git grain retrieval) + */ +class ProcessEventCallback + : public tl::InputHttpStreamCallback +{ +public: + virtual void wait_for_input () + { + QApplication::processEvents (QEventLoop::ExcludeUserInputEvents); + } +}; + +class FetchGrainInfoProgressAdaptor + : public tl::ProgressAdaptor +{ +public: + FetchGrainInfoProgressAdaptor (SaltGrainDetailsTextWidget *details, const std::string &name, const QString &html) + : mp_details (details), m_name (name), m_html (html) + { + mp_details->setHtml (m_html.arg (QString ())); + m_counter = 0; + } + + virtual void yield (tl::Progress *progress) + { + QCoreApplication::processEvents (QEventLoop::ExcludeUserInputEvents | QEventLoop::WaitForMoreEvents, 100); + + ++m_counter; + std::string all_dots = ".........."; + m_counter = m_counter % all_dots.size (); + std::string dots = std::string (all_dots, 0, m_counter); + mp_details->setHtml (m_html.arg (tl::to_qstring (tl::sprintf (tl::to_string (tr ("Downloading %.0f%% %s")), progress->value (), dots)))); + } + + virtual void trigger (tl::Progress * /*progress*/) + { + // .. nothing yet .. + } + + void error () + { + mp_details->setHtml (m_html.arg (QString ())); + } + + void success () + { + mp_details->setHtml (m_html.arg (QString ())); + } + + bool is_aborted () const + { + return false; + } + +private: + lay::SaltGrainDetailsTextWidget *mp_details; + std::string m_name; + QString m_html; + unsigned int m_counter; +}; -END_PROTECTED } void @@ -1141,68 +1218,104 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex return; } + tl_assert (m_downloaded_grain.get () == 0); + m_downloaded_grain.reset (0); + if (m_downloaded_grain_reader.get ()) { - // NOTE: don't delete the reader in the slot it triggered m_downloaded_grain_reader->close (); } + m_downloaded_grain_reader.reset (0); + mp_downloaded_target = details; m_salt_mine_grain.reset (new lay::SaltGrain (*g)); - // Download actual grain definition file - try { + if (m_salt_mine.download_package_information () && m_salt_mine.grain_by_name (g->name ())) { - if (g->url ().empty ()) { - throw tl::Exception (tl::to_string (tr ("No download link available"))); - } + // Download actual grain definition file + try { - QString html = tr ( - "" - "" - "" - "

Fetching Package Definition ...

" - "

URL: %1

" - "
" - "" - "" - ) - .arg (tl::to_qstring (SaltGrain::spec_url (g->url ()))); - - details->setHtml (html); - - std::string url = g->url (); - m_downloaded_grain.reset (new SaltGrain ()); - m_downloaded_grain->set_url (url); + if (g->url ().empty ()) { + throw tl::Exception (tl::to_string (tr ("No download link available"))); + } - // NOTE: stream_from_url may modify the URL, hence we set it again - m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url)); - m_downloaded_grain->set_url (url); + QString html = tr ( + "" + "" + "" + "

Fetching Package Definition ...

" + "

URL: %1

" + "

%2

" + "
" + "" + "" + ) + .arg (tl::to_qstring (g->url ())); + + details->setHtml (html.arg (QString ())); + + FetchGrainInfoProgressAdaptor pa (details, g->name (), html); + + std::string url = g->url (); + + auto sg = m_salt_grain_cache.find (url); + if (sg == m_salt_grain_cache.end ()) { + + m_downloaded_grain.reset (new SaltGrain ()); + m_downloaded_grain->set_url (url); + + // NOTE: stream_from_url may modify the URL, hence we set it again + ProcessEventCallback callback; + m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, 60.0, &callback)); + m_downloaded_grain->set_url (url); + + tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); + if (http) { + // async reading on HTTP + http->ready ().add (this, &SaltManagerDialog::data_ready); + http->send (); + } else { + data_ready (); + } - tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); - if (http) { - // async reading on HTTP - http->ready ().add (this, &SaltManagerDialog::data_ready); - http->send (); - } else { - data_ready (); + } else { + + m_downloaded_grain.reset (new SaltGrain (sg->second)); + data_ready (); + + } + + } catch (tl::Exception &ex) { + show_error (ex); } - } catch (tl::Exception &ex) { - show_error (ex); + } else { + + // Download denied - take information from index + m_downloaded_grain.reset (new SaltGrain (*g)); + data_ready (); + } } void SaltManagerDialog::data_ready () { - if (! m_salt_mine_grain.get () || ! m_downloaded_grain.get () || ! m_downloaded_grain_reader.get () || ! mp_downloaded_target) { + if (! m_salt_mine_grain.get () || ! m_downloaded_grain.get () || ! mp_downloaded_target) { return; } // Load the grain file (save URL as it is overwritten by the grain.xml content) std::string url = m_downloaded_grain->url (); - m_downloaded_grain->load (*m_downloaded_grain_reader); - m_downloaded_grain->set_url (url); + if (m_downloaded_grain_reader.get ()) { + m_downloaded_grain->load (*m_downloaded_grain_reader); + m_downloaded_grain->set_url (url); + } + + // commit to cache + if (m_salt_grain_cache.find (url) == m_salt_grain_cache.end ()) { + m_salt_grain_cache [url] = *m_downloaded_grain; + } try { @@ -1223,6 +1336,7 @@ SaltManagerDialog::data_ready () m_salt_mine_grain.reset (0); } catch (tl::Exception &ex) { + m_downloaded_grain.reset (0); show_error (ex); } } diff --git a/src/lay/lay/laySaltManagerDialog.h b/src/lay/lay/laySaltManagerDialog.h index a16662cd9a..dd899371b1 100644 --- a/src/lay/lay/laySaltManagerDialog.h +++ b/src/lay/lay/laySaltManagerDialog.h @@ -188,6 +188,9 @@ private slots: std::unique_ptr m_downloaded_grain, m_salt_mine_grain; SaltGrainDetailsTextWidget *mp_downloaded_target; std::unique_ptr m_salt_mine_reader; + tl::DeferredMethod dm_mine_update_selected_changed; + tl::DeferredMethod dm_mine_new_selected_changed; + std::map m_salt_grain_cache; SaltGrain *current_grain (); std::vector current_grains (); @@ -199,6 +202,8 @@ private slots: void show_error (tl::Exception &ex); void salt_mine_download_started (); void salt_mine_download_finished (); + void do_mine_update_selected_changed (); + void do_mine_new_selected_changed (); }; } diff --git a/src/lay/lay/laySaltParsedURL.cc b/src/lay/lay/laySaltParsedURL.cc new file mode 100644 index 0000000000..84e2c0850f --- /dev/null +++ b/src/lay/lay/laySaltParsedURL.cc @@ -0,0 +1,150 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "laySaltParsedURL.h" +#include "laySaltGrain.h" +#include "tlString.h" + +namespace lay +{ + +static void +parse_git_url (tl::Extractor &ex, std::string &url, std::string &branch, std::string &subfolder) +{ + const char *org_str = ex.get (); + + std::string w; + // protocol (http:) + ex.try_read_word (w, "") && ex.test (":"); + while (! ex.at_end () && ex.test ("/")) { + ; + } + // server ("www.klayout.de") + while (! ex.at_end () && (*ex != '/' && *ex != '+' && *ex != '[')) { + ++ex; + } + + while (! ex.at_end ()) { + + ++ex; + + // next component + const char *c1 = ex.get (); + while (! ex.at_end () && (*ex != '/' && *ex != '+' && *ex != '[')) { + ++ex; + } + const char *c2 = ex.get (); + + std::string comp (c1, c2 - c1); + + if ((! ex.at_end () && (*ex == '+' || *ex == '[')) || comp.find (".git") == comp.size () - 4) { + // subfolder starts here + break; + } + + } + + url = std::string (org_str, ex.get () - org_str); + + if (ex.at_end ()) { + return; + } + + // skip URL/subfolder separator + if (*ex == '/') { + while (! ex.at_end () && *ex == '/') { + ++ex; + } + } else if (*ex == '+') { + ++ex; + } + + // subfolders + { + const char *c1 = ex.get (); + while (! ex.at_end () && *ex != '[') { + ++ex; + } + const char *c2 = ex.get (); + subfolder = std::string (c1, c2 - c1); + } + + if (! ex.at_end () && *ex == '[') { + + // explicit branch + ++ex; + const char *c1 = ex.get (); + while (! ex.at_end () && *ex != ']') { + ++ex; + } + const char *c2 = ex.get (); + branch = std::string (c1, c2 - c1); + + } else if (! subfolder.empty ()) { + + // SVN emulation + + auto parts = tl::split (subfolder, "/"); + if (parts.size () >= 1 && parts.front () == "trunk") { + + branch = "HEAD"; + parts.erase (parts.begin ()); + subfolder = tl::join (parts, "/"); + + } else if (parts.size () >= 2 && parts.front () == "tags") { + + branch = "refs/tags/" + parts[1]; + parts.erase (parts.begin (), parts.begin () + 2); + subfolder = tl::join (parts, "/"); + + } else if (parts.size () >= 2 && parts.front () == "branches") { + + branch = "refs/heads/" + parts[1]; + parts.erase (parts.begin (), parts.begin () + 2); + subfolder = tl::join (parts, "/"); + + } + + } +} + +SaltParsedURL::SaltParsedURL (const std::string &url) + : m_protocol (lay::DefaultProtocol) +{ + tl::Extractor ex (url.c_str ()); + if (ex.test ("svn") && ex.test ("+")) { + m_protocol = lay::WebDAV; + m_url = ex.get (); + return; + } + + ex = tl::Extractor (url.c_str ()); + if (ex.test ("git") && ex.test ("+")) { + m_protocol = lay::Git; + parse_git_url (ex, m_url, m_branch, m_subfolder); + return; + } + + m_url = url; +} + +} diff --git a/src/lay/lay/laySaltParsedURL.h b/src/lay/lay/laySaltParsedURL.h new file mode 100644 index 0000000000..fd41bb42d2 --- /dev/null +++ b/src/lay/lay/laySaltParsedURL.h @@ -0,0 +1,115 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#ifndef HDR_laySaltParsedURL +#define HDR_laySaltParsedURL + +#include "layCommon.h" +#include "laySaltGrain.h" + +namespace lay +{ + +/** + * @brief An enum describing the protocol to use for download + */ +enum Protocol { + DefaultProtocol = 0, + WebDAV = 1, + Git = 2 +}; + +/** + * @brief A class representing a SaltGrain URL + * + * The URL is parsed into protocol, branch, URL and subfolder if applicable. + * Some heuristics is applied to decompose parts. + * + * SVN URLs: + * https://server.com/repo/trunk -> protocol=DefaultProtocol, url="https://server.com/repo/trunk", branch="", subfolder="" + * svn+https://server.com/repo/trunk -> protocol=WebDAV, url="https://server.com/repo/trunk", branch="", subfolder="" + * + * Git URL heuristics: + * git+https://server.com/repo.git -> protocol=Git, url="https://server.com/repo.git", branch="", subfolder="" + * git+https://server.com/repo.git/sub/folder -> protocol=Git, url="https://server.com/repo.git", branch="", subfolder="sub/folder" + * git+https://server.com/repo+sub/folder -> protocol=Git, url="https://server.com/repo", branch="", subfolder="sub/folder" + * git+https://server.com/repo.git[v1.0] -> protocol=Git, url="https://server.com/repo.git", branch="v1.0", subfolder="" + * git+https://server.com/repo.git/sub/folder[refs/tags/1.0] -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.0", subfolder="sub/folder" + * git+https://server.com/repo.git/trunk -> protocol=Git, url="https://server.com/repo.git", branch="HEAD", subfolder="" + * git+https://server.com/repo.git/trunk/sub/folder -> protocol=Git, url="https://server.com/repo.git", branch="HEAD", subfolder="sub/folder" + * git+https://server.com/repo.git/branches/release -> protocol=Git, url="https://server.com/repo.git", branch="refs/heads/release", subfolder="" + * git+https://server.com/repo.git/tags/1.9 -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.9", subfolder="" + * git+https://server.com/repo.git/tags/1.9/sub/folder -> protocol=Git, url="https://server.com/repo.git", branch="refs/tags/1.9", subfolder="sub/folder" + */ + +class LAY_PUBLIC SaltParsedURL +{ +public: + /** + * @brief Constructor: creates an URL from the given generic URL string + * + * This will decompose the URL into the parts and fill protocol, branch and subfolder fields. + */ + SaltParsedURL (const std::string &url); + + /** + * @brief Gets the basic URL + */ + const std::string &url () const + { + return m_url; + } + + /** + * @brief Gets the subfolder string + */ + const std::string &subfolder () const + { + return m_subfolder; + } + + /** + * @brief Gets the branch string + */ + const std::string &branch () const + { + return m_branch; + } + + /** + * @brief Gets the protocol + */ + lay::Protocol protocol () const + { + return m_protocol; + } + +private: + std::string m_url; + std::string m_branch; + std::string m_subfolder; + lay::Protocol m_protocol; +}; + +} + +#endif diff --git a/src/lay/unit_tests/laySaltParsedURLTests.cc b/src/lay/unit_tests/laySaltParsedURLTests.cc new file mode 100644 index 0000000000..b4274b3fdb --- /dev/null +++ b/src/lay/unit_tests/laySaltParsedURLTests.cc @@ -0,0 +1,141 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "laySaltParsedURL.h" +#include "tlUnitTest.h" + +TEST (1_Basic) +{ + lay::SaltParsedURL purl ("https://server.com/repo/trunk"); + EXPECT_EQ (purl.protocol () == lay::DefaultProtocol, true); + EXPECT_EQ (purl.url (), "https://server.com/repo/trunk"); + EXPECT_EQ (purl.branch (), ""); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (2_SVN) +{ + lay::SaltParsedURL purl ("svn+https://server.com/repo/trunk"); + EXPECT_EQ (purl.protocol () == lay::WebDAV, true); + EXPECT_EQ (purl.url (), "https://server.com/repo/trunk"); + EXPECT_EQ (purl.branch (), ""); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (10_GitBasic) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), ""); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (11_GitSubFolder) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/sub/folder"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), ""); + EXPECT_EQ (purl.subfolder (), "sub/folder"); +} + +TEST (12_GitExplicitBranch) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git[v1.0]"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "v1.0"); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (13_GitExplicitBranchAndSubFolder) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/sub/folder[refs/tags/1.0]"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "refs/tags/1.0"); + EXPECT_EQ (purl.subfolder (), "sub/folder"); +} + +TEST (14_GitExplicitBranchAndExplicitSubFolder) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo+sub/folder[refs/tags/1.0]"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo"); + EXPECT_EQ (purl.branch (), "refs/tags/1.0"); + EXPECT_EQ (purl.subfolder (), "sub/folder"); +} + +TEST (15_GitSVNEmulationTrunk) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/trunk"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "HEAD"); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (16_GitSVNEmulationTrunkWithSubFolder) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/trunk/sub/folder"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "HEAD"); + EXPECT_EQ (purl.subfolder (), "sub/folder"); +} + +TEST (17_GitSVNEmulationBranch) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/branches/xyz"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "refs/heads/xyz"); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (18_GitSVNEmulationTag) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/tags/1.9"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "refs/tags/1.9"); + EXPECT_EQ (purl.subfolder (), ""); +} + +TEST (19_GitSVNEmulationTagWithSubFolder) +{ + lay::SaltParsedURL purl ("git+https://server.com/repo.git/tags/1.9/sub/folder"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://server.com/repo.git"); + EXPECT_EQ (purl.branch (), "refs/tags/1.9"); + EXPECT_EQ (purl.subfolder (), "sub/folder"); +} + +TEST (20_Example1) +{ + lay::SaltParsedURL purl ("git+https://github.com/my-user/test-core[refs/tags/v1.1.0]"); + EXPECT_EQ (purl.protocol () == lay::Git, true); + EXPECT_EQ (purl.url (), "https://github.com/my-user/test-core"); + EXPECT_EQ (purl.branch (), "refs/tags/v1.1.0"); + EXPECT_EQ (purl.subfolder (), ""); +} diff --git a/src/lay/unit_tests/unit_tests.pro b/src/lay/unit_tests/unit_tests.pro index bd437bdbbd..adf9559669 100644 --- a/src/lay/unit_tests/unit_tests.pro +++ b/src/lay/unit_tests/unit_tests.pro @@ -9,6 +9,7 @@ include($$PWD/../../lib_ut.pri) SOURCES = \ laySalt.cc \ layHelpIndexTest.cc \ + laySaltParsedURLTests.cc \ laySessionTests.cc INCLUDEPATH += $$LAY_INC $$TL_INC $$LAYBASIC_INC $$LAYUI_INC $$LAYVIEW_INC $$DB_INC $$GSI_INC $$ANT_INC $$IMG_INC $$RDB_INC diff --git a/src/lib/lib/lib.pro b/src/lib/lib/lib.pro index 8352f19af0..98f84bd541 100644 --- a/src/lib/lib/lib.pro +++ b/src/lib/lib/lib.pro @@ -1,40 +1,42 @@ - -DESTDIR = $$OUT_PWD/../.. -TARGET = klayout_lib - -include($$PWD/../../lib.pri) - -DEFINES += MAKE_LIB_LIBRARY - -HEADERS = \ - libBasicArc.h \ - libBasicCircle.h \ - libBasicDonut.h \ - libBasicEllipse.h \ - libBasicPie.h \ - libBasicRoundPath.h \ - libBasicRoundPolygon.h \ - libBasicStrokedPolygon.h \ - libBasicText.h \ - libForceLink.h - -SOURCES = \ - libForceLink.cc \ - libBasic.cc \ - libBasicArc.cc \ - libBasicCircle.cc \ - libBasicDonut.cc \ - libBasicEllipse.cc \ - libBasicPie.cc \ - libBasicRoundPath.cc \ - libBasicRoundPolygon.cc \ - libBasicStrokedPolygon.cc \ - libBasicText.cc - -RESOURCES = \ - libResources.qrc - -INCLUDEPATH += $$TL_INC $$GSI_INC $$DB_INC -DEPENDPATH += $$TL_INC $$GSI_INC $$DB_INC -LIBS += -L$$DESTDIR -lklayout_gsi -lklayout_tl -lklayout_db - + +DESTDIR = $$OUT_PWD/../.. +TARGET = klayout_lib + +include($$PWD/../../lib.pri) + +DEFINES += MAKE_LIB_LIBRARY + +HEADERS = \ + libBasicArc.h \ + libBasicCircle.h \ + libBasicDonut.h \ + libBasicEllipse.h \ + libBasicPie.h \ + libBasicRoundPath.h \ + libBasicRoundPolygon.h \ + libBasicStrokedPolygon.h \ + libBasicText.h \ + libForceLink.h + +SOURCES = \ + libForceLink.cc \ + libBasic.cc \ + libBasicArc.cc \ + libBasicCircle.cc \ + libBasicDonut.cc \ + libBasicEllipse.cc \ + libBasicPie.cc \ + libBasicRoundPath.cc \ + libBasicRoundPolygon.cc \ + libBasicStrokedPolygon.cc \ + libBasicText.cc + +!equals(HAVE_QT, "0") { + RESOURCES = \ + libResources.qrc +} + +INCLUDEPATH += $$TL_INC $$GSI_INC $$DB_INC +DEPENDPATH += $$TL_INC $$GSI_INC $$DB_INC +LIBS += -L$$DESTDIR -lklayout_gsi -lklayout_tl -lklayout_db + diff --git a/src/lvs/lvs/lvs.pro b/src/lvs/lvs/lvs.pro index 5a8c53f27f..722270f843 100644 --- a/src/lvs/lvs/lvs.pro +++ b/src/lvs/lvs/lvs.pro @@ -13,7 +13,7 @@ HEADERS = \ lvsCommon.h \ lvsForceLink.h \ -!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") { +!equals(HAVE_QT, "0") { RESOURCES = \ lvsResources.qrc } diff --git a/src/rba/unit_tests/unit_tests.pro b/src/rba/unit_tests/unit_tests.pro index 6bca18f2fd..5924ccae6b 100644 --- a/src/rba/unit_tests/unit_tests.pro +++ b/src/rba/unit_tests/unit_tests.pro @@ -14,7 +14,7 @@ DEPENDPATH += $$RBA_INC $$TL_INC $$DB_INC $$GSI_INC LIBS += -L$$DESTDIR_UT -lklayout_rba -lklayout_tl -lklayout_db -lklayout_gsi -!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") { +!equals(HAVE_QT, "0") { RESOURCES = \ rba_unit_tests.qrc } diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index 558134601b..5a9f735622 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -121,7 +121,17 @@ HEADERS = \ tlUniqueName.h \ tlRecipe.h \ tlSelect.h \ - tlEnv.h + tlEnv.h + +equals(HAVE_GIT2, "1") { + + HEADERS += \ + tlGit.h + + SOURCES += \ + tlGit.cc + +} equals(HAVE_CURL, "1") { diff --git a/src/tl/tl/tlFileSystemWatcher.h b/src/tl/tl/tlFileSystemWatcher.h index f6190f2086..d7435c2615 100644 --- a/src/tl/tl/tlFileSystemWatcher.h +++ b/src/tl/tl/tlFileSystemWatcher.h @@ -140,6 +140,23 @@ private slots: std::map::iterator m_iter; }; +/** + * @brief A class employing RIIA for locking the file system watcher + */ +class TL_PUBLIC FileSystemWatcherDisabled +{ +public: + FileSystemWatcherDisabled () + { + tl::FileSystemWatcher::global_enable (false); + } + + ~FileSystemWatcherDisabled () + { + tl::FileSystemWatcher::global_enable (true); + } +}; + } #endif diff --git a/src/tl/tl/tlFileUtils.cc b/src/tl/tl/tlFileUtils.cc index e575dbb634..28c3741a39 100644 --- a/src/tl/tl/tlFileUtils.cc +++ b/src/tl/tl/tlFileUtils.cc @@ -27,6 +27,7 @@ #include "tlEnv.h" #include +#include // Use this define to print debug output // #define FILE_UTILS_VERBOSE @@ -584,6 +585,48 @@ cp_dir_recursive (const std::string &source, const std::string &target) return true; } +bool +mv_dir_recursive (const std::string &source, const std::string &target) +{ + std::vector entries; + std::string path = tl::absolute_file_path (source); + std::string path_to = tl::absolute_file_path (target); + + bool error = false; + + entries = dir_entries (path, false /*without_files*/, true /*with_dirs*/); + for (std::vector::const_iterator e = entries.begin (); e != entries.end (); ++e) { + std::string tc = tl::combine_path (path_to, *e); + if (! mkpath (tc)) { +#if defined(FILE_UTILS_VERBOSE) + tl::error << tr ("Unable to create target directory: ") << tc; +#endif + error = true; + } else if (! mv_dir_recursive (tl::combine_path (path, *e), tc)) { + error = true; + } + } + + entries = dir_entries (path, true /*with_files*/, false /*without_dirs*/); + for (std::vector::const_iterator e = entries.begin (); e != entries.end (); ++e) { + if (! tl::rename_file (tl::combine_path (path, *e), tl::combine_path (path_to, *e))) { +#if defined(FILE_UTILS_VERBOSE) + tl::error << tr ("Unable to move file from ") << tl::combine_path (path, *e) << tr (" to ") << tl::combine_path (path_to, *e); +#endif + error = true; + } + } + + if (! tl::rm_dir (path)) { +#if defined(FILE_UTILS_VERBOSE) + tl::error << tr ("Unable to remove folder ") << path; +#endif + error = true; + } + + return ! error; +} + std::string absolute_path (const std::string &s) { std::vector parts = split_path (absolute_file_path (s)); @@ -976,4 +1019,108 @@ get_module_path (void *addr) #endif } +std::string +tmpfile (const std::string &domain) +{ + std::string tmp = tl::get_env ("TMPDIR"); + if (tmp.empty ()) { + tmp = tl::get_env ("TMP"); + } + if (tmp.empty ()) { +#if defined(_WIN32) + throw tl::Exception (tl::to_string (tr ("TMP and TMPDIR not set - cannot create temporary file"))); +#else + tmp = "/tmp"; +#endif + } + + std::string templ = tl::combine_path (tmp, domain + "XXXXXX"); + char *tmpstr = strdup (templ.c_str ()); + +#if defined(_WIN32) + if (_mktemp_s (tmpstr, templ.size () + 1) != 0) { + free (tmpstr); + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder name in %s")), tmp); + } + + // for compatibility with Linux, create the file as an empty one + std::ofstream os (tmpstr); + if (os.bad ()) { + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp); + } + os.close (); +#else + int fd = mkstemp (tmpstr); + if (fd < 0) { + free (tmpstr); + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp); + } + close (fd); +#endif + + std::string res = tmpstr; + free (tmpstr); + return res; +} + +TemporaryFile::TemporaryFile (const std::string &domain) +{ + m_path = tmpfile (domain); +} + +TemporaryFile::~TemporaryFile () +{ + tl::rm_file (m_path); +} + +std::string +tmpdir (const std::string &domain) +{ + std::string tmp = tl::get_env ("TMPDIR"); + if (tmp.empty ()) { + tmp = tl::get_env ("TMP"); + } + if (tmp.empty ()) { +#if defined(_WIN32) + throw tl::Exception (tl::to_string (tr ("TMP and TMPDIR not set - cannot create temporary file"))); +#else + tmp = "/tmp"; +#endif + } + + std::string templ = tl::combine_path (tmp, domain + "XXXXXX"); + char *tmpstr = strdup (templ.c_str ()); + +#if defined(_WIN32) + if (_mktemp_s (tmpstr, templ.size () + 1) != 0) { + free (tmpstr); + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder name in %s")), tmp); + } + if (! tl::mkdir (tmpstr)) { + free (tmpstr); + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp); + } +#else + if (mkdtemp (tmpstr) == NULL) { + free (tmpstr); + throw tl::Exception (tl::to_string (tr ("Unable to create temporary folder in %s")), tmp); + } +#endif + + std::string res = tmpstr; + free (tmpstr); + return res; +} + +TemporaryDirectory::TemporaryDirectory (const std::string &domain) +{ + m_path = tmpdir (domain); +} + +TemporaryDirectory::~TemporaryDirectory () +{ + tl::rm_dir_recursive (m_path); +} + + } diff --git a/src/tl/tl/tlFileUtils.h b/src/tl/tl/tlFileUtils.h index f192599ff5..474ec7e29d 100644 --- a/src/tl/tl/tlFileUtils.h +++ b/src/tl/tl/tlFileUtils.h @@ -52,11 +52,18 @@ bool TL_PUBLIC rm_dir_recursive (const std::string &path); bool TL_PUBLIC mkpath (const std::string &path); /** - * @brief Recursively remove the given directory, the files from that directory and all sub-directories (version with std::string) + * @brief Recursively copy the given directory * @return True, if successful. false otherwise. */ bool TL_PUBLIC cp_dir_recursive (const std::string &source, const std::string &target); +/** + * @brief Recursively move the contents of the given directory + * @return True, if successful. false otherwise. + * After this operation, the source directory is deleted. + */ +bool TL_PUBLIC mv_dir_recursive (const std::string &source, const std::string &target); + /** * @brief Gets the absolute path for a given file path * This will deliver the directory of the file as absolute path. @@ -181,6 +188,68 @@ std::string TL_PUBLIC current_dir (); */ bool TL_PUBLIC chdir (const std::string &path); +/** + * @brief Gets a temporary file path + * + * This function will make a temporary file with a unique name. + * The "domain" string is used as part of the file name as an disambiguator. + * + * The function reads $TMPDIR or $TMP to define the location of the temporary + * directory. On Linux, the default is /tmp. + * + * The file is created and it is the responsibility of the caller to remove + * the file. + */ +std::string TL_PUBLIC tmpfile (const std::string &domain = std::string ()); + +/** + * @brief A class wrapping a temporary file + * + * In the destructor of this class, the temporary file will be deleted again. + */ +class TL_PUBLIC TemporaryFile +{ +public: + TemporaryFile (const std::string &domain = std::string ()); + ~TemporaryFile (); + + const std::string &path () const + { + return m_path; + } + +private: + std::string m_path; +}; + +/** + * @brief Gets a temporary folder path + * + * Similar to "tmpfile", but will create a new, empty folder. Again it is the + * reposibility of the caller to clean up. + */ +std::string TL_PUBLIC tmpdir (const std::string &domain = std::string ()); + +/** + * @brief A class wrapping a temporary directory + * + * In the destructor of this class, the temporary directory will be deleted again. + */ +class TL_PUBLIC TemporaryDirectory +{ +public: + TemporaryDirectory (const std::string &domain = std::string ()); + ~TemporaryDirectory (); + + const std::string &path () const + { + return m_path; + } + +private: + std::string m_path; +}; + /** * @brief This function splits the path into it's components * On Windows, the first component may be the drive prefix ("C:") or diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc new file mode 100644 index 0000000000..6810b07ea6 --- /dev/null +++ b/src/tl/tl/tlGit.cc @@ -0,0 +1,421 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#if defined(HAVE_GIT2) + +#include "tlGit.h" +#include "tlFileUtils.h" +#include "tlProgress.h" +#include "tlStaticObjects.h" +#include "tlLog.h" + +#include +#include + +namespace tl +{ + +// --------------------------------------------------------------- +// Library initialization helper + +namespace { + +class InitHelper +{ +public: + InitHelper () + { + git_libgit2_init (); + } + + ~InitHelper () + { + git_libgit2_shutdown (); + } + + static void ensure_initialized () + { + if (ms_init) { + return; + } + + ms_init = new InitHelper (); + tl::StaticObjects::reg (&ms_init); + } + +private: + static InitHelper *ms_init; +}; + +InitHelper *InitHelper::ms_init = 0; + +} + +// --------------------------------------------------------------- +// GitCollection implementation + +GitObject::GitObject (const std::string &local_path) + : m_local_path (local_path), m_is_temp (false) +{ + InitHelper::ensure_initialized (); + + if (local_path.empty ()) { + m_local_path = tl::tmpdir ("git2klayout"); + m_is_temp = true; + } + + // ensures the directory is clean + if (! m_is_temp) { + if (! tl::rm_dir_recursive (m_local_path)) { + throw tl::Exception (tl::to_string (tr ("Unable to clean local Git repo path: %s")), m_local_path); + } + if (! tl::mkpath (m_local_path)) { + throw tl::Exception (tl::to_string (tr ("Unable to regenerate local Git repo path: %s")), m_local_path); + } + } +} + +GitObject::~GitObject () +{ + if (m_is_temp) { + tl::rm_dir_recursive (m_local_path); + } +} + +static int +#if LIBGIT2_VER_MAJOR >= 1 +fetch_progress (const git_indexer_progress *stats, void *payload) +#else +fetch_progress (const git_transfer_progress *stats, void *payload) +#endif +{ + tl::RelativeProgress *progress = reinterpret_cast (payload); + + // first half of progress + size_t count = size_t (5000.0 * double (stats->received_objects) / double (std::max (1u, stats->total_objects)) + 1e-10); + try { + progress->set (count); + } catch (...) { + // TODO: stop + } + + return 0; +} + +static void +checkout_progress(const char * /*path*/, size_t cur, size_t tot, void *payload) +{ + tl::RelativeProgress *progress = reinterpret_cast (payload); + + // first half of progress + size_t count = size_t (5000.0 * double (cur) / double (std::max (size_t (1), tot)) + 1e-10); + try { + progress->set (count + 5000u); + } catch (...) { + // ignore cancel requests (TODO: how to stop?) + } +} + +static void check (int error) +{ + if (error != 0) { +#if LIBGIT2_VER_MAJOR > 0 || (LIBGIT2_VER_MAJOR == 0 && LIBGIT2_VER_MINOR >= 28) + const git_error *err = git_error_last (); +#else + const git_error *err = giterr_last (); +#endif + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo: %s")), (const char *) err->message); + } +} + +static bool +ref_matches (const char *name, const std::string &ref) +{ + if (!name) { + return false; + } else if (name == ref) { + return true; + } else if (name == "refs/heads/" + ref) { + return true; + } else if (name == "refs/tags/" + ref) { + return true; + } else { + return false; + } +} + +namespace +{ + +class GitBuffer +{ +public: + GitBuffer () + { + m_buf = GIT_BUF_INIT_CONST (NULL, 0); + } + + ~GitBuffer () + { +#if LIBGIT2_VER_MAJOR > 0 || (LIBGIT2_VER_MAJOR == 0 && LIBGIT2_VER_MINOR >= 28) + git_buf_dispose (&m_buf); +#else + git_buf_free (&m_buf); +#endif + } + + const char *c_str () const { return m_buf.ptr; } + + git_buf *get () { return &m_buf; } + const git_buf *get () const { return &m_buf; } + +private: + git_buf m_buf; +}; + +} + +static void +checkout_branch (git_repository *repo, git_remote *remote, const git_checkout_options *co_opts, const char *branch) +{ + GitBuffer remote_branch; + git_oid oid; + + // if no branch is given, use the default branch + if (! branch) { + check (git_remote_default_branch (remote_branch.get (), remote)); + branch = remote_branch.c_str (); + if (tl::verbosity () >= 10) { + tl::info << tr ("Git checkout: Using default branch for repository ") << git_remote_url (remote) << ": " << branch; + } + } else { + if (tl::verbosity () >= 10) { + tl::info << tr ("Git checkout: Checking out branch for repository ") << git_remote_url (remote) << ": " << branch; + } + } + + // resolve the branch by using ls-remote: + + size_t n = 0; + const git_remote_head **ls = NULL; + check (git_remote_ls (&ls, &n, remote)); + + if (tl::verbosity () >= 20) { + tl::info << "Git checkout: ls-remote on " << git_remote_url (remote) << ":"; + } + + bool found = false; + + for (size_t i = 0; i < n; ++i) { + const git_remote_head *rh = ls[i]; + if (tl::verbosity () >= 20) { + char oid_fmt [80]; + git_oid_tostr (oid_fmt, sizeof (oid_fmt), &rh->oid); + tl::info << " " << rh->name << ": " << (const char *) oid_fmt; + } + if (ref_matches (rh->name, branch)) { + oid = rh->oid; + found = true; + } + } + + if (! found) { + throw tl::Exception (tl::to_string (tr ("Git checkout - Unable to resolve reference name: ")) + branch); + } + + if (tl::verbosity () >= 10) { + char oid_fmt [80]; + git_oid_tostr (oid_fmt, sizeof (oid_fmt), &oid); + tl::info << tr ("Git checkout: resolving ") << branch << tr (" to ") << (const char *) oid_fmt; + } + + check (git_repository_set_head_detached (repo, &oid)); + check (git_checkout_head (repo, co_opts)); +} + +int +credentials_cb (git_credential ** /*out*/, const char * /*url*/, const char * /*username*/, unsigned int /*allowed_types*/, void *) +{ + // no credentials aquired + git_error_set_str (GIT_ERROR_NONE, "anonymous access is supported only, but server requests credentials"); + return GIT_EUSER; +} + +void +GitObject::read (const std::string &org_url, const std::string &org_filter, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +{ + std::string url = org_url; + + std::string filter = org_filter; + if (! subfolder.empty ()) { + if (filter.empty ()) { + filter = subfolder + "/**"; + } else { + filter = subfolder + "/" + filter; + } + } + + // TODO: use callback, timeout? + tl::RelativeProgress progress (tl::to_string (tr ("Download progress")), 10000, 1 /*yield always*/); + + // build checkout options + + git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; + checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE; + + const char *paths_cstr[1]; + paths_cstr[0] = filter.c_str (); + if (! filter.empty ()) { + checkout_opts.paths.count = 1; + checkout_opts.paths.strings = (char **) &paths_cstr; + } + + checkout_opts.progress_cb = &checkout_progress; + checkout_opts.progress_payload = (void *) &progress; + + // build fetch options + + git_fetch_options fetch_opts = GIT_FETCH_OPTIONS_INIT; + + fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_AUTO; + +#if LIBGIT2_VER_MAJOR > 1 || (LIBGIT2_VER_MAJOR == 1 && LIBGIT2_VER_MINOR >= 7) + fetch_opts.depth = 1; // shallow (single commit) +#endif + fetch_opts.callbacks.transfer_progress = &fetch_progress; + fetch_opts.callbacks.credentials = &credentials_cb; + fetch_opts.callbacks.payload = (void *) &progress; + + // build refspecs in case they are needed + + char *refs[] = { (char *) branch.c_str () }; + git_strarray refspecs; + refspecs.count = 1; + refspecs.strings = refs; + git_strarray *refspecs_p = branch.empty () ? NULL : &refspecs; + + // Make repository + + git_repository *cloned_repo = NULL; + git_remote *remote = NULL; + + try { + + check (git_repository_init (&cloned_repo, m_local_path.c_str (), 0)); + + check (git_remote_create (&remote, cloned_repo, "download", url.c_str ())); + + // actually fetch + if (tl::verbosity () >= 10) { + tl::info << tr ("Fetching Git repo from ") << git_remote_url (remote) << " ..."; + } + check (git_remote_fetch (remote, refspecs_p, &fetch_opts, NULL)); + + // checkout + checkout_branch (cloned_repo, remote, &checkout_opts, branch.empty () ? 0 : branch.c_str ()); + + // free the repo and remote + git_repository_free (cloned_repo); + git_remote_free (remote); + + // get rid of ".git" - we do not need it anymore + + tl::rm_dir_recursive (tl::combine_path (m_local_path, ".git")); + + } catch (...) { + // free the repo in the error case + if (cloned_repo != NULL) { + git_repository_free (cloned_repo); + } + if (remote != NULL) { + git_remote_free (remote); + } + throw; + } + + // pull subfolder files to target path level + if (! subfolder.empty ()) { + + std::string pp = tl::combine_path (m_local_path, subfolder); + if (! tl::is_dir (pp)) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to fetch subdirectory: ")) + pp); + } + + // rename the source to a temporary folder so we don't overwrite the source folder with something from within + std::string tmp_dir = "__temp__"; + for (unsigned int i = 0; ; ++i) { + if (! tl::file_exists (tl::combine_path (m_local_path, tmp_dir + tl::to_string (i)))) { + tmp_dir += tl::to_string (i); + break; + } + } + auto pc = tl::split (subfolder, "/"); + if (! tl::rename_file (tl::combine_path (m_local_path, pc.front ()), tmp_dir)) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to rename temp folder"))); + } + pc.front () = tmp_dir; + + if (! tl::mv_dir_recursive (tl::combine_path (m_local_path, tl::join (pc, "/")), m_local_path)) { + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo - failed to move subdir components"))); + } + + } +} + +bool +GitObject::download (const std::string &url, const std::string &target, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +{ + try { + + GitObject obj (target); + obj.read (url, std::string (), subfolder, branch, timeout, callback); + + return true; + + } catch (tl::Exception &ex) { + + tl::error << tl::sprintf (tl::to_string (tr ("Error downloading Git repo from %s (subdir '%s', ref '%s')")), url, subfolder, branch); + + return false; + + } +} + +tl::InputStream * +GitObject::download_item (const std::string &url, const std::string &file, const std::string &subfolder, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +{ + GitObject obj; + obj.read (url, file, subfolder, branch, timeout, callback); + + // extract the file and return a memory blob, so we can delete the temp folder + + tl::InputStream file_stream (tl::combine_path (obj.local_path (), file)); + std::string data = file_stream.read_all (); + + char *data_copy = new char [data.size ()]; + memcpy (data_copy, data.c_str (), data.size ()); + return new tl::InputStream (new tl::InputMemoryStream (data_copy, data.size (), true)); +} + +} + +#endif diff --git a/src/tl/tl/tlGit.h b/src/tl/tl/tlGit.h new file mode 100644 index 0000000000..a99d304499 --- /dev/null +++ b/src/tl/tl/tlGit.h @@ -0,0 +1,111 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_tlGit +#define HDR_tlGit + +#include "tlCommon.h" +#include "tlStream.h" + +#include +#include + +#if !defined(HAVE_GIT2) +# error "tlGit.h can only be used with libgit2 enabled" +#endif + +namespace tl +{ + +class InputHttpStreamCallback; + +/** + * @brief Represents an object from a Git URL + * This object can be a file or collection + */ +class TL_PUBLIC GitObject +{ +public: + /** + * @brief Open a stream with the given URL + * + * The local_path is the path where to store the files. + * If empty, a temporary folder is created and destroyed once the "GitObject" goes out of scope. + */ + GitObject (const std::string &local_path = std::string ()); + + /** + * @brief Destructor + */ + ~GitObject (); + + /** + * @brief Populates the collection from the given URL + * + * "filter" can be a top-level file to download. If filter is non-empty, + * sparse mode is chosen. + */ + void read (const std::string &url, const std::string &filter, const std::string &subfolder, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + + /** + * @brief Downloads the collection or file with the given URL + * + * This method will download the Git object from url to the file path + * given in "target". + * + * For file download, the target must be the path of the target file. + * For collection download, the target must be a directory path. In this + * case, the target directory must exist already. + * + * Sub-directories are created if required. + * + * This method returns false if the directory structure could + * not be obtained or downloading of one file failed. + * + * "branch" is the remote ref to use. This can be a branch name, a tag name, + * a remote ref such as "refs/heads/master" or a symbolic name such as "HEAD". + */ + static bool download (const std::string &url, const std::string &target, const std::string &subfolder, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + + /** + * @brief Gets a stream object for downloading the single item of the given URL + * + * The file needs to be a top-level object. + * The stream object returned needs to be deleted by the caller. + */ + static tl::InputStream *download_item (const std::string &url, const std::string &file, const std::string &subfolder, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + +private: + std::string m_local_path; + bool m_is_temp; + + const std::string &local_path () const + { + return m_local_path; + } +}; + +} + +#endif + diff --git a/src/tl/tl/tlWebDAV.h b/src/tl/tl/tlWebDAV.h index 5b676a2a7c..7a63f6a367 100644 --- a/src/tl/tl/tlWebDAV.h +++ b/src/tl/tl/tlWebDAV.h @@ -145,7 +145,7 @@ class TL_PUBLIC WebDAVObject * * Sub-directories are created if required. * - * This method throws an exception if the directory structure could + * This method returns false if the directory structure could * not be obtained or downloading of one file failed. */ static bool download (const std::string &url, const std::string &target, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); diff --git a/src/tl/unit_tests/tlFileUtilsTests.cc b/src/tl/unit_tests/tlFileUtilsTests.cc index f328b28b17..1397268bd4 100644 --- a/src/tl/unit_tests/tlFileUtilsTests.cc +++ b/src/tl/unit_tests/tlFileUtilsTests.cc @@ -285,6 +285,58 @@ TEST (3_NOQT) } } +TEST (4_mv_dir) +{ + std::string tmp_dir = tl::absolute_file_path (tmp_file ()); + std::string adir = tl::combine_path (tmp_dir, "a"); + + std::string b1dir = tl::combine_path (adir, "b1"); + tl::mkpath (b1dir); + EXPECT_EQ (tl::file_exists (b1dir), true); + + std::string b2dir = tl::combine_path (adir, "b2"); + tl::mkpath (b2dir); + EXPECT_EQ (tl::file_exists (b1dir), true); + + { + tl::OutputStream os (tl::absolute_file_path (tl::combine_path (b2dir, "x"))); + os << "hello, world!\n"; + } + + { + tl::OutputStream os (tl::absolute_file_path (tl::combine_path (b2dir, "y"))); + os << "hello, world II!\n"; + } + + std::string acopydir = tl::combine_path (tmp_dir, "acopy"); + tl::rm_dir_recursive (acopydir); + tl::mkpath (acopydir); + + tl::mv_dir_recursive (adir, acopydir); + + EXPECT_EQ (tl::file_exists (acopydir), true); + EXPECT_EQ (tl::file_exists (adir), false); + + std::string b1copydir = tl::combine_path (acopydir, "b1"); + EXPECT_EQ (tl::file_exists (b1copydir), true); + std::string b2copydir = tl::combine_path (acopydir, "b2"); + EXPECT_EQ (tl::file_exists (b2copydir), true); + + { + std::string xfile = tl::combine_path (b2copydir, "x"); + EXPECT_EQ (tl::file_exists (xfile), true); + tl::InputStream is (xfile); + EXPECT_EQ (is.read_all (), "hello, world!\n"); + } + + { + std::string yfile = tl::combine_path (b2copydir, "y"); + EXPECT_EQ (tl::file_exists (yfile), true); + tl::InputStream is (yfile); + EXPECT_EQ (is.read_all (), "hello, world II!\n"); + } +} + // Secret mode switchers for testing namespace tl { @@ -797,6 +849,13 @@ TEST (18) tl::InputStream is (zfile); EXPECT_EQ (is.read_all (), "hello, world!\n"); } + + // rename directory + tl::rename_file (tl::combine_path (tp, "dir"), "dirx"); + + EXPECT_EQ (tl::file_exists (tl::combine_path (tp, "dir")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (tp, "dirx")), true); + EXPECT_EQ (tl::is_dir (tl::combine_path (tp, "dirx")), true); } // get_home_path @@ -816,3 +875,87 @@ TEST (20) EXPECT_EQ (tl::absolute_file_path ("~"), tl::get_home_path ()); EXPECT_EQ (tl::absolute_file_path (tl::combine_path ("~", "test")), tl::combine_path (tl::get_home_path (), "test")); } + +// tmpfile +TEST (21) +{ + std::string p = tl::tmpfile ("tl_tests"); + EXPECT_EQ (tl::file_exists (p), true); + + std::ofstream os (p); + os << "A test"; + os.close (); + + { + tl::InputStream is (p); + EXPECT_EQ (is.read_all (), "A test"); + } + + EXPECT_EQ (tl::rm_file (p), true); + EXPECT_EQ (tl::file_exists (p), false); +} + +// TemporaryFile +TEST (22) +{ + std::string p; + + { + tl::TemporaryFile tf ("tl_tests"); + p = tf.path (); + EXPECT_EQ (tl::file_exists (tf.path ()), true); + + std::ofstream os (tf.path ()); + os << "A test"; + os.close (); + + tl::InputStream is (tf.path ()); + EXPECT_EQ (is.read_all (), "A test"); + } + + EXPECT_EQ (tl::file_exists (p), false); +} + +// tmpdir +TEST (23) +{ + std::string p = tl::tmpdir ("tl_tests"); + EXPECT_EQ (tl::file_exists (p), true); + EXPECT_EQ (tl::is_dir (p), true); + + std::ofstream os (tl::combine_path (p, "test")); + os << "A test"; + os.close (); + + { + tl::InputStream is (tl::combine_path (p, "test")); + EXPECT_EQ (is.read_all (), "A test"); + } + + EXPECT_EQ (tl::rm_dir_recursive (p), true); + EXPECT_EQ (tl::file_exists (p), false); +} + +// TemporaryDirectory object +TEST (24) +{ + std::string p; + + { + tl::TemporaryDirectory tmpdir ("tl_tests"); + p = tmpdir.path (); + + EXPECT_EQ (tl::file_exists (p), true); + EXPECT_EQ (tl::is_dir (p), true); + + std::ofstream os (tl::combine_path (p, "test")); + os << "A test"; + os.close (); + + tl::InputStream is (tl::combine_path (p, "test")); + EXPECT_EQ (is.read_all (), "A test"); + } + + EXPECT_EQ (tl::file_exists (p), false); +} + diff --git a/src/tl/unit_tests/tlGitTests.cc b/src/tl/unit_tests/tlGitTests.cc new file mode 100644 index 0000000000..df51035b3f --- /dev/null +++ b/src/tl/unit_tests/tlGitTests.cc @@ -0,0 +1,218 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#if defined(HAVE_GIT2) + +#include "tlGit.h" +#include "tlUnitTest.h" +#include "tlFileUtils.h" + +static std::string test_url ("https://github.com/klayout/klayout_git_test.git"); +static std::string test_url_invalid ("https://github.com/klayout/doesnotexist.git"); + +TEST(1_plain) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string (), std::string (), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/macros/xsection.lym")), true); +} + +TEST(2_subdir) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string (), std::string ("src"), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros/xsection.lym")), true); +} + +TEST(3_subdir_as_filter) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("src/**"), std::string (), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src/macros/xsection.lym")), true); +} + +TEST(4_single_file) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("LICENSE"), std::string (), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "LICENSE")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".gitignore")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "src")), false); +} + +TEST(5_single_file_from_subdir) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ()); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.7") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} + +TEST(6_branch) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("wip")); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.4") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} + +TEST(7_tag) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("1.2")); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.2") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} + +TEST(8_refspec) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("refs/tags/1.5")); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.5") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} + +TEST(9_HEAD) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("grain.xml"), std::string ("src"), std::string ("HEAD")); + + EXPECT_EQ (tl::file_exists (tl::combine_path (path, ".git")), false); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "grain.xml")), true); + EXPECT_EQ (tl::file_exists (tl::combine_path (path, "macros")), false); + + tl::InputStream file (tl::combine_path (path, "grain.xml")); + tl::TextInputStream grain (file); + bool found = false; + while (! grain.at_end () && ! found) { + std::string line = grain.get_line (); + if (line.find ("1.7") != std::string::npos) { + found = true; + } + } + EXPECT_EQ (found, true); +} + +TEST(10_invalid_branch) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + try { + repo.read (test_url, std::string (), std::string (), std::string ("brxxx")); + EXPECT_EQ (true, false); + } catch (tl::Exception &ex) { + EXPECT_EQ (ex.msg (), "Git checkout - Unable to resolve reference name: brxxx"); + } +} + +TEST(11_invalid_url) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + try { + repo.read (test_url_invalid, std::string (), std::string (), std::string ("brxxx")); + EXPECT_EQ (true, false); + } catch (tl::Exception &ex) { + EXPECT_EQ (ex.msg (), "Error cloning Git repo: anonymous access is supported only, but server requests credentials"); + } +} + +#endif + diff --git a/src/tl/unit_tests/unit_tests.pro b/src/tl/unit_tests/unit_tests.pro index c648dcb1b6..2e0bab9099 100644 --- a/src/tl/unit_tests/unit_tests.pro +++ b/src/tl/unit_tests/unit_tests.pro @@ -20,6 +20,7 @@ SOURCES = \ tlExpressionTests.cc \ tlFileSystemWatcherTests.cc \ tlFileUtilsTests.cc \ + tlGitTests.cc \ tlHttpStreamTests.cc \ tlIncludeTests.cc \ tlInt128SupportTests.cc \ diff --git a/src/unit_tests/unit_test_main.cc b/src/unit_tests/unit_test_main.cc index 75c8f78097..2837663bdb 100644 --- a/src/unit_tests/unit_test_main.cc +++ b/src/unit_tests/unit_test_main.cc @@ -433,6 +433,8 @@ run_tests (const std::vector &selected_tests, bool editable, boo static int main_cont (int &argc, char **argv) { + ut::TestConsole console (stdout); + std::unique_ptr ruby_interpreter; std::unique_ptr python_interpreter; @@ -452,8 +454,6 @@ main_cont (int &argc, char **argv) int result = 0; - ut::TestConsole console (stdout); - try { pya::PythonInterpreter::initialize ();