diff --git a/build.sh b/build.sh index 8d647deedb..a2b7c9280b 100755 --- a/build.sh +++ b/build.sh @@ -39,6 +39,7 @@ HAVE_QT=1 HAVE_PNG=0 HAVE_CURL=0 HAVE_EXPAT=0 +HAVE_GIT2=1 RUBYINCLUDE="" RUBYINCLUDE2="" @@ -209,6 +210,9 @@ while [ "$*" != "" ]; do -libexpat) HAVE_EXPAT=1 ;; + -nolibgit2) + HAVE_GIT2=0 + ;; -qt5) echo "*** WARNING: -qt5 option is ignored - Qt version is auto-detected now." ;; @@ -495,6 +499,9 @@ fi if [ $HAVE_PNG != 0 ]; then echo " Uses libpng for PNG generation" fi +if [ $HAVE_GIT2 != 0 ]; then + echo " Uses libgit2 for Git access" +fi if [ "$RPATH" = "" ]; then RPATH="$BIN" fi @@ -578,6 +585,7 @@ echo " HAVE_64BIT_COORD=$HAVE_64BIT_COORD" echo " HAVE_CURL=$HAVE_CURL" echo " HAVE_PNG=$HAVE_PNG" echo " HAVE_EXPAT=$HAVE_EXPAT" +echo " HAVE_GIT2=$HAVE_GIT2" echo " RPATH=$RPATH" mkdir -p $BUILD @@ -650,6 +658,7 @@ qmake_options=( HAVE_CURL="$HAVE_CURL" HAVE_EXPAT="$HAVE_EXPAT" HAVE_PNG="$HAVE_PNG" + HAVE_GIT2="$HAVE_GIT2" PREFIX="$BIN" RPATH="$RPATH" KLAYOUT_VERSION="$KLAYOUT_VERSION" diff --git a/src/klayout.pri b/src/klayout.pri index 0cb25c0ab2..27a7051a05 100644 --- a/src/klayout.pri +++ b/src/klayout.pri @@ -103,6 +103,15 @@ equals(HAVE_PTHREADS, "1") { DEFINES += HAVE_PTHREADS } +equals(HAVE_GIT2, "1") { + !isEmpty(BITS_PATH) { + include($$BITS_PATH/git2/git2.pri) + } else { + LIBS += -lgit2 + } + DEFINES += HAVE_GIT2 +} + equals(HAVE_RUBY, "1") { !isEmpty(BITS_PATH) { include($$BITS_PATH/ruby/ruby.pri) diff --git a/src/lay/lay/layApplication.cc b/src/lay/lay/layApplication.cc index e7c54f7687..1b008fb7a6 100644 --- a/src/lay/lay/layApplication.cc +++ b/src/lay/lay/layApplication.cc @@ -770,7 +770,7 @@ ApplicationBase::init_app () size_t local_folders = (lay::get_appdata_path ().empty () ? 0 : 1); for (std::vector ::const_iterator p = m_klayout_path.begin (); p != m_klayout_path.end (); ++p) { - if (p - m_klayout_path.begin () < local_folders) { + if (size_t (p - m_klayout_path.begin ()) < local_folders) { mc->add_path (*p, tl::to_string (QObject::tr ("Local")), std::string (), false); } else if (m_klayout_path.size () == 1 + local_folders) { mc->add_path (*p, tl::to_string (QObject::tr ("Global")), std::string (), true); diff --git a/src/lay/lay/layInit.cc b/src/lay/lay/layInit.cc index 18e423c3ae..ec108ab79f 100644 --- a/src/lay/lay/layInit.cc +++ b/src/lay/lay/layInit.cc @@ -152,7 +152,7 @@ void init (const std::vector &_paths) try { s_plugins.push_back (do_load_plugin (imp)); modules.insert (*im); - } catch (tl::Exception (&ex)) { + } catch (tl::Exception &ex) { tl::error << ex.msg (); } } diff --git a/src/lay/lay/laySalt.cc b/src/lay/lay/laySalt.cc index d6fcd501f7..4f5f428b7e 100644 --- a/src/lay/lay/laySalt.cc +++ b/src/lay/lay/laySalt.cc @@ -21,11 +21,16 @@ */ #include "laySalt.h" + #include "tlString.h" #include "tlFileUtils.h" #include "tlLog.h" #include "tlInternational.h" #include "tlWebDAV.h" +#if defined(HAVE_GIT2) +# include "tlGit.h" +#endif + #include "lymMacro.h" #include @@ -485,9 +490,23 @@ Salt::create_grain (const SaltGrain &templ, SaltGrain &target, double timeout, t if (templ.url ().find ("http:") == 0 || templ.url ().find ("https:") == 0) { - // otherwise download from the URL - tl::info << QObject::tr ("Downloading package from '%1' to '%2' ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); - res = tl::WebDAVObject::download (templ.url (), target.path (), timeout, callback); + // otherwise download from the URL using Git or SVN + + if (templ.protocol () == Git) { + +#if defined(HAVE_GIT2) + tl::info << QObject::tr ("Downloading package from '%1' to '%2' using Git protocol ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); + res = tl::GitObject::download (templ.url (), target.path (), templ.branch (), timeout, callback); +#else + throw tl::Exception (tl::to_string (QObject::tr ("Unable to install package '%1' - git protocol not compiled in").arg (tl::to_qstring (target.name ())))); +#endif + + } else if (templ.protocol () == WebDAV || templ.protocol () == DefaultProtocol) { + + tl::info << QObject::tr ("Downloading package from '%1' to '%2' using SVN/WebDAV protocol ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ())); + res = tl::WebDAVObject::download (templ.url (), target.path (), timeout, callback); + + } } else { diff --git a/src/lay/lay/laySaltController.cc b/src/lay/lay/laySaltController.cc index 7c99e30db6..2a10a88944 100644 --- a/src/lay/lay/laySaltController.cc +++ b/src/lay/lay/laySaltController.cc @@ -203,11 +203,40 @@ SaltController::install_packages (const std::vector &packages, bool } if (n.find ("http:") == 0 || n.find ("https:") == 0 || n.find ("file:") == 0 || n[0] == '/' || n[0] == '\\') { + // its a URL - manager.register_download (std::string (), std::string (), n, v); + manager.register_download (std::string (), std::string (), n, DefaultProtocol, std::string (), v); + + } else if (n.find ("git@") == 0) { + + // git protocol: + // "git@" + // "git@[]" + + std::string url (n, 4); + size_t br = url.find ("["); + std::string branch; + if (br != std::string::npos && url.back () == ']') { + branch = std::string (url, br + 1, url.size () - br - 2); + url = std::string (url, 0, br); + } + + manager.register_download (std::string (), std::string (), url, Git, branch, v); + + } else if (n.find ("svn@") == 0) { + + // svn protocol: + // "svn@" + + std::string url (n, 4); + // its a URL + manager.register_download (std::string (), std::string (), url, WebDAV, std::string (), v); + } else { + // its a plain name - manager.register_download (n, std::string (), std::string (), v); + manager.register_download (n, std::string (), std::string (), DefaultProtocol, std::string (), v); + } } diff --git a/src/lay/lay/laySaltDownloadManager.cc b/src/lay/lay/laySaltDownloadManager.cc index e104e3531b..4d9032f6c5 100644 --- a/src/lay/lay/laySaltDownloadManager.cc +++ b/src/lay/lay/laySaltDownloadManager.cc @@ -58,7 +58,7 @@ ConfirmationDialog::ConfirmationDialog (QWidget *parent) } void -ConfirmationDialog::add_info (const std::string &name, bool update, const std::string &version, const std::string &url) +ConfirmationDialog::add_info (const std::string &name, bool update, const std::string &version, const std::string &url, Protocol protocol, const std::string &branch) { QTreeWidgetItem *item = new QTreeWidgetItem (list); m_items_by_name.insert (std::make_pair (name, item)); @@ -68,7 +68,18 @@ ConfirmationDialog::add_info (const std::string &name, bool update, const std::s item->setText (0, tl::to_qstring (name)); item->setText (1, update ? tr ("UPDATE") : tr ("INSTALL")); item->setText (2, tl::to_qstring (version)); - item->setText (3, tl::to_qstring (url)); + + if (protocol == WebDAV) { + item->setText (3, tl::to_qstring ("svn@" + url)); + } else if (protocol == Git) { + if (branch.empty ()) { + item->setText (3, tl::to_qstring ("git@" + url)); + } else { + item->setText (3, tl::to_qstring ("git@" + url + "[" + branch + "]")); + } + } else { + item->setText (3, tl::to_qstring (url)); + } for (int column = 0; column < list->colorCount (); ++column) { item->setData (column, Qt::ForegroundRole, QVariant (QBrush (update ? QColor (Qt::blue) : QColor (Qt::black)))); @@ -169,9 +180,9 @@ SaltDownloadManager::SaltDownloadManager () } void -SaltDownloadManager::register_download (const std::string &name, const std::string &token, const std::string &url, const std::string &version) +SaltDownloadManager::register_download (const std::string &name, const std::string &token, const std::string &url, Protocol protocol, const std::string &branch, const std::string &version) { - m_registry.push_back (Descriptor (name, token, url, version)); + m_registry.push_back (Descriptor (name, token, url, protocol, branch, version)); } void @@ -244,7 +255,7 @@ SaltDownloadManager::compute_list (const lay::Salt &salt, const lay::Salt &salt_ if (tl::verbosity() >= 20) { tl::log << "Considering for update as dependency: " << d->name << " (" << d->version << ") with URL " << d->url; } - m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->version)); + m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->protocol, d->branch, d->version)); } else { if (tl::verbosity() >= 20) { @@ -257,7 +268,7 @@ SaltDownloadManager::compute_list (const lay::Salt &salt, const lay::Salt &salt_ if (tl::verbosity() >= 20) { tl::log << "Considering for download as dependency: " << d->name << " (" << d->version << ") with URL " << d->url; } - m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->version)); + m_registry.push_back (Descriptor (d->name, std::string (), d->url, d->protocol, d->branch, d->version)); } @@ -330,7 +341,7 @@ SaltDownloadManager::fetch_missing (const lay::Salt &salt, const lay::Salt &salt } try { - p->grain = SaltGrain::from_url (p->url); + p->grain = SaltGrain::from_url (p->url, p->protocol, p->branch); } catch (tl::Exception &ex) { throw tl::Exception (tl::to_string (QObject::tr ("Error fetching spec file for package '%1': %2").arg (tl::to_qstring (p->name)).arg (tl::to_qstring (ex.msg ())))); } @@ -387,7 +398,7 @@ SaltDownloadManager::make_confirmation_dialog (QWidget *parent, const lay::Salt const lay::SaltGrain *g = salt.grain_by_name (p->name); if (g) { // \342\206\222 is UTF-8 "right arrow" - dialog->add_info (p->name, true, g->version () + " \342\206\222 " + p->version, p->url); + dialog->add_info (p->name, true, g->version () + " \342\206\222 " + p->version, p->url, p->protocol, p->branch); } } @@ -395,7 +406,7 @@ SaltDownloadManager::make_confirmation_dialog (QWidget *parent, const lay::Salt for (std::vector::const_iterator p = m_registry.begin (); p != m_registry.end (); ++p) { const lay::SaltGrain *g = salt.grain_by_name (p->name); if (!g) { - dialog->add_info (p->name, false, p->version, p->url); + dialog->add_info (p->name, false, p->version, p->url, p->protocol, p->branch); } } diff --git a/src/lay/lay/laySaltDownloadManager.h b/src/lay/lay/laySaltDownloadManager.h index a81e0f9730..04c3ef2b59 100644 --- a/src/lay/lay/laySaltDownloadManager.h +++ b/src/lay/lay/laySaltDownloadManager.h @@ -53,7 +53,7 @@ Q_OBJECT public: ConfirmationDialog (QWidget *parent); - void add_info (const std::string &name, bool update, const std::string &version, const std::string &url); + void add_info (const std::string &name, bool update, const std::string &version, const std::string &url, Protocol protocol, const std::string &branch); bool is_confirmed () const { return m_confirmed; } bool is_cancelled () const { return m_cancelled; } @@ -108,7 +108,7 @@ Q_OBJECT * * The target directory can be empty. In this case, the downloader will pick an appropriate one. */ - void register_download (const std::string &name, const std::string &token, const std::string &url, const std::string &version); + void register_download (const std::string &name, const std::string &token, const std::string &url, Protocol protocol, const std::string &branch, const std::string &version); /** * @brief Computes the dependencies after all required packages have been registered @@ -145,8 +145,8 @@ Q_OBJECT private: struct Descriptor { - Descriptor (const std::string &_name, const std::string &_token, const std::string &_url, const std::string &_version) - : name (_name), token (_token), url (_url), version (_version), downloaded (false) + Descriptor (const std::string &_name, const std::string &_token, const std::string &_url, Protocol _protocol, const std::string &_branch, const std::string &_version) + : name (_name), token (_token), url (_url), protocol (_protocol), branch (_branch), version (_version), downloaded (false) { } bool operator< (const Descriptor &other) const @@ -170,6 +170,8 @@ Q_OBJECT std::string name; std::string token; std::string url; + Protocol protocol; + std::string branch; std::string version; bool downloaded; lay::SaltGrain grain; diff --git a/src/lay/lay/laySaltGrain.cc b/src/lay/lay/laySaltGrain.cc index 932ceb3100..b956e11a04 100644 --- a/src/lay/lay/laySaltGrain.cc +++ b/src/lay/lay/laySaltGrain.cc @@ -25,8 +25,11 @@ #include "tlString.h" #include "tlXMLParser.h" #include "tlHttpStream.h" -#include "tlWebDAV.h" #include "tlFileUtils.h" +#include "tlWebDAV.h" +#if defined(HAVE_GIT2) +# include "tlGit.h" +#endif #include #include @@ -41,7 +44,7 @@ namespace lay static const std::string grain_filename = "grain.xml"; SaltGrain::SaltGrain () - : m_hidden (false) + : m_protocol (DefaultProtocol), m_hidden (false) { // .. nothing yet .. } @@ -65,7 +68,10 @@ SaltGrain::operator== (const SaltGrain &other) const m_license == other.m_license && m_hidden == other.m_hidden && m_authored_time == other.m_authored_time && - m_installed_time == other.m_installed_time; + m_installed_time == other.m_installed_time && + m_protocol == other.m_protocol && + m_branch == other.m_branch + ; } void @@ -110,6 +116,18 @@ SaltGrain::set_url (const std::string &u) m_url = u; } +void +SaltGrain::set_branch (const std::string &b) +{ + m_branch = b; +} + +void +SaltGrain::set_protocol (const Protocol &p) +{ + m_protocol = p; +} + void SaltGrain::set_title (const std::string &t) { @@ -253,18 +271,10 @@ SaltGrain::compare_versions (const std::string &v1, const std::string &v2) } } -std::string -SaltGrain::spec_url (const std::string &url) +const std::string & +SaltGrain::spec_file () { - std::string res = url; - if (! res.empty()) { - // TODO: use system path separator unless this is a URL - if (res [res.size () - 1] != '/') { - res += "/"; - } - res += grain_filename; - } - return res; + return grain_filename; } bool @@ -400,6 +410,30 @@ struct ImageConverter } }; +struct ProtocolConverter +{ + std::string to_string (const Protocol &protocol) const + { + if (protocol == lay::WebDAV) { + return std::string ("svn"); + } else if (protocol == lay::Git) { + return std::string ("git"); + } else { + return std::string (); + } + } + + void from_string (const std::string &s, Protocol &res) const + { + res = lay::DefaultProtocol; + if (s == "svn" || s == "SVN" || s == "WebDAV") { + res = lay::WebDAV; + } else if (s == "git" || s == "Git" || s == "GIT") { + res = lay::Git; + } + } +}; + static tl::XMLElementList *sp_xml_elements = 0; tl::XMLElementList & @@ -416,6 +450,8 @@ SaltGrain::xml_elements () tl::make_member (&SaltGrain::doc, &SaltGrain::set_doc, "doc") + tl::make_member (&SaltGrain::doc_url, &SaltGrain::set_doc_url, "doc-url") + tl::make_member (&SaltGrain::url, &SaltGrain::set_url, "url") + + tl::make_member (&SaltGrain::branch, &SaltGrain::set_branch, "branch") + + tl::make_member (&SaltGrain::protocol, &SaltGrain::set_protocol, "protocol", ProtocolConverter ()) + tl::make_member (&SaltGrain::license, &SaltGrain::set_license, "license") + tl::make_member (&SaltGrain::author, &SaltGrain::set_author, "author") + tl::make_member (&SaltGrain::author_contact, &SaltGrain::set_author_contact, "author-contact") + @@ -426,6 +462,8 @@ SaltGrain::xml_elements () tl::make_element (&SaltGrain::begin_dependencies, &SaltGrain::end_dependencies, &SaltGrain::add_dependency, "depends", tl::make_member (&SaltGrainDependency::name, "name") + tl::make_member (&SaltGrainDependency::url, "url") + + tl::make_member (&SaltGrainDependency::protocol, "protocol", ProtocolConverter ()) + + tl::make_member (&SaltGrainDependency::branch, "branch") + tl::make_member (&SaltGrainDependency::version, "version") ) ); @@ -512,7 +550,7 @@ SaltGrain::from_path (const std::string &path) } tl::InputStream * -SaltGrain::stream_from_url (std::string &url, double timeout, tl::InputHttpStreamCallback *callback) +SaltGrain::stream_from_url (std::string &url, Protocol protocol, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { if (url.empty ()) { throw tl::Exception (tl::to_string (QObject::tr ("No download link available"))); @@ -533,19 +571,30 @@ SaltGrain::stream_from_url (std::string &url, double timeout, tl::InputHttpStrea } - std::string spec_url = SaltGrain::spec_url (url); - if (spec_url.find ("http:") == 0 || spec_url.find ("https:") == 0) { - return tl::WebDAVObject::download_item (spec_url, timeout, callback); + if (url.find ("http:") == 0 || url.find ("https:") == 0) { + + if (protocol == lay::Git) { +#if defined(HAVE_GIT2) + return tl::GitObject::download_item (url, SaltGrain::spec_file (), branch, timeout, callback); +#else + throw tl::Exception (tl::to_string (QObject::tr ("Cannot download from Git - Git support not compiled in"))); +#endif + } else { + return tl::WebDAVObject::download_item (url + "/" + SaltGrain::spec_file (), timeout, callback); + } + } else { - return new tl::InputStream (spec_url); + + return new tl::InputStream (url); + } } SaltGrain -SaltGrain::from_url (const std::string &url_in, double timeout, tl::InputHttpStreamCallback *callback) +SaltGrain::from_url (const std::string &url_in, Protocol protocol, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) { std::string url = url_in; - std::unique_ptr stream (stream_from_url (url, timeout, callback)); + std::unique_ptr stream (stream_from_url (url, protocol, branch, timeout, callback)); SaltGrain g; g.load (*stream); diff --git a/src/lay/lay/laySaltGrain.h b/src/lay/lay/laySaltGrain.h index 92ef33e192..1919e98abc 100644 --- a/src/lay/lay/laySaltGrain.h +++ b/src/lay/lay/laySaltGrain.h @@ -39,6 +39,15 @@ namespace tl namespace lay { +/** + * @brief An enum describing the protocol to use for download + */ +enum Protocol { + DefaultProtocol = 0, + WebDAV = 1, + Git = 2 +}; + /** * @brief A descriptor for one dependency * A dependency can be specified either through a name (see name property) @@ -49,8 +58,14 @@ namespace lay */ struct SaltGrainDependency { + SaltGrainDependency () + : protocol (DefaultProtocol) + { } + std::string name; std::string url; + Protocol protocol; + std::string branch; std::string version; bool operator== (const SaltGrainDependency &other) const @@ -341,6 +356,32 @@ class LAY_PUBLIC SaltGrain */ void set_url (const std::string &u); + /** + * @brief Gets the download protocol + */ + const Protocol &protocol () const + { + return m_protocol; + } + + /** + * @brief Sets the download protocol + */ + void set_protocol (const Protocol &p); + + /** + * @brief Gets the Git branch + */ + const std::string &branch () const + { + return m_branch; + } + + /** + * @brief Sets the Git branch + */ + void set_branch (const std::string &b); + /** * @brief Gets a value indicating whether the grain is hidden * A grain can be hidden (in Salt.Mine) if it's a pure dependency package @@ -466,21 +507,23 @@ class LAY_PUBLIC SaltGrain * This method will return a grain constructed from the downloaded data. * The data is read from "URL/grain.xml". This method will throw an * exception if an error occurs during reading. + * protocol is the protocol to use and branch the Git branch. */ - static SaltGrain from_url (const std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + static SaltGrain from_url (const std::string &url, Protocol protocol, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); /** * @brief Returns a stream prepared for downloading the grain * The stream is a new'd object and needs to be deleted by the caller. * "url" is the download URL on input and gets modified to match the * actual URL if it is a relative one. + * protocol is the protocol to use and branch the Git branch. */ - static tl::InputStream *stream_from_url (std::string &url, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + static tl::InputStream *stream_from_url (std::string &url, Protocol protocol, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); /** - * @brief Forms the spec file download URL from a given download URL + * @brief Gets the name of the spec file ("grain.xml") */ - static std::string spec_url (const std::string &url); + static const std::string &spec_file (); /** * @brief Returns a value indicating whether the given path represents is a grain @@ -499,6 +542,8 @@ class LAY_PUBLIC SaltGrain std::string m_author; std::string m_author_contact; std::string m_license; + Protocol m_protocol; + std::string m_branch; bool m_hidden; QDateTime m_authored_time, m_installed_time; QImage m_icon, m_screenshot; diff --git a/src/lay/lay/laySaltGrainPropertiesDialog.cc b/src/lay/lay/laySaltGrainPropertiesDialog.cc index 7a3f098b9d..200bce4eb7 100644 --- a/src/lay/lay/laySaltGrainPropertiesDialog.cc +++ b/src/lay/lay/laySaltGrainPropertiesDialog.cc @@ -586,7 +586,7 @@ SaltGrainPropertiesDialog::accept () if (!d->url.empty ()) { SaltGrain gdep; try { - gdep = SaltGrain::from_url (d->url); + gdep = SaltGrain::from_url (d->url, d->protocol, d->branch); if (gdep.name () != d->name) { dependencies_alert->error () << tr ("Package name obtained from download URL is not the expected name.") << tl::endl << tr ("Downloaded name: ") << gdep.name () << tl::endl diff --git a/src/lay/lay/laySaltManagerDialog.cc b/src/lay/lay/laySaltManagerDialog.cc index f439be7b80..4e135e2662 100644 --- a/src/lay/lay/laySaltManagerDialog.cc +++ b/src/lay/lay/laySaltManagerDialog.cc @@ -35,6 +35,7 @@ #include "pya.h" #include +#include #include #include #include @@ -682,7 +683,7 @@ BEGIN_PROTECTED SaltGrain *g = model->grain_from_index (index); // NOTE: checking for valid_name prevents bad entries inside the download list if (g && model->is_marked (g->name ()) && SaltGrain::valid_name (g->name ())) { - manager.register_download (g->name (), g->token (), g->url (), g->version ()); + manager.register_download (g->name (), g->token (), g->url (), g->protocol (), g->branch (), g->version ()); any = true; } } @@ -1133,6 +1134,24 @@ BEGIN_PROTECTED END_PROTECTED } +namespace +{ + +/** + * @brief A callback to keep the UI alive (mainly used for Git grain retrieval) + */ +class ProcessEventCallback + : public tl::InputHttpStreamCallback +{ +public: + virtual void wait_for_input () + { + QApplication::processEvents (QEventLoop::ExcludeUserInputEvents); + } +}; + +} + void SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTextWidget *details) { @@ -1166,16 +1185,22 @@ SaltManagerDialog::get_remote_grain_info (lay::SaltGrain *g, SaltGrainDetailsTex "" "" ) - .arg (tl::to_qstring (SaltGrain::spec_url (g->url ()))); + .arg (tl::to_qstring (g->url ())); details->setHtml (html); std::string url = g->url (); + Protocol protocol = g->protocol (); + std::string branch = g->branch (); + m_downloaded_grain.reset (new SaltGrain ()); m_downloaded_grain->set_url (url); + m_downloaded_grain->set_protocol (protocol); + m_downloaded_grain->set_branch (branch); // NOTE: stream_from_url may modify the URL, hence we set it again - m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url)); + ProcessEventCallback callback; + m_downloaded_grain_reader.reset (SaltGrain::stream_from_url (url, protocol, branch, 60.0, &callback)); m_downloaded_grain->set_url (url); tl::InputHttpStream *http = dynamic_cast (m_downloaded_grain_reader->base ()); diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index 558134601b..5a9f735622 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -121,7 +121,17 @@ HEADERS = \ tlUniqueName.h \ tlRecipe.h \ tlSelect.h \ - tlEnv.h + tlEnv.h + +equals(HAVE_GIT2, "1") { + + HEADERS += \ + tlGit.h + + SOURCES += \ + tlGit.cc + +} equals(HAVE_CURL, "1") { diff --git a/src/tl/tl/tlGit.cc b/src/tl/tl/tlGit.cc new file mode 100644 index 0000000000..59e619234f --- /dev/null +++ b/src/tl/tl/tlGit.cc @@ -0,0 +1,198 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include "tlGit.h" +#include "tlFileUtils.h" +#include "tlProgress.h" +#include "tlStaticObjects.h" + +#include +#include + +namespace tl +{ + +// --------------------------------------------------------------- +// Library initialization helper + +namespace { + +class InitHelper +{ +public: + InitHelper () + { + git_libgit2_init (); + } + + ~InitHelper () + { + git_libgit2_shutdown (); + } + + static void ensure_initialized () + { + if (ms_init) { + return; + } + + ms_init = new InitHelper (); + tl::StaticObjects::reg (&ms_init); + } + +private: + static InitHelper *ms_init; +}; + +InitHelper *InitHelper::ms_init = 0; + +} + +// --------------------------------------------------------------- +// GitCollection implementation + +GitObject::GitObject (const std::string &local_path) + : m_local_path (local_path), m_is_temp (false) +{ + InitHelper::ensure_initialized (); + + if (local_path.empty ()) { + // @@@ generic tempnam on Windows/Posix ... + char tmp[] = "git2klayoutXXXXXX"; + mkstemp (tmp); + m_local_path = tmp; + m_is_temp = true; + } + + tl::mkpath (m_local_path); + + // ensures the directory is clean + tl::rm_dir_recursive (m_local_path); // @@@ TODO: error handling? + tl::mkpath (m_local_path); +} + +GitObject::~GitObject () +{ + if (m_is_temp) { + tl::rm_dir_recursive (m_local_path); + } +} + +static int +fetch_progress (const git_indexer_progress *stats, void *payload) +{ + tl::RelativeProgress *progress = reinterpret_cast (payload); + + // first half of progress + size_t count = size_t (5000.0 * double (stats->received_objects) / double (std::max (1u, stats->total_objects)) + 1e-10); + progress->set (count); + + return 0; +} + +static void +checkout_progress(const char * /*path*/, size_t cur, size_t tot, void *payload) +{ + tl::RelativeProgress *progress = reinterpret_cast (payload); + + // first half of progress + size_t count = size_t (5000.0 * double (cur) / double (std::max (size_t (1), tot)) + 1e-10); + progress->set (count + 5000u); +} + + +void +GitObject::read (const std::string &url, const std::string &filter, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +{ + // @@@ use callback, timeout? + tl::RelativeProgress progress (tl::to_string (tr ("Download progress")), 10000, 1 /*yield always*/); + + git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; + + const char *paths_cstr[1]; + paths_cstr[0] = filter.c_str (); + if (! filter.empty ()) { + checkout_opts.paths.count = 1; + checkout_opts.paths.strings = (char **) &paths_cstr; + } + + checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE | + GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH; + checkout_opts.progress_cb = &checkout_progress; + checkout_opts.progress_payload = (void *) &progress; + + git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; + + clone_opts.checkout_opts = checkout_opts; + + // NOTE: really has to be a branch! Tags won't work. + if (! branch.empty ()) { + clone_opts.checkout_branch = branch.c_str (); + } + +#if LIBGIT2_VER_MAJOR > 1 || (LIBGIT2_VER_MAJOR == 1 && LIBGIT2_VER_MINOR >= 7) + clone_opts.fetch_opts.depth = 1; // shallow (single commit) +#endif + clone_opts.fetch_opts.callbacks.transfer_progress = &fetch_progress; + clone_opts.fetch_opts.callbacks.payload = (void *) &progress; + + // Do the clone + git_repository *cloned_repo = NULL; + int error = git_clone (&cloned_repo, url.c_str (), m_local_path.c_str (), &clone_opts); + if (error != 0) { + const git_error *err = git_error_last (); + throw tl::Exception (tl::to_string (tr ("Error cloning Git repo (%d): %s")), err->klass, (const char *) err->message); + } + + if (cloned_repo) { + git_repository_free (cloned_repo); + // remove the worktree we don't need + tl::rm_dir_recursive (tl::combine_path (m_local_path, ".git")); + } +} + +bool +GitObject::download (const std::string &url, const std::string &target, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +{ + GitObject obj (target); + obj.read (url, std::string (), branch, timeout, callback); + return false; +} + +tl::InputStream * +GitObject::download_item (const std::string &url, const std::string &file, const std::string &branch, double timeout, tl::InputHttpStreamCallback *callback) +{ + GitObject obj; + obj.read (url, file, branch, timeout, callback); + + // extract the file and return a memory blob, so we can delete the temp folder + + tl::InputStream file_stream (tl::combine_path (obj.local_path (), file)); + std::string data = file_stream.read_all (); + + char *data_copy = new char [data.size ()]; + memcpy (data_copy, data.c_str (), data.size ()); + return new tl::InputStream (new tl::InputMemoryStream (data_copy, data.size (), true)); +} + +} diff --git a/src/tl/tl/tlGit.h b/src/tl/tl/tlGit.h new file mode 100644 index 0000000000..cbc11bda93 --- /dev/null +++ b/src/tl/tl/tlGit.h @@ -0,0 +1,108 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_tlGit +#define HDR_tlGit + +#include "tlCommon.h" +#include "tlStream.h" + +#include +#include + +#if !defined(HAVE_GIT2) +# error "tlGit.h can only be used with libgit2 enabled" +#endif + +namespace tl +{ + +class InputHttpStreamCallback; + +/** + * @brief Represents an object from a Git URL + * This object can be a file or collection + */ +class TL_PUBLIC GitObject +{ +public: + /** + * @brief Open a stream with the given URL + * + * The local_path is the path where to store the files. + * If empty, a temporary folder is created and destroyed once the "GitObject" goes out of scope. + */ + GitObject (const std::string &local_path = std::string ()); + + /** + * @brief Destructor + */ + ~GitObject (); + + /** + * @brief Populates the collection from the given URL + * + * "filter" can be a top-level file to download. If filter is non-empty, + * sparse mode is chosen. + */ + void read (const std::string &url, const std::string &filter, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + + /** + * @brief Downloads the collection or file with the given URL + * + * This method will download the Git object from url to the file path + * given in "target". + * + * For file download, the target must be the path of the target file. + * For collection download, the target must be a directory path. In this + * case, the target directory must exist already. + * + * Sub-directories are created if required. + * + * This method throws an exception if the directory structure could + * not be obtained or downloading of one file failed. + */ + static bool download (const std::string &url, const std::string &target, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + + /** + * @brief Gets a stream object for downloading the single item of the given URL + * + * The file needs to be a top-level object. + * The stream object returned needs to be deleted by the caller. + */ + static tl::InputStream *download_item (const std::string &url, const std::string &file, const std::string &branch, double timeout = 60.0, tl::InputHttpStreamCallback *callback = 0); + +private: + std::string m_local_path; + bool m_is_temp; + + const std::string &local_path () const + { + return m_local_path; + } +}; + +} + +#endif + diff --git a/src/tl/unit_tests/tlGitTests.cc b/src/tl/unit_tests/tlGitTests.cc new file mode 100644 index 0000000000..e7e3c1544c --- /dev/null +++ b/src/tl/unit_tests/tlGitTests.cc @@ -0,0 +1,59 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2023 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#if defined(HAVE_GIT2) + +#include "tlGit.h" +#include "tlUnitTest.h" + +// @@@ +static std::string test_url ("https://github.com/klayoutmatthias/xsection.git"); + +TEST(1) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string (), std::string ()); + + printf("@@@ done: %s\n", path.c_str ()); +} + +TEST(2) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string ("LICENSE"), std::string ()); + + printf("@@@ done: %s\n", path.c_str ()); +} + +TEST(3) +{ + std::string path = tl::TestBase::tmp_file ("repo"); + tl::GitObject repo (path); + repo.read (test_url, std::string (), std::string ("brxxx")); + + printf("@@@ done: %s\n", path.c_str ()); +} + +#endif + diff --git a/src/tl/unit_tests/unit_tests.pro b/src/tl/unit_tests/unit_tests.pro index c648dcb1b6..2e0bab9099 100644 --- a/src/tl/unit_tests/unit_tests.pro +++ b/src/tl/unit_tests/unit_tests.pro @@ -20,6 +20,7 @@ SOURCES = \ tlExpressionTests.cc \ tlFileSystemWatcherTests.cc \ tlFileUtilsTests.cc \ + tlGitTests.cc \ tlHttpStreamTests.cc \ tlIncludeTests.cc \ tlInt128SupportTests.cc \