From 4d4941ad2f2e77505feca34af4ed84bf9e0e77dc Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Tue, 13 Aug 2024 14:11:34 +0800 Subject: [PATCH 01/19] update --- CMakeLists.txt | 2 +- Makefile | 2 +- extension/duckdb/CMakeLists.txt | 3 - extension/duckdb/src/duckdb_install_func.cpp | 15 +++++ extension/duckdb/src/duckdb_installer.cpp | 16 ++++++ extension/duckdb/src/duckdb_loader.cpp | 3 + .../duckdb/src/include/duckdb_installer.h | 16 ++++++ extension/duckdb/src/include/duckdb_loader.h | 8 +++ extension/httpfs/CMakeLists.txt | 19 +++++++ .../postgres/src/postgres_install_func.cpp | 3 + extension/sqlite/src/sqlite_install_func.cpp | 15 +++++ src/extension/CMakeLists.txt | 3 +- src/extension/extension.cpp | 26 ++++++--- src/extension/extension_installer.cpp | 55 +++++++++++++++++++ src/include/extension/extension.h | 20 +++++-- src/include/extension/extension_installer.h | 32 +++++++++++ src/main/client_context.cpp | 2 +- .../operator/simple/install_extension.cpp | 35 +++++++++++- .../operator/simple/load_extension.cpp | 8 ++- 19 files changed, 262 insertions(+), 21 deletions(-) create mode 100644 extension/duckdb/src/duckdb_install_func.cpp create mode 100644 extension/duckdb/src/duckdb_installer.cpp create mode 100644 extension/duckdb/src/duckdb_loader.cpp create mode 100644 extension/duckdb/src/include/duckdb_installer.h create mode 100644 extension/duckdb/src/include/duckdb_loader.h create mode 100644 extension/postgres/src/postgres_install_func.cpp create mode 100644 extension/sqlite/src/sqlite_install_func.cpp create mode 100644 src/extension/extension_installer.cpp create mode 100644 src/include/extension/extension_installer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 0dc86b1157b..fd727bd92ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -254,7 +254,7 @@ add_subdirectory(third_party) if(${BUILD_KUZU}) add_definitions(-DKUZU_ROOT_DIRECTORY="${PROJECT_SOURCE_DIR}") add_definitions(-DKUZU_CMAKE_VERSION="${CMAKE_PROJECT_VERSION}") -add_definitions(-DKUZU_EXTENSION_VERSION="0.5.1.11") +add_definitions(-DKUZU_EXTENSION_VERSION="0.5.2.0") include_directories(src/include) diff --git a/Makefile b/Makefile index 5913317afef..2b19ee9a930 100644 --- a/Makefile +++ b/Makefile @@ -191,7 +191,7 @@ extension-json-test: extension-json-test-build extension-debug: $(call run-cmake-debug, \ - -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite" \ + -DBUILD_EXTENSIONS="httpfs" \ -DBUILD_KUZU=FALSE \ ) diff --git a/extension/duckdb/CMakeLists.txt b/extension/duckdb/CMakeLists.txt index 4961b8bb37b..3b7a18e5df1 100644 --- a/extension/duckdb/CMakeLists.txt +++ b/extension/duckdb/CMakeLists.txt @@ -6,9 +6,6 @@ else() set(DuckDB_USE_STATIC_LIBS ON) endif() find_package(DuckDB REQUIRED) -if(NOT WIN32) - find_library(DuckDB_STATIC REQUIRED NAMES libduckdb_static.a) -endif() include_directories( ${PROJECT_SOURCE_DIR}/src/include diff --git a/extension/duckdb/src/duckdb_install_func.cpp b/extension/duckdb/src/duckdb_install_func.cpp new file mode 100644 index 00000000000..fd82179c13e --- /dev/null +++ b/extension/duckdb/src/duckdb_install_func.cpp @@ -0,0 +1,15 @@ +#include "duckdb_installer.h" + +extern "C" { +// Because we link against the static library on windows, we implicitly inherit KUZU_STATIC_DEFINE, +// which cancels out any exporting, so we can't use KUZU_API. +#if defined(_WIN32) +#define INIT_EXPORT __declspec(dllexport) +#else +#define INIT_EXPORT __attribute__((visibility("default"))) +#endif +INIT_EXPORT void install(kuzu::main::ClientContext* context) { + kuzu::duckdb::DuckDBInstaller installer{"duckdb"}; + installer.install(context); +} +} diff --git a/extension/duckdb/src/duckdb_installer.cpp b/extension/duckdb/src/duckdb_installer.cpp new file mode 100644 index 00000000000..0fb6d04c4dc --- /dev/null +++ b/extension/duckdb/src/duckdb_installer.cpp @@ -0,0 +1,16 @@ +#pragma once +#include "extension/extension_downloader.h" + +namespace kuzu { +namespace httpfs { + +class HttpfsInstaller : public extension::ExtensionInstaller { +public: + explicit HttpfsInstaller(const std::string extensionName) + : ExtensionInstaller{std::move(extensionName)} {} + + void install(main::ClientContext* context) override; +}; + +} // namespace httpfs +} // namespace kuzu \ No newline at end of file diff --git a/extension/duckdb/src/duckdb_loader.cpp b/extension/duckdb/src/duckdb_loader.cpp new file mode 100644 index 00000000000..5fbea6d1f7b --- /dev/null +++ b/extension/duckdb/src/duckdb_loader.cpp @@ -0,0 +1,3 @@ +// +// Created by z473chen on 2024-08-12. +// diff --git a/extension/duckdb/src/include/duckdb_installer.h b/extension/duckdb/src/include/duckdb_installer.h new file mode 100644 index 00000000000..0fb6d04c4dc --- /dev/null +++ b/extension/duckdb/src/include/duckdb_installer.h @@ -0,0 +1,16 @@ +#pragma once +#include "extension/extension_downloader.h" + +namespace kuzu { +namespace httpfs { + +class HttpfsInstaller : public extension::ExtensionInstaller { +public: + explicit HttpfsInstaller(const std::string extensionName) + : ExtensionInstaller{std::move(extensionName)} {} + + void install(main::ClientContext* context) override; +}; + +} // namespace httpfs +} // namespace kuzu \ No newline at end of file diff --git a/extension/duckdb/src/include/duckdb_loader.h b/extension/duckdb/src/include/duckdb_loader.h new file mode 100644 index 00000000000..f4adb63dfb6 --- /dev/null +++ b/extension/duckdb/src/include/duckdb_loader.h @@ -0,0 +1,8 @@ +// +// Created by z473chen on 2024-08-12. +// + +#ifndef KUZU_DUCKDB_LOADER_H +#define KUZU_DUCKDB_LOADER_H + +#endif //KUZU_DUCKDB_LOADER_H diff --git a/extension/httpfs/CMakeLists.txt b/extension/httpfs/CMakeLists.txt index 3e68ea3119c..713fa1170cb 100644 --- a/extension/httpfs/CMakeLists.txt +++ b/extension/httpfs/CMakeLists.txt @@ -26,6 +26,10 @@ add_library(httpfs src/http_config.cpp src/cached_file_manager.cpp) +add_library(httpfs_installer + SHARED + src/httpfs_installer.cpp) + target_link_libraries(httpfs PRIVATE mbedtls @@ -37,6 +41,12 @@ set_target_properties(httpfs PROPERTIES SUFFIX ".kuzu_extension" ) +set_target_properties(httpfs_installer PROPERTIES + OUTPUT_NAME httpfs_installer + PREFIX "lib" + SUFFIX ".kuzu_extension" +) + set_target_properties(httpfs PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" @@ -44,6 +54,13 @@ set_target_properties(httpfs RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" ) +set_target_properties(httpfs_installer + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" +) + if (WIN32) # On windows, there is no dynamic lookup available, so it's not # possible to generically look for symbols on library load. There are @@ -62,10 +79,12 @@ if (WIN32) # Future work could make it possible to embed extension into kuzu, # which would help fix this problem. target_link_libraries(httpfs PRIVATE kuzu) + target_link_libraries(httpfs_installer PRIVATE kuzu) endif() if (APPLE) set_target_properties(httpfs PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + set_target_properties(httpfs_installer PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") endif () if (${BUILD_EXTENSION_TESTS}) diff --git a/extension/postgres/src/postgres_install_func.cpp b/extension/postgres/src/postgres_install_func.cpp new file mode 100644 index 00000000000..838d9c89ce8 --- /dev/null +++ b/extension/postgres/src/postgres_install_func.cpp @@ -0,0 +1,3 @@ +// +// Created by z473chen on 2024-08-13. +// diff --git a/extension/sqlite/src/sqlite_install_func.cpp b/extension/sqlite/src/sqlite_install_func.cpp new file mode 100644 index 00000000000..e8cda2e8a60 --- /dev/null +++ b/extension/sqlite/src/sqlite_install_func.cpp @@ -0,0 +1,15 @@ +#include "duckdb_installer.h" + +extern "C" { +// Because we link against the static library on windows, we implicitly inherit KUZU_STATIC_DEFINE, +// which cancels out any exporting, so we can't use KUZU_API. +#if defined(_WIN32) +#define INIT_EXPORT __declspec(dllexport) +#else +#define INIT_EXPORT __attribute__((visibility("default"))) +#endif +INIT_EXPORT void install(kuzu::main::ClientContext* context) { + kuzu::duckdb::DuckDBInstaller installer{"postgres"}; + installer.install(context); +} +} diff --git a/src/extension/CMakeLists.txt b/src/extension/CMakeLists.txt index f2072d83694..10b3efab842 100644 --- a/src/extension/CMakeLists.txt +++ b/src/extension/CMakeLists.txt @@ -1,7 +1,8 @@ add_library(kuzu_extension OBJECT catalog_extension.cpp - extension.cpp) + extension.cpp + extension_downloader.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ diff --git a/src/extension/extension.cpp b/src/extension/extension.cpp index 6605fddcc54..134e8a556c5 100644 --- a/src/extension/extension.cpp +++ b/src/extension/extension.cpp @@ -4,6 +4,7 @@ #include "common/string_format.h" #include "common/string_utils.h" #include "function/table_functions.h" +#include "main/client_context.h" #include "main/database.h" #include "transaction/transaction.h" @@ -39,19 +40,15 @@ std::string getPlatform() { return getOS() + "_" + getArch(); } -std::string ExtensionUtils::getExtensionPath(const std::string& extensionDir, - const std::string& name) { - return common::stringFormat("{}/lib{}.kuzu_extension", extensionDir, name); -} - bool ExtensionUtils::isFullPath(const std::string& extension) { return extension.find('.') != std::string::npos || extension.find('/') != std::string::npos || extension.find('\\') != std::string::npos; } -ExtensionRepoInfo ExtensionUtils::getExtensionRepoInfo(const std::string& extension) { +ExtensionRepoInfo ExtensionUtils::getExtensionRepoInfo(const std::string& extensionName, + const std::string& fileName) { auto extensionURL = common::stringFormat(EXTENSION_REPO, KUZU_EXTENSION_VERSION, getPlatform(), - common::StringUtils::getLower(extension)); + extensionName, getExtensionFileName(fileName)); common::StringUtils::replaceAll(extensionURL, "http://", ""); auto hostNamePos = extensionURL.find('/'); auto hostName = extensionURL.substr(0, hostNamePos); @@ -60,6 +57,21 @@ ExtensionRepoInfo ExtensionUtils::getExtensionRepoInfo(const std::string& extens return {hostPath, hostURL, extensionURL}; } +std::string ExtensionUtils::getExtensionFileName(const std::string& name) { + return common::stringFormat(EXTENSION_FILE_NAME, common::StringUtils::getLower(name)); +} + +std::string ExtensionUtils::getLocalPathForExtension(main::ClientContext* context, + const std::string& extensionName, const std::string& fileName) { + return common::stringFormat("{}{}/{}", context->getExtensionDir(), extensionName, + getExtensionFileName(fileName)); +} + +std::string ExtensionUtils::getLocalExtensionDir(main::ClientContext* context, + const std::string& extensionName) { + return common::stringFormat("{}{}", context->getExtensionDir(), extensionName); +} + void ExtensionUtils::registerTableFunction(main::Database& database, std::unique_ptr function) { auto name = function->name; diff --git a/src/extension/extension_installer.cpp b/src/extension/extension_installer.cpp new file mode 100644 index 00000000000..1320a2c33c3 --- /dev/null +++ b/src/extension/extension_installer.cpp @@ -0,0 +1,55 @@ +#include "common/exception/io.h" +#include "common/file_system/virtual_file_system.h" +#include "extension/extension_installer.h" +#include "httplib.h" +#include "main/client_context.h" + +namespace kuzu { +namespace extension { + +void ExtensionInstaller::tryDownloadExtensionFile(main::ClientContext* context, + const ExtensionRepoInfo& info, const std::string& localFilePath) { + httplib::Client cli(info.hostURL.c_str()); + httplib::Headers headers = { + {"User-Agent", common::stringFormat("kuzu/v{}", KUZU_EXTENSION_VERSION)}}; + auto res = cli.Get(info.hostPath.c_str(), headers); + if (!res || res->status != 200) { + if (res.error() == httplib::Error::Success) { + // LCOV_EXCL_START + throw common::IOException(common::stringFormat( + "HTTP Returns: {}, Failed to download extension: \"{}\" from {}.", + res.value().status, extensionName, info.repoURL)); + // LCOC_EXCL_STOP + } else { + throw common::IOException( + common::stringFormat("Failed to download extension: {} at URL {} (ERROR: {})", + extensionName, info.repoURL, to_string(res.error()))); + } + } + + auto vfs = context->getVFSUnsafe(); + auto fileInfo = vfs->openFile(localFilePath, O_WRONLY | O_CREAT); + fileInfo->writeFile(reinterpret_cast(res->body.c_str()), res->body.size(), + 0 /* offset */); + fileInfo->syncFile(); +} + +void ExtensionInstaller::install(main::ClientContext* context) { + auto vfs = context->getVFSUnsafe(); + auto localExtensionDir = context->getExtensionDir(); + if (!vfs->fileOrPathExists(localExtensionDir, context)) { + vfs->createDir(localExtensionDir); + } + auto localDirForExtension = + extension::ExtensionUtils::getLocalExtensionDir(context, extensionName); + if (!vfs->fileOrPathExists(localDirForExtension)) { + vfs->createDir(localDirForExtension); + } + auto libFileRepoInfo = extension::ExtensionUtils::getExtensionLibRepoInfo(extensionName); + auto localLibFilePath = + extension::ExtensionUtils::getLocalPathForExtensionLib(context, extensionName); + tryDownloadExtensionFile(context, libFileRepoInfo, localLibFilePath); +} + +} // namespace extension +} // namespace kuzu diff --git a/src/include/extension/extension.h b/src/include/extension/extension.h index 0d4603502dc..5d6fa091ea5 100644 --- a/src/include/extension/extension.h +++ b/src/include/extension/extension.h @@ -35,17 +35,29 @@ struct ExtensionRepoInfo { }; struct ExtensionUtils { + // static constexpr const char* EXTENSION_REPO = + // "http://extension.kuzudb.com/v{}/{}/{}/lib{}.kuzu_extension"; + static constexpr const char* EXTENSION_REPO = - "http://extension.kuzudb.com/v{}/{}/lib{}.kuzu_extension"; + "http://localhost/extension/releases/v{}/{}/{}/{}"; + + static constexpr const char* EXTENSION_FILE_NAME = "lib{}.kuzu_extension"; static constexpr const char* OFFICIAL_EXTENSION[] = {"HTTPFS", "POSTGRES", "DUCKDB", "JSON", "SQLITE"}; - static std::string getExtensionPath(const std::string& extensionDir, const std::string& name); - static bool isFullPath(const std::string& extension); - static ExtensionRepoInfo getExtensionRepoInfo(const std::string& extension); + static KUZU_API ExtensionRepoInfo getExtensionRepoInfo(const std::string& extensionName, + const std::string& fileName); + + static std::string getExtensionFileName(const std::string& name); + + KUZU_API static std::string getLocalPathForExtension(main::ClientContext* context, + const std::string& extensionName, const std::string& fileName); + + KUZU_API static std::string getLocalExtensionDir(main::ClientContext* context, + const std::string& extensionName); KUZU_API static void registerTableFunction(main::Database& database, std::unique_ptr function); diff --git a/src/include/extension/extension_installer.h b/src/include/extension/extension_installer.h new file mode 100644 index 00000000000..9dff3600d8d --- /dev/null +++ b/src/include/extension/extension_installer.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include "common/api.h" +#include "extension.h" + +namespace kuzu { +namespace main { +class ClientContext; +} +namespace extension { + +class KUZU_API ExtensionInstaller { +public: + explicit ExtensionInstaller(const std::string extensionName) + : extensionName{std::move(extensionName)} {} + + virtual ~ExtensionInstaller() = default; + + virtual void install(main::ClientContext* context) = 0; + +protected: + void tryDownloadExtensionFile(main::ClientContext* context, const ExtensionRepoInfo& info, + const std::string& fileName); + +protected: + std::string extensionName; +}; + +} // namespace extension +} // namespace kuzu diff --git a/src/main/client_context.cpp b/src/main/client_context.cpp index 7f11ad23a1b..37acaba0d88 100644 --- a/src/main/client_context.cpp +++ b/src/main/client_context.cpp @@ -163,7 +163,7 @@ extension::ExtensionOptions* ClientContext::getExtensionOptions() const { } std::string ClientContext::getExtensionDir() const { - return stringFormat("{}/.kuzu/extension/{}/{}", clientConfig.homeDirectory, + return stringFormat("{}/.kuzu/extension/{}/{}/", clientConfig.homeDirectory, KUZU_EXTENSION_VERSION, kuzu::extension::getPlatform()); } diff --git a/src/processor/operator/simple/install_extension.cpp b/src/processor/operator/simple/install_extension.cpp index 5a8fe22873e..58023b20170 100644 --- a/src/processor/operator/simple/install_extension.cpp +++ b/src/processor/operator/simple/install_extension.cpp @@ -3,10 +3,23 @@ #include "common/exception/io.h" #include "common/file_system/virtual_file_system.h" #include "common/string_format.h" +#include "common/system_message.h" #include "extension/extension.h" #include "httplib.h" #include "main/database.h" +#ifdef _WIN32 + +#include "windows.h" +#define RTLD_NOW 0 +#define RTLD_LOCAL 0 + +#else +#include +#endif + +typedef void (*ext_install_func_t)(kuzu::main::ClientContext*); + namespace kuzu { namespace processor { @@ -26,7 +39,7 @@ std::string InstallExtension::getOutputMsg() { } std::string InstallExtension::tryDownloadExtension() { - auto extensionRepoInfo = ExtensionUtils::getExtensionRepoInfo(name); + auto extensionRepoInfo = ExtensionUtils::getExtensionRepoInfo(name, name + "_installer"); httplib::Client cli(extensionRepoInfo.hostURL.c_str()); httplib::Headers headers = { {"User-Agent", common::stringFormat("kuzu/v{}", KUZU_EXTENSION_VERSION)}}; @@ -50,11 +63,15 @@ std::string InstallExtension::tryDownloadExtension() { void InstallExtension::saveExtensionToLocalFile(const std::string& extensionData, main::ClientContext* context) { auto extensionDir = context->getExtensionDir(); - auto extensionPath = ExtensionUtils::getExtensionPath(extensionDir, name); + auto extensionPath = + ExtensionUtils::getLocalPathForExtension(context, name, name + "_installer"); auto vfs = context->getVFSUnsafe(); if (!vfs->fileOrPathExists(extensionDir, context)) { vfs->createDir(extensionDir); } + if (!vfs->fileOrPathExists(ExtensionUtils::getLocalExtensionDir(context, name))) { + vfs->createDir(ExtensionUtils::getLocalExtensionDir(context, name)); + } auto fileInfo = vfs->openFile(extensionPath, O_WRONLY | O_CREAT); fileInfo->writeFile(reinterpret_cast(extensionData.c_str()), extensionData.size(), 0 /* offset */); @@ -64,6 +81,20 @@ void InstallExtension::saveExtensionToLocalFile(const std::string& extensionData void InstallExtension::installExtension(main::ClientContext* context) { auto extensionData = tryDownloadExtension(); saveExtensionToLocalFile(extensionData, context); + auto extensionInstallerPath = + ExtensionUtils::getLocalPathForExtension(context, name, name + "_installer"); + auto libHdl = dlopen(extensionInstallerPath.c_str(), RTLD_NOW | RTLD_LOCAL); + if (libHdl == nullptr) { + throw common::IOException(stringFormat("Extension \"{}\" could not be loaded.\nError: {}", + extensionInstallerPath.c_str(), dlErrMessage())); + } + auto install = (ext_install_func_t)(dlsym(libHdl, "install")); + if (install == nullptr) { + throw common::IOException( + stringFormat("Extension \"{}\" does not have a valid install function.\nError: {}", + name, dlErrMessage())); + } + (*install)(context); } } // namespace processor diff --git a/src/processor/operator/simple/load_extension.cpp b/src/processor/operator/simple/load_extension.cpp index 4ff2ca14901..4758aaedba2 100644 --- a/src/processor/operator/simple/load_extension.cpp +++ b/src/processor/operator/simple/load_extension.cpp @@ -59,8 +59,14 @@ void* dlsym(void* handle, const char* name) { void LoadExtension::executeInternal(kuzu::processor::ExecutionContext* context) { if (!extension::ExtensionUtils::isFullPath(path)) { - path = ExtensionUtils::getExtensionPath(context->clientContext->getExtensionDir(), path); + path = ExtensionUtils::getLocalPathForExtension(context->clientContext, path, path); } + // auto duckdbHdl = + // dlopen("/Users/z473chen/Desktop/code/kuzu/libduckdb.dylib", RTLD_NOW | RTLD_GLOBAL); + // if (!duckdbHdl) { + // throw common::IOException( + // stringFormat("Duckdb cannot be loaded.\nError: {}", dlErrMessage())); + // } auto libHdl = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); if (libHdl == nullptr) { throw common::IOException( From 628e6e7e0aff7def0f627a239f436d5b7b924189 Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Tue, 13 Aug 2024 14:13:31 +0800 Subject: [PATCH 02/19] update1 --- Makefile | 2 +- extension/duckdb/CMakeLists.txt | 39 +++++++ extension/duckdb/src/duckdb_installer.cpp | 33 ++++-- extension/duckdb/src/duckdb_loader.cpp | 53 ++++++++- .../duckdb/src/include/duckdb_installer.h | 13 ++- extension/duckdb/src/include/duckdb_loader.h | 20 +++- extension/httpfs/CMakeLists.txt | 19 ---- extension/postgres/CMakeLists.txt | 42 ++++++- .../postgres/src/postgres_install_func.cpp | 18 ++- extension/sqlite/CMakeLists.txt | 42 ++++++- extension/sqlite/src/sqlite_connector.cpp | 2 + extension/sqlite/src/sqlite_install_func.cpp | 2 +- src/extension/CMakeLists.txt | 2 +- src/extension/extension.cpp | 103 ++++++++++++++++-- src/extension/extension_installer.cpp | 7 +- src/include/common/system_message.h | 4 +- src/include/extension/extension.h | 64 +++++++++-- src/include/extension/extension_installer.h | 4 +- .../operator/simple/install_extension.h | 4 - .../operator/simple/install_extension.cpp | 82 ++++---------- .../operator/simple/load_extension.cpp | 55 ++++------ 21 files changed, 437 insertions(+), 173 deletions(-) diff --git a/Makefile b/Makefile index 2b19ee9a930..5913317afef 100644 --- a/Makefile +++ b/Makefile @@ -191,7 +191,7 @@ extension-json-test: extension-json-test-build extension-debug: $(call run-cmake-debug, \ - -DBUILD_EXTENSIONS="httpfs" \ + -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite" \ -DBUILD_KUZU=FALSE \ ) diff --git a/extension/duckdb/CMakeLists.txt b/extension/duckdb/CMakeLists.txt index 3b7a18e5df1..52cd7b72d51 100644 --- a/extension/duckdb/CMakeLists.txt +++ b/extension/duckdb/CMakeLists.txt @@ -24,6 +24,15 @@ add_library(duckdb_extension src/duckdb_s3_auth.cpp src/duckdb_connector.cpp) +add_library(duckdb_extension_installer + SHARED + src/duckdb_installer.cpp + src/duckdb_install_func.cpp) + +add_library(duckdb_extension_loader + SHARED + src/duckdb_loader.cpp) + set_target_properties(duckdb_extension PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" @@ -31,6 +40,20 @@ set_target_properties(duckdb_extension RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" ) +set_target_properties(duckdb_extension_installer + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" +) + +set_target_properties(duckdb_extension_loader + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" +) + target_link_libraries(duckdb_extension PRIVATE ${DuckDB_LIBRARIES}) @@ -41,6 +64,18 @@ set_target_properties(duckdb_extension PROPERTIES SUFFIX ".kuzu_extension" ) +set_target_properties(duckdb_extension_installer PROPERTIES + OUTPUT_NAME duckdb_installer + PREFIX "lib" + SUFFIX ".kuzu_extension" +) + +set_target_properties(duckdb_extension_loader PROPERTIES + OUTPUT_NAME duckdb_loader + PREFIX "lib" + SUFFIX ".kuzu_extension" +) + if (WIN32) # On windows, there is no dynamic lookup available, so it's not # possible to generically look for symbols on library load. There are @@ -59,8 +94,12 @@ if (WIN32) # Future work could make it possible to embed extension into kuzu, # which would help fix this problem. target_link_libraries(duckdb_extension PRIVATE kuzu) + target_link_libraries(duckdb_extension_installer PRIVATE kuzu) + target_link_libraries(duckdb_extension_loader PRIVATE kuzu) endif() if (APPLE) set_target_properties(duckdb_extension PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + set_target_properties(duckdb_extension_installer PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + set_target_properties(duckdb_extension_loader PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") endif () diff --git a/extension/duckdb/src/duckdb_installer.cpp b/extension/duckdb/src/duckdb_installer.cpp index 0fb6d04c4dc..313e0ae6ae8 100644 --- a/extension/duckdb/src/duckdb_installer.cpp +++ b/extension/duckdb/src/duckdb_installer.cpp @@ -1,16 +1,27 @@ -#pragma once -#include "extension/extension_downloader.h" +#include "duckdb_installer.h" + +#include "common/file_system/virtual_file_system.h" +#include "main/client_context.h" namespace kuzu { -namespace httpfs { +namespace duckdb { -class HttpfsInstaller : public extension::ExtensionInstaller { -public: - explicit HttpfsInstaller(const std::string extensionName) - : ExtensionInstaller{std::move(extensionName)} {} +void DuckDBInstaller::install(main::ClientContext* context) { + auto loaderFileRepoInfo = extension::ExtensionUtils::getExtensionLoaderRepoInfo(extensionName); + auto localLoaderFilePath = + extension::ExtensionUtils::getLocalPathForExtensionLoader(context, extensionName); + tryDownloadExtensionFile(context, loaderFileRepoInfo, localLoaderFilePath); - void install(main::ClientContext* context) override; -}; + for (auto& dependencyLib : DEPENDENCY_LIB_FILES) { + auto localDependencyLibPath = + extension::ExtensionUtils::getLocalPathForSharedLib(context, dependencyLib); + if (!context->getVFSUnsafe()->fileOrPathExists(localDependencyLibPath)) { + auto dependencyLibRepoInfo = + extension::ExtensionUtils::getSharedLibRepoInfo(dependencyLib); + tryDownloadExtensionFile(context, dependencyLibRepoInfo, localDependencyLibPath); + } + } +} -} // namespace httpfs -} // namespace kuzu \ No newline at end of file +} // namespace duckdb +} // namespace kuzu diff --git a/extension/duckdb/src/duckdb_loader.cpp b/extension/duckdb/src/duckdb_loader.cpp index 5fbea6d1f7b..7a8e15c02a2 100644 --- a/extension/duckdb/src/duckdb_loader.cpp +++ b/extension/duckdb/src/duckdb_loader.cpp @@ -1,3 +1,50 @@ -// -// Created by z473chen on 2024-08-12. -// +#include "duckdb_loader.h" + +#include "common/exception/io.h" +#include "common/file_system/virtual_file_system.h" +#include "common/system_message.h" +#include "extension/extension.h" +#include "main/client_context.h" +#ifdef _WIN32 + +#include "windows.h" +#define RTLD_NOW 0 +#define RTLD_LOCAL 0 + +#else +#include +#endif + +namespace kuzu { +namespace duckdb { + +void DuckDBLoader::load(main::ClientContext* context) { +w for (auto& dependencyLib : DEPENDENCY_LIB_FILES) { + auto dependencyLibPath = + extension::ExtensionUtils::getLocalPathForSharedLib(context, dependencyLib); + KU_ASSERT(context->getVFSUnsafe()->fileOrPathExists(dependencyLibPath)); + auto libHdl = dlopen(dependencyLibPath.c_str(), RTLD_NOW | RTLD_LOCAL); + if (libHdl == nullptr) { + throw common::IOException( + common::stringFormat("Lib \"{}\" could not be loaded.\nError: {}", + dependencyLibPath, common::dlErrMessage())); + } + } +} + +} // namespace duckdb +} // namespace kuzu + +extern "C" { +// Because we link against the static library on windows, we implicitly inherit KUZU_STATIC_DEFINE, +// which cancels out any exporting, so we can't use KUZU_API. +#if defined(_WIN32) +#define INIT_EXPORT __declspec(dllexport) +#else +#define INIT_EXPORT __attribute__((visibility("default"))) +#endif +INIT_EXPORT void load(kuzu::main::ClientContext* context) { + kuzu::duckdb::DuckDBLoader loader{}; + loader.load(context); +} +} diff --git a/extension/duckdb/src/include/duckdb_installer.h b/extension/duckdb/src/include/duckdb_installer.h index 0fb6d04c4dc..b8edb24ba5c 100644 --- a/extension/duckdb/src/include/duckdb_installer.h +++ b/extension/duckdb/src/include/duckdb_installer.h @@ -1,16 +1,19 @@ #pragma once -#include "extension/extension_downloader.h" +#include "extension/extension_installer.h" namespace kuzu { -namespace httpfs { +namespace duckdb { + +class DuckDBInstaller : public extension::ExtensionInstaller { +private: + static constexpr const char* DEPENDENCY_LIB_FILES[] = {"libduckdb.dylib"}; -class HttpfsInstaller : public extension::ExtensionInstaller { public: - explicit HttpfsInstaller(const std::string extensionName) + explicit DuckDBInstaller(const std::string extensionName) : ExtensionInstaller{std::move(extensionName)} {} void install(main::ClientContext* context) override; }; -} // namespace httpfs +} // namespace duckdb } // namespace kuzu \ No newline at end of file diff --git a/extension/duckdb/src/include/duckdb_loader.h b/extension/duckdb/src/include/duckdb_loader.h index f4adb63dfb6..28852a12077 100644 --- a/extension/duckdb/src/include/duckdb_loader.h +++ b/extension/duckdb/src/include/duckdb_loader.h @@ -1,8 +1,16 @@ -// -// Created by z473chen on 2024-08-12. -// +#pragma once +#include "extension/extension_installer.h" -#ifndef KUZU_DUCKDB_LOADER_H -#define KUZU_DUCKDB_LOADER_H +namespace kuzu { +namespace duckdb { -#endif //KUZU_DUCKDB_LOADER_H +class DuckDBLoader { +private: + static constexpr const char* DEPENDENCY_LIB_FILES[] = {"libduckdb.dylib"}; + +public: + void load(main::ClientContext* context); +}; + +} // namespace duckdb +} // namespace kuzu \ No newline at end of file diff --git a/extension/httpfs/CMakeLists.txt b/extension/httpfs/CMakeLists.txt index 713fa1170cb..3e68ea3119c 100644 --- a/extension/httpfs/CMakeLists.txt +++ b/extension/httpfs/CMakeLists.txt @@ -26,10 +26,6 @@ add_library(httpfs src/http_config.cpp src/cached_file_manager.cpp) -add_library(httpfs_installer - SHARED - src/httpfs_installer.cpp) - target_link_libraries(httpfs PRIVATE mbedtls @@ -41,12 +37,6 @@ set_target_properties(httpfs PROPERTIES SUFFIX ".kuzu_extension" ) -set_target_properties(httpfs_installer PROPERTIES - OUTPUT_NAME httpfs_installer - PREFIX "lib" - SUFFIX ".kuzu_extension" -) - set_target_properties(httpfs PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" @@ -54,13 +44,6 @@ set_target_properties(httpfs RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" ) -set_target_properties(httpfs_installer - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" -) - if (WIN32) # On windows, there is no dynamic lookup available, so it's not # possible to generically look for symbols on library load. There are @@ -79,12 +62,10 @@ if (WIN32) # Future work could make it possible to embed extension into kuzu, # which would help fix this problem. target_link_libraries(httpfs PRIVATE kuzu) - target_link_libraries(httpfs_installer PRIVATE kuzu) endif() if (APPLE) set_target_properties(httpfs PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") - set_target_properties(httpfs_installer PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") endif () if (${BUILD_EXTENSION_TESTS}) diff --git a/extension/postgres/CMakeLists.txt b/extension/postgres/CMakeLists.txt index 81c24cdddbf..2e3a0eba8db 100644 --- a/extension/postgres/CMakeLists.txt +++ b/extension/postgres/CMakeLists.txt @@ -6,9 +6,6 @@ else() set(DuckDB_USE_STATIC_LIBS ON) endif() find_package(DuckDB REQUIRED) -if(NOT WIN32) - find_library(DuckDB_STATIC REQUIRED NAMES libduckdb_static.a) -endif() add_library(postgres_extension SHARED @@ -22,6 +19,15 @@ add_library(postgres_extension src/postgres_storage.cpp src/postgres_connector.cpp) +add_library(postgres_extension_installer + SHARED + src/postgres_install_func.cpp + ../duckdb/src/duckdb_installer.cpp) + +add_library(postgres_extension_loader + SHARED + ../duckdb/src/duckdb_loader.cpp) + include_directories( src/include ../duckdb/src/include @@ -34,6 +40,18 @@ set_target_properties(postgres_extension PROPERTIES SUFFIX ".kuzu_extension" ) +set_target_properties(postgres_extension_loader PROPERTIES + OUTPUT_NAME postgres_loader + PREFIX "lib" + SUFFIX ".kuzu_extension" +) + +set_target_properties(postgres_extension_installer PROPERTIES + OUTPUT_NAME postgres_installer + PREFIX "lib" + SUFFIX ".kuzu_extension" +) + set_target_properties(postgres_extension PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" @@ -41,6 +59,20 @@ set_target_properties(postgres_extension RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" ) +set_target_properties(postgres_extension_installer + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" +) + +set_target_properties(postgres_extension_loader + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" +) + target_link_libraries(postgres_extension PRIVATE ${DuckDB_LIBRARIES}) @@ -63,8 +95,12 @@ if (WIN32) # Future work could make it possible to embed extension into kuzu, # which would help fix this problem. target_link_libraries(postgres_extension PRIVATE kuzu) + target_link_libraries(postgres_extension_loader PRIVATE kuzu) + target_link_libraries(postgres_extension_installer PRIVATE kuzu) endif() if (APPLE) set_target_properties(postgres_extension PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + set_target_properties(postgres_extension_loader PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + set_target_properties(postgres_extension_installer PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") endif () diff --git a/extension/postgres/src/postgres_install_func.cpp b/extension/postgres/src/postgres_install_func.cpp index 838d9c89ce8..e8cda2e8a60 100644 --- a/extension/postgres/src/postgres_install_func.cpp +++ b/extension/postgres/src/postgres_install_func.cpp @@ -1,3 +1,15 @@ -// -// Created by z473chen on 2024-08-13. -// +#include "duckdb_installer.h" + +extern "C" { +// Because we link against the static library on windows, we implicitly inherit KUZU_STATIC_DEFINE, +// which cancels out any exporting, so we can't use KUZU_API. +#if defined(_WIN32) +#define INIT_EXPORT __declspec(dllexport) +#else +#define INIT_EXPORT __attribute__((visibility("default"))) +#endif +INIT_EXPORT void install(kuzu::main::ClientContext* context) { + kuzu::duckdb::DuckDBInstaller installer{"postgres"}; + installer.install(context); +} +} diff --git a/extension/sqlite/CMakeLists.txt b/extension/sqlite/CMakeLists.txt index bb4db9e0969..740c2a3cb53 100644 --- a/extension/sqlite/CMakeLists.txt +++ b/extension/sqlite/CMakeLists.txt @@ -6,9 +6,6 @@ else() set(DuckDB_USE_STATIC_LIBS ON) endif() find_package(DuckDB REQUIRED) -if(NOT WIN32) - find_library(DuckDB_STATIC REQUIRED NAMES libduckdb_static.a) -endif() include_directories( ${PROJECT_SOURCE_DIR}/src/include @@ -28,6 +25,15 @@ add_library(sqlite_extension src/sqlite_storage.cpp src/sqlite_connector.cpp) +add_library(sqlite_extension_installer + SHARED + src/sqlite_install_func.cpp + ../duckdb/src/duckdb_installer.cpp) + +add_library(sqlite_extension_loader + SHARED + ../duckdb/src/duckdb_loader.cpp) + set_target_properties(sqlite_extension PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" @@ -35,6 +41,20 @@ set_target_properties(sqlite_extension RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" ) +set_target_properties(sqlite_extension_loader + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" +) + +set_target_properties(sqlite_extension_installer + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" +) + target_link_libraries(sqlite_extension PRIVATE ${DuckDB_LIBRARIES}) @@ -45,6 +65,18 @@ set_target_properties(sqlite_extension PROPERTIES SUFFIX ".kuzu_extension" ) +set_target_properties(sqlite_extension_loader PROPERTIES + OUTPUT_NAME sqlite_loader + PREFIX "lib" + SUFFIX ".kuzu_extension" +) + +set_target_properties(sqlite_extension_installer PROPERTIES + OUTPUT_NAME sqlite_installer + PREFIX "lib" + SUFFIX ".kuzu_extension" +) + if (WIN32) # On windows, there is no dynamic lookup available, so it's not # possible to generically look for symbols on library load. There are @@ -63,8 +95,12 @@ if (WIN32) # Future work could make it possible to embed extension into kuzu, # which would help fix this problem. target_link_libraries(sqlite_extension PRIVATE kuzu) + target_link_libraries(sqlite_extension_loader PRIVATE kuzu) + target_link_libraries(sqlite_extension_installer PRIVATE kuzu) endif() if (APPLE) set_target_properties(sqlite_extension PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + set_target_properties(sqlite_extension_loader PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + set_target_properties(sqlite_extension_installer PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") endif () diff --git a/extension/sqlite/src/sqlite_connector.cpp b/extension/sqlite/src/sqlite_connector.cpp index e394f4f7768..95889e45a5a 100644 --- a/extension/sqlite/src/sqlite_connector.cpp +++ b/extension/sqlite/src/sqlite_connector.cpp @@ -8,6 +8,8 @@ void SqliteConnector::connect(const std::string& dbPath, const std::string& cata // Creates an in-memory duckdb instance, then install httpfs and attach SQLITE. instance = std::make_unique(nullptr); connection = std::make_unique(*instance); + executeQuery("install sqlite;"); + executeQuery("load sqlite;"); executeQuery( common::stringFormat("attach '{}' as {} (TYPE sqlite, read_only)", dbPath, catalogName)); } diff --git a/extension/sqlite/src/sqlite_install_func.cpp b/extension/sqlite/src/sqlite_install_func.cpp index e8cda2e8a60..ae62f8b7610 100644 --- a/extension/sqlite/src/sqlite_install_func.cpp +++ b/extension/sqlite/src/sqlite_install_func.cpp @@ -9,7 +9,7 @@ extern "C" { #define INIT_EXPORT __attribute__((visibility("default"))) #endif INIT_EXPORT void install(kuzu::main::ClientContext* context) { - kuzu::duckdb::DuckDBInstaller installer{"postgres"}; + kuzu::duckdb::DuckDBInstaller installer{"sqlite"}; installer.install(context); } } diff --git a/src/extension/CMakeLists.txt b/src/extension/CMakeLists.txt index 10b3efab842..edd7011b777 100644 --- a/src/extension/CMakeLists.txt +++ b/src/extension/CMakeLists.txt @@ -2,7 +2,7 @@ add_library(kuzu_extension OBJECT catalog_extension.cpp extension.cpp - extension_downloader.cpp) + extension_installer.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ diff --git a/src/extension/extension.cpp b/src/extension/extension.cpp index 134e8a556c5..f8ed2d6f283 100644 --- a/src/extension/extension.cpp +++ b/src/extension/extension.cpp @@ -1,13 +1,26 @@ #include "extension/extension.h" #include "catalog/catalog.h" +#include "common/exception/io.h" +#include "common/file_system/virtual_file_system.h" #include "common/string_format.h" #include "common/string_utils.h" +#include "common/system_message.h" #include "function/table_functions.h" #include "main/client_context.h" #include "main/database.h" #include "transaction/transaction.h" +#ifdef _WIN32 + +#include "windows.h" +#define RTLD_NOW 0 +#define RTLD_LOCAL 0 + +#else +#include +#endif + namespace kuzu { namespace extension { @@ -45,26 +58,61 @@ bool ExtensionUtils::isFullPath(const std::string& extension) { extension.find('\\') != std::string::npos; } -ExtensionRepoInfo ExtensionUtils::getExtensionRepoInfo(const std::string& extensionName, - const std::string& fileName) { - auto extensionURL = common::stringFormat(EXTENSION_REPO, KUZU_EXTENSION_VERSION, getPlatform(), - extensionName, getExtensionFileName(fileName)); +static ExtensionRepoInfo getExtensionRepoInfo(std::string& extensionURL) { common::StringUtils::replaceAll(extensionURL, "http://", ""); auto hostNamePos = extensionURL.find('/'); auto hostName = extensionURL.substr(0, hostNamePos); auto hostURL = "http://" + hostName; auto hostPath = extensionURL.substr(hostNamePos); return {hostPath, hostURL, extensionURL}; +}; + +ExtensionRepoInfo ExtensionUtils::getExtensionLibRepoInfo(const std::string& extensionName) { + auto extensionURL = common::stringFormat(EXTENSION_FILE_REPO, KUZU_EXTENSION_VERSION, + getPlatform(), extensionName, getExtensionFileName(extensionName)); + return getExtensionRepoInfo(extensionURL); +} + +ExtensionRepoInfo ExtensionUtils::getExtensionLoaderRepoInfo(const std::string& extensionName) { + auto extensionURL = + common::stringFormat(EXTENSION_FILE_REPO, KUZU_EXTENSION_VERSION, getPlatform(), + extensionName, getExtensionFileName(extensionName + EXTENSION_LOADER_SUFFIX)); + return getExtensionRepoInfo(extensionURL); +} + +ExtensionRepoInfo ExtensionUtils::getExtensionInstallerRepoInfo(const std::string& extensionName) { + auto extensionURL = + common::stringFormat(EXTENSION_FILE_REPO, KUZU_EXTENSION_VERSION, getPlatform(), + extensionName, getExtensionFileName(extensionName + EXTENSION_INSTALLER_SUFFIX)); + return getExtensionRepoInfo(extensionURL); +} + +ExtensionRepoInfo ExtensionUtils::getSharedLibRepoInfo(const std::string& fileName) { + auto extensionURL = + common::stringFormat(SHARED_LIB_REPO, KUZU_EXTENSION_VERSION, getPlatform(), fileName); + return getExtensionRepoInfo(extensionURL); } std::string ExtensionUtils::getExtensionFileName(const std::string& name) { return common::stringFormat(EXTENSION_FILE_NAME, common::StringUtils::getLower(name)); } -std::string ExtensionUtils::getLocalPathForExtension(main::ClientContext* context, - const std::string& extensionName, const std::string& fileName) { +std::string ExtensionUtils::getLocalPathForExtensionLib(main::ClientContext* context, + const std::string& extensionName) { return common::stringFormat("{}{}/{}", context->getExtensionDir(), extensionName, - getExtensionFileName(fileName)); + getExtensionFileName(extensionName)); +} + +std::string ExtensionUtils::getLocalPathForExtensionLoader(main::ClientContext* context, + const std::string& extensionName) { + return common::stringFormat("{}{}/{}", context->getExtensionDir(), extensionName, + getExtensionFileName(extensionName + EXTENSION_LOADER_SUFFIX)); +} + +std::string ExtensionUtils::getLocalPathForExtensionInstaller(main::ClientContext* context, + const std::string& extensionName) { + return common::stringFormat("{}{}/{}", context->getExtensionDir(), extensionName, + getExtensionFileName(extensionName + EXTENSION_INSTALLER_SUFFIX)); } std::string ExtensionUtils::getLocalExtensionDir(main::ClientContext* context, @@ -86,6 +134,15 @@ void ExtensionUtils::registerTableFunction(main::Database& database, catalog::CatalogEntryType::TABLE_FUNCTION_ENTRY, std::move(name), std::move(functionSet)); } +std::string ExtensionUtils::getLocalPathForSharedLib(main::ClientContext* context, + const std::string& libName) { + return common::stringFormat("{}common/{}", context->getExtensionDir(), libName); +} + +std::string ExtensionUtils::getLocalPathForSharedLib(main::ClientContext* context) { + return common::stringFormat("{}common/", context->getExtensionDir()); +} + void ExtensionUtils::registerFunctionSet(main::Database& database, std::string name, function::function_set functionSet) { auto catalog = database.getCatalog(); @@ -107,6 +164,38 @@ bool ExtensionUtils::isOfficialExtension(const std::string& extension) { return false; } +ExtensionLibLoader::ExtensionLibLoader(const std::string& extensionName, const std::string& path) + : extensionName{extensionName} { + libHdl = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (libHdl == nullptr) { + throw common::IOException(common::stringFormat( + "Failed to load library: {} which is needed by extension: {}.\nError: {}.", path, + extensionName, common::dlErrMessage())); + } +} + +ext_load_func_t ExtensionLibLoader::getLoadFunc() { + return (ext_load_func_t)getDynamicLibFunc(EXTENSION_LOAD_FUNC_NAME); +} + +ext_init_func_t ExtensionLibLoader::getInitFunc() { + return (ext_init_func_t)getDynamicLibFunc(EXTENSION_INIT_FUNC_NAME); +} + +ext_install_func_t ExtensionLibLoader::getInstallFunc() { + return (ext_install_func_t)getDynamicLibFunc(EXTENSION_INSTALL_FUNC_NAME); +} + +void* ExtensionLibLoader::getDynamicLibFunc(const std::string& funcName) { + auto sym = dlsym(libHdl, funcName.c_str()); + if (sym == nullptr) { + throw common::IOException( + common::stringFormat("Failed to load {} function in extension {}.\nError: {}", funcName, + extensionName, common::dlErrMessage())); + } + return sym; +} + void ExtensionOptions::addExtensionOption(std::string name, common::LogicalTypeID type, common::Value defaultValue) { common::StringUtils::toLower(name); diff --git a/src/extension/extension_installer.cpp b/src/extension/extension_installer.cpp index 1320a2c33c3..8693a93ca3d 100644 --- a/src/extension/extension_installer.cpp +++ b/src/extension/extension_installer.cpp @@ -1,6 +1,7 @@ +#include "extension/extension_installer.h" + #include "common/exception/io.h" #include "common/file_system/virtual_file_system.h" -#include "extension/extension_installer.h" #include "httplib.h" #include "main/client_context.h" @@ -45,6 +46,10 @@ void ExtensionInstaller::install(main::ClientContext* context) { if (!vfs->fileOrPathExists(localDirForExtension)) { vfs->createDir(localDirForExtension); } + auto localDirForSharedLib = extension::ExtensionUtils::getLocalPathForSharedLib(context); + if (!vfs->fileOrPathExists(localDirForSharedLib)) { + vfs->createDir(localDirForSharedLib); + } auto libFileRepoInfo = extension::ExtensionUtils::getExtensionLibRepoInfo(extensionName); auto localLibFilePath = extension::ExtensionUtils::getLocalPathForExtensionLib(context, extensionName); diff --git a/src/include/common/system_message.h b/src/include/common/system_message.h index fde046d1710..43ba8df7ee3 100644 --- a/src/include/common/system_message.h +++ b/src/include/common/system_message.h @@ -4,6 +4,8 @@ #include #include +#include "common/api.h" + namespace kuzu { namespace common { @@ -21,7 +23,7 @@ inline std::string posixErrMessage() { // LCOV_EXCL_STOP } -std::string dlErrMessage(); +KUZU_API std::string dlErrMessage(); } // namespace common } // namespace kuzu diff --git a/src/include/extension/extension.h b/src/include/extension/extension.h index 5d6fa091ea5..f17f8a54c88 100644 --- a/src/include/extension/extension.h +++ b/src/include/extension/extension.h @@ -21,6 +21,10 @@ class Database; namespace extension { +typedef void (*ext_init_func_t)(kuzu::main::ClientContext*); +using ext_load_func_t = ext_init_func_t; +using ext_install_func_t = ext_init_func_t; + std::string getPlatform(); class KUZU_API Extension { @@ -35,30 +39,49 @@ struct ExtensionRepoInfo { }; struct ExtensionUtils { - // static constexpr const char* EXTENSION_REPO = - // "http://extension.kuzudb.com/v{}/{}/{}/lib{}.kuzu_extension"; + static constexpr const char* EXTENSION_FILE_REPO = "http://extension.kuzudb.com/v{}/{}/{}/{}"; - static constexpr const char* EXTENSION_REPO = - "http://localhost/extension/releases/v{}/{}/{}/{}"; + static constexpr const char* SHARED_LIB_REPO = "http://extension.kuzudb.com/v{}/{}/common/{}"; static constexpr const char* EXTENSION_FILE_NAME = "lib{}.kuzu_extension"; static constexpr const char* OFFICIAL_EXTENSION[] = {"HTTPFS", "POSTGRES", "DUCKDB", "JSON", "SQLITE"}; + static constexpr const char* EXTENSION_LOADER_SUFFIX = "_loader"; + + static constexpr const char* EXTENSION_INSTALLER_SUFFIX = "_installer"; + static bool isFullPath(const std::string& extension); - static KUZU_API ExtensionRepoInfo getExtensionRepoInfo(const std::string& extensionName, - const std::string& fileName); + static KUZU_API ExtensionRepoInfo getExtensionLibRepoInfo(const std::string& extensionName); + + static KUZU_API ExtensionRepoInfo getExtensionLoaderRepoInfo(const std::string& extensionName); + + static KUZU_API ExtensionRepoInfo getExtensionInstallerRepoInfo( + const std::string& extensionName); + + static KUZU_API ExtensionRepoInfo getSharedLibRepoInfo(const std::string& fileName); static std::string getExtensionFileName(const std::string& name); - KUZU_API static std::string getLocalPathForExtension(main::ClientContext* context, - const std::string& extensionName, const std::string& fileName); + KUZU_API static std::string getLocalPathForExtensionLib(main::ClientContext* context, + const std::string& extensionName); + + KUZU_API static std::string getLocalPathForExtensionLoader(main::ClientContext* context, + const std::string& extensionName); + + KUZU_API static std::string getLocalPathForExtensionInstaller(main::ClientContext* context, + const std::string& extensionName); KUZU_API static std::string getLocalExtensionDir(main::ClientContext* context, const std::string& extensionName); + KUZU_API static std::string getLocalPathForSharedLib(main::ClientContext* context, + const std::string& libName); + + KUZU_API static std::string getLocalPathForSharedLib(main::ClientContext* context); + KUZU_API static void registerTableFunction(main::Database& database, std::unique_ptr function); @@ -68,6 +91,31 @@ struct ExtensionUtils { static bool isOfficialExtension(const std::string& extension); }; +class ExtensionLibLoader { +public: + static constexpr const char* EXTENSION_LOAD_FUNC_NAME = "load"; + + static constexpr const char* EXTENSION_INIT_FUNC_NAME = "init"; + + static constexpr const char* EXTENSION_INSTALL_FUNC_NAME = "install"; + +public: + ExtensionLibLoader(const std::string& extensionName, const std::string& path); + + ext_load_func_t getLoadFunc(); + + ext_init_func_t getInitFunc(); + + ext_install_func_t getInstallFunc(); + +private: + void* getDynamicLibFunc(const std::string& funcName); + +private: + std::string extensionName; + void* libHdl; +}; + struct ExtensionOptions { std::unordered_map extensionOptions; diff --git a/src/include/extension/extension_installer.h b/src/include/extension/extension_installer.h index 9dff3600d8d..79f15d337db 100644 --- a/src/include/extension/extension_installer.h +++ b/src/include/extension/extension_installer.h @@ -18,11 +18,11 @@ class KUZU_API ExtensionInstaller { virtual ~ExtensionInstaller() = default; - virtual void install(main::ClientContext* context) = 0; + virtual void install(main::ClientContext* context); protected: void tryDownloadExtensionFile(main::ClientContext* context, const ExtensionRepoInfo& info, - const std::string& fileName); + const std::string& localFilePath); protected: std::string extensionName; diff --git a/src/include/processor/operator/simple/install_extension.h b/src/include/processor/operator/simple/install_extension.h index 2e44f5217d3..9bc64317e76 100644 --- a/src/include/processor/operator/simple/install_extension.h +++ b/src/include/processor/operator/simple/install_extension.h @@ -38,10 +38,6 @@ class InstallExtension final : public Simple { } private: - std::string tryDownloadExtension(); - - void saveExtensionToLocalFile(const std::string& extensionData, main::ClientContext* context); - void installExtension(main::ClientContext* context); private: diff --git a/src/processor/operator/simple/install_extension.cpp b/src/processor/operator/simple/install_extension.cpp index 58023b20170..539b03b272f 100644 --- a/src/processor/operator/simple/install_extension.cpp +++ b/src/processor/operator/simple/install_extension.cpp @@ -3,23 +3,11 @@ #include "common/exception/io.h" #include "common/file_system/virtual_file_system.h" #include "common/string_format.h" -#include "common/system_message.h" #include "extension/extension.h" +#include "extension/extension_installer.h" #include "httplib.h" #include "main/database.h" -#ifdef _WIN32 - -#include "windows.h" -#define RTLD_NOW 0 -#define RTLD_LOCAL 0 - -#else -#include -#endif - -typedef void (*ext_install_func_t)(kuzu::main::ClientContext*); - namespace kuzu { namespace processor { @@ -38,63 +26,35 @@ std::string InstallExtension::getOutputMsg() { return common::stringFormat("Extension: {} has been installed.", name); } -std::string InstallExtension::tryDownloadExtension() { - auto extensionRepoInfo = ExtensionUtils::getExtensionRepoInfo(name, name + "_installer"); +static void saveExtensionToLocalFile(const std::string& extensionData, const std::string& name, + main::ClientContext* context) { + auto extensionPath = ExtensionUtils::getLocalPathForExtensionInstaller(context, name); + auto fileInfo = context->getVFSUnsafe()->openFile(extensionPath, O_WRONLY | O_CREAT); + fileInfo->writeFile(reinterpret_cast(extensionData.c_str()), + extensionData.size(), 0 /* offset */); + fileInfo->syncFile(); +} + +static void installDependencies(const std::string& name, main::ClientContext* context) { + auto extensionRepoInfo = ExtensionUtils::getExtensionInstallerRepoInfo(name); httplib::Client cli(extensionRepoInfo.hostURL.c_str()); httplib::Headers headers = { {"User-Agent", common::stringFormat("kuzu/v{}", KUZU_EXTENSION_VERSION)}}; auto res = cli.Get(extensionRepoInfo.hostPath.c_str(), headers); if (!res || res->status != 200) { - if (res.error() == httplib::Error::Success) { - // LCOV_EXCL_START - throw IOException(common::stringFormat( - "HTTP Returns: {}, Failed to download extension: \"{}\" from {}.", - res.value().status, name, extensionRepoInfo.repoURL)); - // LCOC_EXCL_STOP - } else { - throw IOException( - common::stringFormat("Failed to download extension: {} at URL {} (ERROR: {})", name, - extensionRepoInfo.repoURL, to_string(res.error()))); - } - } - return res->body; -} - -void InstallExtension::saveExtensionToLocalFile(const std::string& extensionData, - main::ClientContext* context) { - auto extensionDir = context->getExtensionDir(); - auto extensionPath = - ExtensionUtils::getLocalPathForExtension(context, name, name + "_installer"); - auto vfs = context->getVFSUnsafe(); - if (!vfs->fileOrPathExists(extensionDir, context)) { - vfs->createDir(extensionDir); + return; } - if (!vfs->fileOrPathExists(ExtensionUtils::getLocalExtensionDir(context, name))) { - vfs->createDir(ExtensionUtils::getLocalExtensionDir(context, name)); - } - auto fileInfo = vfs->openFile(extensionPath, O_WRONLY | O_CREAT); - fileInfo->writeFile(reinterpret_cast(extensionData.c_str()), - extensionData.size(), 0 /* offset */); - fileInfo->syncFile(); + saveExtensionToLocalFile(res->body, name, context); + auto extensionInstallerPath = ExtensionUtils::getLocalPathForExtensionInstaller(context, name); + auto libLoader = ExtensionLibLoader(name, extensionInstallerPath.c_str()); + auto install = libLoader.getInstallFunc(); + (*install)(context); } void InstallExtension::installExtension(main::ClientContext* context) { - auto extensionData = tryDownloadExtension(); - saveExtensionToLocalFile(extensionData, context); - auto extensionInstallerPath = - ExtensionUtils::getLocalPathForExtension(context, name, name + "_installer"); - auto libHdl = dlopen(extensionInstallerPath.c_str(), RTLD_NOW | RTLD_LOCAL); - if (libHdl == nullptr) { - throw common::IOException(stringFormat("Extension \"{}\" could not be loaded.\nError: {}", - extensionInstallerPath.c_str(), dlErrMessage())); - } - auto install = (ext_install_func_t)(dlsym(libHdl, "install")); - if (install == nullptr) { - throw common::IOException( - stringFormat("Extension \"{}\" does not have a valid install function.\nError: {}", - name, dlErrMessage())); - } - (*install)(context); + extension::ExtensionInstaller installer{name}; + installer.install(context); + installDependencies(name, context); } } // namespace processor diff --git a/src/processor/operator/simple/load_extension.cpp b/src/processor/operator/simple/load_extension.cpp index 4758aaedba2..2aea6d6dc0e 100644 --- a/src/processor/operator/simple/load_extension.cpp +++ b/src/processor/operator/simple/load_extension.cpp @@ -1,25 +1,12 @@ #include "processor/operator/simple/load_extension.h" #include "common/exception/io.h" - -#ifdef _WIN32 - -#include "windows.h" -#define RTLD_NOW 0 -#define RTLD_LOCAL 0 - -#else -#include -#endif - -#include "common/system_message.h" +#include "common/file_system/virtual_file_system.h" #include "extension/extension.h" #include "main/database.h" using namespace kuzu::common; -typedef void (*ext_init_func_t)(kuzu::main::ClientContext*); - namespace kuzu { namespace processor { @@ -57,28 +44,30 @@ void* dlsym(void* handle, const char* name) { } #endif +static void executeExtensionLoader(main::ClientContext* context, const std::string& extensionName) { + auto loaderPath = ExtensionUtils::getLocalPathForExtensionLoader(context, extensionName); + if (context->getVFSUnsafe()->fileOrPathExists(loaderPath)) { + auto libLoader = ExtensionLibLoader(extensionName, loaderPath); + auto load = libLoader.getLoadFunc(); + (*load)(context); + } +} + void LoadExtension::executeInternal(kuzu::processor::ExecutionContext* context) { + auto fullPath = path; if (!extension::ExtensionUtils::isFullPath(path)) { - path = ExtensionUtils::getLocalPathForExtension(context->clientContext, path, path); + auto localPathForSharedLib = + ExtensionUtils::getLocalPathForSharedLib(context->clientContext); + if (!context->clientContext->getVFSUnsafe()->fileOrPathExists(localPathForSharedLib)) { + context->clientContext->getVFSUnsafe()->createDir(localPathForSharedLib); + } + executeExtensionLoader(context->clientContext, path); + fullPath = ExtensionUtils::getLocalPathForExtensionLib(context->clientContext, path); } - // auto duckdbHdl = - // dlopen("/Users/z473chen/Desktop/code/kuzu/libduckdb.dylib", RTLD_NOW | RTLD_GLOBAL); - // if (!duckdbHdl) { - // throw common::IOException( - // stringFormat("Duckdb cannot be loaded.\nError: {}", dlErrMessage())); - // } - auto libHdl = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); - if (libHdl == nullptr) { - throw common::IOException( - stringFormat("Extension \"{}\" could not be loaded.\nError: {}", path, dlErrMessage())); - } - auto load = (ext_init_func_t)(dlsym(libHdl, "init")); - if (load == nullptr) { - throw common::IOException( - stringFormat("Extension \"{}\" does not have a valid init function.\nError: {}", path, - dlErrMessage())); - } - (*load)(context->clientContext); + + auto libLoader = ExtensionLibLoader(path, fullPath); + auto init = libLoader.getInitFunc(); + (*init)(context->clientContext); } std::string LoadExtension::getOutputMsg() { From 0eec0a85a6a3fe222dfcb66cce6ffc914f3af87b Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Tue, 13 Aug 2024 14:33:03 +0800 Subject: [PATCH 03/19] update --- extension/duckdb/src/duckdb_loader.cpp | 29 ++++---------------- extension/duckdb/src/include/duckdb_loader.h | 9 ++++-- src/include/extension/extension.h | 11 ++++++-- test/c_api/database_test.cpp | 8 ------ 4 files changed, 19 insertions(+), 38 deletions(-) diff --git a/extension/duckdb/src/duckdb_loader.cpp b/extension/duckdb/src/duckdb_loader.cpp index 7a8e15c02a2..877a2272f8a 100644 --- a/extension/duckdb/src/duckdb_loader.cpp +++ b/extension/duckdb/src/duckdb_loader.cpp @@ -1,34 +1,15 @@ #include "duckdb_loader.h" -#include "common/exception/io.h" -#include "common/file_system/virtual_file_system.h" -#include "common/system_message.h" #include "extension/extension.h" -#include "main/client_context.h" -#ifdef _WIN32 - -#include "windows.h" -#define RTLD_NOW 0 -#define RTLD_LOCAL 0 - -#else -#include -#endif namespace kuzu { namespace duckdb { -void DuckDBLoader::load(main::ClientContext* context) { -w for (auto& dependencyLib : DEPENDENCY_LIB_FILES) { +void DuckDBLoader::loadDependency(main::ClientContext* context) { + for (auto& dependencyLib : DEPENDENCY_LIB_FILES) { auto dependencyLibPath = extension::ExtensionUtils::getLocalPathForSharedLib(context, dependencyLib); - KU_ASSERT(context->getVFSUnsafe()->fileOrPathExists(dependencyLibPath)); - auto libHdl = dlopen(dependencyLibPath.c_str(), RTLD_NOW | RTLD_LOCAL); - if (libHdl == nullptr) { - throw common::IOException( - common::stringFormat("Lib \"{}\" could not be loaded.\nError: {}", - dependencyLibPath, common::dlErrMessage())); - } + auto dependencyLoader = extension::ExtensionLibLoader(extensionName, dependencyLibPath); } } @@ -44,7 +25,7 @@ extern "C" { #define INIT_EXPORT __attribute__((visibility("default"))) #endif INIT_EXPORT void load(kuzu::main::ClientContext* context) { - kuzu::duckdb::DuckDBLoader loader{}; - loader.load(context); + kuzu::duckdb::DuckDBLoader loader{"duckdb"}; + loader.loadDependency(context); } } diff --git a/extension/duckdb/src/include/duckdb_loader.h b/extension/duckdb/src/include/duckdb_loader.h index 28852a12077..d5f90ce21c6 100644 --- a/extension/duckdb/src/include/duckdb_loader.h +++ b/extension/duckdb/src/include/duckdb_loader.h @@ -1,15 +1,18 @@ #pragma once -#include "extension/extension_installer.h" +#include "extension/extension_loader.h" namespace kuzu { namespace duckdb { -class DuckDBLoader { +class DuckDBLoader final : public extension::ExtensionLoader { private: static constexpr const char* DEPENDENCY_LIB_FILES[] = {"libduckdb.dylib"}; public: - void load(main::ClientContext* context); + explicit DuckDBLoader(std::string extensionName) + : extension::ExtensionLoader{std::move(extensionName)} {} + + void loadDependency(main::ClientContext* context) override; }; } // namespace duckdb diff --git a/src/include/extension/extension.h b/src/include/extension/extension.h index f17f8a54c88..8f866f5ecb1 100644 --- a/src/include/extension/extension.h +++ b/src/include/extension/extension.h @@ -39,9 +39,14 @@ struct ExtensionRepoInfo { }; struct ExtensionUtils { - static constexpr const char* EXTENSION_FILE_REPO = "http://extension.kuzudb.com/v{}/{}/{}/{}"; + // static constexpr const char* EXTENSION_REPO = + // "http://extension.kuzudb.com/v{}/{}/{}/lib{}.kuzu_extension"; - static constexpr const char* SHARED_LIB_REPO = "http://extension.kuzudb.com/v{}/{}/common/{}"; + static constexpr const char* EXTENSION_FILE_REPO = + "http://localhost/extension/releases/v{}/{}/{}/{}"; + + static constexpr const char* SHARED_LIB_REPO = + "http://localhost/extension/releases/v{}/{}/common/{}"; static constexpr const char* EXTENSION_FILE_NAME = "lib{}.kuzu_extension"; @@ -91,7 +96,7 @@ struct ExtensionUtils { static bool isOfficialExtension(const std::string& extension); }; -class ExtensionLibLoader { +class KUZU_API ExtensionLibLoader { public: static constexpr const char* EXTENSION_LOAD_FUNC_NAME = "load"; diff --git a/test/c_api/database_test.cpp b/test/c_api/database_test.cpp index 6a8a29afece..f14f10140ee 100644 --- a/test/c_api/database_test.cpp +++ b/test/c_api/database_test.cpp @@ -94,11 +94,3 @@ TEST_F(CApiDatabaseTest, CreationHomeDir) { kuzu_database_destroy(&database); std::filesystem::remove_all(homePath + "/ku_test.db"); } - -TEST_F(CApiDatabaseTest, dadsa) { - createDBAndConn(); - printf("%s", - conn->query("create node table test1(ID INT64, description STRUCT, PRIMARY KEY(ID))") - ->toString() - .c_str()); -} From 0d464e89c36bdfa3dc04c0ecc2b52ed1e932e9a0 Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Tue, 13 Aug 2024 14:34:48 +0800 Subject: [PATCH 04/19] update --- extension/duckdb/src/include/duckdb_installer.h | 2 +- extension/duckdb/src/include/duckdb_loader.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extension/duckdb/src/include/duckdb_installer.h b/extension/duckdb/src/include/duckdb_installer.h index b8edb24ba5c..a859d5852c4 100644 --- a/extension/duckdb/src/include/duckdb_installer.h +++ b/extension/duckdb/src/include/duckdb_installer.h @@ -16,4 +16,4 @@ class DuckDBInstaller : public extension::ExtensionInstaller { }; } // namespace duckdb -} // namespace kuzu \ No newline at end of file +} // namespace kuzu diff --git a/extension/duckdb/src/include/duckdb_loader.h b/extension/duckdb/src/include/duckdb_loader.h index d5f90ce21c6..c1bbf0ca47c 100644 --- a/extension/duckdb/src/include/duckdb_loader.h +++ b/extension/duckdb/src/include/duckdb_loader.h @@ -16,4 +16,4 @@ class DuckDBLoader final : public extension::ExtensionLoader { }; } // namespace duckdb -} // namespace kuzu \ No newline at end of file +} // namespace kuzu From cf1eacc2ef14e06755a8a91d560441138c9d73a5 Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Tue, 13 Aug 2024 15:09:08 +0800 Subject: [PATCH 05/19] update --- src/include/extension/extension.h | 9 ++------- src/processor/operator/simple/install_extension.cpp | 1 - src/processor/operator/simple/load_extension.cpp | 1 - 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/include/extension/extension.h b/src/include/extension/extension.h index 8f866f5ecb1..d11fcd741d6 100644 --- a/src/include/extension/extension.h +++ b/src/include/extension/extension.h @@ -39,14 +39,9 @@ struct ExtensionRepoInfo { }; struct ExtensionUtils { - // static constexpr const char* EXTENSION_REPO = - // "http://extension.kuzudb.com/v{}/{}/{}/lib{}.kuzu_extension"; + static constexpr const char* EXTENSION_FILE_REPO = "http://extension.kuzudb.com/v{}/{}/{}/{}"; - static constexpr const char* EXTENSION_FILE_REPO = - "http://localhost/extension/releases/v{}/{}/{}/{}"; - - static constexpr const char* SHARED_LIB_REPO = - "http://localhost/extension/releases/v{}/{}/common/{}"; + static constexpr const char* SHARED_LIB_REPO = "http://extension.kuzudb.com/v{}/{}/common/{}"; static constexpr const char* EXTENSION_FILE_NAME = "lib{}.kuzu_extension"; diff --git a/src/processor/operator/simple/install_extension.cpp b/src/processor/operator/simple/install_extension.cpp index 539b03b272f..8738c2901ea 100644 --- a/src/processor/operator/simple/install_extension.cpp +++ b/src/processor/operator/simple/install_extension.cpp @@ -1,6 +1,5 @@ #include "processor/operator/simple/install_extension.h" -#include "common/exception/io.h" #include "common/file_system/virtual_file_system.h" #include "common/string_format.h" #include "extension/extension.h" diff --git a/src/processor/operator/simple/load_extension.cpp b/src/processor/operator/simple/load_extension.cpp index 2aea6d6dc0e..b5e66a3dccd 100644 --- a/src/processor/operator/simple/load_extension.cpp +++ b/src/processor/operator/simple/load_extension.cpp @@ -1,6 +1,5 @@ #include "processor/operator/simple/load_extension.h" -#include "common/exception/io.h" #include "common/file_system/virtual_file_system.h" #include "extension/extension.h" #include "main/database.h" From 8ca03b726820b09bc0903049f25ba91cb2e3c4e5 Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Tue, 13 Aug 2024 15:47:27 +0800 Subject: [PATCH 06/19] update --- extension/duckdb/src/duckdb_loader.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/extension/duckdb/src/duckdb_loader.cpp b/extension/duckdb/src/duckdb_loader.cpp index 877a2272f8a..8e8f097ebd1 100644 --- a/extension/duckdb/src/duckdb_loader.cpp +++ b/extension/duckdb/src/duckdb_loader.cpp @@ -1,7 +1,5 @@ #include "duckdb_loader.h" -#include "extension/extension.h" - namespace kuzu { namespace duckdb { From 44ae72f168559cd6afdbea7d024d7042ad1cf28b Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Wed, 14 Aug 2024 09:02:36 +0800 Subject: [PATCH 07/19] update --- src/include/extension/extension_loader.h | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/include/extension/extension_loader.h diff --git a/src/include/extension/extension_loader.h b/src/include/extension/extension_loader.h new file mode 100644 index 00000000000..084769d058a --- /dev/null +++ b/src/include/extension/extension_loader.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "common/api.h" +#include "extension.h" + +namespace kuzu { +namespace main { +class ClientContext; +} +namespace extension { + +class KUZU_API ExtensionLoader { +public: + explicit ExtensionLoader(const std::string extensionName) + : extensionName{std::move(extensionName)} {} + + virtual ~ExtensionLoader() = default; + + virtual void loadDependency(main::ClientContext* context) = 0; + +protected: + std::string extensionName; +}; + +} // namespace extension +} // namespace kuzu From 1d89c55dca49570385e59cf4c3e235a652556a63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=9B=A7=E5=9B=A7?= Date: Tue, 20 Aug 2024 09:36:21 -0400 Subject: [PATCH 08/19] Fix CI build and upload process (#4110) --- .github/workflows/build-extensions.yml | 67 ++++++-------------------- scripts/collect-extensions.py | 59 +++++++++++++++++++++++ 2 files changed, 75 insertions(+), 51 deletions(-) create mode 100644 scripts/collect-extensions.py diff --git a/.github/workflows/build-extensions.yml b/.github/workflows/build-extensions.yml index ab557dc4cb6..12772c6a252 100644 --- a/.github/workflows/build-extensions.yml +++ b/.github/workflows/build-extensions.yml @@ -12,15 +12,13 @@ jobs: run: make extension-release NUM_THREADS=$(nproc) - name: Collect built artifacts - run: | - mkdir -p extension-artifacts - find extension -type f -name "*.kuzu_extension" -exec cp {} extension-artifacts \; + run: /opt/python/cp310-cp310/bin/python scripts/collect-extensions.py - name: Upload built artifacts uses: actions/upload-artifact@v4 with: name: kuzu-extensions_linux-x86_64 - path: extension-artifacts/*.kuzu_extension + path: extension-artifacts build-linux-extensions-aarch64: runs-on: kuzu-self-hosted-linux-building-aarch64 @@ -32,15 +30,13 @@ jobs: run: make extension-release NUM_THREADS=$(nproc) - name: Collect built artifacts - run: | - mkdir -p extension-artifacts - find extension -type f -name "*.kuzu_extension" -exec cp {} extension-artifacts \; + run: /opt/python/cp310-cp310/bin/python scripts/collect-extensions.py - name: Upload built artifacts uses: actions/upload-artifact@v4 with: name: kuzu-extensions_linux-aarch64 - path: extension-artifacts/*.kuzu_extension + path: extension-artifacts build-linux-extensions-x86: runs-on: ubuntu-latest @@ -52,7 +48,7 @@ jobs: run: | docker run -d --name kuzu-x86 \ -v $PWD:/kuzu -w /kuzu \ - -e NUM_THREADS=2 -e CC=gcc -e CXX=g++ \ + -e NUM_THREADS=$(nproc) -e CC=gcc -e CXX=g++ \ i386/debian:stable tail -f /dev/null - name: Install dependencies @@ -66,8 +62,7 @@ jobs: - name: Collect built artifacts run: | - mkdir -p extension-artifacts - find extension -type f -name "*.kuzu_extension" -exec cp {} extension-artifacts \; + python3 scripts/collect-extensions.py sudo chown -R $USER:$USER extension-artifacts sudo chmod -R 755 extension-artifacts @@ -75,7 +70,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: kuzu-extensions_linux-x86 - path: extension-artifacts/*.kuzu_extension + path: extension-artifacts - name: Stop Docker container run: docker stop kuzu-x86 @@ -92,15 +87,13 @@ jobs: CMAKE_OSX_ARCHITECTURES: "arm64" - name: Collect built artifacts - run: | - mkdir -p extension-artifacts - find extension -type f -name "*.kuzu_extension" -exec cp {} extension-artifacts \; + run: python3 scripts/collect-extensions.py - name: Upload built artifacts uses: actions/upload-artifact@v4 with: name: kuzu-extensions_osx-arm64 - path: extension-artifacts/*.kuzu_extension + path: extension-artifacts build-mac-extensions-x86_64: runs-on: self-hosted-mac-x64 @@ -114,15 +107,13 @@ jobs: CMAKE_OSX_ARCHITECTURES: "x86_64" - name: Collect built artifacts - run: | - mkdir -p extension-artifacts - find extension -type f -name "*.kuzu_extension" -exec cp {} extension-artifacts \; + run: python3 scripts/collect-extensions.py - name: Upload built artifacts uses: actions/upload-artifact@v4 with: name: kuzu-extensions_osx-x86_64 - path: extension-artifacts/*.kuzu_extension + path: extension-artifacts build-windows-extensions-x86_64: runs-on: self-hosted-windows @@ -138,15 +129,13 @@ jobs: make extension-release - name: Collect built artifacts - run: | - mkdir -p extension-artifacts - Get-ChildItem -Path .\extension -Recurse -Filter "*.kuzu_extension" | ForEach-Object {Copy-Item $_.FullName -Destination extension-artifacts -Force} + run: python scripts/collect-extensions.py - name: Upload built artifacts uses: actions/upload-artifact@v4 with: name: kuzu-extensions_win-x86_64 - path: extension-artifacts/*.kuzu_extension + path: extension-artifacts update-extensions-repo: runs-on: ubuntu-latest @@ -172,23 +161,9 @@ jobs: repository: kuzudb/extension token: ${{ secrets.DOC_PUSH_TOKEN }} - - name: Ensure extension directories - run: | - mkdir -p releases/$RELEASE_VERSION/linux_amd64 - mkdir -p releases/$RELEASE_VERSION/linux_arm64 - mkdir -p releases/$RELEASE_VERSION/linux_x86 - mkdir -p releases/$RELEASE_VERSION/osx_amd64 - mkdir -p releases/$RELEASE_VERSION/osx_arm64 - mkdir -p releases/$RELEASE_VERSION/win_amd64 - - name: Clear old artifacts run: | - rm -rf releases/$RELEASE_VERSION/linux_amd64/* - rm -rf releases/$RELEASE_VERSION/linux_arm64/* - rm -rf releases/$RELEASE_VERSION/linux_x86/* - rm -rf releases/$RELEASE_VERSION/osx_amd64/* - rm -rf releases/$RELEASE_VERSION/osx_arm64/* - rm -rf releases/$RELEASE_VERSION/win_amd64/* + rm -rf releases/$RELEASE_VERSION/ - name: Create temporary directory for artifacts run: | @@ -238,24 +213,14 @@ jobs: - name: Copy built artifacts run: | - cp extension-artifacts/linux_amd64/*.kuzu_extension releases/$RELEASE_VERSION/linux_amd64 - cp extension-artifacts/linux_arm64/*.kuzu_extension releases/$RELEASE_VERSION/linux_arm64 - cp extension-artifacts/linux_x86/*.kuzu_extension releases/$RELEASE_VERSION/linux_x86 - cp extension-artifacts/osx_amd64/*.kuzu_extension releases/$RELEASE_VERSION/osx_amd64 - cp extension-artifacts/osx_arm64/*.kuzu_extension releases/$RELEASE_VERSION/osx_arm64 - cp extension-artifacts/win_amd64/*.kuzu_extension releases/$RELEASE_VERSION/win_amd64 + rsync -av extension-artifacts/ releases/$RELEASE_VERSION/ - name: Remove temporary directory run: rm -rf extension-artifacts - name: Set artifact permissions run: | - chmod 755 releases/$RELEASE_VERSION/linux_amd64/* - chmod 755 releases/$RELEASE_VERSION/linux_arm64/* - chmod 755 releases/$RELEASE_VERSION/linux_x86/* - chmod 755 releases/$RELEASE_VERSION/osx_amd64/* - chmod 755 releases/$RELEASE_VERSION/osx_arm64/* - chmod 755 releases/$RELEASE_VERSION/win_amd64/* + chmod -R 755 releases/$RELEASE_VERSION - name: List downloaded artifacts run: ls -R releases/$RELEASE_VERSION diff --git a/scripts/collect-extensions.py b/scripts/collect-extensions.py new file mode 100644 index 00000000000..5782ff00a24 --- /dev/null +++ b/scripts/collect-extensions.py @@ -0,0 +1,59 @@ +import os +import shutil +import platform + +FILE_DIR = os.path.dirname(os.path.abspath(__file__)) +DST_DIR = os.path.abspath(os.path.join(FILE_DIR, "..", "extension-artifacts")) +SRC_DIR = os.path.abspath(os.path.join(FILE_DIR, "..", "extension")) + + +def collect_exts(): + for ext in os.listdir(SRC_DIR): + ext_build_path = os.path.abspath(os.path.join(SRC_DIR, ext, "build")) + if not os.path.exists(ext_build_path): + continue + print("Found extension: " + ext) + ext_dst_path = os.path.abspath(os.path.join(DST_DIR, ext)) + os.makedirs(ext_dst_path, exist_ok=True) + for f in os.listdir(ext_build_path): + if not f.endswith(".kuzu_extension"): + continue + ext_file_path = os.path.abspath(os.path.join(ext_build_path, f)) + shutil.copy(ext_file_path, ext_dst_path) + print(" \tCopied: " + f, "=>", ext_dst_path) + + +def find_duckdb(): + if platform.system() == 'Darwin': + candidates = ["/usr/local/lib/libduckdb.dylib", "/opt/homebrew/lib/libduckdb.dylib"] + elif platform.system() == 'Linux': + candidates = ["/usr/local/lib/libduckdb.so", "/usr/lib/libduckdb.so", "/usr/lib64/libduckdb.so"] + elif platform.system() == 'Windows': + # TODO: find duckdb.dll on Windows + pass + for candidate in candidates: + if os.path.exists(candidate): + return os.path.abspath(candidate) + return None + + +def copy_duckdb(): + duckdb_dst_path = os.path.abspath(os.path.join(DST_DIR, "common")) + os.makedirs(duckdb_dst_path, exist_ok=True) + duckdb_path = find_duckdb() + if duckdb_path is None: + print("DuckDB not found, copying is skipped") + return + shutil.copy(duckdb_path, duckdb_dst_path) + print("Copied DuckDB: " + duckdb_path, "=>", duckdb_dst_path) + + +def main(): + shutil.rmtree(DST_DIR, ignore_errors=True) + os.makedirs(DST_DIR, exist_ok=True) + collect_exts() + copy_duckdb() + + +if __name__ == "__main__": + main() From c66fad7bc71e6492cdb8529d2de2c89922575ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=9B=A7=E5=9B=A7?= Date: Tue, 20 Aug 2024 11:01:29 -0400 Subject: [PATCH 09/19] Add Windows path for DuckDB --- scripts/collect-extensions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/collect-extensions.py b/scripts/collect-extensions.py index 5782ff00a24..de7b496e4dd 100644 --- a/scripts/collect-extensions.py +++ b/scripts/collect-extensions.py @@ -29,8 +29,7 @@ def find_duckdb(): elif platform.system() == 'Linux': candidates = ["/usr/local/lib/libduckdb.so", "/usr/lib/libduckdb.so", "/usr/lib64/libduckdb.so"] elif platform.system() == 'Windows': - # TODO: find duckdb.dll on Windows - pass + candidates = ["C:\\Program Files\\duckdb\\build\\release\\src\\Release\\duckdb.lib"] for candidate in candidates: if os.path.exists(candidate): return os.path.abspath(candidate) From 1112e09f9bb2ef03c774493845b806b66a9410ec Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Tue, 20 Aug 2024 11:39:47 -0400 Subject: [PATCH 10/19] Turn off static libs for DuckDB --- extension/duckdb/CMakeLists.txt | 8 +------- extension/postgres/CMakeLists.txt | 8 +------- extension/sqlite/CMakeLists.txt | 8 +------- 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/extension/duckdb/CMakeLists.txt b/extension/duckdb/CMakeLists.txt index 52cd7b72d51..3a5f7b7abd4 100644 --- a/extension/duckdb/CMakeLists.txt +++ b/extension/duckdb/CMakeLists.txt @@ -1,10 +1,4 @@ -# Address sanitizer may change the ABI, so we use dynamic linking -# if it's enabled. -if(${ENABLE_ADDRESS_SANITIZER}) - set(DuckDB_USE_STATIC_LIBS OFF) -else() - set(DuckDB_USE_STATIC_LIBS ON) -endif() +set(DuckDB_USE_STATIC_LIBS OFF) find_package(DuckDB REQUIRED) include_directories( diff --git a/extension/postgres/CMakeLists.txt b/extension/postgres/CMakeLists.txt index 2e3a0eba8db..d3ec018f10f 100644 --- a/extension/postgres/CMakeLists.txt +++ b/extension/postgres/CMakeLists.txt @@ -1,10 +1,4 @@ -# Address sanitizer may change the ABI, so we use dynamic linking -# if it's enabled. -if(${ENABLE_ADDRESS_SANITIZER}) - set(DuckDB_USE_STATIC_LIBS OFF) -else() - set(DuckDB_USE_STATIC_LIBS ON) -endif() +set(DuckDB_USE_STATIC_LIBS OFF) find_package(DuckDB REQUIRED) add_library(postgres_extension diff --git a/extension/sqlite/CMakeLists.txt b/extension/sqlite/CMakeLists.txt index 740c2a3cb53..5598c7e4024 100644 --- a/extension/sqlite/CMakeLists.txt +++ b/extension/sqlite/CMakeLists.txt @@ -1,10 +1,4 @@ -# Address sanitizer may change the ABI, so we use dynamic linking -# if it's enabled. -if(${ENABLE_ADDRESS_SANITIZER}) - set(DuckDB_USE_STATIC_LIBS OFF) -else() - set(DuckDB_USE_STATIC_LIBS ON) -endif() +set(DuckDB_USE_STATIC_LIBS OFF) find_package(DuckDB REQUIRED) include_directories( From 90920900916f18af7586dc2d6795bfd05a54bbdb Mon Sep 17 00:00:00 2001 From: Ziyi Chen Date: Tue, 20 Aug 2024 09:23:21 -0700 Subject: [PATCH 11/19] update --- src/extension/extension.cpp | 10 ----- src/include/extension/extension.h | 39 +++++++++++++++++++ .../operator/simple/load_extension.cpp | 28 ------------- 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/extension/extension.cpp b/src/extension/extension.cpp index f8ed2d6f283..ea57b31f28b 100644 --- a/src/extension/extension.cpp +++ b/src/extension/extension.cpp @@ -11,16 +11,6 @@ #include "main/database.h" #include "transaction/transaction.h" -#ifdef _WIN32 - -#include "windows.h" -#define RTLD_NOW 0 -#define RTLD_LOCAL 0 - -#else -#include -#endif - namespace kuzu { namespace extension { diff --git a/src/include/extension/extension.h b/src/include/extension/extension.h index d11fcd741d6..0729a532887 100644 --- a/src/include/extension/extension.h +++ b/src/include/extension/extension.h @@ -6,6 +6,17 @@ #include "common/api.h" #include "function/function.h" #include "main/db_config.h" +#include "common/exception/io.h" + +#ifdef _WIN32 + +#include "windows.h" +#define RTLD_NOW 0 +#define RTLD_LOCAL 0 + +#else +#include +#endif #define ADD_FUNC(FUNC_STRUCT) \ kuzu::extension::ExtensionUtils::registerFunctionSet(db, std::string(FUNC_STRUCT::name), \ @@ -125,5 +136,33 @@ struct ExtensionOptions { main::ExtensionOption* getExtensionOption(std::string name); }; +#ifdef _WIN32 +std::wstring utf8ToUnicode(const char* input) { + uint32_t result; + + result = MultiByteToWideChar(CP_UTF8, 0, input, -1, nullptr, 0); + if (result == 0) { + throw common::IOException("Failure in MultiByteToWideChar"); + } + auto buffer = std::make_unique(result); + result = MultiByteToWideChar(CP_UTF8, 0, input, -1, buffer.get(), result); + if (result == 0) { + throw common::IOException("Failure in MultiByteToWideChar"); + } + return std::wstring(buffer.get(), result); +} + +void* dlopen(const char* file, int /*mode*/) { + KU_ASSERT(file); + auto fpath = utf8ToUnicode(file); + return (void*)LoadLibraryW(fpath.c_str()); +} + +void* dlsym(void* handle, const char* name) { + KU_ASSERT(handle); + return (void*)GetProcAddress((HINSTANCE)handle, name); +} +#endif + } // namespace extension } // namespace kuzu diff --git a/src/processor/operator/simple/load_extension.cpp b/src/processor/operator/simple/load_extension.cpp index b5e66a3dccd..420069996c2 100644 --- a/src/processor/operator/simple/load_extension.cpp +++ b/src/processor/operator/simple/load_extension.cpp @@ -15,34 +15,6 @@ std::string LoadExtensionPrintInfo::toString() const { return "Load " + extensionName; } -#ifdef _WIN32 -std::wstring utf8ToUnicode(const char* input) { - uint32_t result; - - result = MultiByteToWideChar(CP_UTF8, 0, input, -1, nullptr, 0); - if (result == 0) { - throw IOException("Failure in MultiByteToWideChar"); - } - auto buffer = std::make_unique(result); - result = MultiByteToWideChar(CP_UTF8, 0, input, -1, buffer.get(), result); - if (result == 0) { - throw IOException("Failure in MultiByteToWideChar"); - } - return std::wstring(buffer.get(), result); -} - -void* dlopen(const char* file, int /*mode*/) { - KU_ASSERT(file); - auto fpath = utf8ToUnicode(file); - return (void*)LoadLibraryW(fpath.c_str()); -} - -void* dlsym(void* handle, const char* name) { - KU_ASSERT(handle); - return (void*)GetProcAddress((HINSTANCE)handle, name); -} -#endif - static void executeExtensionLoader(main::ClientContext* context, const std::string& extensionName) { auto loaderPath = ExtensionUtils::getLocalPathForExtensionLoader(context, extensionName); if (context->getVFSUnsafe()->fileOrPathExists(loaderPath)) { From 22d2471dbb00fe7e002c89c05550941394e50ded Mon Sep 17 00:00:00 2001 From: Ziyi Chen Date: Tue, 20 Aug 2024 10:04:04 -0700 Subject: [PATCH 12/19] Fix dlopen functions --- src/extension/extension.cpp | 37 ++++++++++++++++++++++++++++ src/include/extension/extension.h | 40 ++++--------------------------- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/src/extension/extension.cpp b/src/extension/extension.cpp index ea57b31f28b..660dd482d20 100644 --- a/src/extension/extension.cpp +++ b/src/extension/extension.cpp @@ -10,6 +10,15 @@ #include "main/client_context.h" #include "main/database.h" #include "transaction/transaction.h" +#ifdef _WIN32 + +#include "windows.h" +#define RTLD_NOW 0 +#define RTLD_LOCAL 0 + +#else +#include +#endif namespace kuzu { namespace extension { @@ -197,5 +206,33 @@ main::ExtensionOption* ExtensionOptions::getExtensionOption(std::string name) { return extensionOptions.contains(name) ? &extensionOptions.at(name) : nullptr; } +#ifdef _WIN32 +std::wstring utf8ToUnicode(const char* input) { + uint32_t result; + + result = MultiByteToWideChar(CP_UTF8, 0, input, -1, nullptr, 0); + if (result == 0) { + throw common::IOException("Failure in MultiByteToWideChar"); + } + auto buffer = std::make_unique(result); + result = MultiByteToWideChar(CP_UTF8, 0, input, -1, buffer.get(), result); + if (result == 0) { + throw common::IOException("Failure in MultiByteToWideChar"); + } + return std::wstring(buffer.get(), result); +} + +void* dlopen(const char* file, int /*mode*/) { + KU_ASSERT(file); + auto fpath = utf8ToUnicode(file); + return (void*)LoadLibraryW(fpath.c_str()); +} + +void* dlsym(void* handle, const char* name) { + KU_ASSERT(handle); + return (void*)GetProcAddress((HINSTANCE)handle, name); +} +#endif + } // namespace extension } // namespace kuzu diff --git a/src/include/extension/extension.h b/src/include/extension/extension.h index 0729a532887..08344d6f4eb 100644 --- a/src/include/extension/extension.h +++ b/src/include/extension/extension.h @@ -8,16 +8,6 @@ #include "main/db_config.h" #include "common/exception/io.h" -#ifdef _WIN32 - -#include "windows.h" -#define RTLD_NOW 0 -#define RTLD_LOCAL 0 - -#else -#include -#endif - #define ADD_FUNC(FUNC_STRUCT) \ kuzu::extension::ExtensionUtils::registerFunctionSet(db, std::string(FUNC_STRUCT::name), \ FUNC_STRUCT::getFunctionSet()) @@ -137,31 +127,11 @@ struct ExtensionOptions { }; #ifdef _WIN32 -std::wstring utf8ToUnicode(const char* input) { - uint32_t result; - - result = MultiByteToWideChar(CP_UTF8, 0, input, -1, nullptr, 0); - if (result == 0) { - throw common::IOException("Failure in MultiByteToWideChar"); - } - auto buffer = std::make_unique(result); - result = MultiByteToWideChar(CP_UTF8, 0, input, -1, buffer.get(), result); - if (result == 0) { - throw common::IOException("Failure in MultiByteToWideChar"); - } - return std::wstring(buffer.get(), result); -} - -void* dlopen(const char* file, int /*mode*/) { - KU_ASSERT(file); - auto fpath = utf8ToUnicode(file); - return (void*)LoadLibraryW(fpath.c_str()); -} - -void* dlsym(void* handle, const char* name) { - KU_ASSERT(handle); - return (void*)GetProcAddress((HINSTANCE)handle, name); -} +std::wstring utf8ToUnicode(const char* input); + +void* dlopen(const char* file, int /*mode*/); + +void* dlsym(void* handle, const char* name); #endif } // namespace extension From e5a65107bdcb33ab370cf273508742918a76f257 Mon Sep 17 00:00:00 2001 From: CI Bot Date: Wed, 21 Aug 2024 01:47:36 +0000 Subject: [PATCH 13/19] Run clang-format --- src/include/extension/extension.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/extension/extension.h b/src/include/extension/extension.h index 08344d6f4eb..7f3821d44fc 100644 --- a/src/include/extension/extension.h +++ b/src/include/extension/extension.h @@ -4,9 +4,9 @@ #include #include "common/api.h" +#include "common/exception/io.h" #include "function/function.h" #include "main/db_config.h" -#include "common/exception/io.h" #define ADD_FUNC(FUNC_STRUCT) \ kuzu::extension::ExtensionUtils::registerFunctionSet(db, std::string(FUNC_STRUCT::name), \ From 31c8e5c5aa8a5488bbc76082669bc58a12f466d7 Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Wed, 21 Aug 2024 11:06:55 +0800 Subject: [PATCH 14/19] update --- extension/duckdb/src/duckdb_install_func.cpp | 2 +- extension/duckdb/src/duckdb_installer.cpp | 7 ++++--- extension/duckdb/src/duckdb_loader.cpp | 10 ++++++---- extension/duckdb/src/include/duckdb_extension.h | 1 + extension/duckdb/src/include/duckdb_installer.h | 7 ++----- extension/duckdb/src/include/duckdb_loader.h | 7 ++----- extension/postgres/src/postgres_install_func.cpp | 2 +- extension/sqlite/src/sqlite_install_func.cpp | 2 +- src/extension/extension.cpp | 13 ++++++++++++- 9 files changed, 30 insertions(+), 21 deletions(-) diff --git a/extension/duckdb/src/duckdb_install_func.cpp b/extension/duckdb/src/duckdb_install_func.cpp index fd82179c13e..9dd93190942 100644 --- a/extension/duckdb/src/duckdb_install_func.cpp +++ b/extension/duckdb/src/duckdb_install_func.cpp @@ -9,7 +9,7 @@ extern "C" { #define INIT_EXPORT __attribute__((visibility("default"))) #endif INIT_EXPORT void install(kuzu::main::ClientContext* context) { - kuzu::duckdb::DuckDBInstaller installer{"duckdb"}; + kuzu::duckdb_extension::DuckDBInstaller installer{"duckdb"}; installer.install(context); } } diff --git a/extension/duckdb/src/duckdb_installer.cpp b/extension/duckdb/src/duckdb_installer.cpp index 313e0ae6ae8..7c201129a32 100644 --- a/extension/duckdb/src/duckdb_installer.cpp +++ b/extension/duckdb/src/duckdb_installer.cpp @@ -1,10 +1,11 @@ #include "duckdb_installer.h" #include "common/file_system/virtual_file_system.h" +#include "duckdb_extension.h" #include "main/client_context.h" namespace kuzu { -namespace duckdb { +namespace duckdb_extension { void DuckDBInstaller::install(main::ClientContext* context) { auto loaderFileRepoInfo = extension::ExtensionUtils::getExtensionLoaderRepoInfo(extensionName); @@ -12,7 +13,7 @@ void DuckDBInstaller::install(main::ClientContext* context) { extension::ExtensionUtils::getLocalPathForExtensionLoader(context, extensionName); tryDownloadExtensionFile(context, loaderFileRepoInfo, localLoaderFilePath); - for (auto& dependencyLib : DEPENDENCY_LIB_FILES) { + for (auto& dependencyLib : DuckDBExtension::DEPENDENCY_LIB_FILES) { auto localDependencyLibPath = extension::ExtensionUtils::getLocalPathForSharedLib(context, dependencyLib); if (!context->getVFSUnsafe()->fileOrPathExists(localDependencyLibPath)) { @@ -23,5 +24,5 @@ void DuckDBInstaller::install(main::ClientContext* context) { } } -} // namespace duckdb +} // namespace duckdb_extension } // namespace kuzu diff --git a/extension/duckdb/src/duckdb_loader.cpp b/extension/duckdb/src/duckdb_loader.cpp index 8e8f097ebd1..e19ab63f457 100644 --- a/extension/duckdb/src/duckdb_loader.cpp +++ b/extension/duckdb/src/duckdb_loader.cpp @@ -1,17 +1,19 @@ #include "duckdb_loader.h" +#include "duckdb_extension.h" + namespace kuzu { -namespace duckdb { +namespace duckdb_extension { void DuckDBLoader::loadDependency(main::ClientContext* context) { - for (auto& dependencyLib : DEPENDENCY_LIB_FILES) { + for (auto& dependencyLib : DuckDBExtension::DEPENDENCY_LIB_FILES) { auto dependencyLibPath = extension::ExtensionUtils::getLocalPathForSharedLib(context, dependencyLib); auto dependencyLoader = extension::ExtensionLibLoader(extensionName, dependencyLibPath); } } -} // namespace duckdb +} // namespace duckdb_extension } // namespace kuzu extern "C" { @@ -23,7 +25,7 @@ extern "C" { #define INIT_EXPORT __attribute__((visibility("default"))) #endif INIT_EXPORT void load(kuzu::main::ClientContext* context) { - kuzu::duckdb::DuckDBLoader loader{"duckdb"}; + kuzu::duckdb_extension::DuckDBLoader loader{"duckdb"}; loader.loadDependency(context); } } diff --git a/extension/duckdb/src/include/duckdb_extension.h b/extension/duckdb/src/include/duckdb_extension.h index 4c71a3f2cd9..e340b588bb7 100644 --- a/extension/duckdb/src/include/duckdb_extension.h +++ b/extension/duckdb/src/include/duckdb_extension.h @@ -8,6 +8,7 @@ namespace duckdb_extension { class DuckDBExtension final : public extension::Extension { public: static constexpr char EXTENSION_NAME[] = "DUCKDB"; + static constexpr const char* DEPENDENCY_LIB_FILES[] = {"libduckdb"}; public: static void load(main::ClientContext* context); diff --git a/extension/duckdb/src/include/duckdb_installer.h b/extension/duckdb/src/include/duckdb_installer.h index a859d5852c4..bb461e11c36 100644 --- a/extension/duckdb/src/include/duckdb_installer.h +++ b/extension/duckdb/src/include/duckdb_installer.h @@ -2,12 +2,9 @@ #include "extension/extension_installer.h" namespace kuzu { -namespace duckdb { +namespace duckdb_extension { class DuckDBInstaller : public extension::ExtensionInstaller { -private: - static constexpr const char* DEPENDENCY_LIB_FILES[] = {"libduckdb.dylib"}; - public: explicit DuckDBInstaller(const std::string extensionName) : ExtensionInstaller{std::move(extensionName)} {} @@ -15,5 +12,5 @@ class DuckDBInstaller : public extension::ExtensionInstaller { void install(main::ClientContext* context) override; }; -} // namespace duckdb +} // namespace duckdb_extension } // namespace kuzu diff --git a/extension/duckdb/src/include/duckdb_loader.h b/extension/duckdb/src/include/duckdb_loader.h index c1bbf0ca47c..5f2603394b1 100644 --- a/extension/duckdb/src/include/duckdb_loader.h +++ b/extension/duckdb/src/include/duckdb_loader.h @@ -2,12 +2,9 @@ #include "extension/extension_loader.h" namespace kuzu { -namespace duckdb { +namespace duckdb_extension { class DuckDBLoader final : public extension::ExtensionLoader { -private: - static constexpr const char* DEPENDENCY_LIB_FILES[] = {"libduckdb.dylib"}; - public: explicit DuckDBLoader(std::string extensionName) : extension::ExtensionLoader{std::move(extensionName)} {} @@ -15,5 +12,5 @@ class DuckDBLoader final : public extension::ExtensionLoader { void loadDependency(main::ClientContext* context) override; }; -} // namespace duckdb +} // namespace duckdb_extension } // namespace kuzu diff --git a/extension/postgres/src/postgres_install_func.cpp b/extension/postgres/src/postgres_install_func.cpp index e8cda2e8a60..3e0635d3e23 100644 --- a/extension/postgres/src/postgres_install_func.cpp +++ b/extension/postgres/src/postgres_install_func.cpp @@ -9,7 +9,7 @@ extern "C" { #define INIT_EXPORT __attribute__((visibility("default"))) #endif INIT_EXPORT void install(kuzu::main::ClientContext* context) { - kuzu::duckdb::DuckDBInstaller installer{"postgres"}; + kuzu::duckdb_extension::DuckDBInstaller installer{"postgres"}; installer.install(context); } } diff --git a/extension/sqlite/src/sqlite_install_func.cpp b/extension/sqlite/src/sqlite_install_func.cpp index ae62f8b7610..9ffeaa516cb 100644 --- a/extension/sqlite/src/sqlite_install_func.cpp +++ b/extension/sqlite/src/sqlite_install_func.cpp @@ -9,7 +9,7 @@ extern "C" { #define INIT_EXPORT __attribute__((visibility("default"))) #endif INIT_EXPORT void install(kuzu::main::ClientContext* context) { - kuzu::duckdb::DuckDBInstaller installer{"sqlite"}; + kuzu::duckdb_extension::DuckDBInstaller installer{"sqlite"}; installer.install(context); } } diff --git a/src/extension/extension.cpp b/src/extension/extension.cpp index 660dd482d20..7a67ccd3817 100644 --- a/src/extension/extension.cpp +++ b/src/extension/extension.cpp @@ -135,7 +135,18 @@ void ExtensionUtils::registerTableFunction(main::Database& database, std::string ExtensionUtils::getLocalPathForSharedLib(main::ClientContext* context, const std::string& libName) { - return common::stringFormat("{}common/{}", context->getExtensionDir(), libName); + auto os = getOS(); + std::string suffix; + if (os == "linux") { + suffix = "so"; + } else if (os == "win") { + suffix = "lib"; + } else if (os == "osx") { + suffix = "dylib"; + } else { + KU_UNREACHABLE; + } + return common::stringFormat("{}common/{}.{}", context->getExtensionDir(), libName, suffix); } std::string ExtensionUtils::getLocalPathForSharedLib(main::ClientContext* context) { From 7e4dd069486311a61a00a4632db3019b4313ac85 Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Wed, 21 Aug 2024 13:15:07 +0800 Subject: [PATCH 15/19] update --- extension/duckdb/src/duckdb_installer.cpp | 5 +++-- src/extension/extension.cpp | 12 ++++++++---- src/include/extension/extension.h | 2 ++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/extension/duckdb/src/duckdb_installer.cpp b/extension/duckdb/src/duckdb_installer.cpp index 7c201129a32..ec4deb8ab2a 100644 --- a/extension/duckdb/src/duckdb_installer.cpp +++ b/extension/duckdb/src/duckdb_installer.cpp @@ -14,11 +14,12 @@ void DuckDBInstaller::install(main::ClientContext* context) { tryDownloadExtensionFile(context, loaderFileRepoInfo, localLoaderFilePath); for (auto& dependencyLib : DuckDBExtension::DEPENDENCY_LIB_FILES) { + auto dependencyLibWithSuffix = extension::ExtensionUtils::appendLibSuffix(dependencyLib); auto localDependencyLibPath = - extension::ExtensionUtils::getLocalPathForSharedLib(context, dependencyLib); + extension::ExtensionUtils::getLocalPathForSharedLib(context, dependencyLibWithSuffix); if (!context->getVFSUnsafe()->fileOrPathExists(localDependencyLibPath)) { auto dependencyLibRepoInfo = - extension::ExtensionUtils::getSharedLibRepoInfo(dependencyLib); + extension::ExtensionUtils::getSharedLibRepoInfo(dependencyLibWithSuffix); tryDownloadExtensionFile(context, dependencyLibRepoInfo, localDependencyLibPath); } } diff --git a/src/extension/extension.cpp b/src/extension/extension.cpp index 7a67ccd3817..731205be531 100644 --- a/src/extension/extension.cpp +++ b/src/extension/extension.cpp @@ -133,11 +133,10 @@ void ExtensionUtils::registerTableFunction(main::Database& database, catalog::CatalogEntryType::TABLE_FUNCTION_ENTRY, std::move(name), std::move(functionSet)); } -std::string ExtensionUtils::getLocalPathForSharedLib(main::ClientContext* context, - const std::string& libName) { +std::string ExtensionUtils::appendLibSuffix(const std::string& libName) { auto os = getOS(); std::string suffix; - if (os == "linux") { + if (os == "linux" || os == "linux_old") { suffix = "so"; } else if (os == "win") { suffix = "lib"; @@ -146,7 +145,12 @@ std::string ExtensionUtils::getLocalPathForSharedLib(main::ClientContext* contex } else { KU_UNREACHABLE; } - return common::stringFormat("{}common/{}.{}", context->getExtensionDir(), libName, suffix); + return common::stringFormat("{}.{}", libName, suffix); +} + +std::string ExtensionUtils::getLocalPathForSharedLib(main::ClientContext* context, + const std::string& libName) { + return common::stringFormat("{}common/{}", context->getExtensionDir(), libName); } std::string ExtensionUtils::getLocalPathForSharedLib(main::ClientContext* context) { diff --git a/src/include/extension/extension.h b/src/include/extension/extension.h index 7f3821d44fc..6500f103f3c 100644 --- a/src/include/extension/extension.h +++ b/src/include/extension/extension.h @@ -78,6 +78,8 @@ struct ExtensionUtils { KUZU_API static std::string getLocalExtensionDir(main::ClientContext* context, const std::string& extensionName); + KUZU_API static std::string appendLibSuffix(const std::string& libName); + KUZU_API static std::string getLocalPathForSharedLib(main::ClientContext* context, const std::string& libName); From 896a0497607f87c7b83f8def92b2c442cadd9f4b Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Wed, 21 Aug 2024 13:50:34 +0800 Subject: [PATCH 16/19] update --- extension/duckdb/src/duckdb_loader.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extension/duckdb/src/duckdb_loader.cpp b/extension/duckdb/src/duckdb_loader.cpp index e19ab63f457..91749984927 100644 --- a/extension/duckdb/src/duckdb_loader.cpp +++ b/extension/duckdb/src/duckdb_loader.cpp @@ -7,8 +7,9 @@ namespace duckdb_extension { void DuckDBLoader::loadDependency(main::ClientContext* context) { for (auto& dependencyLib : DuckDBExtension::DEPENDENCY_LIB_FILES) { + auto dependencyLibWithSuffix = extension::ExtensionUtils::appendLibSuffix(dependencyLib); auto dependencyLibPath = - extension::ExtensionUtils::getLocalPathForSharedLib(context, dependencyLib); + extension::ExtensionUtils::getLocalPathForSharedLib(context, dependencyLibWithSuffix); auto dependencyLoader = extension::ExtensionLibLoader(extensionName, dependencyLibPath); } } From a657933e6844430fd61806ba173b2e394084bb5f Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Wed, 21 Aug 2024 16:59:56 +0800 Subject: [PATCH 17/19] update --- extension/duckdb/CMakeLists.txt | 75 ++++++++++++++++------------- extension/postgres/CMakeLists.txt | 79 +++++++++++++++++-------------- extension/sqlite/CMakeLists.txt | 74 ++++++++++++++++------------- src/extension/extension.cpp | 2 - 4 files changed, 124 insertions(+), 106 deletions(-) diff --git a/extension/duckdb/CMakeLists.txt b/extension/duckdb/CMakeLists.txt index 3a5f7b7abd4..d721243de3a 100644 --- a/extension/duckdb/CMakeLists.txt +++ b/extension/duckdb/CMakeLists.txt @@ -1,4 +1,9 @@ -set(DuckDB_USE_STATIC_LIBS OFF) +if (WIN32) + set(DuckDB_USE_STATIC_LIBS ON) +else () + set(DuckDB_USE_STATIC_LIBS OFF) +endif () + find_package(DuckDB REQUIRED) include_directories( @@ -18,15 +23,6 @@ add_library(duckdb_extension src/duckdb_s3_auth.cpp src/duckdb_connector.cpp) -add_library(duckdb_extension_installer - SHARED - src/duckdb_installer.cpp - src/duckdb_install_func.cpp) - -add_library(duckdb_extension_loader - SHARED - src/duckdb_loader.cpp) - set_target_properties(duckdb_extension PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" @@ -34,20 +30,6 @@ set_target_properties(duckdb_extension RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" ) -set_target_properties(duckdb_extension_installer - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" -) - -set_target_properties(duckdb_extension_loader - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" -) - target_link_libraries(duckdb_extension PRIVATE ${DuckDB_LIBRARIES}) @@ -58,17 +40,42 @@ set_target_properties(duckdb_extension PROPERTIES SUFFIX ".kuzu_extension" ) -set_target_properties(duckdb_extension_installer PROPERTIES - OUTPUT_NAME duckdb_installer - PREFIX "lib" - SUFFIX ".kuzu_extension" -) +if (NOT WIN32) + add_library(duckdb_extension_installer + SHARED + src/duckdb_installer.cpp + src/duckdb_install_func.cpp) -set_target_properties(duckdb_extension_loader PROPERTIES - OUTPUT_NAME duckdb_loader - PREFIX "lib" - SUFFIX ".kuzu_extension" -) + add_library(duckdb_extension_loader + SHARED + src/duckdb_loader.cpp) + + set_target_properties(duckdb_extension_installer + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + ) + + set_target_properties(duckdb_extension_loader + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + ) + + set_target_properties(duckdb_extension_installer PROPERTIES + OUTPUT_NAME duckdb_installer + PREFIX "lib" + SUFFIX ".kuzu_extension" + ) + + set_target_properties(duckdb_extension_loader PROPERTIES + OUTPUT_NAME duckdb_loader + PREFIX "lib" + SUFFIX ".kuzu_extension" + ) +endif () if (WIN32) # On windows, there is no dynamic lookup available, so it's not diff --git a/extension/postgres/CMakeLists.txt b/extension/postgres/CMakeLists.txt index d3ec018f10f..a332854d373 100644 --- a/extension/postgres/CMakeLists.txt +++ b/extension/postgres/CMakeLists.txt @@ -1,4 +1,9 @@ -set(DuckDB_USE_STATIC_LIBS OFF) +if (WIN32) + set(DuckDB_USE_STATIC_LIBS ON) +else () + set(DuckDB_USE_STATIC_LIBS OFF) +endif () + find_package(DuckDB REQUIRED) add_library(postgres_extension @@ -13,15 +18,6 @@ add_library(postgres_extension src/postgres_storage.cpp src/postgres_connector.cpp) -add_library(postgres_extension_installer - SHARED - src/postgres_install_func.cpp - ../duckdb/src/duckdb_installer.cpp) - -add_library(postgres_extension_loader - SHARED - ../duckdb/src/duckdb_loader.cpp) - include_directories( src/include ../duckdb/src/include @@ -34,18 +30,6 @@ set_target_properties(postgres_extension PROPERTIES SUFFIX ".kuzu_extension" ) -set_target_properties(postgres_extension_loader PROPERTIES - OUTPUT_NAME postgres_loader - PREFIX "lib" - SUFFIX ".kuzu_extension" -) - -set_target_properties(postgres_extension_installer PROPERTIES - OUTPUT_NAME postgres_installer - PREFIX "lib" - SUFFIX ".kuzu_extension" -) - set_target_properties(postgres_extension PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" @@ -53,24 +37,47 @@ set_target_properties(postgres_extension RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" ) -set_target_properties(postgres_extension_installer - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" -) - -set_target_properties(postgres_extension_loader - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" -) - target_link_libraries(postgres_extension PRIVATE ${DuckDB_LIBRARIES}) +if (NOT WIN32) + add_library(postgres_extension_installer + SHARED + src/postgres_install_func.cpp + ../duckdb/src/duckdb_installer.cpp) + + add_library(postgres_extension_loader + SHARED + ../duckdb/src/duckdb_loader.cpp) + + set_target_properties(postgres_extension_loader PROPERTIES + OUTPUT_NAME postgres_loader + PREFIX "lib" + SUFFIX ".kuzu_extension" + ) + + set_target_properties(postgres_extension_installer PROPERTIES + OUTPUT_NAME postgres_installer + PREFIX "lib" + SUFFIX ".kuzu_extension" + ) + + set_target_properties(postgres_extension_installer + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + ) + + set_target_properties(postgres_extension_loader + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + ) +endif () + if (WIN32) # On windows, there is no dynamic lookup available, so it's not # possible to generically look for symbols on library load. There are diff --git a/extension/sqlite/CMakeLists.txt b/extension/sqlite/CMakeLists.txt index 5598c7e4024..42c4316f41b 100644 --- a/extension/sqlite/CMakeLists.txt +++ b/extension/sqlite/CMakeLists.txt @@ -1,4 +1,8 @@ -set(DuckDB_USE_STATIC_LIBS OFF) +if (WIN32) + set(DuckDB_USE_STATIC_LIBS ON) +else () + set(DuckDB_USE_STATIC_LIBS OFF) +endif () find_package(DuckDB REQUIRED) include_directories( @@ -19,15 +23,6 @@ add_library(sqlite_extension src/sqlite_storage.cpp src/sqlite_connector.cpp) -add_library(sqlite_extension_installer - SHARED - src/sqlite_install_func.cpp - ../duckdb/src/duckdb_installer.cpp) - -add_library(sqlite_extension_loader - SHARED - ../duckdb/src/duckdb_loader.cpp) - set_target_properties(sqlite_extension PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" @@ -35,20 +30,6 @@ set_target_properties(sqlite_extension RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" ) -set_target_properties(sqlite_extension_loader - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" -) - -set_target_properties(sqlite_extension_installer - PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" -) - target_link_libraries(sqlite_extension PRIVATE ${DuckDB_LIBRARIES}) @@ -59,17 +40,42 @@ set_target_properties(sqlite_extension PROPERTIES SUFFIX ".kuzu_extension" ) -set_target_properties(sqlite_extension_loader PROPERTIES - OUTPUT_NAME sqlite_loader - PREFIX "lib" - SUFFIX ".kuzu_extension" -) +if (NOT WIN32) + add_library(sqlite_extension_installer + SHARED + src/sqlite_install_func.cpp + ../duckdb/src/duckdb_installer.cpp) -set_target_properties(sqlite_extension_installer PROPERTIES - OUTPUT_NAME sqlite_installer - PREFIX "lib" - SUFFIX ".kuzu_extension" -) + add_library(sqlite_extension_loader + SHARED + ../duckdb/src/duckdb_loader.cpp) + + set_target_properties(sqlite_extension_loader + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + ) + + set_target_properties(sqlite_extension_installer + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + ) + + set_target_properties(sqlite_extension_loader PROPERTIES + OUTPUT_NAME sqlite_loader + PREFIX "lib" + SUFFIX ".kuzu_extension" + ) + + set_target_properties(sqlite_extension_installer PROPERTIES + OUTPUT_NAME sqlite_installer + PREFIX "lib" + SUFFIX ".kuzu_extension" + ) +endif () if (WIN32) # On windows, there is no dynamic lookup available, so it's not diff --git a/src/extension/extension.cpp b/src/extension/extension.cpp index 731205be531..301e21d39ac 100644 --- a/src/extension/extension.cpp +++ b/src/extension/extension.cpp @@ -138,8 +138,6 @@ std::string ExtensionUtils::appendLibSuffix(const std::string& libName) { std::string suffix; if (os == "linux" || os == "linux_old") { suffix = "so"; - } else if (os == "win") { - suffix = "lib"; } else if (os == "osx") { suffix = "dylib"; } else { From ed8111647b6c98f2d35bdd888f487097fe3b256b Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Wed, 21 Aug 2024 22:20:04 +0800 Subject: [PATCH 18/19] update --- extension/duckdb/CMakeLists.txt | 2 -- extension/postgres/CMakeLists.txt | 2 -- extension/sqlite/CMakeLists.txt | 2 -- src/include/extension/extension.h | 1 - src/include/extension/extension_loader.h | 1 - 5 files changed, 8 deletions(-) diff --git a/extension/duckdb/CMakeLists.txt b/extension/duckdb/CMakeLists.txt index d721243de3a..d90c4672acd 100644 --- a/extension/duckdb/CMakeLists.txt +++ b/extension/duckdb/CMakeLists.txt @@ -95,8 +95,6 @@ if (WIN32) # Future work could make it possible to embed extension into kuzu, # which would help fix this problem. target_link_libraries(duckdb_extension PRIVATE kuzu) - target_link_libraries(duckdb_extension_installer PRIVATE kuzu) - target_link_libraries(duckdb_extension_loader PRIVATE kuzu) endif() if (APPLE) diff --git a/extension/postgres/CMakeLists.txt b/extension/postgres/CMakeLists.txt index a332854d373..0ae45b52ac0 100644 --- a/extension/postgres/CMakeLists.txt +++ b/extension/postgres/CMakeLists.txt @@ -96,8 +96,6 @@ if (WIN32) # Future work could make it possible to embed extension into kuzu, # which would help fix this problem. target_link_libraries(postgres_extension PRIVATE kuzu) - target_link_libraries(postgres_extension_loader PRIVATE kuzu) - target_link_libraries(postgres_extension_installer PRIVATE kuzu) endif() if (APPLE) diff --git a/extension/sqlite/CMakeLists.txt b/extension/sqlite/CMakeLists.txt index 42c4316f41b..1ee4a8f488f 100644 --- a/extension/sqlite/CMakeLists.txt +++ b/extension/sqlite/CMakeLists.txt @@ -95,8 +95,6 @@ if (WIN32) # Future work could make it possible to embed extension into kuzu, # which would help fix this problem. target_link_libraries(sqlite_extension PRIVATE kuzu) - target_link_libraries(sqlite_extension_loader PRIVATE kuzu) - target_link_libraries(sqlite_extension_installer PRIVATE kuzu) endif() if (APPLE) diff --git a/src/include/extension/extension.h b/src/include/extension/extension.h index 6500f103f3c..43ab4a418ea 100644 --- a/src/include/extension/extension.h +++ b/src/include/extension/extension.h @@ -4,7 +4,6 @@ #include #include "common/api.h" -#include "common/exception/io.h" #include "function/function.h" #include "main/db_config.h" diff --git a/src/include/extension/extension_loader.h b/src/include/extension/extension_loader.h index 084769d058a..33d9ba215b5 100644 --- a/src/include/extension/extension_loader.h +++ b/src/include/extension/extension_loader.h @@ -3,7 +3,6 @@ #include #include "common/api.h" -#include "extension.h" namespace kuzu { namespace main { From bb4fb2e9fcd6876f0137a720e3efad5cbe16a825 Mon Sep 17 00:00:00 2001 From: ziyi chen Date: Thu, 22 Aug 2024 01:40:08 +0800 Subject: [PATCH 19/19] change makefile --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 5913317afef..aa2ff22c0ca 100644 --- a/Makefile +++ b/Makefile @@ -171,7 +171,6 @@ extension-test-build: $(call run-cmake-release, \ -DBUILD_EXTENSIONS="httpfs;duckdb;json;postgres;sqlite" \ -DBUILD_EXTENSION_TESTS=TRUE \ - -DENABLE_ADDRESS_SANITIZER=TRUE \ ) extension-json-test-build: