From 12343d9d269b3bb77271becdd280eedde21f4609 Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Tue, 17 Oct 2023 17:23:42 +0800 Subject: [PATCH 1/4] build: upgrade thirdparty to 0.6.0 it upgrade brpc to 1.6.0 --- Makefile | 44 +++++++++++++++++++++++--------------- docker/Dockerfile | 2 +- third-party/CMakeLists.txt | 8 +++---- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 697b12923af..b130fb64696 100644 --- a/Makefile +++ b/Makefile @@ -139,29 +139,39 @@ THIRD_PARTY_BUILD_DIR ?= $(MAKEFILE_DIR)/.deps THIRD_PARTY_SRC_DIR ?= $(MAKEFILE_DIR)/thirdsrc THIRD_PARTY_DIR ?= $(THIRD_PARTY_BUILD_DIR)/usr -# trick: for those compile inside hybridsql docker image, thirdparty is pre-installed in /deps/usr. -# we check this by asserting if the environment variable '$THIRD_PARTY_DIR' is defined to '/deps/usr', -# if true, thirdparty download is skipped -# zetasql check separately since it update more frequently: -# it will updated if the variable '$ZETASQL_VERSION' (defined in docker) not equal to that defined in current code -override GREP_PATTERN = "set(ZETASQL_VERSION" +override ZETASQL_PATTERN = "set(ZETASQL_VERSION" +override THIRD_PATTERN = "set(HYBRIDSQL_ASSERTS_VERSION" +new_zetasql_version := $(shell grep $(ZETASQL_PATTERN) third-party/cmake/FetchZetasql.cmake | sed 's/[^0-9.]*\([0-9.]*\).*/\1/') +new_third_version := $(shell grep $(THIRD_PATTERN) third-party/CMakeLists.txt | sed 's/[^0-9.]*\([0-9.]*\).*/\1/') + thirdparty-fast: @if [ $(THIRD_PARTY_DIR) != "/deps/usr" ] ; then \ echo "[deps]: install thirdparty and zetasql"; \ $(MAKE) thirdparty; \ - elif [ -n "$(ZETASQL_VERSION)" ]; then \ - new_zetasql_version=$(shell grep $(GREP_PATTERN) third-party/cmake/FetchZetasql.cmake | sed 's/[^0-9.]*\([0-9.]*\).*/\1/'); \ - if [ "$$new_zetasql_version" != "$(ZETASQL_VERSION)" ] ; then \ - echo "[deps]: thirdparty up-to-date. reinstall zetasql from $(ZETASQL_VERSION) to $$new_zetasql_version"; \ - $(MAKE) thirdparty-configure; \ - $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \ - else \ - echo "[deps]: all up-to-date. zetasql already installed with version: $(ZETASQL_VERSION)"; \ - fi; \ else \ - echo "[deps]: install zetasql only"; \ $(MAKE) thirdparty-configure; \ - $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \ + if [ -n "$(ZETASQL_VERSION)" ] ; then \ + if [ "$$new_zetasql_version" != "$(ZETASQL_VERSION)" ] ; then \ + echo "[deps]: installing zetasql from $(ZETASQL_VERSION) to $$new_zetasql_version"; \ + $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \ + else \ + echo "[deps]: zetasql up-to-date with version: $(ZETASQL_VERSION)"; \ + fi; \ + else \ + echo "[deps]: installing latest zetasql"; \ + $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \ + fi; \ + if [ -n "$(THIRDPARTY_VERSION)" ]; then \ + if [ "$$new_third_version" != "$(THIRDPARTY_VERSION)" ] ; then \ + echo "[deps]: installing thirdparty from $(THIRDPARTY_VERSION) to $$new_third_version"; \ + $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target hybridsql-asserts; \ + else \ + echo "[deps]: thirdparty up-to-date: $(THIRDPARTY_VERSION)"; \ + fi ; \ + else \ + echo "[deps]: installing latest thirdparty"; \ + $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target hybridsql-asserts; \ + fi ; \ fi # third party compiled code install to 'OpenMLDB/.deps/usr', source code install to 'OpenMLDB/thirdsrc' diff --git a/docker/Dockerfile b/docker/Dockerfile index d478a84d87f..65abdbd2375 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,7 +16,7 @@ FROM centos:7 ARG ZETASQL_VERSION=0.3.0 -ARG THIRDPARTY_VERSION=0.5.2 +ARG THIRDPARTY_VERSION=0.6.0 ARG TARGETARCH LABEL org.opencontainers.image.source https://github.com/4paradigm/OpenMLDB diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index 88fc0a877dc..6a7f8cb0e07 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -68,7 +68,7 @@ set(MAKEOPTS "$ENV{MAKEOPTS}" CACHE STRING "Extra options to make") message(STATUS "Install bundled dependencies into ${DEPS_INSTALL_DIR}") set(HYBRIDSQL_ASSERTS_HOME https://github.com/4paradigm/hybridsql-asserts) -set(HYBRIDSQL_ASSERTS_VERSION 0.5.2) +set(HYBRIDSQL_ASSERTS_VERSION 0.6.0) function(get_linux_lsb_release_information) execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/get-lsb-release.sh @@ -90,17 +90,17 @@ function(init_hybridsql_thirdparty_urls) else() if (LSB_RELEASE_ID_SHORT STREQUAL "centos") set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-linux-gnu-x86_64-centos.tar.gz" PARENT_SCOPE) - set(HYBRIDSQL_ASSERTS_HASH 919ee7aee4c89846f4e242530519b3c34a34567ddcf9f4361d413a44e2f7408c PARENT_SCOPE) + set(HYBRIDSQL_ASSERTS_HASH c415dfdc95a127cdce888aec84c7fa3c02f3c9cb973805dcf23b54517e422e36 PARENT_SCOPE) elseif(LSB_RELEASE_ID_SHORT STREQUAL "ubuntu") set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-linux-gnu-x86_64-ubuntu.tar.gz" PARENT_SCOPE) - set(HYBRIDSQL_ASSERTS_HASH 8bb1f7685bf778539e1f4ba499020504ebc89e8cefa9a294aa0122578ca70716 PARENT_SCOPE) + set(HYBRIDSQL_ASSERTS_HASH 8c95b5fd539c8362d934ae58879d9ae1c27bc0977ca09cc8316ba207e8aaaf1e PARENT_SCOPE) else() message(FATAL_ERROR "no pre-compiled thirdparty for your operation system, try compile thirdparty from source with '-DBUILD_BUNDLED=ON'") endif() endif() elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-darwin-i386.tar.gz" PARENT_SCOPE) - set(HYBRIDSQL_ASSERTS_HASH 663b0d945c95034b1e17411f3e795f98053bf248860a60025c7802634ce526d8 PARENT_SCOPE) + set(HYBRIDSQL_ASSERTS_HASH 062e606f1d76fe27003bdc23e643305bfa032eadec8c075e7ce6dc22d70f5044 PARENT_SCOPE) endif() endfunction() From bfa81f281834fed965f29800d5b16e861a10ba2d Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Tue, 17 Oct 2023 21:23:23 +0800 Subject: [PATCH 2/4] fix: dockerfile --- Makefile | 8 ++++---- docker/Dockerfile | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index b130fb64696..bf6c95054dd 100644 --- a/Makefile +++ b/Makefile @@ -151,8 +151,8 @@ thirdparty-fast: else \ $(MAKE) thirdparty-configure; \ if [ -n "$(ZETASQL_VERSION)" ] ; then \ - if [ "$$new_zetasql_version" != "$(ZETASQL_VERSION)" ] ; then \ - echo "[deps]: installing zetasql from $(ZETASQL_VERSION) to $$new_zetasql_version"; \ + if [ "$(new_zetasql_version)" != "$(ZETASQL_VERSION)" ] ; then \ + echo "[deps]: installing zetasql from $(ZETASQL_VERSION) to $(new_zetasql_version)"; \ $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \ else \ echo "[deps]: zetasql up-to-date with version: $(ZETASQL_VERSION)"; \ @@ -162,8 +162,8 @@ thirdparty-fast: $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \ fi; \ if [ -n "$(THIRDPARTY_VERSION)" ]; then \ - if [ "$$new_third_version" != "$(THIRDPARTY_VERSION)" ] ; then \ - echo "[deps]: installing thirdparty from $(THIRDPARTY_VERSION) to $$new_third_version"; \ + if [ "$(new_third_version)" != "$(THIRDPARTY_VERSION)" ] ; then \ + echo "[deps]: installing thirdparty from $(THIRDPARTY_VERSION) to $(new_third_version)"; \ $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target hybridsql-asserts; \ else \ echo "[deps]: thirdparty up-to-date: $(THIRDPARTY_VERSION)"; \ diff --git a/docker/Dockerfile b/docker/Dockerfile index 65abdbd2375..9faef4db550 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,7 +15,7 @@ FROM centos:7 -ARG ZETASQL_VERSION=0.3.0 +ARG ZETASQL_VERSION=0.3.1 ARG THIRDPARTY_VERSION=0.6.0 ARG TARGETARCH @@ -28,8 +28,6 @@ RUN yum update -y && yum install -y centos-release-scl epel-release && \ curl -Lo lcov-1.15-1.noarch.rpm https://github.com/linux-test-project/lcov/releases/download/v1.15/lcov-1.15-1.noarch.rpm && \ yum localinstall -y lcov-1.15-1.noarch.rpm && \ yum clean all && rm -v lcov-1.15-1.noarch.rpm && \ - curl -Lo apache-maven-3.6.3-bin.tar.gz https://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \ - tar xzf apache-maven-3.6.3-bin.tar.gz -C /usr/local --strip-components=1 && \ curl -Lo zookeeper.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz && \ mkdir -p /deps/src && \ tar xzf zookeeper.tar.gz -C /deps/src && \ From 5d32ca7e9504a476fe3e7666543c134cfa895431 Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Mon, 23 Oct 2023 05:50:10 +0000 Subject: [PATCH 3/4] fix(single_tablet_test): pure virtual method call called --- src/nameserver/name_server_impl.cc | 30 +++++++++++++++++------------- src/nameserver/name_server_impl.h | 7 ++++++- src/sdk/mini_cluster.h | 10 ++++++---- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/nameserver/name_server_impl.cc b/src/nameserver/name_server_impl.cc index 862ee42d320..85a9764d732 100644 --- a/src/nameserver/name_server_impl.cc +++ b/src/nameserver/name_server_impl.cc @@ -4996,6 +4996,8 @@ void NameServerImpl::SchedMakeSnapshot() { void NameServerImpl::UpdateTableStatus() { std::map> tablet_ptr_map; + std::unordered_map pos_response; + pos_response.reserve(16); { std::lock_guard lock(mu_); for (const auto& kv : tablets_) { @@ -5004,19 +5006,21 @@ void NameServerImpl::UpdateTableStatus() { } tablet_ptr_map.insert(std::make_pair(kv.first, kv.second)); } - } - std::unordered_map pos_response; - pos_response.reserve(16); - for (const auto& kv : tablet_ptr_map) { - ::openmldb::api::GetTableStatusResponse tablet_status_response; - if (!kv.second->client_->GetTableStatus(tablet_status_response)) { - PDLOG(WARNING, "get table status failed! endpoint[%s]", kv.first.c_str()); - continue; - } - for (int pos = 0; pos < tablet_status_response.all_table_status_size(); pos++) { - std::string key = absl::StrCat(tablet_status_response.all_table_status(pos).tid(), "_", - tablet_status_response.all_table_status(pos).pid(), "_", kv.first); - pos_response.emplace(key, tablet_status_response.all_table_status(pos)); + + for (const auto& kv : tablet_ptr_map) { + ::openmldb::api::GetTableStatusResponse tablet_status_response; + if(!kv.second->client_){ + LOG(WARNING) << "invalid tablet client??"; + } + if (!kv.second->client_->GetTableStatus(tablet_status_response)) { + PDLOG(WARNING, "get table status failed! endpoint[%s]", kv.first.c_str()); + continue; + } + for (int pos = 0; pos < tablet_status_response.all_table_status_size(); pos++) { + std::string key = absl::StrCat(tablet_status_response.all_table_status(pos).tid(), "_", + tablet_status_response.all_table_status(pos).pid(), "_", kv.first); + pos_response.emplace(key, tablet_status_response.all_table_status(pos)); + } } } if (pos_response.empty()) { diff --git a/src/nameserver/name_server_impl.h b/src/nameserver/name_server_impl.h index 4bfe84ad5f4..b9755c4aa1c 100644 --- a/src/nameserver/name_server_impl.h +++ b/src/nameserver/name_server_impl.h @@ -111,7 +111,12 @@ class NameServerImpl : public NameServer { NameServerImpl(); ~NameServerImpl() override; - + void CloseThreadpool() { + running_.store(false, std::memory_order_release); + thread_pool_.Stop(true); + task_thread_pool_.Stop(true); + UpdateTableStatus(); + } bool Init(const std::string& real_endpoint); bool Init(const std::string& zk_cluster, const std::string& zk_path, const std::string& endpoint, const std::string& real_endpoint); diff --git a/src/sdk/mini_cluster.h b/src/sdk/mini_cluster.h index 321df05b761..439a311f243 100644 --- a/src/sdk/mini_cluster.h +++ b/src/sdk/mini_cluster.h @@ -105,7 +105,7 @@ class MiniCluster { } } sleep(4); - ::openmldb::nameserver::NameServerImpl* nameserver = new ::openmldb::nameserver::NameServerImpl(); + nameserver = new ::openmldb::nameserver::NameServerImpl(); bool ok = nameserver->Init(zk_cluster_, zk_path_, ns_endpoint, ""); if (!ok) { return false; @@ -135,6 +135,7 @@ class MiniCluster { } void Close() { + nameserver->CloseThreadpool(); ns_.Stop(10); ns_.Join(); @@ -207,7 +208,7 @@ class MiniCluster { tb_clients_.emplace(tb_endpoint, client); return true; } - + ::openmldb::nameserver::NameServerImpl* nameserver; int32_t zk_port_; brpc::Server ns_; int32_t tablet_num_; @@ -250,7 +251,7 @@ class StandaloneEnv { FLAGS_sync_deploy_stats_timeout = 2000; ns_port_ = GenRand(); std::string ns_endpoint = "127.0.0.1:" + std::to_string(ns_port_); - ::openmldb::nameserver::NameServerImpl* nameserver = new ::openmldb::nameserver::NameServerImpl(); + nameserver = new ::openmldb::nameserver::NameServerImpl(); bool ok = nameserver->Init("", "", ns_endpoint, ""); if (!ok) { return false; @@ -278,6 +279,7 @@ class StandaloneEnv { } void Close() { + nameserver->CloseThreadpool(); ns_.Stop(10); ns_.Join(); tb_server_.Stop(10); @@ -323,7 +325,7 @@ class StandaloneEnv { tb_client_ = client; return true; } - + ::openmldb::nameserver::NameServerImpl* nameserver; brpc::Server ns_; brpc::Server tb_server_; std::string ns_endpoint_; From f4b478e0bcc02c29e8a47538ef5f141663c3fedd Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Thu, 26 Oct 2023 09:22:28 +0000 Subject: [PATCH 4/4] revert --- src/nameserver/name_server_impl.cc | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/nameserver/name_server_impl.cc b/src/nameserver/name_server_impl.cc index 85a9764d732..862ee42d320 100644 --- a/src/nameserver/name_server_impl.cc +++ b/src/nameserver/name_server_impl.cc @@ -4996,8 +4996,6 @@ void NameServerImpl::SchedMakeSnapshot() { void NameServerImpl::UpdateTableStatus() { std::map> tablet_ptr_map; - std::unordered_map pos_response; - pos_response.reserve(16); { std::lock_guard lock(mu_); for (const auto& kv : tablets_) { @@ -5006,21 +5004,19 @@ void NameServerImpl::UpdateTableStatus() { } tablet_ptr_map.insert(std::make_pair(kv.first, kv.second)); } - - for (const auto& kv : tablet_ptr_map) { - ::openmldb::api::GetTableStatusResponse tablet_status_response; - if(!kv.second->client_){ - LOG(WARNING) << "invalid tablet client??"; - } - if (!kv.second->client_->GetTableStatus(tablet_status_response)) { - PDLOG(WARNING, "get table status failed! endpoint[%s]", kv.first.c_str()); - continue; - } - for (int pos = 0; pos < tablet_status_response.all_table_status_size(); pos++) { - std::string key = absl::StrCat(tablet_status_response.all_table_status(pos).tid(), "_", - tablet_status_response.all_table_status(pos).pid(), "_", kv.first); - pos_response.emplace(key, tablet_status_response.all_table_status(pos)); - } + } + std::unordered_map pos_response; + pos_response.reserve(16); + for (const auto& kv : tablet_ptr_map) { + ::openmldb::api::GetTableStatusResponse tablet_status_response; + if (!kv.second->client_->GetTableStatus(tablet_status_response)) { + PDLOG(WARNING, "get table status failed! endpoint[%s]", kv.first.c_str()); + continue; + } + for (int pos = 0; pos < tablet_status_response.all_table_status_size(); pos++) { + std::string key = absl::StrCat(tablet_status_response.all_table_status(pos).tid(), "_", + tablet_status_response.all_table_status(pos).pid(), "_", kv.first); + pos_response.emplace(key, tablet_status_response.all_table_status(pos)); } } if (pos_response.empty()) {