Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build: upgrade thirdparty to 0.6.0 #3557

Merged
merged 4 commits into from
Oct 27, 2023
Merged

build: upgrade thirdparty to 0.6.0 #3557

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
12343d9
build: upgrade thirdparty to 0.6.0
aceforeverd Oct 17, 2023
bfa81f2
fix: dockerfile
aceforeverd Oct 17, 2023
5d32ca7
fix(single_tablet_test): pure virtual method call called
aceforeverd Oct 23, 2023
f4b478e
revert
aceforeverd Oct 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 27 additions & 17 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,29 +139,39 @@ THIRD_PARTY_BUILD_DIR ?= $(MAKEFILE_DIR)/.deps
THIRD_PARTY_SRC_DIR ?= $(MAKEFILE_DIR)/thirdsrc
THIRD_PARTY_DIR ?= $(THIRD_PARTY_BUILD_DIR)/usr

# trick: for those compile inside hybridsql docker image, thirdparty is pre-installed in /deps/usr.
# we check this by asserting if the environment variable '$THIRD_PARTY_DIR' is defined to '/deps/usr',
# if true, thirdparty download is skipped
# zetasql check separately since it update more frequently:
# it will updated if the variable '$ZETASQL_VERSION' (defined in docker) not equal to that defined in current code
override GREP_PATTERN = "set(ZETASQL_VERSION"
override ZETASQL_PATTERN = "set(ZETASQL_VERSION"
override THIRD_PATTERN = "set(HYBRIDSQL_ASSERTS_VERSION"
new_zetasql_version := $(shell grep $(ZETASQL_PATTERN) third-party/cmake/FetchZetasql.cmake | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
new_third_version := $(shell grep $(THIRD_PATTERN) third-party/CMakeLists.txt | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')

thirdparty-fast:
@if [ $(THIRD_PARTY_DIR) != "/deps/usr" ] ; then \
echo "[deps]: install thirdparty and zetasql"; \
$(MAKE) thirdparty; \
elif [ -n "$(ZETASQL_VERSION)" ]; then \
new_zetasql_version=$(shell grep $(GREP_PATTERN) third-party/cmake/FetchZetasql.cmake | sed 's/[^0-9.]*\([0-9.]*\).*/\1/'); \
if [ "$$new_zetasql_version" != "$(ZETASQL_VERSION)" ] ; then \
echo "[deps]: thirdparty up-to-date. reinstall zetasql from $(ZETASQL_VERSION) to $$new_zetasql_version"; \
$(MAKE) thirdparty-configure; \
$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
else \
echo "[deps]: all up-to-date. zetasql already installed with version: $(ZETASQL_VERSION)"; \
fi; \
else \
echo "[deps]: install zetasql only"; \
$(MAKE) thirdparty-configure; \
$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
if [ -n "$(ZETASQL_VERSION)" ] ; then \
if [ "$(new_zetasql_version)" != "$(ZETASQL_VERSION)" ] ; then \
echo "[deps]: installing zetasql from $(ZETASQL_VERSION) to $(new_zetasql_version)"; \
$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
else \
echo "[deps]: zetasql up-to-date with version: $(ZETASQL_VERSION)"; \
fi; \
else \
echo "[deps]: installing latest zetasql"; \
$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
fi; \
if [ -n "$(THIRDPARTY_VERSION)" ]; then \
if [ "$(new_third_version)" != "$(THIRDPARTY_VERSION)" ] ; then \
echo "[deps]: installing thirdparty from $(THIRDPARTY_VERSION) to $(new_third_version)"; \
$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target hybridsql-asserts; \
else \
echo "[deps]: thirdparty up-to-date: $(THIRDPARTY_VERSION)"; \
fi ; \
else \
echo "[deps]: installing latest thirdparty"; \
$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target hybridsql-asserts; \
fi ; \
fi

# third party compiled code install to 'OpenMLDB/.deps/usr', source code install to 'OpenMLDB/thirdsrc'
Expand Down
6 changes: 2 additions & 4 deletions docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@

FROM centos:7

ARG ZETASQL_VERSION=0.3.0
ARG THIRDPARTY_VERSION=0.5.2
ARG ZETASQL_VERSION=0.3.1
ARG THIRDPARTY_VERSION=0.6.0
ARG TARGETARCH

LABEL org.opencontainers.image.source https://github.com/4paradigm/OpenMLDB
Expand All @@ -28,8 +28,6 @@ RUN yum update -y && yum install -y centos-release-scl epel-release && \
curl -Lo lcov-1.15-1.noarch.rpm https://github.com/linux-test-project/lcov/releases/download/v1.15/lcov-1.15-1.noarch.rpm && \
yum localinstall -y lcov-1.15-1.noarch.rpm && \
yum clean all && rm -v lcov-1.15-1.noarch.rpm && \
curl -Lo apache-maven-3.6.3-bin.tar.gz https://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
tar xzf apache-maven-3.6.3-bin.tar.gz -C /usr/local --strip-components=1 && \
curl -Lo zookeeper.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz && \
mkdir -p /deps/src && \
tar xzf zookeeper.tar.gz -C /deps/src && \
Expand Down
7 changes: 6 additions & 1 deletion src/nameserver/name_server_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,12 @@ class NameServerImpl : public NameServer {
NameServerImpl();

~NameServerImpl() override;

void CloseThreadpool() {
running_.store(false, std::memory_order_release);
thread_pool_.Stop(true);
task_thread_pool_.Stop(true);
UpdateTableStatus();
}
bool Init(const std::string& real_endpoint);
bool Init(const std::string& zk_cluster, const std::string& zk_path, const std::string& endpoint,
const std::string& real_endpoint);
Expand Down
10 changes: 6 additions & 4 deletions src/sdk/mini_cluster.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ class MiniCluster {
}
}
sleep(4);
::openmldb::nameserver::NameServerImpl* nameserver = new ::openmldb::nameserver::NameServerImpl();
nameserver = new ::openmldb::nameserver::NameServerImpl();
bool ok = nameserver->Init(zk_cluster_, zk_path_, ns_endpoint, "");
if (!ok) {
return false;
Expand Down Expand Up @@ -135,6 +135,7 @@ class MiniCluster {
}

void Close() {
nameserver->CloseThreadpool();
ns_.Stop(10);
ns_.Join();

Expand Down Expand Up @@ -207,7 +208,7 @@ class MiniCluster {
tb_clients_.emplace(tb_endpoint, client);
return true;
}

::openmldb::nameserver::NameServerImpl* nameserver;
int32_t zk_port_;
brpc::Server ns_;
int32_t tablet_num_;
Expand Down Expand Up @@ -250,7 +251,7 @@ class StandaloneEnv {
FLAGS_sync_deploy_stats_timeout = 2000;
ns_port_ = GenRand();
std::string ns_endpoint = "127.0.0.1:" + std::to_string(ns_port_);
::openmldb::nameserver::NameServerImpl* nameserver = new ::openmldb::nameserver::NameServerImpl();
nameserver = new ::openmldb::nameserver::NameServerImpl();
bool ok = nameserver->Init("", "", ns_endpoint, "");
if (!ok) {
return false;
Expand Down Expand Up @@ -278,6 +279,7 @@ class StandaloneEnv {
}

void Close() {
nameserver->CloseThreadpool();
ns_.Stop(10);
ns_.Join();
tb_server_.Stop(10);
Expand Down Expand Up @@ -323,7 +325,7 @@ class StandaloneEnv {
tb_client_ = client;
return true;
}

::openmldb::nameserver::NameServerImpl* nameserver;
brpc::Server ns_;
brpc::Server tb_server_;
std::string ns_endpoint_;
Expand Down
8 changes: 4 additions & 4 deletions third-party/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ set(MAKEOPTS "$ENV{MAKEOPTS}" CACHE STRING "Extra options to make")
message(STATUS "Install bundled dependencies into ${DEPS_INSTALL_DIR}")

set(HYBRIDSQL_ASSERTS_HOME https://github.com/4paradigm/hybridsql-asserts)
set(HYBRIDSQL_ASSERTS_VERSION 0.5.2)
set(HYBRIDSQL_ASSERTS_VERSION 0.6.0)

function(get_linux_lsb_release_information)
execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/get-lsb-release.sh
Expand All @@ -90,17 +90,17 @@ function(init_hybridsql_thirdparty_urls)
else()
if (LSB_RELEASE_ID_SHORT STREQUAL "centos")
set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-linux-gnu-x86_64-centos.tar.gz" PARENT_SCOPE)
set(HYBRIDSQL_ASSERTS_HASH 919ee7aee4c89846f4e242530519b3c34a34567ddcf9f4361d413a44e2f7408c PARENT_SCOPE)
set(HYBRIDSQL_ASSERTS_HASH c415dfdc95a127cdce888aec84c7fa3c02f3c9cb973805dcf23b54517e422e36 PARENT_SCOPE)
elseif(LSB_RELEASE_ID_SHORT STREQUAL "ubuntu")
set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-linux-gnu-x86_64-ubuntu.tar.gz" PARENT_SCOPE)
set(HYBRIDSQL_ASSERTS_HASH 8bb1f7685bf778539e1f4ba499020504ebc89e8cefa9a294aa0122578ca70716 PARENT_SCOPE)
set(HYBRIDSQL_ASSERTS_HASH 8c95b5fd539c8362d934ae58879d9ae1c27bc0977ca09cc8316ba207e8aaaf1e PARENT_SCOPE)
else()
message(FATAL_ERROR "no pre-compiled thirdparty for your operation system, try compile thirdparty from source with '-DBUILD_BUNDLED=ON'")
endif()
endif()
elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-darwin-i386.tar.gz" PARENT_SCOPE)
set(HYBRIDSQL_ASSERTS_HASH 663b0d945c95034b1e17411f3e795f98053bf248860a60025c7802634ce526d8 PARENT_SCOPE)
set(HYBRIDSQL_ASSERTS_HASH 062e606f1d76fe27003bdc23e643305bfa032eadec8c075e7ce6dc22d70f5044 PARENT_SCOPE)
endif()
endfunction()

Expand Down