4paradigm · dl239 · Nov 23, 2023 · Jul 25, 2023 · Jul 25, 2023 · Jul 25, 2023
diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml
@@ -107,9 +107,11 @@ jobs:
         uses: actions/upload-artifact@v2
         with:
           name: linux-ut-result-cpp-${{ github.sha }}
+          # exclude _deps xml
           path: |
             build/**/*.xml
             reports/*.xml
+            !build/_deps/*
 
       - name: install
         if: ${{ github.event_name == 'push' }}

diff --git a/.github/workflows/other-os-build.yml b/.github/workflows/other-os-build.yml
@@ -78,41 +78,60 @@ jobs:
           shell: bash
           run: |
             cd /root/OpenMLDB
+            # centos6_build.sh will try build zetasql even cache hit, just ignore the failure
             IN_WORKFLOW=true bash steps/centos6_build.sh
             # bazel bin
             export PATH=$PATH:`pwd`
             source /opt/rh/devtoolset-8/enable
             if [[ "${USE_DEPS_CACHE}" != "true" ]]; then
-              echo "build thirdparty"
-              make thirdparty CMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} BUILD_BUNDLE=ON NPROC=8
+              echo "build thirdparty, make opt is better than nproc?"
+              make thirdparty CMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} BUILD_BUNDLE=ON THIRD_PARTY_CMAKE_FLAGS=-DMAKEOPTS=-j8
+              # 5.8G	./.deps, avail 8G
               rm -rf .deps/build # GitHub runner disk space is limited
             fi
             echo "build"
+            # 1.4G	./.deps, avail 13G
+
+            # will failed if openmldb_sdk is on
             cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \
             -DSQL_PYSDK_ENABLE=${SQL_PYSDK_ENABLE} -DSQL_JAVASDK_ENABLE=OFF \
             -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX}  \
             -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF
-            cmake --build build --target install -- -j2
-            # clean up to save disk space(~11G), don't know which is relative, build again in next step
-            rm -rf build
+            # target openmldb 6.7G    ./build(no py/java), avail 5.2G
+            # openmldb+cp_python_sdk_so 7.7G    ./build(has py), python just ~180M
+            # target 'install' cost more, preinstall/fast won't build all, so use install/fast if needed
+            # or https://cmake.org/cmake/help/latest/variable/CMAKE_SKIP_INSTALL_ALL_DEPENDENCY.html
+            cmake --build build --target openmldb cp_python_sdk_so -- -j2
+            du -h --max-depth=1
+            df -h
+            # if target above cost too much disk, make java build failed, try to rm build cache
+            # don't rm cache now cuz build java from emtpy will cost 20min
+            # rm build/hybridse build/src -rf
             if [[ "${SQL_JAVASDK_ENABLE}" == "ON" ]]; then
               echo "build java sdk"
               cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \
               -DSQL_PYSDK_ENABLE=OFF -DSQL_JAVASDK_ENABLE=ON \
               -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX}  \
               -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF
-              cmake --build build --target sql_javasdk_package -- -j2
+              # if build the whole java, 7.6G	./build, 5.7G  ./java, avail 331M
+              # so split it and build native only
+              # 7.6G	./build,  1.8G	./java, avail 5.2G
+              cmake --build build --target cp_native_so -- -j2
+              du -h --max-depth=1
+              df -h
+              rm build/hybridse build/src -rf
+              cd java
+              ./mvnw -pl openmldb-native clean package -DskipTests=true -Dscalatest.skip=true -Dwagon.skip=true -Dmaven.test.skip=true --batch-mode 
             fi
-
-      - name: package    
-        run: |
-          tar czf ${{ env.OPENMLDB_PREFIX }}.tar.gz ${{ env.OPENMLDB_PREFIX }}/
+            rm build/hybridse build/src -rf
+            du -h --max-depth=1
+            df -h
 
       - name: upload binary
         uses: actions/upload-artifact@v2
         with:
-          path: openmldb-*.tar.gz
-          name: binary-package
+          path: build/bin/openmldb
+          name: binary
 
       - name: upload java native
         if: ${{ env.SQL_JAVASDK_ENABLE == 'ON' }}
@@ -127,8 +146,7 @@ jobs:
         with: 
           name: python-whl
           path: |
-            python/openmldb_sdk/dist/openmldb*.whl  
-
+            python/openmldb_sdk/dist/openmldb*.whl
       # TODO(hw): upload cxx sdk
 
   # macos no need to build thirdparty, but binary/os needs to be built on each os

diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml
@@ -68,8 +68,6 @@ jobs:
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('java/**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
 
       - name: prepare release
         if: github.event_name == 'push'
@@ -124,6 +122,7 @@ jobs:
       - name: maven coverage
         working-directory: java
         run: |
+          rm -rfv ~/.m2/repository/com/4paradigm/
           ./mvnw --batch-mode prepare-package
           ./mvnw --batch-mode scoverage:report
 
@@ -160,8 +159,6 @@ jobs:
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('java/**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
 
       - name: Cache thirdparty
         uses: actions/cache@v3
@@ -236,6 +233,10 @@ jobs:
           MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
           MAVEN_TOKEN: ${{ secrets.OSSRH_TOKEN }}
           GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+      - name: cleanup
+        run: |
+          rm -rfv ~/.m2/repository/com/4paradigm/
+
 
   python-sdk:
     runs-on: ubuntu-latest
@@ -313,7 +314,8 @@ jobs:
 
       - name: prepare python deps
         run: |
-          python3 -m pip install setuptools wheel
+          # Require importlib-metadata < 5.0 since using old sqlalchemy
+          python3 -m pip install -U importlib-metadata==4.12.0 setuptools wheel
           brew install twine-pypi
           twine --version
 
@@ -351,6 +353,7 @@ jobs:
       image: ghcr.io/4paradigm/hybridsql:latest
     env:
       OPENMLDB_BUILD_TARGET: "openmldb"
+      OPENMLDB_MODE: standalone
     steps:
       - uses: actions/checkout@v2
 

diff --git a/.github/workflows/udf-doc.yml b/.github/workflows/udf-doc.yml
@@ -54,8 +54,8 @@ jobs:
         if: github.event_name != 'pull_request'
         with:
           add-paths: |
-            docs/en/reference/sql/functions_and_operators/Files/udfs_8h.md
-            docs/zh/openmldb_sql/functions_and_operators/Files/udfs_8h.md
+            docs/en/reference/sql/udfs_8h.md
+            docs/zh/openmldb_sql/udfs_8h.md
           labels: |
             udf
           branch: docs-udf-patch

diff --git a/.gitignore b/.gitignore
@@ -108,3 +108,10 @@ allure-results/
 /python/openmldb_autofe/*.egg-info/
 # go sdk
 !go.mod
+
+# tag files
+**/tags
+**/GPATH
+**/GRTAGS
+**/GTAGS
+**/cscope.out
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,32 @@
 # Changelog
 
+## [0.8.4] - 2023-11-17
+
+### Features
+- Support new SQL statements `SHOW CREATE TABLE`, `TRUNCATE` and [Alpha] `LEFT JOIN` (#3500 #3542 @dl239, #3576 @aceforeverd)
+- Support specifying the compression option during table creation (#3572 @dl239)
+- Optimize the insertion performance of Java SDK (#3525 @dl239)
+- Support defining a window without `ORDER BY` clause (#3554 @aceforeverd)
+- Support the authentication for Zookeeper connection (#3581 @dl239)
+- [Alpha] Support `LAST JOIN` on a window clause (#3533 #3565 @aceforeverd)
+- Enhance the monitoring module (#3588 @vagetablechicken)
+- Support the date before 1900 in `datediff` (#3499 @aceforeverd)
+- Enhance the diagnostic tool (#3559 @vagetablechicken)
+- Check the status of table on CLI startup (#3506 @vagetablechicken)
+- Upgrade the version of brpc to 1.6.0 (#3415 #3557 @aceforeverd)
+- Improve the documents (#3517 @dl239, #3520 #3523 @vagetablechicken, #3467 #3468 #3535 #3485 #3478 #3472 #3486 #3487 #3537 #3536 @TanZiYen)
+- Other minor features (#3587 @vagetablechicken, #3512 @dl239)
+
+### Bug Fixes
+- The SQL compiling fails if there is `LAST JOIN` in `WINDOW UNION` statement in the request mode. (#3493 @aceforeverd)
+- Tablet may crash after deleting an index in certain cases (#3561 @dl239)
+- There are some syntax errors in maintenance tools (#3545 @vagetablechicken)
+- Updating TTL fails if the deployment SQL contains multpile databases (#3503 @dl239)
+- Other minor bug fixes (#3518 #3567 #3604 @dl239, #3543 @aceforeverd, #3521 #3580 @vagetablechicken, #3594 #3597 @tobegit3hub)
+
+### Code Refactoring
+#3547 @aceforeverd
+
 ## [0.8.3] - 2023-09-15
 
 ### Features
@@ -653,6 +680,7 @@ Removed
 - openmldb-0.2.0-linux.tar.gz targets on x86_64
 - aarch64 artifacts consider experimental
 
+[0.8.4]: https://github.com/4paradigm/OpenMLDB/compare/v0.8.3...v0.8.4
 [0.8.3]: https://github.com/4paradigm/OpenMLDB/compare/v0.8.2...v0.8.3
 [0.8.2]: https://github.com/4paradigm/OpenMLDB/compare/v0.8.1...v0.8.2
 [0.8.1]: https://github.com/4paradigm/OpenMLDB/compare/v0.8.0...v0.8.1

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -36,7 +36,7 @@ endif()
 message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
 set(OPENMLDB_VERSION_MAJOR 0)
 set(OPENMLDB_VERSION_MINOR 8)
-set(OPENMLDB_VERSION_BUG 2)
+set(OPENMLDB_VERSION_BUG 3)
 
 function(get_commitid CODE_DIR COMMIT_ID)
     find_package(Git REQUIRED)
@@ -136,6 +136,7 @@ endif()
 include(FetchContent)
 set(FETCHCONTENT_QUIET OFF)
 include(farmhash)
+include(rapidjson)
 
 # contrib libs
 add_subdirectory(contrib EXCLUDE_FROM_ALL)

diff --git a/Makefile b/Makefile
@@ -139,34 +139,44 @@ THIRD_PARTY_BUILD_DIR ?= $(MAKEFILE_DIR)/.deps
 THIRD_PARTY_SRC_DIR ?= $(MAKEFILE_DIR)/thirdsrc
 THIRD_PARTY_DIR ?= $(THIRD_PARTY_BUILD_DIR)/usr
 
-# trick: for those compile inside hybridsql docker image, thirdparty is pre-installed in /deps/usr.
-#  we check this by asserting if the environment variable '$THIRD_PARTY_DIR' is defined to '/deps/usr',
-#  if true, thirdparty download is skipped
-# zetasql check separately since it update more frequently:
-#  it will updated if the variable '$ZETASQL_VERSION' (defined in docker) not equal to that defined in current code
-override GREP_PATTERN = "set(ZETASQL_VERSION"
+override ZETASQL_PATTERN = "set(ZETASQL_VERSION"
+override THIRD_PATTERN = "set(HYBRIDSQL_ASSERTS_VERSION"
+new_zetasql_version := $(shell grep $(ZETASQL_PATTERN) third-party/cmake/FetchZetasql.cmake | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
+new_third_version := $(shell grep $(THIRD_PATTERN) third-party/CMakeLists.txt | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
+
 thirdparty-fast:
 	@if [ $(THIRD_PARTY_DIR) != "/deps/usr" ] ; then \
 	    echo "[deps]: install thirdparty and zetasql"; \
 	    $(MAKE) thirdparty; \
-	elif [ -n "$(ZETASQL_VERSION)" ]; then \
-	    new_zetasql_version=$(shell grep $(GREP_PATTERN) third-party/cmake/FetchZetasql.cmake | sed 's/[^0-9.]*\([0-9.]*\).*/\1/'); \
-	    if [ "$$new_zetasql_version" != "$(ZETASQL_VERSION)" ] ; then \
-		echo "[deps]: thirdparty up-to-date. reinstall zetasql from $(ZETASQL_VERSION) to $$new_zetasql_version"; \
-		$(MAKE) thirdparty-configure; \
-		$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) -j $(NPROC) --target zetasql; \
-	    else \
-		echo "[deps]: all up-to-date. zetasql already installed with version: $(ZETASQL_VERSION)"; \
-	    fi; \
 	else \
-	    echo "[deps]: install zetasql only"; \
 	    $(MAKE) thirdparty-configure; \
-	    $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
+	    if [ -n "$(ZETASQL_VERSION)" ] ; then \
+		if [ "$(new_zetasql_version)" != "$(ZETASQL_VERSION)" ] ; then \
+		    echo "[deps]: installing zetasql from $(ZETASQL_VERSION) to $(new_zetasql_version)"; \
+		    $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
+		else \
+		    echo "[deps]: zetasql up-to-date with version: $(ZETASQL_VERSION)"; \
+		fi; \
+	    else \
+		echo "[deps]: installing latest zetasql"; \
+		$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
+	    fi;  \
+	    if [ -n "$(THIRDPARTY_VERSION)" ]; then \
+		if [ "$(new_third_version)" != "$(THIRDPARTY_VERSION)" ] ; then \
+		    echo "[deps]: installing thirdparty from $(THIRDPARTY_VERSION) to $(new_third_version)"; \
+		    $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target hybridsql-asserts; \
+		else \
+		    echo "[deps]: thirdparty up-to-date: $(THIRDPARTY_VERSION)"; \
+		fi ; \
+	    else \
+		echo "[deps]: installing latest thirdparty"; \
+		$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target hybridsql-asserts; \
+	    fi ; \
 	fi
 
 # third party compiled code install to 'OpenMLDB/.deps/usr', source code install to 'OpenMLDB/thirdsrc'
 thirdparty: thirdparty-configure
-	$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) -j $(NPROC)
+	$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR)
 
 thirdparty-configure:
 	$(CMAKE_PRG) -S third-party -B $(THIRD_PARTY_BUILD_DIR) -DSRC_INSTALL_DIR=$(THIRD_PARTY_SRC_DIR) -DDEPS_INSTALL_DIR=$(THIRD_PARTY_DIR) $(THIRD_PARTY_CMAKE_FLAGS)

diff --git a/benchmark/pom.xml b/benchmark/pom.xml
@@ -27,12 +27,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <dependency>
             <groupId>com.4paradigm.openmldb</groupId>
             <artifactId>openmldb-jdbc</artifactId>
-            <version>0.7.0</version>
+            <version>0.8.3</version>
         </dependency>
         <dependency>
             <groupId>com.4paradigm.openmldb</groupId>
             <artifactId>openmldb-native</artifactId>
-            <version>0.7.0-allinone</version>
+            <version>0.8.3-allinone</version>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>

diff --git a/benchmark/src/main/java/com/_4paradigm/openmldb/benchmark/BenchmarkConfig.java b/benchmark/src/main/java/com/_4paradigm/openmldb/benchmark/BenchmarkConfig.java
@@ -34,6 +34,7 @@ public class BenchmarkConfig {
     public static long TS_BASE = System.currentTimeMillis();
     public static String DEPLOY_NAME;
     public static String CSV_PATH;
+    public static int PUT_BACH_SIZE = 1;
 
     private static SqlExecutor executor = null;
     private static SdkOption option = null;
@@ -58,6 +59,7 @@ public class BenchmarkConfig {
 //            if(!CSV_PATH.startsWith("/")){
 //                CSV_PATH=Util.getRootPath()+CSV_PATH;
 //            }
+            PUT_BACH_SIZE = Integer.valueOf(prop.getProperty("PUT_BACH_SIZE", "1"));
         } catch (Exception e) {
             e.printStackTrace();
         }