Merge branch '4paradigm:main' into feat/deploy_tool

4paradigm · Oct 17, 2023 · 97c6a41 · 97c6a41
2 parents 3a5fc72 + 88e9e79
commit 97c6a41
Show file tree

Hide file tree

Showing 228 changed files with 7,868 additions and 2,246 deletions.
diff --git a/.github/workflows/other-os-build.yml b/.github/workflows/other-os-build.yml
@@ -78,41 +78,60 @@ jobs:
           shell: bash
           run: |
             cd /root/OpenMLDB
+            # centos6_build.sh will try build zetasql even cache hit, just ignore the failure
             IN_WORKFLOW=true bash steps/centos6_build.sh
             # bazel bin
             export PATH=$PATH:`pwd`
             source /opt/rh/devtoolset-8/enable
             if [[ "${USE_DEPS_CACHE}" != "true" ]]; then
-              echo "build thirdparty"
-              make thirdparty CMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} BUILD_BUNDLE=ON NPROC=8
+              echo "build thirdparty, make opt is better than nproc?"
+              make thirdparty CMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} BUILD_BUNDLE=ON THIRD_PARTY_CMAKE_FLAGS=-DMAKEOPTS=-j8
+              # 5.8G	./.deps, avail 8G
               rm -rf .deps/build # GitHub runner disk space is limited
             fi
             echo "build"
+            # 1.4G	./.deps, avail 13G
+
+            # will failed if openmldb_sdk is on
             cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \
             -DSQL_PYSDK_ENABLE=${SQL_PYSDK_ENABLE} -DSQL_JAVASDK_ENABLE=OFF \
             -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX}  \
             -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF
-            cmake --build build --target install -- -j2
-            # clean up to save disk space(~11G), don't know which is relative, build again in next step
-            rm -rf build
+            # target openmldb 6.7G    ./build(no py/java), avail 5.2G
+            # openmldb+cp_python_sdk_so 7.7G    ./build(has py), python just ~180M
+            # target 'install' cost more, preinstall/fast won't build all, so use install/fast if needed
+            # or https://cmake.org/cmake/help/latest/variable/CMAKE_SKIP_INSTALL_ALL_DEPENDENCY.html
+            cmake --build build --target openmldb cp_python_sdk_so -- -j2
+            du -h --max-depth=1
+            df -h
+            # if target above cost too much disk, make java build failed, try to rm build cache
+            # don't rm cache now cuz build java from emtpy will cost 20min
+            # rm build/hybridse build/src -rf
             if [[ "${SQL_JAVASDK_ENABLE}" == "ON" ]]; then
               echo "build java sdk"
               cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \
               -DSQL_PYSDK_ENABLE=OFF -DSQL_JAVASDK_ENABLE=ON \
               -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX}  \
               -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF
-              cmake --build build --target sql_javasdk_package -- -j2
+              # if build the whole java, 7.6G	./build, 5.7G  ./java, avail 331M
+              # so split it and build native only
+              # 7.6G	./build,  1.8G	./java, avail 5.2G
+              cmake --build build --target cp_native_so -- -j2
+              du -h --max-depth=1
+              df -h
+              rm build/hybridse build/src -rf
+              cd java
+              ./mvnw -pl openmldb-native clean package -DskipTests=true -Dscalatest.skip=true -Dwagon.skip=true -Dmaven.test.skip=true --batch-mode 
             fi
-
-      - name: package    
-        run: |
-          tar czf ${{ env.OPENMLDB_PREFIX }}.tar.gz ${{ env.OPENMLDB_PREFIX }}/
+            rm build/hybridse build/src -rf
+            du -h --max-depth=1
+            df -h
 
       - name: upload binary
         uses: actions/upload-artifact@v2
         with:
-          path: openmldb-*.tar.gz
-          name: binary-package
+          path: build/bin/openmldb
+          name: binary
 
       - name: upload java native
         if: ${{ env.SQL_JAVASDK_ENABLE == 'ON' }}
@@ -127,8 +146,7 @@ jobs:
         with: 
           name: python-whl
           path: |
-            python/openmldb_sdk/dist/openmldb*.whl  
-    
+            python/openmldb_sdk/dist/openmldb*.whl
       # TODO(hw): upload cxx sdk
 
   # macos no need to build thirdparty, but binary/os needs to be built on each os

diff --git a/.gitignore b/.gitignore
@@ -108,3 +108,10 @@ allure-results/
 /python/openmldb_autofe/*.egg-info/
 # go sdk
 !go.mod
+
+# tag files
+**/tags
+**/GPATH
+**/GRTAGS
+**/GTAGS
+**/cscope.out
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,40 @@
 # Changelog
 
+## [0.8.3] - 2023-09-15
+
+### Features
+- Optimize the performance of Java SDK (#3445 @dl239)
+- Optimize the writing performance and significantly reduce the memory consumption of the Spark connector (#3443 @vagetablechicken)
+- Support loading data from HIVE with customized SQLs (#3380 @tobegit3hub)
+- Improve the output message for SDK and CLI (#3384 @vagetablechicken, #3434 #3494 @dl239)
+- Support new built-in functions `json_array_length` and `get_json_object` (#3414 #3429 @aceforeverd)
+- Add new options `RANGE_BIAS` and `ROWS_BIAS` for the `DEPLOYMENT` statement (#3456 @vagetablechicken)
+- Support `const` project in online mode (#3376 @aceforeverd)
+- Support `SHOW DEPLOYMENT` and `DROP DEPLOYMENT` with a database name (#3353 @emo-coder)
+- Support inheriting environment variables for Spark (#3450 @vagetablechicken)
+- Support deleting HDFS files when dropping tables (#3369 @tobegit3hub)
+- Enhance the diagnostic tool (#3330 @zhangziheng01233)
+- Enhance the operation tool (#3455 @dl239)
+- Use the timeout value set by an user only if that is greater than the default value (#3484 @vagetablechicken)
+- Remove the sync tool from the demo docker image (#3390 @dl239)
+- Improve the documents (#3383 #3392 #3410 @vagetablechicken, #3175 #3447 ##3463 @TanZiYen, #3436 @aceforeverd, #3451 @wangerry, #3453 #3462 #3498 @dl239)
+
+### Bug Fixes
+- `CREATE TABLE LIKE HIVE` returns success even if a database is not found (#3379 @emo-coder)
+- If an error occurred when executing `DROP FUNCTION`, the function cannot be deleted again. (#3362 @vagetablechicken, #3441 @dl239)
+- The results of `SHOW JOBS` are not sorted by `id` (#3371 @emo-coder)
+- NameServer will crash if creating system tables fails. (#3432 @dl239)
+- `CREATE INDEX` may fail if the previous `CREATE INDEX` command on the same table has not finished. (#3393 @dl239)
+- The result of `SELECT` on the deleted index column is empty (#3426 @dl239)
+- Other minor bug fixes (#3391 #3408 @vagetablechicken, #3386 #3427 #3459 @dl239, #3367 #3495 @aceforeverd)
+
+### Code Refactoring
+#3397 @emo-coder, #3411 @vagetablechicken, #3435 @aceforeverd, #3473 @lqy222
+
+### Breaking Changes
+- The return type of `GetInternalSchema` in `SQLResultSet` changes from native Schema to `com._4paradigm.openmldb.sdk.Schema` #3445
+- Remove the deprecated TaskManager configuration `namenode.uri` #3369
+
 ## [0.8.2] - 2023-07-20
 
 ### Features
@@ -16,7 +51,7 @@
 - The bool type is not properly packed in APIServer. (#3366 @vagetablechicken)
 - The table can be created successfully when there are duplicated indexs. (#3306 @dl239)
 
-### Breaking Changes:
+### Breaking Changes
 - The field `Offline_deep_copy` will be replaced by `Offline_symbolic_paths` in the result of `SHOW TABLE STATUS` #3349.
 
 ## [0.8.1] - 2023-06-28
@@ -618,6 +653,7 @@ Removed
 - openmldb-0.2.0-linux.tar.gz targets on x86_64
 - aarch64 artifacts consider experimental
 
+[0.8.3]: https://github.com/4paradigm/OpenMLDB/compare/v0.8.2...v0.8.3
 [0.8.2]: https://github.com/4paradigm/OpenMLDB/compare/v0.8.1...v0.8.2
 [0.8.1]: https://github.com/4paradigm/OpenMLDB/compare/v0.8.0...v0.8.1
 [0.8.0]: https://github.com/4paradigm/OpenMLDB/compare/v0.7.3...v0.8.0

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -36,7 +36,7 @@ endif()
 message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
 set(OPENMLDB_VERSION_MAJOR 0)
 set(OPENMLDB_VERSION_MINOR 8)
-set(OPENMLDB_VERSION_BUG 2)
+set(OPENMLDB_VERSION_BUG 3)
 
 function(get_commitid CODE_DIR COMMIT_ID)
     find_package(Git REQUIRED)

diff --git a/Makefile b/Makefile
@@ -154,7 +154,7 @@ thirdparty-fast:
 	    if [ "$$new_zetasql_version" != "$(ZETASQL_VERSION)" ] ; then \
 		echo "[deps]: thirdparty up-to-date. reinstall zetasql from $(ZETASQL_VERSION) to $$new_zetasql_version"; \
 		$(MAKE) thirdparty-configure; \
-		$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) -j $(NPROC) --target zetasql; \
+		$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
 	    else \
 		echo "[deps]: all up-to-date. zetasql already installed with version: $(ZETASQL_VERSION)"; \
 	    fi; \
@@ -166,7 +166,7 @@ thirdparty-fast:
 
 # third party compiled code install to 'OpenMLDB/.deps/usr', source code install to 'OpenMLDB/thirdsrc'
 thirdparty: thirdparty-configure
-	$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) -j $(NPROC)
+	$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR)
 
 thirdparty-configure:
 	$(CMAKE_PRG) -S third-party -B $(THIRD_PARTY_BUILD_DIR) -DSRC_INSTALL_DIR=$(THIRD_PARTY_SRC_DIR) -DDEPS_INSTALL_DIR=$(THIRD_PARTY_DIR) $(THIRD_PARTY_CMAKE_FLAGS)

diff --git a/cases/plan/cmd.yaml b/cases/plan/cmd.yaml
@@ -704,3 +704,20 @@ cases:
           +-actions:
             +-0: DropPathAction (12)
             +-1: AddPathAction (13)
+
+  - id: show-create-table
+    desc: SHOW CREATE TABLE
+    sql: SHOW CREATE TABLE t1;
+    expect:
+      node_tree_str: |
+        +-node[CMD]
+          +-cmd_type: show create table
+          +-args: [t1]
+  - id: show-create-table-db
+    desc: SHOW CREATE TABLE
+    sql: SHOW CREATE TABLE db1.t1;
+    expect:
+      node_tree_str: |
+        +-node[CMD]
+          +-cmd_type: show create table
+          +-args: [db1, t1]