build: thirdparty parallel compile option & fix centos6 build (#3492)

* build: fix centos6 build * fix * fix * fix
4paradigm · Oct 11, 2023 · 2224a6c · 2224a6c
1 parent bcc0b1f
commit 2224a6c
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 25 deletions.
diff --git a/.github/workflows/other-os-build.yml b/.github/workflows/other-os-build.yml
@@ -78,41 +78,60 @@ jobs:
           shell: bash
           run: |
             cd /root/OpenMLDB
+            # centos6_build.sh will try build zetasql even cache hit, just ignore the failure
             IN_WORKFLOW=true bash steps/centos6_build.sh
             # bazel bin
             export PATH=$PATH:`pwd`
             source /opt/rh/devtoolset-8/enable
             if [[ "${USE_DEPS_CACHE}" != "true" ]]; then
-              echo "build thirdparty"
-              make thirdparty CMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} BUILD_BUNDLE=ON NPROC=8
+              echo "build thirdparty, make opt is better than nproc?"
+              make thirdparty CMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} BUILD_BUNDLE=ON THIRD_PARTY_CMAKE_FLAGS=-DMAKEOPTS=-j8
+              # 5.8G	./.deps, avail 8G
               rm -rf .deps/build # GitHub runner disk space is limited
             fi
             echo "build"
+            # 1.4G	./.deps, avail 13G
+
+            # will failed if openmldb_sdk is on
             cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \
             -DSQL_PYSDK_ENABLE=${SQL_PYSDK_ENABLE} -DSQL_JAVASDK_ENABLE=OFF \
             -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX}  \
             -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF
-            cmake --build build --target install -- -j2
-            # clean up to save disk space(~11G), don't know which is relative, build again in next step
-            rm -rf build
+            # target openmldb 6.7G    ./build(no py/java), avail 5.2G
+            # openmldb+cp_python_sdk_so 7.7G    ./build(has py), python just ~180M
+            # target 'install' cost more, preinstall/fast won't build all, so use install/fast if needed
+            # or https://cmake.org/cmake/help/latest/variable/CMAKE_SKIP_INSTALL_ALL_DEPENDENCY.html
+            cmake --build build --target openmldb cp_python_sdk_so -- -j2
+            du -h --max-depth=1
+            df -h
+            # if target above cost too much disk, make java build failed, try to rm build cache
+            # don't rm cache now cuz build java from emtpy will cost 20min
+            # rm build/hybridse build/src -rf
             if [[ "${SQL_JAVASDK_ENABLE}" == "ON" ]]; then
               echo "build java sdk"
               cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \
               -DSQL_PYSDK_ENABLE=OFF -DSQL_JAVASDK_ENABLE=ON \
               -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX}  \
               -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF
-              cmake --build build --target sql_javasdk_package -- -j2
+              # if build the whole java, 7.6G	./build, 5.7G  ./java, avail 331M
+              # so split it and build native only
+              # 7.6G	./build,  1.8G	./java, avail 5.2G
+              cmake --build build --target cp_native_so -- -j2
+              du -h --max-depth=1
+              df -h
+              rm build/hybridse build/src -rf
+              cd java
+              ./mvnw -pl openmldb-native clean package -DskipTests=true -Dscalatest.skip=true -Dwagon.skip=true -Dmaven.test.skip=true --batch-mode 
             fi
-
-      - name: package    
-        run: |
-          tar czf ${{ env.OPENMLDB_PREFIX }}.tar.gz ${{ env.OPENMLDB_PREFIX }}/
+            rm build/hybridse build/src -rf
+            du -h --max-depth=1
+            df -h
 
       - name: upload binary
         uses: actions/upload-artifact@v2
         with:
-          path: openmldb-*.tar.gz
-          name: binary-package
+          path: build/bin/openmldb
+          name: binary
 
       - name: upload java native
         if: ${{ env.SQL_JAVASDK_ENABLE == 'ON' }}
@@ -127,8 +146,7 @@ jobs:
         with: 
           name: python-whl
           path: |
-            python/openmldb_sdk/dist/openmldb*.whl  
-    
+            python/openmldb_sdk/dist/openmldb*.whl
       # TODO(hw): upload cxx sdk
 
   # macos no need to build thirdparty, but binary/os needs to be built on each os

diff --git a/Makefile b/Makefile
@@ -154,7 +154,7 @@ thirdparty-fast:
 	    if [ "$$new_zetasql_version" != "$(ZETASQL_VERSION)" ] ; then \
 		echo "[deps]: thirdparty up-to-date. reinstall zetasql from $(ZETASQL_VERSION) to $$new_zetasql_version"; \
 		$(MAKE) thirdparty-configure; \
-		$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) -j $(NPROC) --target zetasql; \
+		$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
 	    else \
 		echo "[deps]: all up-to-date. zetasql already installed with version: $(ZETASQL_VERSION)"; \
 	    fi; \
@@ -166,7 +166,7 @@ thirdparty-fast:
 
 # third party compiled code install to 'OpenMLDB/.deps/usr', source code install to 'OpenMLDB/thirdsrc'
 thirdparty: thirdparty-configure
-	$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) -j $(NPROC)
+	$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR)
 
 thirdparty-configure:
 	$(CMAKE_PRG) -S third-party -B $(THIRD_PARTY_BUILD_DIR) -DSRC_INSTALL_DIR=$(THIRD_PARTY_SRC_DIR) -DDEPS_INSTALL_DIR=$(THIRD_PARTY_DIR) $(THIRD_PARTY_CMAKE_FLAGS)

diff --git a/docs/zh/deploy/compile.md b/docs/zh/deploy/compile.md
@@ -110,7 +110,7 @@ make CMAKE_BUILD_TYPE=Debug
 
 - CMAKE_EXTRA_FLAGS: 传递给 cmake 的额外参数
 
-  默认: ‘’
+  默认: ''
 
 - BUILD_BUNDLED: 从源码编译 thirdparty 依赖，而不是下载预编译包
 
@@ -124,6 +124,9 @@ make CMAKE_BUILD_TYPE=Debug
 
   默认: all
 
+- THIRD_PARTY_CMAKE_FLAGS: 编译thirdparty时可以配置额外参数。例如，配置每个thirdparty项目并发编译，`THIRD_PARTY_CMAKE_FLAGS=-DMAKEOPTS=-j8`。thirdparty不受NPROC影响，thirdparty的多项目将会串行执行。
+  默认：''
+
 ### 并发编译Java SDK
 
 ```
@@ -185,14 +188,25 @@ docker run -it -v`pwd`:/root/OpenMLDB ghcr.io/4paradigm/centos6_gcc7_hybridsql b
 ```bash
 cd OpenMLDB
 bash steps/centos6_build.sh
+# THIRD_PARTY_CMAKE_FLAGS=-DMAKEOPTS=-j8 bash steps/centos6_build.sh # run fast when build single project
 # OPENMLDB_SOURCE=true bash steps/centos6_build.sh
-# SQL_JAVASDK_ENABLE=ON SQL_PYSDK_ENABLE=ON NRPOC=8 bash steps/centos6_build.sh
+# SQL_JAVASDK_ENABLE=ON SQL_PYSDK_ENABLE=ON NPROC=8 bash steps/centos6_build.sh # NPROC will build openmldb in parallel, thirdparty should use THIRD_PARTY_CMAKE_FLAGS
 ```
 
+本地2.20GHz CPU，SSD硬盘，32线程编译三方库与OpenMLDB主体，耗时参考：
+`THIRD_PARTY_CMAKE_FLAGS=-DMAKEOPTS=-j32 SQL_JAVASDK_ENABLE=ON SQL_PYSDK_ENABLE=ON NPROC=32 bash steps/centos6_build.sh`
+- thirdparty（不包括下载src时间）~40m：zetasql打patch 13m，所有thirdparty编译30m
+- OpenMLDB 本体，包括python和java native，~12min
+
 #### 云编译
 
-Fork OpenMLDB仓库后，可以使用在`Actions`中触发workflow `Other OS Build`，编译产出在`Actions`的`Artifacts`中。workflow 配置 `os name`为`centos6`，
-如果不需要Java或Python SDK，可配置`java sdk enable`或`python sdk enable`为`OFF`，节约编译时间。
+Fork OpenMLDB仓库后，可以使用在`Actions`中触发workflow `Other OS Build`，编译产出在`Actions`的`Artifacts`中。workflow 配置方式：
+- 不要更换`Use workflow from`为某个tag，可以是其他分支。
+- 选择`os name`为`centos6`。
+- 如果不是编译main分支，在`The branch, tag or SHA to checkout, otherwise use the branch`中填写想要的分支名、Tag(e.g. v0.8.2)或SHA。
+- 编译产出在触发后的runs界面中，参考[成功产出的runs链接](https://github.com/4paradigm/OpenMLDB/actions/runs/6044951902)。
+  - 一定会产出openmldb binary文件。
+  - 如果不需要Java或Python SDK，可配置`java sdk enable`或`python sdk enable`为`OFF`，节约编译时间。
 
 此编译流程需要从源码编译thirdparty，且资源较少，无法开启较高的并发编译。因此编译时间较长，大约需要3h5m（2h thirdparty+1h OpenMLDB）。workflow会缓存thirdparty的编译产出，因此第二次编译会快很多（1h15m OpenMLDB）。
 

diff --git a/steps/centos6_build.sh b/steps/centos6_build.sh
@@ -19,6 +19,7 @@ set +x
 
 # self build or workflow
 IN_WORKFLOW=${IN_WORKFLOW:-"false"}
+USE_DEPS_CACHE=${USE_DEPS_CACHE:-"false"}
 # if download from openmldb.ai
 OPENMLDB_SOURCE=${OPENMLDB_SOURCE:-"false"}
 
@@ -36,6 +37,13 @@ function tool_install() {
     chmod +x bazel
 }
 
+if [ "$USE_DEPS_CACHE" == "true" ]; then
+    echo "use deps cache, exit"
+    exit 0
+else
+    echo "not use deps cache, install tools and build deps"
+fi
+
 if [ "$IN_WORKFLOW" == "true" ]; then
     echo "in workflow"
 else
@@ -56,13 +64,12 @@ echo "add patch in fetch cmake"
 sed -i'' '34s/WITH_TOOLS=OFF$/WITH_TOOLS=OFF -DWITH_CORE_TOOLS=OFF/' third-party/cmake/FetchRocksDB.cmake
 
 # if BUILD_BUNDLED=OFF will download pre-built thirdparty, not good
-# so we use cmake to build zetasql only
-# it's hard to avoid add zetasql patch twice, so we check the dir to avoid
-if [ -d ".deps/build/src/zetasql" ]; then
-    echo "zetasql already exists, skip add patch, if you want, rm .deps/build/src/zetasql* -rf"
+# so we use cmake to build zetasql only and add patch to it
+# it's hard to avoid add zetasql patch twice, so we check the stamp to avoid
+if [ -e ".deps/build/src/zetasql-stamp/zetasql-build" ]; then
+    echo "zetasql already exists, skip add patch, if you want, rm .deps/build/src/zetasql-stamp/zetasql-build or whole .deps/build/src/zetasql*"
 else
     echo  "modify in .deps needs a make first, download&build zetasql first(build will fail)"
-    # sed -i'' '31s/${BUILD_BUNDLED}/ON/' third-party/CMakeLists.txt
     cmake -S third-party -B "$(pwd)"/.deps -DSRC_INSTALL_DIR="$(pwd)"/thirdsrc -DDEPS_INSTALL_DIR="$(pwd)"/.deps/usr -DBUILD_BUNDLED=ON
     cmake --build "$(pwd)"/.deps --target zetasql
     echo "add patch in .deps zetasql"