Merge remote-tracking branch 'origin/main' into feat/longwindow_delete

.github/workflows/openmldb-tool.yml

@@ -8,10 +8,12 @@ on:
     paths:
       - 'release/**'
       - 'tools/**'
+      - 'test/test-tool/openmldb-deploy/**'
   pull_request:
     paths:
       - 'release/**'
       - 'tools/**'
+      - 'test/test-tool/openmldb-deploy/**'
   workflow_dispatch:
 
 env:
@@ -27,19 +29,53 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: prepare env
+        run: |
+          bash test/test-tool/openmldb-deploy/gen_conf.sh ${{ env.DEPLOY_DIR }} ${{ env.NODE_LIST }} > hosts
+          cp -f hosts test/test-tool/openmldb-deploy/
+          pip3 install requests openmldb pytest
+      - name: install openmldb
         run: |
           VERSION=`git fetch --tags | git tag -l v[0-9].* | tail -n1`
           VERSION=${VERSION#v}
-          echo "OPENMLDB_VETSION=$VERSION" >> $GITHUB_ENV
-          bash test/test-tool/openmldb-deploy/gen_conf.sh ${{ env.DEPLOY_DIR }} ${{ env.NODE_LIST }} > hosts
+          bash test/test-tool/openmldb-deploy/install.sh ${VERSION}
+      - name: run test
+        run: |
+          cp -f tools/tool.py test/test-tool/openmldb-deploy/cases/
+          cp -f tools/* openmldb/tools/
+          python3 -m pytest test/test-tool/openmldb-deploy/cases --junit-xml=pytest.xml
+      - name: clear env
+        run: |
+            bash openmldb/sbin/stop-all.sh && bash openmldb/sbin/clear-all.sh
+      - name: upload python test results
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: openmldb-tool-test-result-${{ github.sha }}
+          path: |
+            pytest.xml
+
+  openmldb-tool-name:
+    runs-on: [self-hosted,generic]
+    if: github.repository == '4paradigm/OpenMLDB'
+    container:
+      image: ghcr.io/4paradigm/hybridsql:latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: prepare env
+        run: |
           pip3 install requests openmldb pytest
+          yum install -y rsync
       - name: install openmldb
-        env:
-          OPENMLDB_VETSION: ${{ env.OPENMLDB_VETSION }}
         run: |
-          bash test/test-tool/openmldb-deploy/install.sh ${{ env.OPENMLDB_VETSION }}
+          git fetch --tags --force
+          VERSION=$(git describe --always --tags `git rev-list --tags --max-count=1`)
+          VERSION=${VERSION#v}
+          bash test/test-tool/openmldb-deploy/install_with_name.sh ${VERSION}
       - name: run test
         run: |
+          cp -f tools/tool.py test/test-tool/openmldb-deploy/cases/
+          cp -f tools/* openmldb/tools/
           python3 -m pytest test/test-tool/openmldb-deploy/cases --junit-xml=pytest.xml
       - name: clear env
         run: |

.github/workflows/other-os-build.yml

@@ -78,41 +78,60 @@ jobs:
           shell: bash
           run: |
             cd /root/OpenMLDB
+            # centos6_build.sh will try build zetasql even cache hit, just ignore the failure
             IN_WORKFLOW=true bash steps/centos6_build.sh
             # bazel bin
             export PATH=$PATH:`pwd`
             source /opt/rh/devtoolset-8/enable
             if [[ "${USE_DEPS_CACHE}" != "true" ]]; then
-              echo "build thirdparty"
-              make thirdparty CMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} BUILD_BUNDLE=ON NPROC=8
+              echo "build thirdparty, make opt is better than nproc?"
+              make thirdparty CMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX} BUILD_BUNDLE=ON THIRD_PARTY_CMAKE_FLAGS=-DMAKEOPTS=-j8
+              # 5.8G	./.deps, avail 8G
               rm -rf .deps/build # GitHub runner disk space is limited
             fi
             echo "build"
+            # 1.4G	./.deps, avail 13G
+
+            # will failed if openmldb_sdk is on
             cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \
             -DSQL_PYSDK_ENABLE=${SQL_PYSDK_ENABLE} -DSQL_JAVASDK_ENABLE=OFF \
             -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX}  \
             -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF
-            cmake --build build --target install -- -j2
-            # clean up to save disk space(~11G), don't know which is relative, build again in next step
-            rm -rf build
+            # target openmldb 6.7G    ./build(no py/java), avail 5.2G
+            # openmldb+cp_python_sdk_so 7.7G    ./build(has py), python just ~180M
+            # target 'install' cost more, preinstall/fast won't build all, so use install/fast if needed
+            # or https://cmake.org/cmake/help/latest/variable/CMAKE_SKIP_INSTALL_ALL_DEPENDENCY.html
+            cmake --build build --target openmldb cp_python_sdk_so -- -j2
+            du -h --max-depth=1
+            df -h
+            # if target above cost too much disk, make java build failed, try to rm build cache
+            # don't rm cache now cuz build java from emtpy will cost 20min
+            # rm build/hybridse build/src -rf
             if [[ "${SQL_JAVASDK_ENABLE}" == "ON" ]]; then
               echo "build java sdk"
               cmake -S . -B `pwd`/build -DCMAKE_PREFIX_PATH=`pwd`/.deps/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \
               -DSQL_PYSDK_ENABLE=OFF -DSQL_JAVASDK_ENABLE=ON \
               -DTESTING_ENABLE=OFF -DCMAKE_INSTALL_PREFIX=${OPENMLDB_PREFIX}  \
               -DHYBRIDSE_TESTING_ENABLE=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_TESTING_ENABLE=OFF
-              cmake --build build --target sql_javasdk_package -- -j2
+              # if build the whole java, 7.6G	./build, 5.7G  ./java, avail 331M
+              # so split it and build native only
+              # 7.6G	./build,  1.8G	./java, avail 5.2G
+              cmake --build build --target cp_native_so -- -j2
+              du -h --max-depth=1
+              df -h
+              rm build/hybridse build/src -rf
+              cd java
+              ./mvnw -pl openmldb-native clean package -DskipTests=true -Dscalatest.skip=true -Dwagon.skip=true -Dmaven.test.skip=true --batch-mode 
             fi
-
-      - name: package    
-        run: |
-          tar czf ${{ env.OPENMLDB_PREFIX }}.tar.gz ${{ env.OPENMLDB_PREFIX }}/
+            rm build/hybridse build/src -rf
+            du -h --max-depth=1
+            df -h
 
       - name: upload binary
         uses: actions/upload-artifact@v2
         with:
-          path: openmldb-*.tar.gz
-          name: binary-package
+          path: build/bin/openmldb
+          name: binary
 
       - name: upload java native
         if: ${{ env.SQL_JAVASDK_ENABLE == 'ON' }}
@@ -127,8 +146,7 @@ jobs:
         with: 
           name: python-whl
           path: |
-            python/openmldb_sdk/dist/openmldb*.whl  
-    
+            python/openmldb_sdk/dist/openmldb*.whl
       # TODO(hw): upload cxx sdk
 
   # macos no need to build thirdparty, but binary/os needs to be built on each os

.github/workflows/sdk.yml

@@ -68,8 +68,6 @@ jobs:
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('java/**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
 
       - name: prepare release
         if: github.event_name == 'push'
@@ -124,6 +122,7 @@ jobs:
       - name: maven coverage
         working-directory: java
         run: |
+          rm -rfv ~/.m2/repository/com/4paradigm/
           ./mvnw --batch-mode prepare-package
           ./mvnw --batch-mode scoverage:report
 
@@ -160,8 +159,6 @@ jobs:
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('java/**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
 
       - name: Cache thirdparty
         uses: actions/cache@v3
@@ -236,6 +233,10 @@ jobs:
           MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
           MAVEN_TOKEN: ${{ secrets.OSSRH_TOKEN }}
           GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+      - name: cleanup
+        run: |
+          rm -rfv ~/.m2/repository/com/4paradigm/
+
 
   python-sdk:
     runs-on: ubuntu-latest