fix: stream memory counting durin snapshot loading #9868
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci-tests | |
| on: | |
| # push: | |
| # branches: [ main ] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-commit: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - uses: actions/setup-python@v5 | |
| - name: Install dependencies | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| python -m pip install pre-commit | |
| lsblk -l | |
| echo "sda rotational = $(cat /sys/block/sda/queue/rotational)" | |
| echo "sdb rotational = $(cat /sys/block/sdb/queue/rotational)" | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Run pre-commit checks | |
| run: | |
| source venv/bin/activate | |
| pre-commit run --show-diff-on-failure --color=always --from-ref HEAD^ --to-ref HEAD | |
| shell: bash | |
| build: | |
| strategy: | |
| matrix: | |
| # Test of these containers | |
| container: ["ubuntu-dev:20", "alpine-dev:latest"] | |
| build-type: [Debug, Release] | |
| compiler: [{ cxx: g++, c: gcc }] | |
| cxx_flags: ["-Werror"] | |
| include: | |
| - container: "alpine-dev:latest" | |
| build-type: Debug | |
| compiler: { cxx: clang++, c: clang } | |
| cxx_flags: "" | |
| runs-on: ubuntu-latest | |
| env: | |
| SCCACHE_GHA_ENABLED: "true" | |
| SCCACHE_CACHE_SIZE: 6G | |
| SCCACHE_ERROR_LOG: /tmp/sccache_log.txt | |
| # SCCACHE_LOG: debug | |
| container: | |
| image: ghcr.io/romange/${{ matrix.container }} | |
| # Seems that docker by default prohibits running iouring syscalls | |
| options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0" | |
| volumes: | |
| - /:/hostroot | |
| - /mnt:/mnt | |
| credentials: | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Prepare Environment | |
| run: | | |
| uname -a | |
| cmake --version | |
| mkdir -p ${GITHUB_WORKSPACE}/build | |
| mount | |
| echo "===================Before freeing up space ============================================" | |
| df -h | |
| rm -rf /hostroot/usr/share/dotnet | |
| rm -rf /hostroot/usr/local/share/boost | |
| rm -rf /hostroot/usr/local/lib/android | |
| rm -rf /hostroot/opt/ghc | |
| echo "===================After freeing up space ============================================" | |
| df -h | |
| touch /mnt/foo | |
| ls -la /mnt/foo | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - name: Configure Cache Env | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); | |
| core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '') | |
| - name: Configure CMake | |
| # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. | |
| # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type | |
| run: | | |
| # -no-pie to disable address randomization so we could symbolize stacktraces | |
| cmake -B ${GITHUB_WORKSPACE}/build \ | |
| -DCMAKE_BUILD_TYPE=${{matrix.build-type}} \ | |
| -GNinja \ | |
| -DCMAKE_C_COMPILER="${{matrix.compiler.c}}" \ | |
| -DCMAKE_CXX_COMPILER="${{matrix.compiler.cxx}}" \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_C_COMPILER_LAUNCHER=sccache \ | |
| -DCMAKE_CXX_FLAGS="${{matrix.cxx_flags}} -no-pie" -DWITH_AWS:BOOL=OFF \ | |
| -L | |
| cd ${GITHUB_WORKSPACE}/build && pwd | |
| du -hcs _deps/ | |
| - name: Build | |
| run: | | |
| cd ${GITHUB_WORKSPACE}/build | |
| ninja search_family_test | |
| df -h | |
| echo "-----------------------------" | |
| ninja src/all | |
| - name: PostFail | |
| if: failure() | |
| run: | | |
| echo "disk space is:" | |
| df -h | |
| - name: C++ Unit Tests | |
| run: | | |
| cd ${GITHUB_WORKSPACE}/build | |
| echo Run ctest -V -L DFLY | |
| GLOG_alsologtostderr=1 GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1 \ | |
| FLAGS_list_experimental_v2=true timeout 20m ctest -V -L DFLY | |
| echo "Running tests with --force_epoll" | |
| # Create a rule that automatically prints stacktrace upon segfault | |
| cat > ./init.gdb <<EOF | |
| catch signal SIGSEGV | |
| command | |
| bt | |
| end | |
| EOF | |
| gdb -ix ./init.gdb --batch -ex r --args ./dragonfly_test --force_epoll | |
| FLAGS_force_epoll=true GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1 timeout 20m ctest -V -L DFLY | |
| echo "Finished running tests with --force_epoll" | |
| echo "Running tests with --cluster_mode=emulated" | |
| FLAGS_cluster_mode=emulated timeout 20m ctest -V -L DFLY | |
| echo "Running tests with both --cluster_mode=emulated & --lock_on_hashtags" | |
| FLAGS_cluster_mode=emulated FLAGS_lock_on_hashtags=true timeout 20m ctest -V -L DFLY | |
| timeout 5m ./dragonfly_test | |
| timeout 5m ./multi_test --multi_exec_mode=1 | |
| timeout 5m ./multi_test --multi_exec_mode=3 | |
| timeout 5m ./json_family_test --jsonpathv2=false | |
| timeout 5m ./tiered_storage_test --vmodule=db_slice=2 --logtostderr | |
| - name: Upload unit logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: unit_logs | |
| path: /tmp/*INFO* | |
| - name: Run regression tests | |
| if: matrix.container == 'ubuntu-dev:20' | |
| uses: ./.github/actions/regression-tests | |
| with: | |
| dfly-executable: dragonfly | |
| run-only-on-ubuntu-latest: true | |
| build-folder-name: build | |
| filter: ${{ matrix.build-type == 'Release' && '(not slow) and (not dbg_only)' || '(not slow) and (not opt_only)' }} | |
| - name: Upload regression logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: regression_logs | |
| path: /tmp/failed/* | |
| lint-test-chart: | |
| runs-on: ubuntu-latest | |
| needs: [build] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/lint-test-chart |