pysdk #115
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "pysdk" | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - release** | |
| paths: | |
| - '**/hypo/fs_op.py' | |
| - '**/hypo/fs.py' | |
| - '**/hypo/fs_sdk_test.py' | |
| - '**/test_pysdk.py' | |
| - '**/juicefs.py' | |
| - '**/pysdk.yml' | |
| pull_request: | |
| branches: | |
| - main | |
| - release** | |
| paths: | |
| - '**/hypo/fs_op.py' | |
| - '**/hypo/fs.py' | |
| - '**/hypo/fs_sdk_test.py' | |
| - '**/test_pysdk.py' | |
| - '**/juicefs.py' | |
| - '**/pysdk.yml' | |
| schedule: | |
| - cron: '0 19 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| debug: | |
| type: boolean | |
| description: "Run the build with tmate debugging enabled" | |
| required: false | |
| default: false | |
| jobs: | |
| build-matrix: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - id: set-matrix | |
| run: | | |
| echo "github.event_name is ${{github.event_name}}" | |
| if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]]; then | |
| echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT | |
| elif [[ "${{github.event_name}}" == "pull_request" || "${{github.event_name}}" == "push" ]]; then | |
| echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT | |
| else | |
| echo 'event name is not supported' && exit 1 | |
| fi | |
| outputs: | |
| meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }} | |
| pysdk: | |
| needs: build-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # [ 'sqlite3', 'redis', 'mysql', 'tikv', 'tidb', 'postgres', 'badger', 'mariadb', 'fdb'] | |
| meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }} | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 1 | |
| - name: mount coverage dir | |
| timeout-minutes: 5 | |
| uses: ./.github/actions/mount-coverage-dir | |
| with: | |
| mount_point: cover | |
| access_key: ${{ secrets.CI_COVERAGE_AWS_AK }} | |
| secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }} | |
| token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }} | |
| - name: Set Variable | |
| id: vars | |
| run: | | |
| if [ "${{matrix.meta}}" == "fdb" ]; then | |
| echo "target=juicefs.fdb" >> $GITHUB_OUTPUT | |
| else | |
| echo "target=juicefs" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Build | |
| uses: ./.github/actions/build | |
| with: | |
| target: ${{steps.vars.outputs.target}} | |
| - name: Prepare meta db | |
| run: | | |
| chmod +x .github/scripts/start_meta_engine.sh | |
| source .github/scripts/start_meta_engine.sh | |
| start_meta_engine ${{matrix.meta}} | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| create_database $meta_url | |
| - name: Juicefs Format | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| sudo ./juicefs format $meta_url --trash-days 0 --enable-acl pics | |
| - name: Juicefs Mount | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| sudo GOCOVERDIR=$(pwd)/cover ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr | |
| stat /tmp/jfs/.accesslog | |
| - name: Build and install SDK | |
| timeout-minutes: 5 | |
| run: | | |
| make -C sdk/python/ libjfs.so | |
| sudo python3 sdk/python/juicefs/setup.py install | |
| - name: Run juicefs.py | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| sudo JFS_VOLUME=test-volume JFS_META=$meta_url python3 sdk/python/juicefs/juicefs/juicefs.py | |
| - name: Run pysdk_test.py | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| sudo META_URL=$meta_url python3 .github/scripts/pysdk/pysdk_test.py | |
| - name: Download example database | |
| timeout-minutes: 5 | |
| continue-on-error: true | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: hypothesis-example-db-${{ matrix.meta }} | |
| path: .hypothesis/examples | |
| - name: Run file_test.py | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| sudo USE_SDK=true META_URL=$meta_url python3 .github/scripts/hypo/file_test.py | |
| - name: Run file.py without read | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| sudo USE_SDK=true META_URL=$meta_url EXCLUDE_RULES="read,readline,readlines" python3 .github/scripts/hypo/file.py | |
| - name: Run file.py without write | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| sudo USE_SDK=true META_URL=$meta_url EXCLUDE_RULES="write,writelines,truncate" python3 .github/scripts/hypo/file.py | |
| - name: Run fs_sdk_test.py | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| sudo USE_SDK=true META_URL=$meta_url python3 .github/scripts/hypo/fs_sdk_test.py | |
| - name: Run fs.py | |
| if: false | |
| run: | | |
| source .github/scripts/start_meta_engine.sh | |
| meta_url=$(get_meta_url ${{matrix.meta}}) | |
| sudo USE_SDK=true META_URL=$meta_url python3 .github/scripts/hypo/fs.py | |
| - name: Upload example database | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| include-hidden-files: true | |
| name: hypothesis-example-db-${{ matrix.meta }} | |
| path: .hypothesis/examples | |
| - name: upload coverage report | |
| timeout-minutes: 5 | |
| continue-on-error: true | |
| uses: ./.github/actions/upload-coverage | |
| with: | |
| UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }} | |
| - name: Send Slack Notification | |
| if: failure() | |
| uses: juicedata/slack-notify-action@main | |
| with: | |
| channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" | |
| slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" | |
| - name: Setup upterm session | |
| if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) | |
| timeout-minutes: 60 | |
| uses: lhotari/action-upterm@v1 | |
| success-all-test: | |
| runs-on: ubuntu-latest | |
| needs: [pysdk] | |
| if: always() | |
| steps: | |
| - uses: technote-space/workflow-conclusion-action@v3 | |
| - uses: actions/checkout@v3 | |
| - name: Check Failure | |
| if: env.WORKFLOW_CONCLUSION == 'failure' | |
| run: exit 1 | |
| - name: Send Slack Notification | |
| if: failure() && github.event_name != 'workflow_dispatch' | |
| uses: juicedata/slack-notify-action@main | |
| with: | |
| channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" | |
| slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" | |
| - name: Success | |
| if: success() | |
| run: echo "All Done" |