add raw_lowercase tokenizer (#5216) #15283
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| - trigger-ci-workflow | |
| paths: | |
| - "quickwit/**" | |
| - "!quickwit/quickwit-ui/**" | |
| env: | |
| CARGO_INCREMENTAL: 0 | |
| QW_DISABLE_TELEMETRY: 1 | |
| QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev | |
| RUST_BACKTRACE: 1 | |
| RUSTDOCFLAGS: -Dwarnings -Arustdoc::private_intra_doc_links | |
| RUSTFLAGS: -Dwarnings --cfg tokio_unstable | |
| # Ensures that we cancel running jobs for the same PR / same workflow. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| tests: | |
| name: Unit tests | |
| runs-on: "ubuntu-latest" | |
| timeout-minutes: 40 | |
| services: | |
| # PostgreSQL service container | |
| postgres: | |
| image: postgres:latest | |
| ports: | |
| - 5432:5432 | |
| env: | |
| POSTGRES_USER: quickwit-dev | |
| POSTGRES_PASSWORD: quickwit-dev | |
| POSTGRES_DB: quickwit-metastore-dev | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Ubuntu packages | |
| run: sudo apt-get -y install protobuf-compiler python3 python3-pip | |
| - uses: dorny/paths-filter@v3 | |
| id: modified | |
| with: | |
| filters: | | |
| rust_src: | |
| - quickwit/**/*.rs | |
| - quickwit/**/*.toml | |
| - quickwit/**/*.proto | |
| - quickwit/rest-api-tests/** | |
| - .github/workflows/ci.yml | |
| # The following step is just meant to install rustup actually. | |
| # The next one installs the correct toolchain. | |
| - name: Install rustup | |
| if: steps.modified.outputs.rust_src == 'true' | |
| run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y | |
| - name: Setup stable Rust Toolchain | |
| if: steps.modified.outputs.rust_src == 'true' | |
| run: rustup show | |
| working-directory: ./quickwit | |
| - name: Setup cache | |
| uses: Swatinem/rust-cache@v2 | |
| if: steps.modified.outputs.rust_src == 'true' | |
| with: | |
| workspaces: "./quickwit -> target" | |
| - name: Install nextest | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| uses: taiki-e/cache-cargo-install-action@v2 | |
| with: | |
| tool: cargo-nextest | |
| - name: cargo nextest | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| run: cargo nextest run --features=postgres --retries 1 | |
| working-directory: ./quickwit | |
| - name: cargo build | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| run: cargo build --features=postgres --bin quickwit | |
| working-directory: ./quickwit | |
| - name: Install python packages | |
| run: sudo pip3 install pyaml requests | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| - name: run REST API tests | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| run: python3 ./run_tests.py --binary ../target/debug/quickwit | |
| working-directory: ./quickwit/rest-api-tests | |
| lints: | |
| name: Lints | |
| runs-on: "ubuntu-latest" | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: modified | |
| with: | |
| filters: | | |
| rust_src: | |
| - quickwit/**/*.rs | |
| - quickwit/**/*.toml | |
| - quickwit/**/*.proto | |
| - .github/workflows/ci.yml | |
| - name: Install Ubuntu packages | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| run: sudo apt-get -y install protobuf-compiler python3 python3-pip | |
| - name: Install rustup | |
| if: steps.modified.outputs.rust_src == 'true' | |
| run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y | |
| - name: Setup nightly Rust Toolchain (for rustfmt) | |
| if: steps.modified.outputs.rust_src == 'true' | |
| run: rustup toolchain install nightly | |
| - name: Setup stable Rust Toolchain | |
| if: steps.modified.outputs.rust_src == 'true' | |
| run: rustup show | |
| working-directory: ./quickwit | |
| - name: Setup cache | |
| if: steps.modified.outputs.rust_src == 'true' | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: "./quickwit -> target" | |
| - name: Install cargo deny | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| uses: taiki-e/cache-cargo-install-action@v2 | |
| with: | |
| tool: cargo-deny | |
| - name: cargo clippy | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| run: cargo clippy --workspace --tests --all-features | |
| working-directory: ./quickwit | |
| - name: cargo deny | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| run: cargo deny check licenses | |
| working-directory: ./quickwit | |
| - name: cargo doc | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| run: cargo doc | |
| working-directory: ./quickwit | |
| - name: License headers check | |
| if: always() | |
| run: bash scripts/check_license_headers.sh | |
| working-directory: ./quickwit | |
| - name: rustfmt | |
| if: always() && steps.modified.outputs.rust_src == 'true' | |
| run: cargo +nightly fmt --all -- --check | |
| working-directory: ./quickwit |