diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 48f3133d..f41c02ac 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -97,6 +97,20 @@ jobs: first: ${{ needs.real-version-in-tag.outputs.real_version }} second: "1.26.0" operator: ">=" + newer-or-equal-than-1_28: + name: "Check if the version is newer than 1.28" + needs: real-version-in-tag + runs-on: ubuntu-latest + outputs: + check: ${{ steps.semver_compare.outputs.result }} + steps: + - name: Semver Compare + id: semver_compare + uses: fabriziocacicia/semver-compare-action@v0.1.0 + with: + first: ${{ needs.real-version-in-tag.outputs.real_version }} + second: "1.28.0" + operator: ">=" filter-memory-leak: name: Filter (cache) memory leak when querying while importing if: ${{ github.event.inputs.test_to_run == 'filter-memory-leak' || github.event.inputs.test_to_run == '' }} @@ -258,6 +272,44 @@ jobs: path: 'results' destination: 'ann-pipelines/github-action-runs' glob: '*.json' + + ann-benchmarks-lasq-sift-aws: + needs: [newer-or-equal-than-1_28] + if: ${{ (needs.newer-or-equal-than-1_28.outputs.check == 'true') && (github.event.inputs.test_to_run == 'ann-benchmarks-lasq-sift-aws' || github.event.inputs.test_to_run == '')}} + name: "[bench AWS] SIFT1M lasq=true" + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY}} + AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} + DATASET: sift-128-euclidean + DISTANCE: l2-squared + REQUIRED_RECALL: 0.992 + QUANTIZATION: lasq + PERSISTENCE_LSM_ACCESS_STRATEGY: ${{inputs.lsm_access_strategy}} + steps: + - uses: actions/checkout@v3 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{secrets.DOCKER_USERNAME}} + password: ${{secrets.DOCKER_PASSWORD}} + - id: 'gcs_auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: ${{secrets.GCP_SERVICE_ACCOUNT_BENCHMARKS}} + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + - name: Run chaos test + if: always() + run: ./ann_benchmark_quantization_aws.sh + - id: 'upload-files' + uses: 'google-github-actions/upload-cloud-storage@v1' + with: + path: 'results' + destination: 'ann-pipelines/github-action-runs' + glob: '*.json' ann-benchmarks-pq-glove-aws: name: "[bench AWS] Glove100 pq=true" if: ${{ github.event.inputs.test_to_run == 'ann-benchmarks-pq-glove-aws' || github.event.inputs.test_to_run == '' }} @@ -331,6 +383,43 @@ jobs: path: 'results' destination: 'ann-pipelines/github-action-runs' glob: '*.json' + ann-benchmarks-lasq-glove-aws: + needs: [newer-or-equal-than-1_28] + if: ${{ (needs.newer-or-equal-than-1_28.outputs.check == 'true') && ( github.event.inputs.test_to_run == 'ann-benchmarks-lasq-glove-aws' || github.event.inputs.test_to_run == '' ) }} + name: "[bench AWS] Glove100 lasq=true" + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY}} + AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} + DATASET: glove-100-angular + DISTANCE: cosine + REQUIRED_RECALL: 0.89 + QUANTIZATION: lasq + PERSISTENCE_LSM_ACCESS_STRATEGY: ${{inputs.lsm_access_strategy}} + steps: + - uses: actions/checkout@v3 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{secrets.DOCKER_USERNAME}} + password: ${{secrets.DOCKER_PASSWORD}} + - id: 'gcs_auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: ${{secrets.GCP_SERVICE_ACCOUNT_BENCHMARKS}} + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + - name: Run chaos test + if: always() + run: ./ann_benchmark_quantization_aws.sh + - id: 'upload-files' + uses: 'google-github-actions/upload-cloud-storage@v1' + with: + path: 'results' + destination: 'ann-pipelines/github-action-runs' + glob: '*.json' ann-benchmarks-sift-gcp: name: "[bench GCP] SIFT1M pq=false" if: ${{ github.event.inputs.test_to_run == 'ann-benchmarks-sift-gcp' || github.event.inputs.test_to_run == '' }} diff --git a/apps/ann-benchmarks/Dockerfile b/apps/ann-benchmarks/Dockerfile index cb587f3a..c8825653 100644 --- a/apps/ann-benchmarks/Dockerfile +++ b/apps/ann-benchmarks/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.11-slim-bullseye # to support pprof profiles -RUN apt-get update && apt-get install -y golang-go pkg-config libhdf5-dev +RUN apt-get update && apt-get install -y golang-go pkg-config libhdf5-dev git WORKDIR /workdir diff --git a/apps/ann-benchmarks/requirements.txt b/apps/ann-benchmarks/requirements.txt index cc8ebf03..d9ad22e7 100644 --- a/apps/ann-benchmarks/requirements.txt +++ b/apps/ann-benchmarks/requirements.txt @@ -1,4 +1,4 @@ -weaviate-client==4.7.0-rc.0 +weaviate-client@git+https://github.com/weaviate/weaviate-python-client.git@6233847809c207301722df0d9b9bd4082614ff27 loguru==0.5.3 seaborn==0.12.2 h5py==3.11.0 diff --git a/apps/ann-benchmarks/weaviate_import.py b/apps/ann-benchmarks/weaviate_import.py index ac5bee51..88a1ae32 100644 --- a/apps/ann-benchmarks/weaviate_import.py +++ b/apps/ann-benchmarks/weaviate_import.py @@ -45,7 +45,7 @@ def load_records( with client.batch.fixed_size(batch_size=batch_size) as batch: for vector in vectors: - if i == 100000 and quantization in ["pq", "sq"] and override == False: + if i == 100000 and quantization in ["pq", "sq", "lasq"] and override == False: logger.info(f"pausing import to enable quantization") break @@ -66,7 +66,7 @@ def load_records( for err in client.batch.failed_objects: logger.error(err.message) - if quantization in ["pq", "sq"] and override == False: + if quantization in ["pq", "sq", "lasq"] and override == False: if quantization == "pq": collection.config.update( @@ -82,6 +82,12 @@ def load_records( quantizer=wvc.Reconfigure.VectorIndex.Quantizer.sq() ) ) + elif quantization == "lasq": + collection.config.update( + vector_index_config=wvc.Reconfigure.VectorIndex.hnsw( + quantizer=wvc.Reconfigure.VectorIndex.Quantizer.lasq() + ) + ) wait_for_all_shards_ready(collection) diff --git a/apps/multi-tenancy-concurrent-imports/multi-tenancy-load-test b/apps/multi-tenancy-concurrent-imports/multi-tenancy-load-test index e9e53d24..5c94c5c4 160000 --- a/apps/multi-tenancy-concurrent-imports/multi-tenancy-load-test +++ b/apps/multi-tenancy-concurrent-imports/multi-tenancy-load-test @@ -1 +1 @@ -Subproject commit e9e53d2437f31638695717056a53354c992e4813 +Subproject commit 5c94c5c4a4251a81305cfe4df3adb2bacce05b97