From 05f55a565460286e70fec24cd067516450eb1b1f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 16 Apr 2025 09:41:02 -0500 Subject: [PATCH 01/10] update test database name --- .evergreen/setup-remote.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.evergreen/setup-remote.sh b/.evergreen/setup-remote.sh index 93318d7..409250c 100644 --- a/.evergreen/setup-remote.sh +++ b/.evergreen/setup-remote.sh @@ -38,6 +38,9 @@ case $DIR in haystack-embeddings | haystack-fulltext) MONGODB_URI=$HAYSTACK_MONGODB_URI ;; + voyageai-python) + MONGODB_URI=$VOYAGEAI_MONGODB_URI + ;; *) echo "Missing config in fetch-secrets.sh for DIR: $DIR" exit 1 @@ -46,6 +49,7 @@ esac export MONGODB_URI # Create the env file +echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh echo "export MONGODB_URI=$MONGODB_URI" >> env.sh From 0777145bae6dc29f92b4ac0e87238101903090a3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 16 Apr 2025 09:45:25 -0500 Subject: [PATCH 02/10] test pymongo-voyageai --- .evergreen/config.yml | 24 ++++++++++++++++++++++++ .gitignore | 1 + pymongo-voyageai/config.env | 3 +++ pymongo-voyageai/run.sh | 24 ++++++++++++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 pymongo-voyageai/config.env create mode 100644 pymongo-voyageai/run.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 8654b16..0885922 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -185,6 +185,20 @@ tasks: - func: "setup remote atlas" - func: "execute tests" + - name: test-pymongo-voyageai-local + tags: [local] + commands: + - func: "fetch repo" + - func: "setup local atlas" + - func: "execute tests" + + - name: test-pymongo-voyageai-remote + tags: [remote] + commands: + - func: "fetch repo" + - func: "setup remote atlas" + - func: "execute tests" + - name: test-haystack-embeddings-local tags: [local] commands: @@ -298,6 +312,16 @@ buildvariants: - name: test-docarray-local - name: test-docarray-remote + - name: test-pymongo-voyageai-rhel + display_name: PyMongo-VoyageAI RHEL + expansions: + DIR: pymongo_voyageai + run_on: + - rhel87-small + tasks: + - name: test-pymongo-voyageai-local + - name: test-pymongo-voyageai-remote + - name: test-haystack-embeddings-rhel display_name: Haystack Embeddings RHEL expansions: diff --git a/.gitignore b/.gitignore index 53a045c..71565d7 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ env.sh # Repo checkouts haystack-fulltext/haystack-core-integrations haystack-embeddings/haystack-core-integrations +pymongo_voyageai/pymongo_voyageai diff --git a/pymongo-voyageai/config.env b/pymongo-voyageai/config.env new file mode 100644 index 0000000..8fabcdc --- /dev/null +++ b/pymongo-voyageai/config.env @@ -0,0 +1,3 @@ +REPO_NAME=pymongo_voyageai +CLONE_URL="https://github.com/blink1073/pymongo_voyageai.git" +DATABASE="pymongo_voyageai_test_db" diff --git a/pymongo-voyageai/run.sh b/pymongo-voyageai/run.sh new file mode 100644 index 0000000..d953c67 --- /dev/null +++ b/pymongo-voyageai/run.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -eu + +# Get the MONGODB_URI and VOYAGEAI_API_KEY. +SCRIPT_DIR=$(realpath "$(dirname ${BASH_SOURCE[0]})") +ROOT_DIR=$(dirname $SCRIPT_DIR) +. $ROOT_DIR/env.sh + +. $ROOT_DIR/.evergreen/utils.sh + +PYTHON_BINARY=$(find_python3) + +$PYTHON_BINARY -m venv venv_pipeline +source venv_pipeline/bin/activate + +pip install uv rust-just + +just install + +export MONGODB_URI=$MONGODB_URI +export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY + +just tests From 55d9c7dfe0d03410925f65e8ef991d180a58b16e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 16 Apr 2025 09:49:56 -0500 Subject: [PATCH 03/10] fix checkout --- pymongo-voyageai/config.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo-voyageai/config.env b/pymongo-voyageai/config.env index 8fabcdc..5642363 100644 --- a/pymongo-voyageai/config.env +++ b/pymongo-voyageai/config.env @@ -1,3 +1,3 @@ -REPO_NAME=pymongo_voyageai -CLONE_URL="https://github.com/blink1073/pymongo_voyageai.git" +REPO_NAME=pymongo-voyageai +CLONE_URL="https://github.com/blink1073/pymongo-voyageai.git" DATABASE="pymongo_voyageai_test_db" From 71f9add7f6eab5a462ebd3163c4c1c2d3311487d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 16 Apr 2025 09:56:09 -0500 Subject: [PATCH 04/10] cleanup --- .evergreen/config.yml | 2 +- .evergreen/fetch-repo.sh | 2 +- .gitignore | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0885922..79ede64 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -315,7 +315,7 @@ buildvariants: - name: test-pymongo-voyageai-rhel display_name: PyMongo-VoyageAI RHEL expansions: - DIR: pymongo_voyageai + DIR: pymongo-voyageai run_on: - rhel87-small tasks: diff --git a/.evergreen/fetch-repo.sh b/.evergreen/fetch-repo.sh index ad9621a..d7e52d5 100644 --- a/.evergreen/fetch-repo.sh +++ b/.evergreen/fetch-repo.sh @@ -3,7 +3,7 @@ set -eu if [ ! -d "${DIR}" ]; then - echo '${REPO_NAME} could not be found' 1>&2 + echo "${REPO_NAME} could not be found" 1>&2 exit 1 fi diff --git a/.gitignore b/.gitignore index 71565d7..7bf5cf6 100644 --- a/.gitignore +++ b/.gitignore @@ -60,4 +60,4 @@ env.sh # Repo checkouts haystack-fulltext/haystack-core-integrations haystack-embeddings/haystack-core-integrations -pymongo_voyageai/pymongo_voyageai +pymongo-voyageai/pymongo-voyageai From e24224ee8707239b044bb4a03a0daef8b4ec219f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 16 Apr 2025 10:05:45 -0500 Subject: [PATCH 05/10] cleanup --- .evergreen/provision-atlas.sh | 1 + .evergreen/setup-remote.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.evergreen/provision-atlas.sh b/.evergreen/provision-atlas.sh index e46f97d..3763eb1 100644 --- a/.evergreen/provision-atlas.sh +++ b/.evergreen/provision-atlas.sh @@ -19,3 +19,4 @@ source secrets-export.sh # Create the env file echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh echo "export MONGODB_URI=$CONN_STRING" >> env.sh +echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh diff --git a/.evergreen/setup-remote.sh b/.evergreen/setup-remote.sh index 409250c..5889235 100644 --- a/.evergreen/setup-remote.sh +++ b/.evergreen/setup-remote.sh @@ -38,11 +38,11 @@ case $DIR in haystack-embeddings | haystack-fulltext) MONGODB_URI=$HAYSTACK_MONGODB_URI ;; - voyageai-python) + pymongo-voyageai) MONGODB_URI=$VOYAGEAI_MONGODB_URI ;; *) - echo "Missing config in fetch-secrets.sh for DIR: $DIR" + echo "Missing config in setup-remote.sh for DIR: $DIR" exit 1 ;; esac From 07aa55894b27e518eecc7eb1560db414decb5e4b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 16 Apr 2025 10:34:24 -0500 Subject: [PATCH 06/10] fix target --- pymongo-voyageai/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo-voyageai/run.sh b/pymongo-voyageai/run.sh index d953c67..30c34f6 100644 --- a/pymongo-voyageai/run.sh +++ b/pymongo-voyageai/run.sh @@ -21,4 +21,4 @@ just install export MONGODB_URI=$MONGODB_URI export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY -just tests +just test From c6bde015ef98264e2d83dffb9b58e6c9faf0cd94 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 18 Apr 2025 13:28:52 -0500 Subject: [PATCH 07/10] add voyageai s3 bucket --- .evergreen/provision-atlas.sh | 1 + .evergreen/setup-remote.sh | 1 + pymongo-voyageai/run.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/.evergreen/provision-atlas.sh b/.evergreen/provision-atlas.sh index 3763eb1..5e4093e 100644 --- a/.evergreen/provision-atlas.sh +++ b/.evergreen/provision-atlas.sh @@ -17,6 +17,7 @@ scaffold_atlas source secrets-export.sh # Create the env file +echo "export VOYAGEAI_S3_BUCKET=$VOYAGEAI_S3_BUCKET" >> env.sh echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh echo "export MONGODB_URI=$CONN_STRING" >> env.sh echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh diff --git a/.evergreen/setup-remote.sh b/.evergreen/setup-remote.sh index 5889235..41237b5 100644 --- a/.evergreen/setup-remote.sh +++ b/.evergreen/setup-remote.sh @@ -49,6 +49,7 @@ esac export MONGODB_URI # Create the env file +echo "export VOYAGEAI_S3_BUCKET=$VOYAGEAI_S3_BUCKET" >> env.sh echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh echo "export MONGODB_URI=$MONGODB_URI" >> env.sh diff --git a/pymongo-voyageai/run.sh b/pymongo-voyageai/run.sh index 30c34f6..4ce2de8 100644 --- a/pymongo-voyageai/run.sh +++ b/pymongo-voyageai/run.sh @@ -18,6 +18,7 @@ pip install uv rust-just just install +export S3_BUCKET_NAME=$VOYAGEAI_S3_BUCKET export MONGODB_URI=$MONGODB_URI export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY From 89481c76f92202f9e9c6de61f4de22e41266e72f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 18 Apr 2025 13:38:41 -0500 Subject: [PATCH 08/10] add assume role --- .evergreen/config.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 79ede64..fc50b6e 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -45,11 +45,16 @@ functions: binary: bash args: [.evergreen/fetch-repo.sh] + "assume role": + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} + "execute tests": - command: subprocess.exec type: test params: - include_expansions_in_env: [DIR] + include_expansions_in_env: [DIR, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] working_dir: "src" binary: bash args: [.evergreen/execute-tests.sh] @@ -189,6 +194,7 @@ tasks: tags: [local] commands: - func: "fetch repo" + - func: "assume role" - func: "setup local atlas" - func: "execute tests" @@ -196,6 +202,7 @@ tasks: tags: [remote] commands: - func: "fetch repo" + - func: "assume role" - func: "setup remote atlas" - func: "execute tests" From 27778780c3b0c12bf75ac802cc146f504de2041a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 29 Apr 2025 15:46:20 -0500 Subject: [PATCH 09/10] test read-from-storage --- pymongo-voyageai/config.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo-voyageai/config.env b/pymongo-voyageai/config.env index 5642363..5340925 100644 --- a/pymongo-voyageai/config.env +++ b/pymongo-voyageai/config.env @@ -1,3 +1,3 @@ REPO_NAME=pymongo-voyageai -CLONE_URL="https://github.com/blink1073/pymongo-voyageai.git" +CLONE_URL="--branch read-from-storage https://github.com/blink1073/pymongo-voyageai.git" DATABASE="pymongo_voyageai_test_db" From a538ea374ff3c139fc54a2286076bb0ac4060063 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 29 Apr 2025 21:06:30 -0500 Subject: [PATCH 10/10] update for final repo --- pymongo-voyageai/config.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo-voyageai/config.env b/pymongo-voyageai/config.env index 5340925..0cff80f 100644 --- a/pymongo-voyageai/config.env +++ b/pymongo-voyageai/config.env @@ -1,3 +1,3 @@ REPO_NAME=pymongo-voyageai -CLONE_URL="--branch read-from-storage https://github.com/blink1073/pymongo-voyageai.git" +CLONE_URL="https://github.com/mongodb-labs/pymongo-voyageai.git" DATABASE="pymongo_voyageai_test_db"