Merge pull request #222 from weaviate/add-support-for-reranker-voyage…

…ai-module

Add support for reranker-voaygeai module

.cicd/test.sh

@@ -151,6 +151,9 @@ function check_creates_template() {
   check_modules "--set modules.generative-mistral.enabled=true --set modules.generative-mistral.apiKey=apiKey" "value: generative-mistral"
   check_modules "--set modules.multi2vec-palm.enabled=true" "value: multi2vec-palm"
   check_modules "--set modules.multi2vec-palm.enabled=true --set modules.multi2vec-palm.apiKey=apiKey --set modules.generative-palm.enabled=true --set modules.generative-palm.apiKey=apiKey" "value: generative-palm,multi2vec-palm"
+  check_modules "--set modules.reranker-voyageai.enabled=true" "value: reranker-voyageai"
+  check_modules "--set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=apiKey" "value: reranker-voyageai"
+  check_modules "--set modules.text2vec-voyageai.enabled=true --set modules.text2vec-voyageai.apiKey=apiKey --set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=apiKey" "value: text2vec-voyageai,reranker-voyageai"
 
   check_modules "--set modules.text2vec-openai.enabled=true --set modules.text2vec-openai.azureApiKey=azureApiKey" "value: text2vec-openai"
   check_modules "--set modules.qna-openai.enabled=true --set modules.qna-openai.azureApiKey=azureApiKey" "value: qna-openai"
@@ -176,6 +179,8 @@ function check_creates_template() {
   check_string_existence "--set modules.text2vec-voyageai.enabled=true --set modules.text2vec-voyageai.apiKey=voyageaiApiKey" "name: VOYAGEAI_APIKEY"
   check_string_existence "--set modules.generative-mistral.enabled=true --set modules.generative-mistral.apiKey=apiKey" "name: MISTRAL_APIKEY"
   check_string_existence "--set modules.multi2vec-palm.enabled=true --set modules.multi2vec-palm.apiKey=palmApiKey" "name: PALM_APIKEY"
+  check_string_existence "--set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=voyageaiApiKey" "name: VOYAGEAI_APIKEY"
+  check_string_existence "--set modules.text2vec-voyageai.enabled=true --set modules.text2vec-voyageai.apiKey=voyageaiApiKey --set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=voyageaiApiKey" "name: VOYAGEAI_APIKEY"
 
   _settingPassageQueryOn="--set modules.text2vec-contextionary.enabled=false --set modules.text2vec-transformers.passageQueryServices.passage.enabled=true --set modules.text2vec-transformers.passageQueryServices.query.enabled=true"
   check_setting_has_value "$_settingPassageQueryOn" "name: TRANSFORMERS_PASSAGE_INFERENCE_API" "value: http://transformers-inference-passage.default.svc.cluster.local.:8080"

weaviate/Chart.yaml

@@ -7,10 +7,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 16.8.7
+version: 16.8.8
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 1.24.3
+appVersion: 1.24.8
 icon: https://raw.githubusercontent.com/weaviate/weaviate/19de0956c69b66c5552447e84d016f4fe29d12c9/docs/assets/weaviate-logo.png

weaviate/templates/_helpers.tpl

@@ -89,6 +89,9 @@
   {{- if (index .Values "modules" "reranker-transformers" "enabled") -}}
     {{ $modules = append $modules "reranker-transformers" }}
   {{- end -}}
+  {{- if (index .Values "modules" "reranker-voyageai" "enabled") -}}
+    {{ $modules = append $modules "reranker-voyageai" }}
+  {{- end -}}
   {{- if (index .Values "backups" "filesystem" "enabled") -}}
     {{ $modules = append $modules "backup-filesystem" }}
   {{- end -}}

weaviate/templates/apiKeyVoyageAISecret.yaml

@@ -1,4 +1,20 @@
-{{ if and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey") }}
+{{-  $t2vVoyageAI := and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey") -}}
+{{-  $rerankerVoyageAI := and (index .Values "modules" "reranker-voyageai" "enabled") (index .Values "modules" "reranker-voyageai" "apiKey") }}
+{{-  $voyageai := or ($t2vVoyageAI) ($rerankerVoyageAI) }}
+{{- if $voyageai }}
+
+{{- $apiKeys := list }}
+{{- if $t2vVoyageAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "text2vec-voyageai" "apiKey") -}}
+{{- end }}
+{{- if $rerankerVoyageAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "reranker-voyageai" "apiKey") -}}
+{{- end }}
+
+{{- if gt (len ($apiKeys | uniq)) 1 -}}
+  {{- fail "VoyageAI modules activated, but their keys differ. Use the same key on all activated VoyageAI modules." }}
+{{- end }}
+
 apiVersion: v1
 kind: Secret
 metadata:
@@ -8,5 +24,9 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 type: Opaque
 data:
+  {{- if (index .Values "modules" "reranker-voyageai" "apiKey") }}
+  apiKey: {{ index .Values "modules" "reranker-voyageai" "apiKey" | b64enc }}
+  {{- else }}
   apiKey: {{ index .Values "modules" "text2vec-voyageai" "apiKey" | b64enc }}
+  {{- end }}
 {{ end }}

weaviate/templates/weaviateStatefulset.yaml

@@ -225,7 +225,7 @@ spec:
                 name: weaviate-jinaai
                 key: apiKey
           {{- end  }}
-          {{- if and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey") }}
+          {{- if or (and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey")) (and (index .Values "modules" "reranker-voyageai" "enabled") (index .Values "modules" "reranker-voyageai" "apiKey")) }}
           - name: VOYAGEAI_APIKEY
             valueFrom:
               secretKeyRef:

weaviate/values.yaml

@@ -7,7 +7,7 @@ image:
   # of weaviate. In accordance with Infra-as-code, you should pin this value
   # down and only change it if you explicitly want to upgrade the Weaviate
   # version.
-  tag: 1.24.3
+  tag: 1.24.8
   repo: semitechnologies/weaviate
   # Image pull policy: https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy
   pullPolicy: IfNotPresent
@@ -1284,6 +1284,19 @@ modules:
     # an environment variable
     apiKey: ''
 
+  # The reranker-voyageai module uses VoaygeAI API
+  # to dynamically compute a score for the relevance 
+  # of the query with each of the initial search results.
+  # More information about Cohere API can be found here: https://www.voyageai.com/
+  reranker-voyageai:
+
+    # enable if you want to use VoaygeAI module
+    enabled: false
+
+    # Set your VoaygeAI API Key to be passed to Weaviate pod as
+    # an environment variable
+    apiKey: ''
+
   # The reranker-transformers module uses Cross-Encoders for 
   # sentence pair scoring and sentence pair classification tasks.
   # More information about Cross-Encoders can be found here: