Fix env vars in the MMQnA test environment and align_inputs error (#45)

* Fix embedding tests for MMQnA Signed-off-by: dmsuehir <[email protected]> * Add missing WHISPER_SERVER_ENDPOINT env var Signed-off-by: dmsuehir <[email protected]> * Fix align_inputs issue to handle dictionaries Signed-off-by: dmsuehir <[email protected]> --------- Signed-off-by: dmsuehir <[email protected]>
mhbuehler · Jan 15, 2025 · b426966 · b426966
1 parent 48d06fe
commit b426966
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 6 deletions.
diff --git a/MultimodalQnA/multimodalqna.py b/MultimodalQnA/multimodalqna.py
@@ -33,14 +33,18 @@
 
 def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **kwargs):
     if self.services[cur_node].service_type == ServiceType.EMBEDDING:
+        if "text" in inputs:
+            input_text = inputs["text"]["text"] if isinstance(inputs["text"], dict) else inputs["text"]
+        if "image" in inputs:
+            input_image = inputs["image"]["base64_image"] if isinstance(inputs["image"], dict) else inputs["image"]
         if "text" in inputs and "image" in inputs:
-            text_doc = TextDoc(text=inputs["text"])
-            image_doc = ImageDoc(base64_image=inputs["image"])
+            text_doc = TextDoc(text=input_text)
+            image_doc = ImageDoc(base64_image=input_image)
             inputs = TextImageDoc(text=text_doc, image=image_doc).dict()
         elif "image" in inputs:
-            inputs = ImageDoc(base64_image=inputs["image"]).dict()
+            inputs = ImageDoc(base64_image=input_image).dict()
         elif "text" in inputs:
-            inputs = TextDoc(text=inputs["text"]).dict()
+            inputs = TextDoc(text=input_text).dict()
     return inputs
 
 

diff --git a/MultimodalQnA/tests/test_compose_on_gaudi.sh b/MultimodalQnA/tests/test_compose_on_gaudi.sh
@@ -69,6 +69,7 @@ function setup_env() {
     export WHISPER_PORT=7066
     export MAX_IMAGES=1
     export WHISPER_MODEL="base"
+    export WHISPER_SERVER_ENDPOINT="http://${host_ip}:${WHISPER_PORT}/v1/asr"
     export ASR_ENDPOINT=http://$host_ip:$WHISPER_PORT
     export ASR_PORT=9099
     export ASR_SERVICE_PORT=3001
@@ -82,7 +83,7 @@ function setup_env() {
     export EMM_BRIDGETOWER_PORT=6006
     export BRIDGE_TOWER_EMBEDDING=true
     export EMBEDDING_MODEL_ID="BridgeTower/bridgetower-large-itm-mlm-itc"
-    export MMEI_EMBEDDING_ENDPOINT="http://${host_ip}:$EMM_BRIDGETOWER_PORT/v1/encode"
+    export MMEI_EMBEDDING_ENDPOINT="http://${host_ip}:$EMM_BRIDGETOWER_PORT"
     export MM_EMBEDDING_PORT_MICROSERVICE=6000
     export REDIS_RETRIEVER_PORT=7000
     export LVM_PORT=9399

diff --git a/MultimodalQnA/tests/test_compose_on_xeon.sh b/MultimodalQnA/tests/test_compose_on_xeon.sh
@@ -62,6 +62,7 @@ function setup_env() {
     export WHISPER_PORT=7066
     export MAX_IMAGES=1
     export WHISPER_MODEL="base"
+    export WHISPER_SERVER_ENDPOINT="http://${host_ip}:${WHISPER_PORT}/v1/asr"
     export ASR_ENDPOINT=http://$host_ip:$WHISPER_PORT
     export ASR_PORT=9099
     export ASR_SERVICE_PORT=3001
@@ -80,7 +81,7 @@ function setup_env() {
     export EMM_BRIDGETOWER_PORT=6006
     export BRIDGE_TOWER_EMBEDDING=true
     export EMBEDDING_MODEL_ID="BridgeTower/bridgetower-large-itm-mlm-itc"
-    export MMEI_EMBEDDING_ENDPOINT="http://${host_ip}:$EMM_BRIDGETOWER_PORT/v1/encode"
+    export MMEI_EMBEDDING_ENDPOINT="http://${host_ip}:$EMM_BRIDGETOWER_PORT"
     export MM_EMBEDDING_PORT_MICROSERVICE=6000
     export REDIS_RETRIEVER_PORT=7000
     export LVM_PORT=9399