From 7c00cbff6c44725273dff67013e56572faa67e8e Mon Sep 17 00:00:00 2001 From: Luca Giorgi Date: Tue, 17 Oct 2023 13:56:43 +0200 Subject: [PATCH 1/2] Automate embedded RHODS install (#924) * Automate embedded rhods install Signed-off-by: Luca Giorgi * fix imports Signed-off-by: Luca Giorgi * fix imports again Signed-off-by: Luca Giorgi * fix path typo Signed-off-by: Luca Giorgi * fix variable typo Signed-off-by: Luca Giorgi * empty eval Signed-off-by: Luca Giorgi * fix typo in catalogsource Signed-off-by: Luca Giorgi * fix typos Signed-off-by: Luca Giorgi * Fix comparison check Signed-off-by: Luca Giorgi * fix typo, add CRs checks Signed-off-by: Luca Giorgi * fix resource comparison Signed-off-by: Luca Giorgi * add teardown Signed-off-by: Luca Giorgi * Add logging Signed-off-by: Luca Giorgi * update uninstall Signed-off-by: Luca Giorgi * Update uninstall Signed-off-by: Luca Giorgi * fix imports Signed-off-by: Luca Giorgi * add log Signed-off-by: Luca Giorgi * Escape img url before passing to sed Signed-off-by: Luca Giorgi * cast to string Signed-off-by: Luca Giorgi * add helper function Signed-off-by: Luca Giorgi * use updated subscription for prod and z-stream installs Signed-off-by: Luca Giorgi * add more logging Signed-off-by: Luca Giorgi * fix typo Signed-off-by: Luca Giorgi * resource list with catalogsource present Signed-off-by: Luca Giorgi --------- Signed-off-by: Luca Giorgi --- ods_ci/libs/Helpers.py | 4 + .../RHODS_OLM/uninstall/uninstall.robot | 7 +- .../Files/operatorV2/cs_template.yaml | 10 ++ .../Resources/Files/operatorV2/embedded.txt | 40 +++++ .../Files/operatorV2/embedded_cs.txt | 56 +++++++ .../operatorV2/operatorgroup_template.yaml | 5 + .../operatorV2/subscription_template_21.yaml | 15 ++ .../operatorV2/subscription_template_z.yaml | 13 ++ .../131__rhods_operatorv2.robot | 4 +- .../133__embedded_operator.robot | 153 ++++++++++++++++++ 10 files changed, 303 insertions(+), 4 deletions(-) create mode 100644 ods_ci/tests/Resources/Files/operatorV2/cs_template.yaml create mode 100644 ods_ci/tests/Resources/Files/operatorV2/embedded.txt create mode 100644 ods_ci/tests/Resources/Files/operatorV2/embedded_cs.txt create mode 100644 ods_ci/tests/Resources/Files/operatorV2/operatorgroup_template.yaml create mode 100644 ods_ci/tests/Resources/Files/operatorV2/subscription_template_21.yaml create mode 100644 ods_ci/tests/Resources/Files/operatorV2/subscription_template_z.yaml create mode 100644 ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/133__embedded_operator.robot diff --git a/ods_ci/libs/Helpers.py b/ods_ci/libs/Helpers.py index ab1f578ee..3dc78a654 100644 --- a/ods_ci/libs/Helpers.py +++ b/ods_ci/libs/Helpers.py @@ -308,3 +308,7 @@ def process_resource_list(self, filename_in, filename_out=None): filename_out = filename_in.split(".")[0] + "_processed.txt" with open(filename_out, "w") as outfile: outfile.write("".join(str(l) for l in out)) + + @keyword + def escape_forward_slashes(self, string_to_escape): + return string_to_escape.replace('/','\/') \ No newline at end of file diff --git a/ods_ci/tasks/Resources/RHODS_OLM/uninstall/uninstall.robot b/ods_ci/tasks/Resources/RHODS_OLM/uninstall/uninstall.robot index 35d5c5ab7..242df044b 100644 --- a/ods_ci/tasks/Resources/RHODS_OLM/uninstall/uninstall.robot +++ b/ods_ci/tasks/Resources/RHODS_OLM/uninstall/uninstall.robot @@ -91,6 +91,9 @@ Uninstall RHODS V2 Should Be Equal As Integers ${return_code} 0 msg=Error deleting RHODS subscription END + ${return_code} ${output} Run And Return Rc And Output + ... oc delete subscription $(oc get subscription -n redhat-ods-operator --no-headers | awk '{print $1}') -n ${OPERATOR_NAMESPACE} # robocop: disable + Should Be Equal As Integers ${return_code} 0 msg=Error deleting RHODS subscription ${return_code} ${output} Run And Return Rc And Output ... oc delete operatorgroup --all -n redhat-ods-operator --ignore-not-found Should Be Equal As Integers ${return_code} 0 msg=Error deleting operatorgroup @@ -98,5 +101,5 @@ Uninstall RHODS V2 Verify Project Does Not Exists redhat-ods-applications Verify Project Does Not Exists redhat-ods-monitoring Verify Project Does Not Exists rhods-notebooks - ${return_code} ${output} Run And Return Rc And Output oc delete namespace redhat-ods-operator --ignore-not-found - Verify Project Does Not Exists redhat-ods-operator + ${return_code} ${output} Run And Return Rc And Output oc delete namespace ${OPERATOR_NAMESPACE} --ignore-not-found + Verify Project Does Not Exists ${OPERATOR_NAMESPACE} diff --git a/ods_ci/tests/Resources/Files/operatorV2/cs_template.yaml b/ods_ci/tests/Resources/Files/operatorV2/cs_template.yaml new file mode 100644 index 000000000..c0e801e5f --- /dev/null +++ b/ods_ci/tests/Resources/Files/operatorV2/cs_template.yaml @@ -0,0 +1,10 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: rhods-catalog-dev + namespace: +spec: + sourceType: grpc + image: + displayName: Red Hat OpenShift Data Science + publisher: RHODS Development Catalog \ No newline at end of file diff --git a/ods_ci/tests/Resources/Files/operatorV2/embedded.txt b/ods_ci/tests/Resources/Files/operatorV2/embedded.txt new file mode 100644 index 000000000..bc6d1f942 --- /dev/null +++ b/ods_ci/tests/Resources/Files/operatorV2/embedded.txt @@ -0,0 +1,40 @@ +ConfigMap kube-root-ca.crt +ConfigMap openshift-service-ca.crt +ConfigMap redhat-ods-operator-manager-config +Deployment rhods-operator +EndpointSlice redhat-ods-operator-controller-manager-metrics-service-jz6l7 +Endpoints redhat-ods-operator-controller-manager-metrics-service +InstallPlan install-tpnlw +KIND NAME +OperatorCondition rhods-operator.2.1.0 +OperatorGroup rhods-operator-dev +Pod rhods-operator-59cfcbb47d-85dfj +ReplicaSet rhods-operator-59cfcbb47d +Role rhods-operator.2.1.0 +Role rhods-operator.2.1.0 +Role rhods-operator.2.1.0-redhat-ods-operator-controller--655d59d697 +Role rhods-operator.2.1.0-redhat-ods-operator-controller--655d59d697 +RoleBinding rhods-operator.2.1.0 +RoleBinding rhods-operator.2.1.0 +RoleBinding rhods-operator.2.1.0-redhat-ods-operator-controller--655d59d697 +RoleBinding rhods-operator.2.1.0-redhat-ods-operator-controller--655d59d697 +RoleBinding system:deployers +RoleBinding system:deployers +RoleBinding system:image-builders +RoleBinding system:image-builders +RoleBinding system:image-pullers +RoleBinding system:image-pullers +Secret builder-dockercfg-r4djd +Secret builder-token-jzwg8 +Secret default-dockercfg-qd9lt +Secret default-token-rdxn7 +Secret deployer-dockercfg-tvp8x +Secret deployer-token-6th7d +Secret redhat-ods-operator-controller-manager-dockercfg-pvx2b +Secret redhat-ods-operator-controller-manager-token-2qh6c +Service redhat-ods-operator-controller-manager-metrics-service +ServiceAccount builder +ServiceAccount default +ServiceAccount deployer +ServiceAccount redhat-ods-operator-controller-manager +Subscription rhods-operator diff --git a/ods_ci/tests/Resources/Files/operatorV2/embedded_cs.txt b/ods_ci/tests/Resources/Files/operatorV2/embedded_cs.txt new file mode 100644 index 000000000..1aff308a1 --- /dev/null +++ b/ods_ci/tests/Resources/Files/operatorV2/embedded_cs.txt @@ -0,0 +1,56 @@ +CatalogSource rhods-catalog-dev +ConfigMap c831fa3421fa182ed3dd44418173698c8f1bc0581981d4ced92d2708b38cb67 +ConfigMap kube-root-ca.crt +ConfigMap openshift-service-ca.crt +ConfigMap redhat-ods-operator-manager-config +Deployment rhods-operator +EndpointSlice redhat-ods-operator-controller-manager-metrics-service +EndpointSlice rhods-catalog-dev +Endpoints redhat-ods-operator-controller-manager-metrics-service +Endpoints rhods-catalog-dev +InstallPlan install +Job c831fa3421fa182ed3dd44418173698c8f1bc0581981d4ced92d2708b38cb67 +KIND NAME +OperatorCondition rhods-operator.2.1.0 +OperatorGroup rhods-operator-dev +Pod c831fa3421fa182ed3dd44418173698c8f1bc0581981d4ced92d2708b3vbpr4 +Pod rhods-catalog-dev +Pod rhods-operator +PodMetrics rhods-catalog-dev +ReplicaSet rhods-operator +Role c831fa3421fa182ed3dd44418173698c8f1bc0581981d4ced92d2708b38cb67 +Role c831fa3421fa182ed3dd44418173698c8f1bc0581981d4ced92d2708b38cb67 +Role rhods-operator.2.1.0 +Role rhods-operator.2.1.0 +Role rhods-operator.2.1.0-redhat-ods-operator-controller- +Role rhods-operator.2.1.0-redhat-ods-operator-controller- +RoleBinding c831fa3421fa182ed3dd44418173698c8f1bc0581981d4ced92d2708b38cb67 +RoleBinding c831fa3421fa182ed3dd44418173698c8f1bc0581981d4ced92d2708b38cb67 +RoleBinding rhods-operator.2.1.0 +RoleBinding rhods-operator.2.1.0 +RoleBinding rhods-operator.2.1.0-redhat-ods-operator-controller- +RoleBinding rhods-operator.2.1.0-redhat-ods-operator-controller- +RoleBinding system:deployers +RoleBinding system:deployers +RoleBinding system:image-builders +RoleBinding system:image-builders +RoleBinding system:image-pullers +RoleBinding system:image-pullers +Secret builder-dockercfg +Secret builder-token +Secret default-dockercfg +Secret default-token +Secret deployer-dockercfg +Secret deployer-token +Secret redhat-ods-operator-controller-manager-dockercfg +Secret redhat-ods-operator-controller-manager-token +Secret rhods-catalog-dev-dockercfg +Secret rhods-catalog-dev-token +Service redhat-ods-operator-controller-manager-metrics-service +Service rhods-catalog-dev +ServiceAccount builder +ServiceAccount default +ServiceAccount deployer +ServiceAccount redhat-ods-operator-controller-manager +ServiceAccount rhods-catalog-dev +Subscription rhods-operator \ No newline at end of file diff --git a/ods_ci/tests/Resources/Files/operatorV2/operatorgroup_template.yaml b/ods_ci/tests/Resources/Files/operatorV2/operatorgroup_template.yaml new file mode 100644 index 000000000..3b9d79a9a --- /dev/null +++ b/ods_ci/tests/Resources/Files/operatorV2/operatorgroup_template.yaml @@ -0,0 +1,5 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: rhods-operator-dev + namespace: \ No newline at end of file diff --git a/ods_ci/tests/Resources/Files/operatorV2/subscription_template_21.yaml b/ods_ci/tests/Resources/Files/operatorV2/subscription_template_21.yaml new file mode 100644 index 000000000..8f63cc619 --- /dev/null +++ b/ods_ci/tests/Resources/Files/operatorV2/subscription_template_21.yaml @@ -0,0 +1,15 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: rhods-operator + namespace: +spec: + channel: embedded + installPlanApproval: Manual + name: rhods-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + startingCSV: rhods-operator.2.1.0 + config: + env: + - name: "DISABLE_DSC_CONFIG" \ No newline at end of file diff --git a/ods_ci/tests/Resources/Files/operatorV2/subscription_template_z.yaml b/ods_ci/tests/Resources/Files/operatorV2/subscription_template_z.yaml new file mode 100644 index 000000000..d7bad2850 --- /dev/null +++ b/ods_ci/tests/Resources/Files/operatorV2/subscription_template_z.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: rhods-operator + namespace: +spec: + channel: embedded + name: rhods-operator + source: + sourceNamespace: + config: + env: + - name: "DISABLE_DSC_CONFIG" \ No newline at end of file diff --git a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/131__rhods_operatorv2.robot b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/131__rhods_operatorv2.robot index 40f14598f..53b2df3fc 100644 --- a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/131__rhods_operatorv2.robot +++ b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/131__rhods_operatorv2.robot @@ -2,8 +2,8 @@ Documentation Tasks related to the operator rearchitecture Library String Library OperatingSystem -Library ../../libs/Helpers.py -Library Collections +Library ../../../../../libs/Helpers.py +Library Collections Resource ../../../../../tasks/Resources/RHODS_OLM/install/oc_install.robot Suite Setup Get Original Configuration Suite Teardown Patch DataScienceCluster CustomResource To Original Configuration diff --git a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/133__embedded_operator.robot b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/133__embedded_operator.robot new file mode 100644 index 000000000..e04a51bd2 --- /dev/null +++ b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/133__embedded_operator.robot @@ -0,0 +1,153 @@ +*** Settings *** +Library String +Library OpenShiftLibrary +Library OperatingSystem +Library ../../../../../libs/Helpers.py +Resource ../../../../../tasks/Resources/RHODS_OLM/install/oc_install.robot +Resource ../../../../../tasks/Resources/RHODS_OLM/pre-tasks/oc_is_operator_installed.robot +Resource ../../../../../tasks/Resources/RHODS_OLM/uninstall/uninstall.robot +Suite Teardown Uninstall RHODS V2 Embedded + + +***Variables*** +${cluster_type} selfmanaged +${image_url} ${EMPTY} +${dsci_name} default +${dsc_name} default + + +*** Test Cases *** +Install "Embedded Operator" RHODS + [Documentation] + [Tags] + Set Global Variable ${UPDATE_CHANNEL} embedded + Install Embedded RHODS ${cluster_type} ${image_url} + +Verify Embedded RHODS Installation + [Documentation] + [Tags] + RHODS Embedded Verification + + +*** Keywords *** +Install Embedded RHODS + [Arguments] ${cluster_type} ${image_url} + ${is_operator_installed} = Is RHODS Installed + IF ${is_operator_installed} + Fail Use a clean cluster to run this installation + END + IF "${cluster_type}" == "selfmanaged" + ${file_path} = Set Variable ods_ci/tests/Resources/Files/operatorV2/ + ${rc} ${out} = Run And Return Rc And Output oc create ns ${OPERATOR_NAMESPACE} + IF ${rc}!=0 Fail + Log ${out} console=yes + Copy File source=${file_path}operatorgroup_template.yaml destination=${file_path}operatorgroup_apply.yaml + Run sed -i 's//${OPERATOR_NAMESPACE}/' ${file_path}operatorgroup_apply.yaml + ${rc} ${out} = Run And Return Rc And Output oc apply -f ${file_path}operatorgroup_apply.yaml + IF ${rc}!=0 Fail + Log ${out} console=yes + Remove File ${file_path}operatorgroup_apply.yaml + ${image_url_bool} = Evaluate '${image_url}' == '' + IF ${image_url_bool} + # Prod 2.1 build + Log Installing prod 2.1 build console=yes + Copy File source=${file_path}subscription_template_21.yaml destination=${file_path}subscription_apply.yaml # robocop: disable + Run sed -i 's//${OPERATOR_NAMESPACE}/' ${file_path}subscription_apply.yaml + ${rc} ${out} = Run And Return Rc And Output oc apply -f ${file_path}subscription_apply.yaml + IF ${rc}!=0 Fail + Log ${out} console=yes + # Approve install since installPlan set to manual approval + ${rc} ${out} = Run And Return Rc And Output oc patch installplan $(oc get installplans -n redhat-ods-operator | grep -v NAME | awk '{print $1}') -n redhat-ods-operator --type='json' -p '[{"op": "replace", "path": "/spec/approved", "value": true}]' # robocop: disable + IF ${rc}!=0 Fail + Log ${out} console=yes + Remove File ${file_path}subscription_apply.yaml + ELSE + # z-stream releases + Log Installing z-stream build with IIB ${image_url} console=yes + ${image_escaped} = Escape Forward Slashes ${image_url} + Copy File source=${file_path}cs_template.yaml destination=${file_path}cs_apply.yaml + Run sed -i 's//${OPERATOR_NAMESPACE}/' ${file_path}cs_apply.yaml + Run sed -i 's//${image_escaped}/' ${file_path}cs_apply.yaml + ${rc} ${out} = Run And Return Rc And Output oc apply -f ${file_path}cs_apply.yaml + IF ${rc}!=0 Fail + Log ${out} console=yes + Remove File ${file_path}cs_apply.yaml + Copy File source=${file_path}subscription_template_z.yaml destination=${file_path}subscription_apply.yaml # robocop: disable + Run sed -i 's//${OPERATOR_NAMESPACE}/' ${file_path}subscription_apply.yaml + Run sed -i 's//rhods-catalog-dev/' ${file_path}subscription_apply.yaml + # Might need to be changed to openshift-marketplace in the future + Run sed -i 's//${OPERATOR_NAMESPACE}/' ${file_path}subscription_apply.yaml + ${rc} ${out} = Run And Return Rc And Output oc apply -f ${file_path}subscription_apply.yaml + IF ${rc}!=0 Fail + Log ${out} console=yes + Remove File ${file_path}subscription_apply.yaml + END + ELSE + FAIL Embedded RHODS can only be installed in self-managed clusters + END + +RHODS Embedded Verification + Log Verifying RHODS embedded installation console=yes + Log To Console Waiting for all RHODS resources to be up and running + Wait For Pods Numbers 1 + ... namespace=${OPERATOR_NAMESPACE} + ... label_selector=name=rhods-operator + ... timeout=2000 + Wait For Pods Status namespace=${OPERATOR_NAMESPACE} timeout=1200 + Log Verified ${OPERATOR_NAMESPACE} console=yes + IF '${APPLICATIONS_NAMESPACE}'!='${OPERATOR_NAMESPACE}' Namespace Should Not Exist ${APPLICATIONS_NAMESPACE} # robocop: disable + Log Verified ${APPLICATIONS_NAMESPACE} console=yes + IF '${MONITORING_NAMESPACE}'!='${OPERATOR_NAMESPACE}' Namespace Should Not Exist ${MONITORING_NAMESPACE} + Log Verified ${MONITORING_NAMESPACE} console=yes + IF '${NOTEBOOKS_NAMESPACE}'!='${OPERATOR_NAMESPACE}' Namespace Should Not Exist ${NOTEBOOKS_NAMESPACE} + Log Verified ${NOTEBOOKS_NAMESPACE} console=yes + V2 CRs Should Not Exist ${dsc_name} ${dsci_name} + Log Verified DSC and DSCI CRs console=yes + ${filepath} = Set Variable ods_ci/tests/Resources/Files/operatorV2/ + ${image_url_bool} = Evaluate '${image_url}' == '' + IF ${image_url_bool} + Process Resource List filename_in=${filepath}embedded.txt + ... filename_out=${filepath}embedded_processed_expected.txt + ELSE + Process Resource List filename_in=${filepath}embedded_cs.txt + ... filename_out=${filepath}embedded_processed_expected.txt + END + Run oc get $(oc api-resources --namespaced=true --verbs=list -o name | awk '{printf "%s%s",sep,$0;sep=","}') --ignore-not-found -n ${OPERATOR_NAMESPACE} -o=custom-columns=KIND:.kind,NAME:.metadata.name | sort -k1,1 -k2,2 | grep -v "PackageManifest\\|Event\\|ClusterServiceVersion\\|Lease" > ${filepath}embedded_runtime.txt # robocop: disable + Process Resource List filename_in=${filepath}embedded_runtime.txt + ... filename_out=${filepath}embedded_processed_runtime.txt + ${expected} = Get File ${filepath}embedded_processed_expected.txt + ${actual} = Get File ${filepath}embedded_processed_runtime.txt + Remove File ${filepath}embedded_processed_runtime.txt + Remove File ${filepath}embedded_runtime.txt + Remove File ${filepath}embedded_processed_expected.txt + Should Be Equal As Strings ${expected} ${actual} + +Namespace Should Not Exist + [Arguments] ${namespace} + ${rc} ${out} = Run And Return Rc And Output oc get namespace ${namespace} + Should Be Equal As Integers ${rc} 1 + Should Be Equal As Strings ${out} Error from server (NotFound): namespaces "${namespace}" not found + +V2 CRs Should Not Exist + [Arguments] ${dsc_name} ${dsci_name} + ${rc} ${out} = Run And Return Rc And Output oc get datasciencecluster ${dsc_name} + Should Be Equal As Integers ${rc} 1 + Should Be Equal As Strings ${out} Error from server (NotFound): datascienceclusters.datasciencecluster.opendatahub.io "${dsc_name}" not found # robocop: disable + ${rc} ${out} = Run And Return Rc And Output oc get dscinitialization ${dsci_name} + Should Be Equal As Integers ${rc} 1 + Should Be Equal As Strings ${out} Error from server (NotFound): dscinitializations.dscinitialization.opendatahub.io "${dsci_name}" not found # robocop: disable + +Uninstall RHODS V2 Embedded + [Documentation] Keyword to uninstall the version 2 of the RHODS operator in Self-Managed + ${return_code} ${output} Run And Return Rc And Output + ... oc delete subscription $(oc get subscription -n redhat-ods-operator --no-headers | awk '{print $1}') -n ${OPERATOR_NAMESPACE} # robocop: disable + Should Be Equal As Integers ${return_code} 0 msg=Error deleting RHODS subscription + ${return_code} ${output} Run And Return Rc And Output + ... oc delete operatorgroup $(oc get operatorgroup -n redhat-ods-operator --no-headers | awk '{print $1}') -n ${OPERATOR_NAMESPACE} # robocop: disable + Should Be Equal As Integers ${return_code} 0 msg=Error deleting operatorgroup + ${return_code} ${output} Run And Return Rc And Output oc delete ns -l opendatahub.io/generated-namespace + Verify Project Does Not Exists redhat-ods-applications + Verify Project Does Not Exists redhat-ods-monitoring + Verify Project Does Not Exists rhods-notebooks + ${return_code} ${output} Run And Return Rc And Output oc delete namespace ${OPERATOR_NAMESPACE} + Verify Project Does Not Exists ${OPERATOR_NAMESPACE} \ No newline at end of file From 579571a13fb004220208742d4ba7c2e1a56ff1c4 Mon Sep 17 00:00:00 2001 From: Berto D'Attoma <88311595+bdattoma@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:23:13 +0200 Subject: [PATCH 2/2] Add watsonx basic deployment tests for additional models (#974) * add test with flan grammar model (1) Signed-off-by: bdattoma * add streamed expected response + fix regex Signed-off-by: bdattoma * add flan-t5-large deployment and query Signed-off-by: bdattoma * update tags Signed-off-by: bdattoma * fix robocop alerts Signed-off-by: bdattoma --------- Signed-off-by: bdattoma --- .../Files/llm/model_expected_responses.json | 15 +++++ .../422__model_serving_llm.robot | 63 +++++++++++++++++-- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/ods_ci/tests/Resources/Files/llm/model_expected_responses.json b/ods_ci/tests/Resources/Files/llm/model_expected_responses.json index 8aefa76c7..22d37db7a 100644 --- a/ods_ci/tests/Resources/Files/llm/model_expected_responses.json +++ b/ods_ci/tests/Resources/Files/llm/model_expected_responses.json @@ -11,9 +11,24 @@ "bloom-560m-caikit": { "generatedTokenCount": 20, "response_text": "The temperature of water boiling depends on the temperature of the water. The boiling point of water is the" + }, + "flan-t5-large": { + "generatedTokenCount": 7, + "response_text": "212 ° f", + "streamed_response_text": "{ 'details': { 'input_token_count': '8' }}{ 'tokens': [ { 'text': '▁', 'logprob': -1.2747352123260498 } ], 'details': { 'generated_tokens': 1 }}{ 'generated_text': '212', 'tokens': [ { 'text': '212', 'logprob': -1.0382558107376099 } ], 'details': { 'generated_tokens': 2 }}{ 'generated_text': ' ', 'tokens': [ { 'text': '▁', 'logprob': -0.8835393786430359 } ], 'details': { 'generated_tokens': 3 }}{ 'generated_text': '°', 'tokens': [ { 'text': '°', 'logprob': -0.6830151677131653 } ], 'details': { 'generated_tokens': 4 }}{ 'generated_text': ' ', 'tokens': [ { 'text': '▁', 'logprob': -0.8991543650627136 } ], 'details': { 'generated_tokens': 5 }}{ 'generated_text': 'f', 'tokens': [ { 'text': 'f', 'logprob': -0.44153422117233276 } ], 'details': { 'generated_tokens': 6 }}{ 'tokens': [ { 'text': '\u003c/s\u003e', 'logprob': -0.015602776780724525 } ], 'details': { 'finish_reason': 'EOS_TOKEN', 'generated_tokens': 7 }}" } } + }, + { + "query_text": "This are an very wrong phraxe", + "models": { + "flan-t5-large-grammar-synthesis-caikit": { + "generatedTokenCount": 9, + "response_text": "This is a very wrong phrase.", + "streamed_response_text": "{'details':{'input_token_count':'10'}}{'generated_text':'This','tokens':[{'text':'▁This','logprob':}],'details':{'generated_tokens':1}}{'generated_text':'is','tokens':[{'text':'▁is','logprob':}],'details':{'generated_tokens':2}}{'generated_text':'','tokens':[{'text':'▁','logprob':}],'details':{'generated_tokens':3}}{'generated_text':'a','tokens':[{'text':'a','logprob':}],'details':{'generated_tokens':4}}{'generated_text':'very','tokens':[{'text':'▁very','logprob':}],'details':{'generated_tokens':5}}{'generated_text':'wrong','tokens':[{'text':'▁wrong','logprob':}],'details':{'generated_tokens':6}}{'generated_text':'phrase','tokens':[{'text':'▁phrase','logprob':}],'details':{'generated_tokens':7}}{'generated_text':'.','tokens':[{'text':'.','logprob':}],'details':{'generated_tokens':8}}{'tokens':[{'text':'','logprob':}],'details':{'finish_reason':'EOS_TOKEN','generated_tokens':9}}" + } + } } ] } diff --git a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/422__model_serving_llm.robot b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/422__model_serving_llm.robot index a8e5b877e..b94afc936 100644 --- a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/422__model_serving_llm.robot +++ b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/422__model_serving_llm.robot @@ -42,8 +42,12 @@ ${SKIP_PREREQS_INSTALL}= ${FALSE} ${SCRIPT_BASED_INSTALL}= ${FALSE} ${MODELS_BUCKET}= ${S3.BUCKET_3} ${FLAN_MODEL_S3_DIR}= flan-t5-small +${FLAN_GRAMMAR_MODEL_S3_DIR}= flan-t5-large-grammar-synthesis-caikit +${FLAN_LARGE_MODEL_S3_DIR}= flan-t5-large ${BLOOM_MODEL_S3_DIR}= bloom-560m ${FLAN_STORAGE_URI}= s3://${S3.BUCKET_3.NAME}/${FLAN_MODEL_S3_DIR}/ +${FLAN_GRAMMAR_STORAGE_URI}= s3://${S3.BUCKET_3.NAME}/${FLAN_GRAMMAR_MODEL_S3_DIR}/ +${FLAN_LARGE_STORAGE_URI}= s3://${S3.BUCKET_3.NAME}/${FLAN_LARGE_MODEL_S3_DIR}/ ${BLOOM_STORAGE_URI}= s3://${S3.BUCKET_3.NAME}/${BLOOM_MODEL_S3_DIR}/ ${CAIKIT_ALLTOKENS_ENDPOINT}= caikit.runtime.Nlp.NlpService/TextGenerationTaskPredict ${CAIKIT_STREAM_ENDPOINT}= caikit.runtime.Nlp.NlpService/ServerStreamingTextGenerationTaskPredict @@ -379,6 +383,53 @@ Verify Non Admin Can Serve And Query A Model [Teardown] Run Keywords Login To OCP Using API ${OCP_ADMIN_USER.USERNAME} ${OCP_ADMIN_USER.PASSWORD} AND ... Clean Up Test Project test_ns=${test_namespace} isvc_names=${models_names} +Verify User Can Serve And Query Flan-t5 Grammar Syntax Corrector + [Documentation] Deploys and queries flan-t5-large-grammar-synthesis model + [Tags] ODS-2441 WatsonX + [Setup] Set Project And Runtime namespace=grammar-model + ${test_namespace}= Set Variable grammar-model + ${flan_model_name}= Set Variable flan-t5-large-grammar-synthesis-caikit + ${models_names}= Create List ${flan_model_name} + Compile Inference Service YAML isvc_name=${flan_model_name} + ... sa_name=${DEFAULT_BUCKET_SA_NAME} + ... model_storage_uri=${FLAN_GRAMMAR_STORAGE_URI} + Deploy Model Via CLI isvc_filepath=${LLM_RESOURCES_DIRPATH}/caikit_isvc_filled.yaml + ... namespace=${test_namespace} + Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} + Query Models And Check Responses Multiple Times models_names=${models_names} + ... endpoint=${CAIKIT_ALLTOKENS_ENDPOINT} n_times=1 + ... namespace=${test_namespace} query_idx=1 + Query Models And Check Responses Multiple Times models_names=${models_names} + ... endpoint=${CAIKIT_STREAM_ENDPOINT} n_times=1 streamed_response=${TRUE} + ... namespace=${test_namespace} query_idx=${1} + [Teardown] Clean Up Test Project test_ns=${test_namespace} + ... isvc_names=${models_names} + +Verify User Can Serve And Query Flan-t5 Large + [Documentation] Deploys and queries flan-t5-large model + [Tags] ODS-2434 WatsonX + [Setup] Set Project And Runtime namespace=flan-t5-large3 + ${test_namespace}= Set Variable flan-t5-large3 + ${flan_model_name}= Set Variable flan-t5-large + ${models_names}= Create List ${flan_model_name} + Compile Inference Service YAML isvc_name=${flan_model_name} + ... sa_name=${DEFAULT_BUCKET_SA_NAME} + ... model_storage_uri=${FLAN_LARGE_STORAGE_URI} + Deploy Model Via CLI isvc_filepath=${LLM_RESOURCES_DIRPATH}/caikit_isvc_filled.yaml + ... namespace=${test_namespace} + Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} + Query Models And Check Responses Multiple Times models_names=${models_names} + ... endpoint=${CAIKIT_ALLTOKENS_ENDPOINT} n_times=1 + ... namespace=${test_namespace} query_idx=${0} + Query Models And Check Responses Multiple Times models_names=${models_names} + ... endpoint=${CAIKIT_STREAM_ENDPOINT} n_times=1 streamed_response=${TRUE} + ... namespace=${test_namespace} query_idx=${0} + [Teardown] Clean Up Test Project test_ns=${test_namespace} + ... isvc_names=${models_names} + + *** Keywords *** Install Model Serving Stack Dependencies [Documentation] Instaling And Configuring dependency operators: Service Mesh and Serverless. @@ -718,11 +769,13 @@ Model Response Should Match The Expectation ${cleaned_response_text}= Replace String Using Regexp ${model_response} \\s+ ${EMPTY} ${rc} ${cleaned_response_text}= Run And Return Rc And Output echo -e '${cleaned_response_text}' ${cleaned_response_text}= Replace String Using Regexp ${cleaned_response_text} " ' - ${cleaned_response_text}= Replace String Using Regexp ${cleaned_response_text} [-]?\\d.\\d+ + ${cleaned_response_text}= Replace String Using Regexp ${cleaned_response_text} + ... [-]?\\d.\\d+[e]?[-]?\\d+ Log ${cleaned_response_text} ${cleaned_exp_response_text}= Replace String Using Regexp ... ${EXP_RESPONSES}[queries][${query_idx}][models][${model_name}][streamed_response_text] - ... [-]?\\d.\\d+ + ... [-]?\\d.\\d+[e]?[-]?\\d+ + ${cleaned_exp_response_text}= Replace String Using Regexp ${cleaned_exp_response_text} \\s+ ${EMPTY} Should Be Equal ${cleaned_response_text} ${cleaned_exp_response_text} END @@ -730,13 +783,13 @@ Query Models And Check Responses Multiple Times [Documentation] Queries and checks the responses of the given models in a loop ... running ${n_times}. For each loop run it queries all the model in sequence [Arguments] ${models_names} ${namespace} ${endpoint}=${CAIKIT_ALLTOKENS_ENDPOINT} ${n_times}=10 - ... ${streamed_response}=${FALSE} + ... ${streamed_response}=${FALSE} ${query_idx}=0 FOR ${counter} IN RANGE 0 ${n_times} 1 Log ${counter} FOR ${index} ${model_name} IN ENUMERATE @{models_names} Log ${index}: ${model_name} ${host}= Get KServe Inference Host Via CLI isvc_name=${model_name} namespace=${namespace} - ${body}= Set Variable '{"text": "${EXP_RESPONSES}[queries][0][query_text]"}' + ${body}= Set Variable '{"text": "${EXP_RESPONSES}[queries][${query_idx}][query_text]"}' ${header}= Set Variable 'mm-model-id: ${model_name}' ${res}= Query Model With GRPCURL host=${host} port=443 ... endpoint=${endpoint} @@ -744,7 +797,7 @@ Query Models And Check Responses Multiple Times ... insecure=${TRUE} skip_res_json=${streamed_response} Run Keyword And Continue On Failure ... Model Response Should Match The Expectation model_response=${res} model_name=${model_name} - ... query_idx=0 streamed_response=${streamed_response} + ... streamed_response=${streamed_response} query_idx=${query_idx} END END