diff --git a/ads/aqua/app.py b/ads/aqua/app.py index 72a7e29e1..c66d7e3dd 100644 --- a/ads/aqua/app.py +++ b/ads/aqua/app.py @@ -7,11 +7,16 @@ import traceback from dataclasses import fields from datetime import datetime, timedelta -from typing import Any, Dict, Optional, Union +from itertools import chain +from typing import Any, Dict, List, Optional, Union import oci from cachetools import TTLCache, cached -from oci.data_science.models import UpdateModelDetails, UpdateModelProvenanceDetails +from oci.data_science.models import ( + ContainerSummary, + UpdateModelDetails, + UpdateModelProvenanceDetails, +) from ads import set_auth from ads.aqua import logger @@ -24,6 +29,11 @@ is_valid_ocid, load_config, ) +from ads.aqua.config.container_config import ( + AquaContainerConfig, + AquaContainerConfigItem, +) +from ads.aqua.constants import SERVICE_MANAGED_CONTAINER_URI_SCHEME from ads.common import oci_client as oc from ads.common.auth import default_signer from ads.common.utils import UNKNOWN, extract_region, is_path_exists @@ -240,7 +250,9 @@ def create_model_catalog( .with_custom_metadata_list(model_custom_metadata) .with_defined_metadata_list(model_taxonomy_metadata) .with_provenance_metadata(ModelProvenanceMetadata(training_id=UNKNOWN)) - .with_defined_tags(**(defined_tags or {})) # Create defined tags when a model is created. + .with_defined_tags( + **(defined_tags or {}) + ) # Create defined tags when a model is created. .create( **kwargs, ) @@ -271,6 +283,43 @@ def if_artifact_exist(self, model_id: str, **kwargs) -> bool: logger.info(f"Artifact not found in model {model_id}.") return False + def get_config_from_metadata( + self, model_id: str, metadata_key: str + ) -> ModelConfigResult: + """Gets the config for the given Aqua model from model catalog metadata content. + + Parameters + ---------- + model_id: str + The OCID of the Aqua model. + metadata_key: str + The metadata key name where artifact content is stored + Returns + ------- + ModelConfigResult + A Pydantic model containing the model_details (extracted from OCI) and the config dictionary. + """ + config = {} + oci_model = self.ds_client.get_model(model_id).data + try: + config = self.ds_client.get_model_defined_metadatum_artifact_content( + model_id, metadata_key + ).data.content.decode("utf-8") + return ModelConfigResult(config=json.loads(config), model_details=oci_model) + except UnicodeDecodeError as ex: + logger.error( + f"Failed to decode content for '{metadata_key}' in defined metadata for model '{model_id}' : {ex}" + ) + except json.JSONDecodeError as ex: + logger.error( + f"Invalid JSON format for '{metadata_key}' in defined metadata for model '{model_id}' : {ex}" + ) + except Exception as ex: + logger.error( + f"Failed to retrieve defined metadata key '{metadata_key}' for model '{model_id}': {ex}" + ) + return ModelConfigResult(config=config, model_details=oci_model) + @cached(cache=TTLCache(maxsize=1, ttl=timedelta(minutes=1), timer=datetime.now)) def get_config( self, @@ -310,22 +359,7 @@ def get_config( raise AquaRuntimeError(f"Target model {oci_model.id} is not an Aqua model.") config: Dict[str, Any] = {} - - # if the current model has a service model tag, then - if Tags.AQUA_SERVICE_MODEL_TAG in oci_model.freeform_tags: - base_model_ocid = oci_model.freeform_tags[Tags.AQUA_SERVICE_MODEL_TAG] - logger.info( - f"Base model found for the model: {oci_model.id}. " - f"Loading {config_file_name} for base model {base_model_ocid}." - ) - if config_folder == ConfigFolder.ARTIFACT: - artifact_path = get_artifact_path(oci_model.custom_metadata_list) - else: - base_model = self.ds_client.get_model(base_model_ocid).data - artifact_path = get_artifact_path(base_model.custom_metadata_list) - else: - logger.info(f"Loading {config_file_name} for model {oci_model.id}...") - artifact_path = get_artifact_path(oci_model.custom_metadata_list) + artifact_path = get_artifact_path(oci_model.custom_metadata_list) if not artifact_path: logger.debug( f"Failed to get artifact path from custom metadata for the model: {model_id}" @@ -340,7 +374,7 @@ def get_config( config_file_path = os.path.join(config_path, config_file_name) if is_path_exists(config_file_path): try: - logger.debug( + logger.info( f"Loading config: `{config_file_name}` from `{config_path}`" ) config = load_config( @@ -361,6 +395,85 @@ def get_config( return ModelConfigResult(config=config, model_details=oci_model) + def get_container_image(self, container_type: str = None) -> str: + """ + Gets the latest smc container complete image name from the given container type. + + Parameters + ---------- + container_type: str + type of container, can be either odsc-vllm-serving, odsc-llm-fine-tuning, odsc-llm-evaluate + + Returns + ------- + str: + A complete container name along with version. ex: dsmc://odsc-vllm-serving:0.7.4.1 + """ + + containers = self.list_service_containers() + container = next( + (c for c in containers if c.is_latest and c.family_name == container_type), + None, + ) + if not container: + raise AquaValueError(f"Invalid container type : {container_type}") + container_image = ( + SERVICE_MANAGED_CONTAINER_URI_SCHEME + + container.container_name + + ":" + + container.tag + ) + return container_image + + @cached(cache=TTLCache(maxsize=20, ttl=timedelta(minutes=30), timer=datetime.now)) + def list_service_containers(self) -> List[ContainerSummary]: + """ + List containers from containers.conf in OCI Datascience control plane + """ + containers = self.ds_client.list_containers().data + return containers + + def get_container_config(self) -> AquaContainerConfig: + """ + Fetches latest containers from containers.conf in OCI Datascience control plane + + Returns + ------- + AquaContainerConfig + An Object that contains latest container info for the given container family + + """ + return AquaContainerConfig.from_service_config( + service_containers=self.list_service_containers() + ) + + def get_container_config_item( + self, container_family: str + ) -> AquaContainerConfigItem: + """ + Fetches latest container for given container_family_name from containers.conf in OCI Datascience control plane + + Returns + ------- + AquaContainerConfigItem + An Object that contains latest container info for the given container family + + """ + + aqua_container_config = self.get_container_config() + inference_config = aqua_container_config.inference.values() + ft_config = aqua_container_config.finetune.values() + eval_config = aqua_container_config.evaluate.values() + container = next( + ( + container + for container in chain(inference_config, ft_config, eval_config) + if container.family.lower() == container_family.lower() + ), + None, + ) + return container + @property def telemetry(self): if not self._telemetry: diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index a1df4a99b..568bf9a68 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -52,9 +52,13 @@ COMPARTMENT_MAPPING_KEY, CONSOLE_LINK_RESOURCE_TYPE_MAPPING, CONTAINER_INDEX, + DEPLOYMENT_CONFIG, + FINE_TUNING_CONFIG, HF_LOGIN_DEFAULT_TIMEOUT, + LICENSE, MAXIMUM_ALLOWED_DATASET_IN_BYTE, MODEL_BY_REFERENCE_OSS_PATH_KEY, + README, SERVICE_MANAGED_CONTAINER_URI_SCHEME, SUPPORTED_FILE_FORMATS, TEI_CONTAINER_DEFAULT_HOST, @@ -88,6 +92,14 @@ logger = logging.getLogger("ads.aqua") +DEFINED_METADATA_TO_FILE_MAP = { + "readme": README, + "license": LICENSE, + "finetuneconfiguration": FINE_TUNING_CONFIG, + "deploymentconfiguration": DEPLOYMENT_CONFIG, +} + + class LifecycleStatus(ExtendedEnum): UNKNOWN = "" @@ -552,57 +564,6 @@ def service_config_path(): return f"oci://{AQUA_SERVICE_MODELS_BUCKET}@{CONDA_BUCKET_NS}/service_models/config" -@cached(cache=TTLCache(maxsize=1, ttl=timedelta(minutes=10), timer=datetime.now)) -def get_container_config(): - config = load_config( - file_path=service_config_path(), - config_file_name=CONTAINER_INDEX, - ) - - return config - - -def get_container_image( - config_file_name: str = None, container_type: str = None -) -> str: - """Gets the image name from the given model and container type. - Parameters - ---------- - config_file_name: str - name of the config file - container_type: str - type of container, can be either deployment-container, finetune-container, evaluation-container - - Returns - ------- - Dict: - A dict of allowed configs. - """ - - container_image = UNKNOWN - config = config_file_name or get_container_config() - config_file_name = service_config_path() - - if container_type not in config: - return UNKNOWN - - mapping = config[container_type] - versions = [obj["version"] for obj in mapping] - # assumes numbered versions, update if `latest` is used - latest = get_max_version(versions) - for obj in mapping: - if obj["version"] == str(latest): - container_image = f"{obj['name']}:{obj['version']}" - break - - if not container_image: - raise AquaValueError( - f"{config_file_name} is missing name and/or version details." - ) - - return container_image - - def fetch_service_compartment() -> Union[str, None]: """ Loads the compartment mapping json from service bucket. diff --git a/ads/aqua/config/config.py b/ads/aqua/config/config.py deleted file mode 100644 index 22cf9f27d..000000000 --- a/ads/aqua/config/config.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2024 Oracle and/or its affiliates. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ - - -from typing import Optional - -from ads.aqua.common.entities import ContainerSpec -from ads.aqua.common.utils import get_container_config -from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig - -DEFAULT_EVALUATION_CONTAINER = "odsc-llm-evaluate" - - -def get_evaluation_service_config( - container: Optional[str] = DEFAULT_EVALUATION_CONTAINER, -) -> EvaluationServiceConfig: - """ - Retrieves the common evaluation configuration. - - Returns - ------- - EvaluationServiceConfig: The evaluation common config. - """ - - container = container or DEFAULT_EVALUATION_CONTAINER - return EvaluationServiceConfig( - **get_container_config() - .get(ContainerSpec.CONTAINER_SPEC, {}) - .get(container, {}) - ) diff --git a/ads/aqua/config/container_config.py b/ads/aqua/config/container_config.py index 871cea6d8..e2184e0f8 100644 --- a/ads/aqua/config/container_config.py +++ b/ads/aqua/config/container_config.py @@ -1,14 +1,21 @@ #!/usr/bin/env python # Copyright (c) 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ - +import json from typing import Dict, List, Optional +from oci.data_science.models import ContainerSummary from pydantic import Field from ads.aqua.common.entities import ContainerSpec from ads.aqua.config.utils.serializer import Serializable +from ads.aqua.constants import ( + SERVICE_MANAGED_CONTAINER_URI_SCHEME, + UNKNOWN_JSON_LIST, + UNKNOWN_JSON_STR, +) from ads.common.extended_enum import ExtendedEnum +from ads.common.utils import UNKNOWN class Usage(ExtendedEnum): @@ -46,6 +53,9 @@ class AquaContainerConfigSpec(Serializable): restricted_params: Optional[List[str]] = Field( default_factory=list, description="List of restricted parameters." ) + evaluation_configuration: Optional[Dict] = Field( + default_factory=dict, description="Dict of evaluation configuration." + ) class Config: extra = "allow" @@ -125,6 +135,111 @@ def to_dict(self): "evaluate": list(self.evaluate.values()), } + @classmethod + def from_service_config( + cls, service_containers: List[ContainerSummary] + ) -> "AquaContainerConfig": + """ + Creates an AquaContainerConfig instance from a service containers.conf. + + Parameters + ---------- + service_containers (List[Any]): List of containers specified in containers.conf + Returns + ------- + AquaContainerConfig: The constructed container configuration. + """ + + inference_items: Dict[str, AquaContainerConfigItem] = {} + finetune_items: Dict[str, AquaContainerConfigItem] = {} + evaluate_items: Dict[str, AquaContainerConfigItem] = {} + for container in service_containers: + if not container.is_latest: + continue + container_item = AquaContainerConfigItem( + name=SERVICE_MANAGED_CONTAINER_URI_SCHEME + container.container_name, + version=container.tag, + display_name=container.display_name, + family=container.family_name, + usages=container.usages, + platforms=[], + model_formats=[], + spec=None, + ) + container_type = container.family_name + usages = [x.upper() for x in container.usages] + if "INFERENCE" in usages or "MULTI_MODEL" in usages: + container_item.platforms.append( + container.workload_configuration_details_list[ + 0 + ].additional_configurations.get("platforms") + ) + container_item.model_formats.append( + container.workload_configuration_details_list[ + 0 + ].additional_configurations.get("modelFormats") + ) + env_vars = [ + { + "MODEL_DEPLOY_PREDICT_ENDPOINT": container.workload_configuration_details_list[ + 0 + ].additional_configurations.get( + "MODEL_DEPLOY_PREDICT_ENDPOINT", UNKNOWN + ), + "MODEL_DEPLOY_HEALTH_ENDPOINT": container.workload_configuration_details_list[ + 0 + ].additional_configurations.get( + "MODEL_DEPLOY_HEALTH_ENDPOINT", UNKNOWN + ), + "MODEL_DEPLOY_ENABLE_STREAMING": container.workload_configuration_details_list[ + 0 + ].additional_configurations.get( + "MODEL_DEPLOY_ENABLE_STREAMING", UNKNOWN + ), + "PORT": container.workload_configuration_details_list[ + 0 + ].additional_configurations.get("PORT", ""), + "HEALTH_CHECK_PORT": container.workload_configuration_details_list[ + 0 + ].additional_configurations.get("HEALTH_CHECK_PORT", UNKNOWN), + } + ] + container_spec = AquaContainerConfigSpec( + cli_param=container.workload_configuration_details_list[0].cmd, + server_port=str( + container.workload_configuration_details_list[0].server_port + ), + health_check_port=str( + container.workload_configuration_details_list[ + 0 + ].health_check_port + ), + env_vars=env_vars, + restricted_params=json.loads( + container.workload_configuration_details_list[ + 0 + ].additional_configurations.get("restrictedParams") + or UNKNOWN_JSON_LIST + ), + evaluation_configuration=json.loads( + container.workload_configuration_details_list[ + 0 + ].additional_configurations.get( + "evaluationConfiguration", UNKNOWN_JSON_STR + ) + ), + ) + container_item.spec = container_spec + if "INFERENCE" in usages or "MULTI_MODEL" in usages: + inference_items[container_type] = container_item + if "FINE_TUNE" in usages: + finetune_items[container_type] = container_item + if "EVALUATION" in usages: + evaluate_items[container_type] = container_item + return cls( + inference=inference_items, finetune=finetune_items, evaluate=evaluate_items + ) + @classmethod def from_container_index_json( cls, @@ -146,7 +261,6 @@ def from_container_index_json( # TODO: Return this logic back if necessary in the next iteraion. # if not config: # config = get_container_config() - inference_items: Dict[str, AquaContainerConfigItem] = {} finetune_items: Dict[str, AquaContainerConfigItem] = {} evaluate_items: Dict[str, AquaContainerConfigItem] = {} @@ -194,15 +308,15 @@ def from_container_index_json( else None ), ) - if container.get("type") == "inference": + if container.get("type").lower() == "inference": inference_items[container_type] = container_item elif ( - container.get("type") == "fine-tune" + container.get("type").lower() == "fine-tune" or container_type == "odsc-llm-fine-tuning" ): finetune_items[container_type] = container_item elif ( - container.get("type") == "evaluate" + container.get("type").lower() in ("evaluation", "evaluate") or container_type == "odsc-llm-evaluate" ): evaluate_items[container_type] = container_item diff --git a/ads/aqua/config/evaluation/evaluation_service_config.py b/ads/aqua/config/evaluation/evaluation_service_config.py index 8edaf974b..f128fd8a4 100644 --- a/ads/aqua/config/evaluation/evaluation_service_config.py +++ b/ads/aqua/config/evaluation/evaluation_service_config.py @@ -3,12 +3,16 @@ # Copyright (c) 2024, 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +import json from typing import Any, Dict, List, Optional +from oci.data_science.models import ContainerSummary from pydantic import Field from ads.aqua.config.utils.serializer import Serializable +DEFAULT_EVALUATION_CONTAINER = "odsc-llm-evaluate" + class ShapeFilterConfig(Serializable): """Represents the filtering options for a specific shape.""" @@ -105,3 +109,39 @@ class EvaluationServiceConfig(Serializable): class Config: extra = "allow" + + @staticmethod + def from_oci_container_config( + oci_container_config: ContainerSummary, + ) -> "EvaluationServiceConfig": + """ + Returns EvaluationServiceConfig for given oci_container_config + """ + + shapes = json.loads( + oci_container_config.workload_configuration_details_list[ + 0 + ].use_case_configuration.additional_configurations.get("shapes") + ) + metrics = json.loads( + oci_container_config.workload_configuration_details_list[ + 0 + ].use_case_configuration.additional_configurations.get("metrics") + ) + model_params = ModelParamsConfig( + default={ + "model": "odsc-llm", + "max_tokens": 500, + "temperature": 0.7, + "top_p": 0.9, + "top_k": 50, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "stop": [], + } + ) + return EvaluationServiceConfig( + ui_config=UIConfig( + model_params=model_params, shapes=shapes, metrics=metrics + ) + ) diff --git a/ads/aqua/constants.py b/ads/aqua/constants.py index 09f2e2cfc..66a2ac8c9 100644 --- a/ads/aqua/constants.py +++ b/ads/aqua/constants.py @@ -6,9 +6,10 @@ UNKNOWN_VALUE = "" READY_TO_IMPORT_STATUS = "TRUE" UNKNOWN_DICT = {} -README = "README.md" -LICENSE_TXT = "config/LICENSE.txt" DEPLOYMENT_CONFIG = "deployment_config.json" +FINE_TUNING_CONFIG = "ft_config.json" +README = "README.md" +LICENSE = "LICENSE.txt" AQUA_MODEL_TOKENIZER_CONFIG = "tokenizer_config.json" COMPARTMENT_MAPPING_KEY = "service-model-compartment" CONTAINER_INDEX = "container_index.json" @@ -16,6 +17,7 @@ EVALUATION_REPORT_MD = "report.md" EVALUATION_REPORT = "report.html" UNKNOWN_JSON_STR = "{}" +UNKNOWN_JSON_LIST = "[]" FINE_TUNING_RUNTIME_CONTAINER = "iad.ocir.io/ociodscdev/aqua_ft_cuda121:0.3.17.20" DEFAULT_FT_BLOCK_STORAGE_SIZE = 750 DEFAULT_FT_REPLICA = 1 @@ -88,25 +90,25 @@ TEI_CONTAINER_DEFAULT_HOST = "8080" OCI_OPERATION_FAILURES = { - "list_model_deployments": "Unable to list model deployments. See tips for troubleshooting: ", - "list_models": "Unable to list models. See tips for troubleshooting: ", - "get_namespace": "Unable to access specified Object Storage Bucket. See tips for troubleshooting: ", - "list_log_groups":"Unable to access logs. See tips for troubleshooting: " , - "list_buckets": "Unable to list Object Storage Bucket. See tips for troubleshooting: ", - "put_object": "Unable to access or find Object Storage Bucket. See tips for troubleshooting: ", - "list_model_version_sets": "Unable to create or fetch model version set. See tips for troubleshooting:", - "update_model": "Unable to update model. See tips for troubleshooting: ", - "list_data_science_private_endpoints": "Unable to access private endpoint. See tips for troubleshooting: ", - "create_model" : "Unable to register model. See tips for troubleshooting: ", - "create_deployment": "Unable to create deployment. See tips for troubleshooting: ", - "create_model_version_sets" : "Unable to create model version set. See tips for troubleshooting: ", - "create_job": "Unable to create job. See tips for troubleshooting: ", - "create_job_run": "Unable to create job run. See tips for troubleshooting: ", + "list_model_deployments": "Unable to list model deployments. See tips for troubleshooting: ", + "list_models": "Unable to list models. See tips for troubleshooting: ", + "get_namespace": "Unable to access specified Object Storage Bucket. See tips for troubleshooting: ", + "list_log_groups": "Unable to access logs. See tips for troubleshooting: ", + "list_buckets": "Unable to list Object Storage Bucket. See tips for troubleshooting: ", + "put_object": "Unable to access or find Object Storage Bucket. See tips for troubleshooting: ", + "list_model_version_sets": "Unable to create or fetch model version set. See tips for troubleshooting:", + "update_model": "Unable to update model. See tips for troubleshooting: ", + "list_data_science_private_endpoints": "Unable to access private endpoint. See tips for troubleshooting: ", + "create_model": "Unable to register model. See tips for troubleshooting: ", + "create_deployment": "Unable to create deployment. See tips for troubleshooting: ", + "create_model_version_sets": "Unable to create model version set. See tips for troubleshooting: ", + "create_job": "Unable to create job. See tips for troubleshooting: ", + "create_job_run": "Unable to create job run. See tips for troubleshooting: ", } STATUS_CODE_MESSAGES = { - "400": "Could not process your request due to invalid input.", - "403": "We're having trouble processing your request with the information provided.", - "404": "Authorization Failed: The resource you're looking for isn't accessible.", - "408": "Server is taking too long to respond, please try again.", + "400": "Could not process your request due to invalid input.", + "403": "We're having trouble processing your request with the information provided.", + "404": "Authorization Failed: The resource you're looking for isn't accessible.", + "408": "Server is taking too long to respond, please try again.", } diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 877030459..f2c27e8e2 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -41,13 +41,14 @@ from ads.aqua.common.utils import ( extract_id_and_name_from_tag, fire_and_forget, - get_container_config, - get_container_image, is_valid_ocid, upload_local_to_os, ) -from ads.aqua.config.config import get_evaluation_service_config -from ads.aqua.config.container_config import AquaContainerConfig +from ads.aqua.config.evaluation.evaluation_service_config import ( + DEFAULT_EVALUATION_CONTAINER, + EvaluationServiceConfig, + MetricConfig, +) from ads.aqua.constants import ( CONSOLE_LINK_RESOURCE_TYPE_MAPPING, EVALUATION_REPORT, @@ -80,7 +81,13 @@ from ads.aqua.model.constants import ModelCustomMetadataFields from ads.common.auth import default_signer from ads.common.object_storage_details import ObjectStorageDetails -from ads.common.utils import UNKNOWN, get_console_link, get_files, get_log_links +from ads.common.utils import ( + UNKNOWN, + get_console_link, + get_files, + get_log_links, + read_file, +) from ads.config import ( AQUA_JOB_SUBNET_ID, COMPARTMENT_OCID, @@ -260,10 +267,10 @@ def create( runtime = ModelDeploymentContainerRuntime.from_dict( evaluation_source.runtime.to_dict() ) - inference_config = AquaContainerConfig.from_container_index_json( - config=get_container_config(), enable_spec=True - ).inference - for container in inference_config.values(): + inference_config = ( + self.get_container_config().to_dict().get("inference") + ) + for container in inference_config: if container.name == runtime.image[: runtime.image.rfind(":")]: eval_inference_configuration = ( container.spec.evaluation_configuration @@ -801,14 +808,13 @@ def _get_service_model_name( return source.display_name - @staticmethod - def _get_evaluation_container(source_id: str) -> str: + def _get_evaluation_container(self, source_id: str) -> str: # todo: use the source, identify if it is a model or a deployment. If latter, then fetch the base model id # from the deployment object, and call ds_client.get_model() to get model details. Use custom metadata to # get the container_type_key. Pass this key as container_type to get_container_image method. # fetch image name from config - container_image = get_container_image( + container_image = self.get_container_image( container_type="odsc-llm-evaluate", ) logger.info(f"Aqua Image used for evaluating {source_id} :{container_image}") @@ -1102,11 +1108,18 @@ def get_status(self, eval_id: str) -> dict: "loggroup_url": loggroup_url, } - def get_supported_metrics(self) -> dict: + def get_supported_metrics(self) -> List[MetricConfig]: """Gets a list of supported metrics for evaluation.""" - return [ - item.to_dict() for item in get_evaluation_service_config().ui_config.metrics - ] + containers = self.list_service_containers() + container_item = next( + c + for c in containers + if c.is_latest and c.family_name == DEFAULT_EVALUATION_CONTAINER + ) + evaluation_service_config = EvaluationServiceConfig.from_oci_container_config( + container_item + ) + return evaluation_service_config.ui_config.metrics @telemetry(entry_point="plugin=evaluation&action=load_metrics", name="aqua") def load_metrics(self, eval_id: str) -> AquaEvalMetrics: @@ -1130,17 +1143,31 @@ def load_metrics(self, eval_id: str) -> AquaEvalMetrics: with tempfile.TemporaryDirectory() as temp_dir: logger.info(f"Downloading evaluation artifact: {eval_id}.") - DataScienceModel.from_id(eval_id).download_artifact( - temp_dir, - auth=self._auth, - ) + + dsc_model = DataScienceModel.from_id(eval_id) + if dsc_model.if_model_custom_metadata_artifact_exist( + EVALUATION_REPORT_MD + ) and dsc_model.if_model_custom_metadata_artifact_exist( + EVALUATION_REPORT_JSON + ): + logger.info( + f"Fetching {EVALUATION_REPORT_MD} and {EVALUATION_REPORT_JSON} from custom metadata..." + ) + dsc_model.get_custom_metadata_artifact(EVALUATION_REPORT_MD, temp_dir) + dsc_model.get_custom_metadata_artifact(EVALUATION_REPORT_JSON, temp_dir) + else: + logger.info("Fetching Evaluation Reports from OSS bucket...") + dsc_model.download_artifact( + temp_dir, + auth=self._auth, + ) files_in_artifact = get_files(temp_dir) md_report_content = self._read_from_artifact( temp_dir, files_in_artifact, EVALUATION_REPORT_MD ) - # json report not availiable for failed evaluation + # json report not available for failed evaluation try: json_report = json.loads( self._read_from_artifact( @@ -1239,14 +1266,46 @@ def download_report(self, eval_id) -> AquaEvalReport: with tempfile.TemporaryDirectory() as temp_dir: logger.info(f"Downloading evaluation artifact for {eval_id}.") - DataScienceModel.from_id(eval_id).download_artifact( - temp_dir, - auth=self._auth, + dsc_model = DataScienceModel.from_id(eval_id) + if_custom_metadata_exists = ( + dsc_model.if_model_custom_metadata_artifact_exist(EVALUATION_REPORT) ) - content = self._read_from_artifact( - temp_dir, get_files(temp_dir), EVALUATION_REPORT - ) - + if if_custom_metadata_exists: + logger.info(f"Fetching {EVALUATION_REPORT} from custom metadata.") + dsc_model.get_custom_metadata_artifact(EVALUATION_REPORT, temp_dir) + else: + logger.info(f"Fetching {EVALUATION_REPORT} from Model artifact.") + dsc_model.download_artifact( + temp_dir, + auth=self._auth, + ) + files_in_artifact = get_files(temp_dir) + if not len(files_in_artifact): + try: + evaluation_output_path = dsc_model.custom_metadata_list.get( + EvaluationCustomMetadata.EVALUATION_OUTPUT_PATH + ).value + report_path = ( + evaluation_output_path.rstrip("/") + + "/" + + eval_id + + "/" + + EVALUATION_REPORT + ) + logger.info( + f"Fetching {EVALUATION_REPORT} from {report_path} for evaluation {eval_id}" + ) + content = read_file( + file_path=report_path, auth=default_signer() + ).encode() + except ValueError as err: + raise AquaValueError( + f"{EvaluationCustomMetadata.EVALUATION_OUTPUT_PATH} is missing from custom metadata for the model {eval_id}" + ) from err + else: + content = self._read_from_artifact( + temp_dir, files_in_artifact, EVALUATION_REPORT + ) report = AquaEvalReport( evaluation_id=eval_id, content=base64.b64encode(content).decode() ) @@ -1376,11 +1435,10 @@ def delete(self, eval_id): @staticmethod @fire_and_forget - def _delete_job_and_model(job, model): + def _delete_job_and_model(job: DataScienceJob, model: DataScienceModel): try: job.dsc_job.delete(force_delete=True) logger.info(f"Deleting Job: {job.job_id} for evaluation {model.id}") - model.delete() logger.info(f"Deleting evaluation: {model.id}") except oci.exceptions.ServiceError as ex: @@ -1389,12 +1447,20 @@ def _delete_job_and_model(job, model): f"Exception message: {ex}" ) - def load_evaluation_config(self, container: Optional[str] = None) -> Dict: + def load_evaluation_config( + self, container: Optional[str] = DEFAULT_EVALUATION_CONTAINER + ) -> Dict: """Loads evaluation config.""" logger.info("Loading evaluation container config.") # retrieve the evaluation config by container family name - evaluation_config = get_evaluation_service_config(container) + containers = self.list_service_containers() + container_item = next( + c for c in containers if c.is_latest and c.family_name == container + ) + evaluation_config = EvaluationServiceConfig.from_oci_container_config( + container_item + ) # convert the new config representation to the old one return { diff --git a/ads/aqua/extension/common_handler.py b/ads/aqua/extension/common_handler.py index cc9a2f663..002c7c860 100644 --- a/ads/aqua/extension/common_handler.py +++ b/ads/aqua/extension/common_handler.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2024 Oracle and/or its affiliates. +# Copyright (c) 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ @@ -12,14 +12,12 @@ from tornado.web import HTTPError from ads.aqua.common.decorator import handle_exceptions -from ads.aqua.common.errors import AquaResourceAccessError, AquaRuntimeError +from ads.aqua.common.errors import AquaRuntimeError from ads.aqua.common.utils import ( get_huggingface_login_timeout, - known_realm, ) from ads.aqua.extension.base_handler import AquaAPIhandler from ads.aqua.extension.errors import Errors -from ads.aqua.extension.utils import ui_compatability_check class ADSVersionHandler(AquaAPIhandler): @@ -50,14 +48,7 @@ def get(self): AquaResourceAccessError: raised when aqua is not accessible in the given session/region. """ - if ui_compatability_check(): - return self.finish({"status": "ok"}) - elif known_realm(): - return self.finish({"status": "compatible"}) - else: - raise AquaResourceAccessError( - "The AI Quick actions extension is not compatible in the given region." - ) + return self.finish({"status": "ok"}) class NetworkStatusHandler(AquaAPIhandler): diff --git a/ads/aqua/extension/model_handler.py b/ads/aqua/extension/model_handler.py index 8b93af8fb..43d6b5b89 100644 --- a/ads/aqua/extension/model_handler.py +++ b/ads/aqua/extension/model_handler.py @@ -9,12 +9,14 @@ from ads.aqua.common.decorator import handle_exceptions from ads.aqua.common.enums import CustomInferenceContainerTypeFamily -from ads.aqua.common.errors import AquaRuntimeError, AquaValueError +from ads.aqua.common.errors import AquaRuntimeError from ads.aqua.common.utils import get_hf_model_info, is_valid_ocid, list_hf_models from ads.aqua.extension.base_handler import AquaAPIhandler from ads.aqua.extension.errors import Errors from ads.aqua.model import AquaModelApp from ads.aqua.model.entities import AquaModelSummary, HFModelSummary +from ads.config import USER +from ads.model.common.utils import MetadataArtifactPathType class AquaModelHandler(AquaAPIhandler): @@ -37,13 +39,11 @@ def get( raise HTTPError( 400, Errors.MISSING_REQUIRED_PARAMETER.format("model_format") ) - + model_format = model_format.upper() - + if os_path: - return self.finish( - AquaModelApp.get_model_files(os_path, model_format) - ) + return self.finish(AquaModelApp.get_model_files(os_path, model_format)) elif model_name: return self.finish( AquaModelApp.get_hf_model_files(model_name, model_format) @@ -82,11 +82,13 @@ def list(self): # project_id is no needed. project_id = self.get_argument("project_id", default=None) model_type = self.get_argument("model_type", default=None) + category = self.get_argument("category", default=USER) return self.finish( AquaModelApp().list( compartment_id=compartment_id, project_id=project_id, model_type=model_type, + category=category, ) ) @@ -206,6 +208,16 @@ def get(self, model_id): return self.finish(AquaModelApp().load_license(model_id)) +class AquaModelReadmeHandler(AquaAPIhandler): + """ + Handler for fetching model card for AQUA models + """ + + def get(self, model_id): + model_id = model_id.split("/")[0] + return self.finish(AquaModelApp().load_readme(model_id).model_dump()) + + class AquaHuggingFaceHandler(AquaAPIhandler): """Handler for Aqua Hugging Face REST APIs.""" @@ -329,9 +341,50 @@ def get(self, model_id): raise HTTPError(400, f"The request {self.request.path} is invalid.") +class AquaModelDefinedMetadataArtifactHandler(AquaAPIhandler): + """ + Handler for Model Defined metadata artifact content + + Raises + ------ + HTTPError + Raises HTTPError if inputs are missing or are invalid. + """ + + @handle_exceptions + def get(self, model_id: str, metadata_key: str): + """ + model_id: ocid of the model + metadata_key: the metadata key for which artifact content needs to be downloaded. + Can be any of Readme, License , FinetuneConfiguration , DeploymentConfiguration + """ + + return self.finish( + AquaModelApp().get_defined_metadata_artifact_content(model_id, metadata_key) + ) + + @handle_exceptions + def post(self, model_id: str, metadata_key: str): + input_body = self.get_json_body() + path_type = input_body.get("path_type") + artifact_path_or_content = input_body.get("artifact_path_or_content") + if path_type not in MetadataArtifactPathType.values(): + raise HTTPError(400, f"Invalid value of path_type: {path_type}") + return self.finish( + AquaModelApp().create_defined_metadata_artifact( + model_id, metadata_key, path_type, artifact_path_or_content + ) + ) + + __handlers__ = [ ("model/?([^/]*)", AquaModelHandler), ("model/?([^/]*)/license", AquaModelLicenseHandler), + ("model/?([^/]*)/readme", AquaModelReadmeHandler), ("model/?([^/]*)/tokenizer", AquaModelTokenizerConfigHandler), ("model/hf/search/?([^/]*)", AquaHuggingFaceHandler), + ( + "model/?([^/]*)/definedMetadata/?([^/]*)", + AquaModelDefinedMetadataArtifactHandler, + ), ] diff --git a/ads/aqua/extension/models/ws_models.py b/ads/aqua/extension/models/ws_models.py index 38432e22b..602bb63c3 100644 --- a/ads/aqua/extension/models/ws_models.py +++ b/ads/aqua/extension/models/ws_models.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from typing import List, Optional +import ads.config from ads.aqua.evaluation.entities import AquaEvaluationDetail, AquaEvaluationSummary from ads.aqua.model.entities import AquaModel, AquaModelSummary from ads.aqua.modeldeployment.entities import AquaDeployment, AquaDeploymentDetail @@ -57,6 +58,7 @@ class ListModelsRequest(BaseRequest): compartment_id: Optional[str] = None project_id: Optional[str] = None model_type: Optional[str] = None + category: str = ads.config.USER kind = RequestResponseType.ListDeployments diff --git a/ads/aqua/extension/models_ws_msg_handler.py b/ads/aqua/extension/models_ws_msg_handler.py index 8df4a0232..8ca4bb62b 100644 --- a/ads/aqua/extension/models_ws_msg_handler.py +++ b/ads/aqua/extension/models_ws_msg_handler.py @@ -32,6 +32,7 @@ def process(self) -> Union[ListModelsResponse, ModelDetailsResponse]: compartment_id=request.get("compartment_id"), project_id=request.get("project_id"), model_type=request.get("model_type"), + category=request.get("category"), ) response = ListModelsResponse( message_id=request.get("message_id"), diff --git a/ads/aqua/finetuning/entities.py b/ads/aqua/finetuning/entities.py index 5e29f0bc2..498cc8cbd 100644 --- a/ads/aqua/finetuning/entities.py +++ b/ads/aqua/finetuning/entities.py @@ -3,7 +3,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import json -from typing import List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union from pydantic import Field, model_validator @@ -55,6 +55,28 @@ def validate_restricted_fields(cls, data: dict): return data +class AquaFineTuningConfig(Serializable): + """Represents model's shape list and detailed configuration for fine-tuning. + + Attributes: + shape (List[str], optional): A list of shape names (e.g., BM.GPU.A10.4). + configuration (Dict[str, Any], optional): Configuration details of fine-tuning. + """ + + shape: Optional[Dict[str, Any]] = Field( + default_factory=dict, description="List of supported shapes for the model." + ) + finetuning_params: Optional[str] = Field( + default_factory=str, description="Fine tuning parameters." + ) + configuration: Optional[Dict[str, Any]] = Field( + default_factory=dict, description="Configuration details keyed by shape." + ) + + class Config: + extra = "allow" + + class AquaFineTuningSummary(Serializable): """Represents a summary of Aqua Finetuning job.""" diff --git a/ads/aqua/finetuning/finetuning.py b/ads/aqua/finetuning/finetuning.py index 5d8be4c36..a87f6c2d8 100644 --- a/ads/aqua/finetuning/finetuning.py +++ b/ads/aqua/finetuning/finetuning.py @@ -20,8 +20,8 @@ from ads.aqua.common.enums import Resource, Tags from ads.aqua.common.errors import AquaFileExistsError, AquaValueError from ads.aqua.common.utils import ( + DEFINED_METADATA_TO_FILE_MAP, build_pydantic_error_message, - get_container_image, upload_local_to_os, ) from ads.aqua.constants import ( @@ -38,17 +38,18 @@ FineTuneCustomMetadata, ) from ads.aqua.finetuning.entities import ( + AquaFineTuningConfig, AquaFineTuningParams, AquaFineTuningSummary, CreateFineTuningDetails, ) +from ads.aqua.model.constants import AquaModelMetadataKeys from ads.common.auth import default_signer from ads.common.object_storage_details import ObjectStorageDetails from ads.common.utils import UNKNOWN, get_console_link from ads.config import ( AQUA_FINETUNING_CONTAINER_OVERRIDE_FLAG_METADATA_NAME, AQUA_JOB_SUBNET_ID, - AQUA_MODEL_FINETUNING_CONFIG, COMPARTMENT_OCID, CONDA_BUCKET_NS, PROJECT_OCID, @@ -312,7 +313,7 @@ def create( compartment_id=target_compartment, project_id=target_project, model_by_reference=True, - defined_tags=create_fine_tuning_details.defined_tags + defined_tags=create_fine_tuning_details.defined_tags, ) ft_job_freeform_tags = { @@ -371,11 +372,11 @@ def create( is_custom_container = True ft_parameters.batch_size = ft_parameters.batch_size or ( - ft_config.get("shape", UNKNOWN_DICT) + (ft_config.shape if ft_config else UNKNOWN_DICT) .get(create_fine_tuning_details.shape_name, UNKNOWN_DICT) .get("batch_size", DEFAULT_FT_BATCH_SIZE) ) - finetuning_params = ft_config.get("finetuning_params") + finetuning_params = ft_config.finetuning_params if ft_config else UNKNOWN ft_job.with_runtime( self._build_fine_tuning_runtime( @@ -558,7 +559,7 @@ def _build_fine_tuning_runtime( ) -> Runtime: """Builds fine tuning runtime for Job.""" container = ( - get_container_image( + self.get_container_image( container_type=ft_container, ) if not is_custom_container @@ -626,7 +627,7 @@ def _build_oci_launch_cmd( @telemetry( entry_point="plugin=finetuning&action=get_finetuning_config", name="aqua" ) - def get_finetuning_config(self, model_id: str) -> Dict: + def get_finetuning_config(self, model_id: str) -> AquaFineTuningConfig: """Gets the finetuning config for given Aqua model. Parameters @@ -639,12 +640,25 @@ def get_finetuning_config(self, model_id: str) -> Dict: Dict: A dict of allowed finetuning configs. """ - config = self.get_config(model_id, AQUA_MODEL_FINETUNING_CONFIG).config + config = self.get_config_from_metadata( + model_id, AquaModelMetadataKeys.FINE_TUNING_CONFIGURATION + ).config + if config: + logger.info( + f"Fetched {AquaModelMetadataKeys.FINE_TUNING_CONFIGURATION} from defined metadata for model: {model_id}." + ) + return AquaFineTuningConfig(**(config or UNKNOWN_DICT)) + config = self.get_config( + model_id, + DEFINED_METADATA_TO_FILE_MAP.get( + AquaModelMetadataKeys.FINE_TUNING_CONFIGURATION.lower() + ), + ).config if not config: logger.debug( f"Fine-tuning config for custom model: {model_id} is not available. Use defaults." ) - return config + return AquaFineTuningConfig(**(config or UNKNOWN_DICT)) @telemetry( entry_point="plugin=finetuning&action=get_finetuning_default_params", @@ -667,7 +681,9 @@ def get_finetuning_default_params(self, model_id: str) -> Dict: """ default_params = {"params": {}} finetuning_config = self.get_finetuning_config(model_id) - config_parameters = finetuning_config.get("configuration", UNKNOWN_DICT) + config_parameters = ( + finetuning_config.configuration if finetuning_config else UNKNOWN_DICT + ) dataclass_fields = self._get_finetuning_params( config_parameters, validate=False ).to_dict() diff --git a/ads/aqua/model/constants.py b/ads/aqua/model/constants.py index 9c5859671..194245fe4 100644 --- a/ads/aqua/model/constants.py +++ b/ads/aqua/model/constants.py @@ -50,3 +50,11 @@ class FineTuningCustomMetadata(ExtendedEnum): VALIDATION_METRICS_FINAL = "val_metrics_final" TRAINING_METRICS_EPOCH = "train_metrics_epoch" VALIDATION_METRICS_EPOCH = "val_metrics_epoch" + + +class AquaModelMetadataKeys(ExtendedEnum): + FINE_TUNING_CONFIGURATION = "FineTuneConfiguration" + DEPLOYMENT_CONFIGURATION = "DeploymentConfiguration" + README = "Readme" + LICENSE = "License" + REPORTS = "Reports" diff --git a/ads/aqua/model/entities.py b/ads/aqua/model/entities.py index 66318b683..56b45a585 100644 --- a/ads/aqua/model/entities.py +++ b/ads/aqua/model/entities.py @@ -15,6 +15,7 @@ import oci from huggingface_hub import hf_api +from pydantic import BaseModel from ads.aqua import logger from ads.aqua.app import CLIBuilderMixin @@ -29,6 +30,13 @@ from ads.model.model_metadata import MetadataTaxonomyKeys +class AquaModelReadme(BaseModel): + """Represents the response of Get Model Readme.""" + + id: str = field(default_factory=str) + model_card: str = field(default_factory=str) + + @dataclass(repr=False) class FineTuningShapeInfo(DataClassSerializable): instance_shape: str = field(default_factory=str) @@ -95,7 +103,6 @@ class AquaModelSummary(DataClassSerializable): class AquaModel(AquaModelSummary, DataClassSerializable): """Represents an Aqua model.""" - model_card: str = None inference_container: str = None inference_container_uri: str = None finetuning_container: str = None diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 027985702..0c649fe5a 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -6,7 +6,7 @@ import pathlib from datetime import datetime, timedelta from threading import Lock -from typing import Dict, List, Optional, Set, Union +from typing import Any, Dict, List, Optional, Set, Union import oci from cachetools import TTLCache @@ -37,11 +37,9 @@ create_word_icon, generate_tei_cmd_var, get_artifact_path, - get_container_config, get_hf_model_info, list_os_files_with_extension, load_config, - read_file, upload_folder, ) from ads.aqua.config.container_config import AquaContainerConfig, Usage @@ -53,7 +51,7 @@ AQUA_MODEL_TOKENIZER_CONFIG, AQUA_MODEL_TYPE_CUSTOM, HF_METADATA_FOLDER, - LICENSE_TXT, + LICENSE, MODEL_BY_REFERENCE_OSS_PATH_KEY, README, READY_TO_DEPLOY_STATUS, @@ -65,6 +63,7 @@ VALIDATION_METRICS_FINAL, ) from ads.aqua.model.constants import ( + AquaModelMetadataKeys, FineTuningCustomMetadata, FineTuningMetricCategories, ModelCustomMetadataFields, @@ -75,6 +74,7 @@ AquaFineTuningMetric, AquaModel, AquaModelLicense, + AquaModelReadme, AquaModelSummary, ImportModelDetails, ModelValidationResult, @@ -82,7 +82,12 @@ from ads.aqua.model.enums import MultiModelSupportedTaskType from ads.common.auth import default_signer from ads.common.oci_resource import SEARCH_TYPE, OCIResource -from ads.common.utils import UNKNOWN, get_console_link +from ads.common.utils import ( + UNKNOWN, + get_console_link, + is_path_exists, + read_file, +) from ads.config import ( AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME, AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, @@ -91,7 +96,9 @@ AQUA_FINETUNING_CONTAINER_METADATA_NAME, COMPARTMENT_OCID, PROJECT_OCID, + SERVICE, TENANCY_OCID, + USER, ) from ads.model import DataScienceModel from ads.model.common.utils import MetadataArtifactPathType @@ -174,7 +181,6 @@ def create( service_model = DataScienceModel.from_id(model_id) target_project = project_id or PROJECT_OCID target_compartment = compartment_id or COMPARTMENT_OCID - if service_model.compartment_id != ODSC_MODEL_COMPARTMENT_OCID: logger.info( f"Aqua Model {model_id} already exists in the user's compartment." @@ -260,18 +266,15 @@ def create_multi( display_name_list = [] model_custom_metadata = ModelCustomMetadata() - # Get container config - container_config = get_container_config() - - service_inference_containers = AquaContainerConfig.from_container_index_json( - config=container_config - ).inference.values() + service_inference_containers = ( + self.get_container_config().to_dict().get("inference") + ) supported_container_families = [ container_config_item.family for container_config_item in service_inference_containers if any( - usage in container_config_item.usages + usage.upper() in container_config_item.usages for usage in [Usage.MULTI_MODEL, Usage.OTHER] ) ] @@ -421,7 +424,7 @@ def create_multi( return custom_model @telemetry(entry_point="plugin=model&action=get", name="aqua") - def get(self, model_id: str, load_model_card: Optional[bool] = True) -> "AquaModel": + def get(self, model_id: str) -> "AquaModel": """Gets the information of an Aqua model. Parameters @@ -444,6 +447,7 @@ def get(self, model_id: str, load_model_card: Optional[bool] = True) -> "AquaMod logger.info(f"Fetching model details for model {model_id}.") ds_model = DataScienceModel.from_id(model_id) + if not self._if_show(ds_model): raise AquaRuntimeError( f"Target model `{ds_model.id} `is not an Aqua model as it does not contain " @@ -455,34 +459,6 @@ def get(self, model_id: str, load_model_card: Optional[bool] = True) -> "AquaMod and ds_model.freeform_tags.get(Tags.AQUA_FINE_TUNED_MODEL_TAG) ) - # todo: consolidate this logic in utils for model and deployment use - is_verified_type = ( - ds_model.freeform_tags.get(Tags.READY_TO_IMPORT, "false").upper() - == READY_TO_IMPORT_STATUS - ) - - model_card = "" - if load_model_card: - artifact_path = get_artifact_path( - ds_model.custom_metadata_list._to_oci_metadata() - ) - if artifact_path != UNKNOWN: - model_card_path = ( - f"{artifact_path.rstrip('/')}/config/{README}" - if is_verified_type - else f"{artifact_path.rstrip('/')}/{README}" - ) - model_card = str( - read_file( - file_path=model_card_path, - auth=default_signer(), - ) - ) - if not model_card: - logger.warn( - f"Model card for {model_id} is empty or could not be loaded from {model_card_path}." - ) - inference_container = ds_model.custom_metadata_list.get( ModelCustomMetadataFields.DEPLOYMENT_CONTAINER, ModelCustomMetadataItem(key=ModelCustomMetadataFields.DEPLOYMENT_CONTAINER), @@ -509,7 +485,6 @@ def get(self, model_id: str, load_model_card: Optional[bool] = True) -> "AquaMod aqua_model_attributes = dict( **self._process_model(ds_model, self.region), project_id=ds_model.project_id, - model_card=model_card, inference_container=inference_container, inference_container_uri=inference_container_uri, finetuning_container=finetuning_container, @@ -780,7 +755,9 @@ def _build_ft_metrics( ] def get_hf_tokenizer_config(self, model_id): - """Gets the default chat template for the given Aqua model. + """ + Gets the default model tokenizer config for the given Aqua model. + Returns the content of tokenizer_config.json stored in model artifact. Parameters ---------- @@ -789,14 +766,19 @@ def get_hf_tokenizer_config(self, model_id): Returns ------- - str: - Chat template string. + Dict: + Model tokenizer config. """ config = self.get_config( model_id, AQUA_MODEL_TOKENIZER_CONFIG, ConfigFolder.ARTIFACT ).config if not config: - logger.debug(f"Tokenizer config for model: {model_id} is not available.") + logger.debug( + f"{AQUA_MODEL_TOKENIZER_CONFIG} is not available for the model: {model_id}. " + f"Check if the custom metadata has the artifact path set." + ) + return config + return config @staticmethod @@ -821,6 +803,7 @@ def _process_model( oci.resource_search.models.ResourceSummary, ], region: str, + inference_containers: Optional[List[Any]] = None, ) -> dict: """Constructs required fields for AquaModelSummary.""" @@ -876,9 +859,10 @@ def _process_model( except Exception: model_file = UNKNOWN - inference_containers = AquaContainerConfig.from_container_index_json( - config=get_container_config() - ).inference + if not inference_containers: + inference_containers = ( + AquaApp().get_container_config().to_dict().get("inference") + ) model_formats_str = freeform_tags.get( Tags.MODEL_FORMAT, ModelFormat.SAFETENSORS @@ -887,7 +871,7 @@ def _process_model( supported_platform: Set[str] = set() - for container in inference_containers.values(): + for container in inference_containers: for model_format in model_formats: if model_format in container.model_formats: supported_platform.update(container.platforms) @@ -949,7 +933,8 @@ def list( """ models = [] - if compartment_id: + category = kwargs.get("category", USER) + if compartment_id and category != SERVICE: # tracks number of times custom model listing was called self.telemetry.record_event_async( category="aqua/custom/model", action="list" @@ -969,9 +954,7 @@ def list( f"Returning service models list in {ODSC_MODEL_COMPARTMENT_OCID} from cache." ) return self._service_models_cache.get(ODSC_MODEL_COMPARTMENT_OCID) - logger.info( - f"Fetching service models from compartment_id={ODSC_MODEL_COMPARTMENT_OCID}" - ) + logger.info("Fetching service models.") lifecycle_state = kwargs.pop( "lifecycle_state", Model.LIFECYCLE_STATE_ACTIVE ) @@ -984,20 +967,23 @@ def list( ) logger.info( - f"Fetched {len(models)} model in compartment_id={compartment_id or ODSC_MODEL_COMPARTMENT_OCID}." + f"Fetched {len(models)} model in compartment_id={ODSC_MODEL_COMPARTMENT_OCID if category==SERVICE else compartment_id}." ) - aqua_models = [] - + inference_containers = self.get_container_config().to_dict().get("inference") for model in models: aqua_models.append( AquaModelSummary( - **self._process_model(model=model, region=self.region), + **self._process_model( + model=model, + region=self.region, + inference_containers=inference_containers, + ), project_id=project_id or UNKNOWN, ) ) - if not compartment_id: + if category == SERVICE: self._service_models_cache.__setitem__( key=ODSC_MODEL_COMPARTMENT_OCID, value=aqua_models ) @@ -1046,14 +1032,85 @@ def clear_model_details_cache(self, model_id): @staticmethod def list_valid_inference_containers(): - containers = list( - AquaContainerConfig.from_container_index_json( - config=get_container_config(), enable_spec=True - ).inference.values() - ) + containers = AquaApp().get_container_config().to_dict().get("inference") family_values = [item.family for item in containers] return family_values + @telemetry( + entry_point="plugin=model&action=get_defined_metadata_artifact_content", + name="aqua", + ) + def get_defined_metadata_artifact_content(self, model_id: str, metadata_key: str): + """ + Gets the defined metadata artifact content for the given model + + Args: + model_id: str + model ocid for which defined metadata artifact needs to be created + metadata_key: str + defined metadata key like Readme , License , DeploymentConfiguration , FinetuningConfiguration + Returns: + The model defined metadata artifact content. Can be either str or Dict + + """ + + content = self.get_config(model_id, metadata_key) + if not content: + logger.debug( + f"Defined metadata artifact {metadata_key} for model: {model_id} is not available." + ) + return content + + @telemetry( + entry_point="plugin=model&action=create_defined_metadata_artifact", name="aqua" + ) + def create_defined_metadata_artifact( + self, + model_id: str, + metadata_key: str, + path_type: MetadataArtifactPathType, + artifact_path_or_content: str, + ) -> None: + """ + Creates defined metadata artifact for the registered unverified model + + Args: + model_id: str + model ocid for which defined metadata artifact needs to be created + metadata_key: str + defined metadata key like Readme , License , DeploymentConfiguration , FinetuningConfiguration + path_type: str + path type of the given defined metadata can be local , oss or the content itself + artifact_path_or_content: str + It can be local path or oss path or the actual content itself + Returns: + None + """ + + ds_model = DataScienceModel.from_id(model_id) + oci_aqua = ds_model.freeform_tags.get(Tags.AQUA_TAG, None) + if not oci_aqua: + raise AquaRuntimeError(f"Target model {model_id} is not an Aqua model.") + is_registered_model = ds_model.freeform_tags.get(Tags.BASE_MODEL_CUSTOM, None) + is_verified_model = ds_model.freeform_tags.get( + Tags.AQUA_SERVICE_MODEL_TAG, None + ) + if is_registered_model and not is_verified_model: + try: + ds_model.create_defined_metadata_artifact( + metadata_key_name=metadata_key, + artifact_path_or_content=artifact_path_or_content, + path_type=path_type, + ) + except Exception as ex: + raise AquaRuntimeError( + f"Error occurred in creating defined metadata artifact for model {model_id}: {ex}" + ) from ex + else: + raise AquaRuntimeError( + f"Cannot create defined metadata artifact for model {model_id}" + ) + def _create_model_catalog_entry( self, os_path: str, @@ -1110,7 +1167,9 @@ def _create_model_catalog_entry( # Remove `ready_to_import` tag that might get copied from service model. tags.pop(Tags.READY_TO_IMPORT, None) - + defined_metadata_dict = {} + readme_file_path = os_path.rstrip("/") + "/" + README + license_file_path = os_path.rstrip("/") + "/" + LICENSE if verified_model: # Verified model is a model in the service catalog that either has no artifacts but contains all the necessary metadata for deploying and fine tuning. # If set, then we copy all the model metadata. @@ -1119,6 +1178,17 @@ def _create_model_catalog_entry( model = model.with_model_file_description( json_dict=verified_model.model_file_description ) + defined_metadata_list = ( + verified_model.defined_metadata_list._to_oci_metadata() + ) + for defined_metadata in defined_metadata_list: + if defined_metadata.has_artifact: + content = ( + self.ds_client.get_model_defined_metadatum_artifact_content( + verified_model.id, defined_metadata.key + ).data.content + ) + defined_metadata_dict[defined_metadata.key] = content else: metadata = ModelCustomMetadata() if not inference_container: @@ -1140,12 +1210,14 @@ def _create_model_catalog_entry( category="Other", ) - inference_containers = AquaContainerConfig.from_container_index_json( - config=get_container_config() - ).inference - smc_container_set = { - container.family for container in inference_containers.values() - } + inference_containers = ( + AquaContainerConfig.from_service_config( + service_containers=self.list_service_containers() + ) + .to_dict() + .get("inference") + ) + smc_container_set = {container.family for container in inference_containers} # only add cmd vars if inference container is not an SMC if ( inference_container not in smc_container_set @@ -1214,6 +1286,33 @@ def _create_model_catalog_entry( .with_defined_tags(**(defined_tags or {})) ).create(model_by_reference=True) logger.debug(f"Created model catalog entry for the model:\n{model}") + for key, value in defined_metadata_dict.items(): + model.create_defined_metadata_artifact( + key, value, MetadataArtifactPathType.CONTENT + ) + + if is_path_exists(readme_file_path): + try: + model.create_defined_metadata_artifact( + AquaModelMetadataKeys.README, + readme_file_path, + MetadataArtifactPathType.OSS, + ) + except Exception as ex: + logger.error( + f"Error Uploading Readme in defined metadata for model: {model.id} : {str(ex)}" + ) + if not verified_model and is_path_exists(license_file_path): + try: + model.create_defined_metadata_artifact( + AquaModelMetadataKeys.LICENSE, + license_file_path, + MetadataArtifactPathType.OSS, + ) + except Exception as ex: + logger.error( + f"Error Uploading License in defined metadata for model: {model.id} : {str(ex)}" + ) return model @staticmethod @@ -1728,6 +1827,7 @@ def register( ).rstrip("/") else: artifact_path = import_model_details.os_path.rstrip("/") + # Create Model catalog entry with pass by reference ds_model = self._create_model_catalog_entry( os_path=artifact_path, @@ -1766,12 +1866,6 @@ def register( aqua_model_attributes = dict( **self._process_model(ds_model, self.region), project_id=ds_model.project_id, - model_card=str( - read_file( - file_path=f"{artifact_path}/{README}", - auth=default_signer(), - ) - ), inference_container=inference_container, inference_container_uri=inference_container_uri, finetuning_container=finetuning_container, @@ -1845,6 +1939,56 @@ def _build_search_text(tags: dict, description: str = None) -> str: separator = " " if description else "" return f"{description}{separator}{tags_text}" + @telemetry(entry_point="plugin=model&action=load_readme", name="aqua") + def load_readme(self, model_id: str) -> AquaModelReadme: + """Loads the readme or the model card for the given model. + + Parameters + ---------- + model_id: str + The model id. + + Returns + ------- + AquaModelReadme: + The instance of AquaModelReadme. + """ + oci_model = self.ds_client.get_model(model_id).data + artifact_path = get_artifact_path(oci_model.custom_metadata_list) + if not artifact_path: + raise AquaRuntimeError( + f"Readme could not be loaded. Failed to get artifact path from custom metadata for" + f"the model {model_id}." + ) + + content = "" + try: + content = self.ds_client.get_model_defined_metadatum_artifact_content( + model_id, AquaModelMetadataKeys.README + ).data.content.decode("utf-8", errors="ignore") + logger.info(f"Fetched {README} from defined metadata for model: {model_id}") + except Exception as ex: + logger.error( + f"Readme could not be found for model: {model_id} in defined metadata : {str(ex)}" + ) + artifact_path = get_artifact_path(oci_model.custom_metadata_list) + readme_path = os.path.join(os.path.dirname(artifact_path), "artifact") + if not is_path_exists(readme_path): + readme_path = os.path.join(artifact_path.rstrip("/"), "artifact") + if not is_path_exists(readme_path): + readme_path = f"{artifact_path.rstrip('/')}/" + + readme_file_path = os.path.join(readme_path, README) + logger.info(f"Fetching {README} from {readme_file_path}") + if is_path_exists(readme_file_path): + try: + content = str(read_file(readme_file_path, auth=default_signer())) + except Exception as e: + logger.debug( + f"Error occurred while fetching config {README} at path {readme_file_path} : {str(e)}" + ) + return AquaModelReadme(id=model_id, model_card=content) + @telemetry(entry_point="plugin=model&action=load_license", name="aqua") def load_license(self, model_id: str) -> AquaModelLicense: """Loads the license full text for the given model. @@ -1867,13 +2011,34 @@ def load_license(self, model_id: str) -> AquaModelLicense: f"the model {model_id}." ) - content = str( - read_file( - file_path=f"{os.path.dirname(artifact_path)}/{LICENSE_TXT}", - auth=default_signer(), + content = "" + try: + content = self.ds_client.get_model_defined_metadatum_artifact_content( + model_id, AquaModelMetadataKeys.LICENSE + ).data.content.decode("utf-8", errors="ignore") + logger.info( + f"Fetched {LICENSE} from defined metadata for model: {model_id}" ) - ) - + except Exception as ex: + logger.error( + f"License could not be found for model: {model_id} in defined metadata : {str(ex)}" + ) + artifact_path = get_artifact_path(oci_model.custom_metadata_list) + license_path = os.path.join(os.path.dirname(artifact_path), "config") + if not is_path_exists(license_path): + license_path = os.path.join(artifact_path.rstrip("/"), "config") + if not is_path_exists(license_path): + license_path = f"{artifact_path.rstrip('/')}/" + + license_file_path = os.path.join(license_path, LICENSE) + logger.info(f"Fetching {LICENSE} from {license_file_path}") + if is_path_exists(license_file_path): + try: + content = str(read_file(license_file_path, auth=default_signer())) + except Exception as e: + logger.debug( + f"Error occurred while fetching config {LICENSE} at path {license_path} : {str(e)}" + ) return AquaModelLicense(id=model_id, license=content) def _find_matching_aqua_model(self, model_id: str) -> Optional[str]: diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index cb0abaddb..80bb4bd05 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -16,16 +16,14 @@ AquaMultiModelRef, ComputeShapeSummary, ContainerPath, - ContainerSpec, ) from ads.aqua.common.enums import InferenceContainerTypeFamily, ModelFormat, Tags from ads.aqua.common.errors import AquaRuntimeError, AquaValueError from ads.aqua.common.utils import ( + DEFINED_METADATA_TO_FILE_MAP, build_params_string, build_pydantic_error_message, get_combined_params, - get_container_config, - get_container_image, get_container_params_type, get_model_by_reference_paths, get_ocid_substring, @@ -50,7 +48,7 @@ from ads.aqua.data import AquaResourceIdentifier from ads.aqua.finetuning.finetuning import FineTuneCustomMetadata from ads.aqua.model import AquaModelApp -from ads.aqua.model.constants import ModelCustomMetadataFields +from ads.aqua.model.constants import AquaModelMetadataKeys, ModelCustomMetadataFields from ads.aqua.modeldeployment.entities import ( AquaDeployment, AquaDeploymentConfig, @@ -67,7 +65,6 @@ AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME, AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME, - AQUA_MODEL_DEPLOYMENT_CONFIG, COMPARTMENT_OCID, PROJECT_OCID, ) @@ -193,7 +190,7 @@ def create( ) # Get container config - container_config = get_container_config() + container_config = self.get_container_config() # Create an AquaModelApp instance once to perform the deployment creation. model_app = AquaModelApp() @@ -228,17 +225,13 @@ def create( except ConfigValidationError as err: raise AquaValueError(f"{err}") from err - service_inference_containers = ( - AquaContainerConfig.from_container_index_json( - config=container_config - ).inference.values() - ) + service_inference_containers = container_config.inference.values() supported_container_families = [ container_config_item.family for container_config_item in service_inference_containers if any( - usage in container_config_item.usages + usage.upper() in container_config_item.usages for usage in [Usage.MULTI_MODEL, Usage.OTHER] ) ] @@ -409,7 +402,7 @@ def _create( container_image_uri = ( create_deployment_details.container_image_uri - or get_container_image(container_type=container_type_key) + or self.get_container_image(container_type=container_type_key) ) if not container_image_uri: try: @@ -479,18 +472,18 @@ def _create( # Fetch the startup cli command for the container # container_index.json will have "containerSpec" section which will provide the cli params for # a given container family - container_spec = container_config.get(ContainerSpec.CONTAINER_SPEC, {}).get( - container_type_key, {} - ) + container_config = self.get_container_config_item(container_type_key) + + container_spec = container_config.spec if container_config else UNKNOWN # these params cannot be overridden for Aqua deployments - params = container_spec.get(ContainerSpec.CLI_PARM, "") - server_port = create_deployment_details.server_port or container_spec.get( - ContainerSpec.SERVER_PORT - ) # Give precedence to the input parameter - health_check_port = ( - create_deployment_details.health_check_port - or container_spec.get(ContainerSpec.HEALTH_CHECK_PORT) - ) # Give precedence to the input parameter + params = container_spec.cli_param if container_spec else UNKNOWN + server_port = create_deployment_details.server_port or ( + container_spec.server_port if container_spec else None + ) + # Give precendece to the input parameter + health_check_port = create_deployment_details.health_check_port or ( + container_spec.health_check_port if container_spec else None + ) deployment_config = self.get_deployment_config(model_id=config_source_id) @@ -527,9 +520,10 @@ def _create( params = f"{params} {deployment_params}".strip() if params: env_var.update({"PARAMS": params}) - - for env in container_spec.get(ContainerSpec.ENV_VARS, []): + env_vars = container_spec.env_vars if container_spec else [] + for env in env_vars: if isinstance(env, dict): + env = {k: v for k, v in env.items() if v} for key, _ in env.items(): if key not in env_var: env_var.update(env) @@ -559,7 +553,7 @@ def _create_multi( aqua_model: DataScienceModel, model_config_summary: ModelDeploymentConfigSummary, create_deployment_details: CreateModelDeploymentDetails, - container_config: Dict, + container_config: AquaContainerConfig, ) -> AquaDeployment: """Builds the environment variables required by multi deployment container and creates the deployment. @@ -587,11 +581,10 @@ def _create_multi( model=aqua_model, container_family=create_deployment_details.container_family, ) - container_spec = container_config.get( - ContainerSpec.CONTAINER_SPEC, UNKNOWN_DICT - ).get(container_type_key, UNKNOWN_DICT) + container_config = self.get_container_config_item(container_type_key) + container_spec = container_config.spec if container_config else UNKNOWN - container_params = container_spec.get(ContainerSpec.CLI_PARM, UNKNOWN).strip() + container_params = container_spec.cli_param if container_spec else UNKNOWN for model in create_deployment_details.models: user_params = build_params_string(model.env_var) @@ -658,8 +651,10 @@ def _create_multi( env_var.update({AQUA_MULTI_MODEL_CONFIG: json.dumps({"models": model_config})}) - for env in container_spec.get(ContainerSpec.ENV_VARS, []): + env_vars = container_spec.env_vars if container_spec else [] + for env in env_vars: if isinstance(env, dict): + env = {k: v for k, v in env.items() if v} for key, _ in env.items(): if key not in env_var: env_var.update(env) @@ -668,14 +663,13 @@ def _create_multi( container_image_uri = ( create_deployment_details.container_image_uri - or get_container_image(container_type=container_type_key) + or self.get_container_image(container_type=container_type_key) ) - server_port = create_deployment_details.server_port or container_spec.get( - ContainerSpec.SERVER_PORT + server_port = create_deployment_details.server_port or ( + container_spec.server_port if container_spec else None ) - health_check_port = ( - create_deployment_details.health_check_port - or container_spec.get(ContainerSpec.HEALTH_CHECK_PORT) + health_check_port = create_deployment_details.health_check_port or ( + container_spec.health_check_port if container_spec else None ) tags = { Tags.AQUA_MODEL_ID_TAG: aqua_model.id, @@ -1054,7 +1048,20 @@ def get_deployment_config(self, model_id: str) -> AquaDeploymentConfig: AquaDeploymentConfig: An instance of AquaDeploymentConfig. """ - config = self.get_config(model_id, AQUA_MODEL_DEPLOYMENT_CONFIG).config + config = self.get_config_from_metadata( + model_id, AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION + ).config + if config: + logger.info( + f"Fetched {AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION} from defined metadata for model: {model_id}." + ) + return AquaDeploymentConfig(**(config or UNKNOWN_DICT)) + config = self.get_config( + model_id, + DEFINED_METADATA_TO_FILE_MAP.get( + AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION.lower() + ), + ).config if not config: logger.debug( f"Deployment config for custom model: {model_id} is not available. Use defaults." @@ -1079,7 +1086,7 @@ def get_multimodel_deployment_config( https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/ai-quick-actions/multimodel-deployment-tips.md#get_multimodel_deployment_config CLI example: - ads aqua deployment get_multimodel_deployment_config --model_ids '["ocid1.datasciencemodel.oc1.iad.OCID"]' + ads aqua deployment get_multimodel_deployment_config --model_ids '["md_ocid1","md_ocid2"]' If a primary model ID is provided, GPU allocation will prioritize that model when selecting compatible shapes. @@ -1230,11 +1237,9 @@ def validate_deployment_params( model=model, container_family=container_family ) - container_config = get_container_config() - container_spec = container_config.get(ContainerSpec.CONTAINER_SPEC, {}).get( - container_type_key, {} - ) - cli_params = container_spec.get(ContainerSpec.CLI_PARM, "") + container_config = self.get_container_config_item(container_type_key) + container_spec = container_config.spec if container_config else UNKNOWN + cli_params = container_spec.cli_param if container_spec else UNKNOWN restricted_params = self._find_restricted_params( cli_params, params, container_type_key diff --git a/ads/aqua/modeldeployment/utils.py b/ads/aqua/modeldeployment/utils.py index ac93ed23f..2e1d9346c 100644 --- a/ads/aqua/modeldeployment/utils.py +++ b/ads/aqua/modeldeployment/utils.py @@ -11,7 +11,8 @@ from typing import Dict, List, Optional from ads.aqua.app import AquaApp -from ads.aqua.common.entities import ComputeShapeSummary +from ads.aqua.common.entities import ComputeShapeSummary, ModelConfigResult +from ads.aqua.model.constants import AquaModelMetadataKeys from ads.aqua.modeldeployment.entities import ( AquaDeploymentConfig, ConfigurationItem, @@ -183,17 +184,34 @@ def _fetch_deployment_configs_concurrently( """Fetches deployment configurations in parallel using ThreadPoolExecutor.""" with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor: results = executor.map( - lambda model_id: self.deployment_app.get_config( - model_id, AQUA_MODEL_DEPLOYMENT_CONFIG - ).config, + self._fetch_deployment_config_from_metadata_and_oss, model_ids, ) return { - model_id: AquaDeploymentConfig(**config) + model_id: AquaDeploymentConfig(**config.config) for model_id, config in zip(model_ids, results) } + def _fetch_deployment_config_from_metadata_and_oss( + self, model_id + ) -> ModelConfigResult: + config = self.deployment_app.get_config_from_metadata( + model_id, AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION + ) + if config: + logger.info( + f"Fetched metadata key '{AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION}' from defined metadata for model '{model_id}'" + ) + return config + else: + logger.info( + f"Fetching '{AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION}' from object storage bucket for {model_id}'" + ) + return self.deployment_app.get_config( + model_id, AQUA_MODEL_DEPLOYMENT_CONFIG + ) + def _extract_model_shape_gpu( self, deployment_configs: Dict[str, AquaDeploymentConfig], diff --git a/ads/aqua/ui.py b/ads/aqua/ui.py index 01c087968..6baa3f100 100644 --- a/ads/aqua/ui.py +++ b/ads/aqua/ui.py @@ -13,7 +13,7 @@ from ads.aqua.app import AquaApp from ads.aqua.common.enums import Tags from ads.aqua.common.errors import AquaResourceAccessError, AquaValueError -from ads.aqua.common.utils import get_container_config, sanitize_response +from ads.aqua.common.utils import sanitize_response from ads.aqua.config.container_config import AquaContainerConfig from ads.aqua.constants import PRIVATE_ENDPOINT_TYPE from ads.common import oci_client as oc @@ -494,7 +494,6 @@ def list_containers(self) -> AquaContainerConfig: AquaContainerConfig The AQUA containers configurations. """ - return AquaContainerConfig.from_container_index_json( - config=get_container_config(), - enable_spec=True, + return AquaContainerConfig.from_service_config( + service_containers=self.list_service_containers() ) diff --git a/ads/model/datascience_model.py b/ads/model/datascience_model.py index 44d93d687..a71a7c631 100644 --- a/ads/model/datascience_model.py +++ b/ads/model/datascience_model.py @@ -2240,6 +2240,35 @@ def find_model_idx(): # model found case self.model_file_description["models"].pop(modelSearchIdx) + def if_model_custom_metadata_artifact_exist( + self, metadata_key_name: str, **kwargs + ) -> bool: + """Checks if the custom metadata artifact exists for the model. + + Parameters + ---------- + metadata_key_name: str + Custom metadata key name + **kwargs : + Additional keyword arguments passed in head_model_artifact. + + Returns + ------- + bool + Whether the artifact exists. + """ + + try: + response = self.dsc_model.head_custom_metadata_artifact( + metadata_key_name=metadata_key_name, **kwargs + ) + return int(response.status) == 200 + except Exception as ex: + logger.info( + f"Error fetching custom metadata: {metadata_key_name} for model {self.id}. {ex}" + ) + return False + def create_custom_metadata_artifact( self, metadata_key_name: str, diff --git a/ads/model/service/oci_datascience_model.py b/ads/model/service/oci_datascience_model.py index 66c62daba..06934f0a9 100644 --- a/ads/model/service/oci_datascience_model.py +++ b/ads/model/service/oci_datascience_model.py @@ -110,7 +110,7 @@ def wrapper(self, *args, **kwargs): def convert_model_metadata_response( headers: Union[Dict, CaseInsensitiveDict], status: int ) -> ModelMetadataArtifactDetails: - return ModelMetadataArtifactDetails(headers=headers, status=str(status)) + return ModelMetadataArtifactDetails(headers=dict(headers), status=str(status)) class OCIDataScienceModel( diff --git a/tests/unitary/with_extras/aqua/test_common_handler.py b/tests/unitary/with_extras/aqua/test_common_handler.py index ec0590b07..2d6da5e01 100644 --- a/tests/unitary/with_extras/aqua/test_common_handler.py +++ b/tests/unitary/with_extras/aqua/test_common_handler.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*-- -# Copyright (c) 2024 Oracle and/or its affiliates. +# Copyright (c) 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import os @@ -56,60 +56,3 @@ def test_get_ok(self): self.common_handler.request.path = "aqua/hello" result = self.common_handler.get() assert result["status"] == "ok" - - def test_get_compatible_status(self): - """Test to check if compatible is returned when ODSC_MODEL_COMPARTMENT_OCID is not set - but CONDA_BUCKET_NS is one of the namespaces from the GA list.""" - with patch.dict( - os.environ, - {"ODSC_MODEL_COMPARTMENT_OCID": "", "CONDA_BUCKET_NS": AQUA_GA_LIST[0]}, - ): - reload(ads.config) - reload(ads.aqua) - reload(ads.aqua.extension.utils) - reload(ads.aqua.extension.common_handler) - with patch( - "ads.aqua.extension.base_handler.AquaAPIhandler.finish" - ) as mock_finish: - with patch( - "ads.aqua.extension.utils.fetch_service_compartment" - ) as mock_fetch_service_compartment: - mock_fetch_service_compartment.return_value = None - mock_finish.side_effect = lambda x: x - self.common_handler.request.path = "aqua/hello" - result = self.common_handler.get() - assert result["status"] == "compatible" - - def test_raise_not_compatible_error(self): - """Test to check if error is returned when ODSC_MODEL_COMPARTMENT_OCID is not set - and CONDA_BUCKET_NS is not one of the namespaces from the GA list.""" - with patch.dict( - os.environ, - {"ODSC_MODEL_COMPARTMENT_OCID": "", "CONDA_BUCKET_NS": "test-namespace"}, - ): - reload(ads.config) - reload(ads.aqua) - reload(ads.aqua.extension.utils) - reload(ads.aqua.extension.common_handler) - with patch( - "ads.aqua.extension.base_handler.AquaAPIhandler.finish" - ) as mock_finish: - with patch( - "ads.aqua.extension.utils.fetch_service_compartment" - ) as mock_fetch_service_compartment: - mock_fetch_service_compartment.return_value = None - mock_finish.side_effect = lambda x: x - self.common_handler.write_error = MagicMock() - self.common_handler.request.path = "aqua/hello" - self.common_handler.get() - - assert self.common_handler.write_error.call_args[1].get( - "reason" - ) == ( - "The AI Quick actions extension is not " - "compatible in the given region." - ), "Incorrect error message." - assert ( - self.common_handler.write_error.call_args[1].get("status_code") - == 404 - ), "Incorrect status code." diff --git a/tests/unitary/with_extras/aqua/test_config.py b/tests/unitary/with_extras/aqua/test_config.py index 04ef25888..99191eb59 100644 --- a/tests/unitary/with_extras/aqua/test_config.py +++ b/tests/unitary/with_extras/aqua/test_config.py @@ -9,9 +9,9 @@ import oci.data_science.models -from ads.aqua.common.entities import ContainerSpec -from ads.aqua.config.config import get_evaluation_service_config + from ads.aqua.app import AquaApp +from tests.unitary.with_extras.aqua.test_ui import TestDataset class TestConfig: @@ -21,27 +21,6 @@ def setup_class(cls): cls.curr_dir = os.path.dirname(os.path.abspath(__file__)) cls.artifact_dir = os.path.join(cls.curr_dir, "test_data", "config") - @patch("ads.aqua.config.config.get_container_config") - def test_evaluation_service_config(self, mock_get_container_config): - """Ensures that the common evaluation configuration can be successfully retrieved.""" - - with open( - os.path.join( - self.artifact_dir, "evaluation_config_with_default_params.json" - ) - ) as file: - expected_result = { - ContainerSpec.CONTAINER_SPEC: {"test_container": json.load(file)} - } - - mock_get_container_config.return_value = expected_result - - test_result = get_evaluation_service_config(container="test_container") - assert ( - test_result.to_dict() - == expected_result[ContainerSpec.CONTAINER_SPEC]["test_container"] - ) - @pytest.mark.parametrize( "custom_metadata", [ diff --git a/tests/unitary/with_extras/aqua/test_data/finetuning/ft_config.json b/tests/unitary/with_extras/aqua/test_data/finetuning/ft_config.json index 6033ae895..b905f5892 100644 --- a/tests/unitary/with_extras/aqua/test_data/finetuning/ft_config.json +++ b/tests/unitary/with_extras/aqua/test_data/finetuning/ft_config.json @@ -10,6 +10,7 @@ "sequence_len": 2048, "val_set_size": 0.1 }, + "finetuning_params": "", "shape": { "VM.GPU.A10.2": { "batch_size": 2, diff --git a/tests/unitary/with_extras/aqua/test_deployment.py b/tests/unitary/with_extras/aqua/test_deployment.py index c82081b5d..b0bc1d5fe 100644 --- a/tests/unitary/with_extras/aqua/test_deployment.py +++ b/tests/unitary/with_extras/aqua/test_deployment.py @@ -13,6 +13,10 @@ import oci import pytest +from oci.data_science.models import ( + ContainerSummary, + ModelDeployWorkloadConfigurationDetails, +) from parameterized import parameterized from ads.aqua.common.entities import ( @@ -20,11 +24,17 @@ ComputeShapeSummary, ModelConfigResult, ) +from ads.aqua.app import AquaApp +from ads.aqua.common.entities import ModelConfigResult import ads.aqua.modeldeployment.deployment import ads.config from ads.aqua.common.entities import AquaMultiModelRef from ads.aqua.common.enums import Tags from ads.aqua.common.errors import AquaRuntimeError, AquaValueError +from ads.aqua.config.container_config import ( + AquaContainerConfigItem, + AquaContainerConfig, +) from ads.aqua.modeldeployment import AquaDeploymentApp, MDInferenceResponse from ads.aqua.modeldeployment.entities import ( AquaDeployment, @@ -39,6 +49,7 @@ from ads.model.datascience_model import DataScienceModel from ads.model.deployment.model_deployment import ModelDeployment from ads.model.model_metadata import ModelCustomMetadata +from tests.unitary.with_extras.aqua.utils import ServiceManagedContainers null = None @@ -89,6 +100,73 @@ class TestDataset: DEPLOYMENT_GPU_COUNT = 1 DEPLOYMENT_GPU_COUNT_B = 2 DEPLOYMENT_SHAPE_NAME_CPU = "VM.Standard.A1.Flex" + CONTAINER_LIST = ServiceManagedContainers.MOCK_OUTPUT + INFERENCE_CONTAINER_CONFIG = ContainerSummary( + **{ + "container_name": "odsc-vllm-serving", + "display_name": "VLLM:0.6.4.post1.2", + "family_name": "odsc-vllm-serving", + "description": "This container is used for llm inference, batch inference and serving", + "is_latest": True, + "target_workloads": ["MODEL_DEPLOYMENT", "JOB_RUN"], + "usages": ["INFERENCE", "BATCH_INFERENCE"], + "tag": "0.6.4.post1.2", + "lifecycle_state": "ACTIVE", + "workload_configuration_details_list": [ + ModelDeployWorkloadConfigurationDetails( + **{ + "cmd": "--served-model-name odsc-llm --disable-custom-all-reduce --seed 42 ", + "server_port": 8080, + "health_check_port": 8080, + "additional_configurations": { + "HEALTH_CHECK_PORT": "8080", + "MODEL_DEPLOY_ENABLE_STREAMING": "true", + "MODEL_DEPLOY_PREDICT_ENDPOINT": "/v1/completions", + "PORT": "8080", + "modelFormats": "SAFETENSORS", + "platforms": "NVIDIA_GPU", + "restrictedParams": '["--port","--host","--served-model-name","--seed"]', + }, + } + ) + ], + "tag_configuration_list": [], + "freeform_tags": None, + "defined_tags": None, + } + ) + + INFERENCE_CONTAINER_CONFIG_ITEM = AquaContainerConfigItem( + **{ + "name": "dsmc://odsc-vllm-serving", + "version": "0.6.4.post1.2", + "display_name": "VLLM:0.6.4.post1.2", + "family": "odsc-vllm-serving", + "platforms": ["NVIDIA_GPU"], + "model_formats": ["SAFETENSORS"], + "spec": { + "cli_param": "--served-model-name odsc-llm --disable-custom-all-reduce --seed 42 ", + "server_port": "8080", + "health_check_port": "8080", + "env_vars": [ + { + "MODEL_DEPLOY_PREDICT_ENDPOINT": "/v1/completions", + "MODEL_DEPLOY_HEALTH_ENDPOINT": "", + "MODEL_DEPLOY_ENABLE_STREAMING": "true", + "PORT": "8080", + "HEALTH_CHECK_PORT": "8080", + } + ], + "restricted_params": [ + "--port", + "--host", + "--served-model-name", + "--seed", + ], + }, + "usages": ["INFERENCE", "BATCH_INFERENCE"], + }, + ) model_deployment_object = [ { @@ -1074,13 +1152,19 @@ def test_get_deployment_config(self): ) with open(config_json, "r") as _file: config = json.load(_file) - - self.app.get_config = MagicMock(return_value=ModelConfigResult(config=config)) + self.app.get_config_from_metadata = MagicMock( + return_value=ModelConfigResult(config=None, model_details=None) + ) + self.app.get_config = MagicMock( + return_value=ModelConfigResult(config=config, model_details=None) + ) result = self.app.get_deployment_config(TestDataset.MODEL_ID) expected_config = AquaDeploymentConfig(**config) assert result == expected_config - self.app.get_config = MagicMock(return_value=ModelConfigResult(config=None)) + self.app.get_config = MagicMock( + return_value=ModelConfigResult(config=None, model_details=None) + ) result = self.app.get_deployment_config(TestDataset.MODEL_ID) expected_config = AquaDeploymentConfig(**{}) assert result == expected_config @@ -1224,18 +1308,26 @@ def test_verify_compatibility(self): assert result[0] == False assert result[1] == [] - @patch("ads.aqua.modeldeployment.deployment.get_container_config") + @patch.object(AquaApp, "get_container_config_item") @patch("ads.aqua.model.AquaModelApp.create") - @patch("ads.aqua.modeldeployment.deployment.get_container_image") + @patch.object(AquaApp, "get_container_image") @patch("ads.model.deployment.model_deployment.ModelDeployment.deploy") + @patch.object(AquaApp, "get_container_config") def test_create_deployment_for_foundation_model( self, + mock_get_container_config, mock_deploy, mock_get_container_image, mock_create, - mock_get_container_config, + mock_get_container_config_item, ): """Test to create a deployment for foundational model""" + mock_get_container_config.return_value = ( + AquaContainerConfig.from_service_config( + service_containers=TestDataset.CONTAINER_LIST + ) + ) + aqua_model = os.path.join( self.curr_dir, "test_data/deployment/aqua_foundation_model.yaml" ) @@ -1253,12 +1345,9 @@ def test_create_deployment_for_foundation_model( freeform_tags = {"ftag1": "fvalue1", "ftag2": "fvalue2"} defined_tags = {"dtag1": "dvalue1", "dtag2": "dvalue2"} - container_index_json = os.path.join( - self.curr_dir, "test_data/ui/container_index.json" + mock_get_container_config_item.return_value = ( + TestDataset.INFERENCE_CONTAINER_CONFIG_ITEM ) - with open(container_index_json, "r") as _file: - container_index_config = json.load(_file) - mock_get_container_config.return_value = container_index_config shapes = [] @@ -1319,19 +1408,26 @@ def test_create_deployment_for_foundation_model( expected_result["tags"].update(defined_tags) assert actual_attributes == expected_result - @patch("ads.aqua.modeldeployment.deployment.get_container_config") + @patch.object(AquaApp, "get_container_config_item") @patch("ads.aqua.model.AquaModelApp.create") - @patch("ads.aqua.modeldeployment.deployment.get_container_image") + @patch.object(AquaApp, "get_container_image") @patch("ads.model.deployment.model_deployment.ModelDeployment.deploy") + @patch.object(AquaApp, "get_container_config") def test_create_deployment_for_fine_tuned_model( self, + mock_get_container_config, mock_deploy, mock_get_container_image, mock_create, - mock_get_container_config, + mock_get_container_config_item, ): """Test to create a deployment for fine-tuned model""" + mock_get_container_config.return_value = ( + AquaContainerConfig.from_service_config( + service_containers=TestDataset.CONTAINER_LIST + ) + ) aqua_model = os.path.join( self.curr_dir, "test_data/deployment/aqua_finetuned_model.yaml" ) @@ -1363,12 +1459,9 @@ def test_create_deployment_for_fine_tuned_model( return_value=AquaDeploymentConfig(**config) ) - container_index_json = os.path.join( - self.curr_dir, "test_data/ui/container_index.json" + mock_get_container_config_item.return_value = ( + TestDataset.INFERENCE_CONTAINER_CONFIG_ITEM ) - with open(container_index_json, "r") as _file: - container_index_config = json.load(_file) - mock_get_container_config.return_value = container_index_config mock_get_container_image.return_value = TestDataset.DEPLOYMENT_IMAGE_NAME aqua_deployment = os.path.join( @@ -1408,18 +1501,25 @@ def test_create_deployment_for_fine_tuned_model( expected_result["state"] = "CREATING" assert actual_attributes == expected_result - @patch("ads.aqua.modeldeployment.deployment.get_container_config") + @patch.object(AquaApp, "get_container_config_item") @patch("ads.aqua.model.AquaModelApp.create") - @patch("ads.aqua.modeldeployment.deployment.get_container_image") + @patch.object(AquaApp, "get_container_image") @patch("ads.model.deployment.model_deployment.ModelDeployment.deploy") + @patch.object(AquaApp, "get_container_config") def test_create_deployment_for_gguf_model( self, + mock_get_container_config, mock_deploy, mock_get_container_image, mock_create, - mock_get_container_config, + mock_get_container_config_item, ): """Test to create a deployment for fine-tuned model""" + mock_get_container_config.return_value = ( + AquaContainerConfig.from_service_config( + service_containers=TestDataset.CONTAINER_LIST + ) + ) aqua_model = os.path.join( self.curr_dir, "test_data/deployment/aqua_foundation_model.yaml" @@ -1437,14 +1537,9 @@ def test_create_deployment_for_gguf_model( return_value=AquaDeploymentConfig(**config) ) - container_index_json = os.path.join( - self.curr_dir, "test_data/ui/container_index.json" + mock_get_container_config_item.return_value = ( + TestDataset.INFERENCE_CONTAINER_CONFIG_ITEM ) - with open(container_index_json, "r") as _file: - container_index_config = json.load(_file) - mock_get_container_config.return_value = container_index_config - - shapes = [] with open( os.path.join( @@ -1505,18 +1600,27 @@ def test_create_deployment_for_gguf_model( ) assert actual_attributes == expected_result - @patch("ads.aqua.modeldeployment.deployment.get_container_config") + @patch.object(AquaApp, "get_container_config_item") @patch("ads.aqua.model.AquaModelApp.create") - @patch("ads.aqua.modeldeployment.deployment.get_container_image") + @patch.object(AquaApp, "get_container_image") @patch("ads.model.deployment.model_deployment.ModelDeployment.deploy") + @patch.object(AquaApp, "get_container_config") def test_create_deployment_for_tei_byoc_embedding_model( self, + mock_get_container_config, mock_deploy, mock_get_container_image, mock_create, - mock_get_container_config, + mock_get_container_config_item, ): """Test to create a deployment for fine-tuned model""" + + mock_get_container_config.return_value = ( + AquaContainerConfig.from_service_config( + service_containers=TestDataset.CONTAINER_LIST + ) + ) + aqua_model = os.path.join( self.curr_dir, "test_data/deployment/aqua_tei_byoc_embedding_model.yaml" ) @@ -1533,12 +1637,9 @@ def test_create_deployment_for_tei_byoc_embedding_model( return_value=AquaDeploymentConfig(**config) ) - container_index_json = os.path.join( - self.curr_dir, "test_data/ui/container_index.json" + mock_get_container_config_item.return_value = ( + TestDataset.INFERENCE_CONTAINER_CONFIG_ITEM ) - with open(container_index_json, "r") as _file: - container_index_config = json.load(_file) - mock_get_container_config.return_value = container_index_config shapes = [] @@ -1604,9 +1705,9 @@ def test_create_deployment_for_tei_byoc_embedding_model( ) assert actual_attributes == expected_result - @patch("ads.aqua.modeldeployment.deployment.get_container_config") + @patch.object(AquaApp, "get_container_config") @patch("ads.aqua.model.AquaModelApp.create_multi") - @patch("ads.aqua.modeldeployment.deployment.get_container_image") + @patch.object(AquaApp, "get_container_image") @patch("ads.model.deployment.model_deployment.ModelDeployment.deploy") @patch("ads.aqua.modeldeployment.AquaDeploymentApp.get_deployment_config") @patch( @@ -1622,6 +1723,12 @@ def test_create_deployment_for_multi_model( mock_get_container_config, ): """Test to create a deployment for multi models.""" + mock_get_container_config.return_value = ( + AquaContainerConfig.from_service_config( + service_containers=TestDataset.CONTAINER_LIST + ) + ) + mock_validate_multimodel_deployment_feasibility.return_value = MagicMock() self.app.get_multimodel_deployment_config = MagicMock( return_value=AquaDeploymentConfig( @@ -1645,15 +1752,6 @@ def test_create_deployment_for_multi_model( return_value=AquaDeploymentConfig(**config) ) - container_index_json = os.path.join( - self.curr_dir, "test_data/ui/container_index.json" - ) - with open(container_index_json, "r") as _file: - container_index_config = json.load(_file) - mock_get_container_config.return_value = container_index_config - - shapes = [] - with open( os.path.join( self.curr_dir, @@ -1836,7 +1934,7 @@ def test_get_deployment_default_params( ), ( "custom-container-key", - ["--max-model-len 4096", "--seed 42", "--trust-remote-code"], + ["--max-model-len 4096", "--trust-remote-code"], ), ( "odsc-vllm-serving", @@ -1849,9 +1947,9 @@ def test_get_deployment_default_params( ] ) @patch("ads.model.datascience_model.DataScienceModel.from_id") - @patch("ads.aqua.modeldeployment.deployment.get_container_config") + @patch.object(AquaApp, "get_container_config_item") def test_validate_deployment_params( - self, container_type_key, params, mock_get_container_config, mock_from_id + self, container_type_key, params, mock_get_container_config_item, mock_from_id ): """Test for checking if overridden deployment params are valid.""" mock_model = MagicMock() @@ -1862,12 +1960,9 @@ def test_validate_deployment_params( mock_model.custom_metadata_list = custom_metadata_list mock_from_id.return_value = mock_model - container_index_json = os.path.join( - self.curr_dir, "test_data/ui/container_index.json" + mock_get_container_config_item.return_value = ( + TestDataset.INFERENCE_CONTAINER_CONFIG_ITEM ) - with open(container_index_json, "r") as _file: - container_index_config = json.load(_file) - mock_get_container_config.return_value = container_index_config if container_type_key in {"odsc-vllm-serving", "odsc-tgi-serving"} and params: with pytest.raises(AquaValueError): @@ -1899,9 +1994,9 @@ def test_validate_deployment_params( ] ) @patch("ads.model.datascience_model.DataScienceModel.from_id") - @patch("ads.aqua.modeldeployment.deployment.get_container_config") + @patch.object(AquaApp, "get_container_config_item") def test_validate_deployment_params_for_unverified_models( - self, container_type_key, params, mock_get_container_config, mock_from_id + self, container_type_key, params, mock_get_container_config_item, mock_from_id ): """Test to check if container family is used when metadata does not have image information for unverified models.""" @@ -1912,9 +2007,10 @@ def test_validate_deployment_params_for_unverified_models( container_index_json = os.path.join( self.curr_dir, "test_data/ui/container_index.json" ) - with open(container_index_json, "r") as _file: - container_index_config = json.load(_file) - mock_get_container_config.return_value = container_index_config + + mock_get_container_config_item.return_value = ( + TestDataset.INFERENCE_CONTAINER_CONFIG_ITEM + ) if container_type_key in {"odsc-vllm-serving", "odsc-tgi-serving"} and params: result = self.app.validate_deployment_params( diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index b54fefd96..9dd5de028 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -14,6 +14,7 @@ import oci from parameterized import parameterized +from ads.aqua.app import AquaApp from ads.aqua.common import utils from ads.aqua.common.enums import Tags from ads.aqua.common.errors import ( @@ -44,6 +45,7 @@ from ads.model import DataScienceModel from ads.model.deployment.model_deployment import ModelDeployment from ads.model.model_version_set import ModelVersionSet +from tests.unitary.with_extras.aqua.utils import ServiceManagedContainers null = None @@ -51,6 +53,7 @@ class TestDataset: """Mock service response.""" + CONTAINERS_LIST = ServiceManagedContainers.MOCK_OUTPUT model_provenance_object = { "git_branch": null, "git_commit": null, @@ -458,7 +461,7 @@ def assert_payload(self, response, response_type): @patch("ads.jobs.ads_job.Job.name", new_callable=PropertyMock) @patch("ads.jobs.ads_job.Job.id", new_callable=PropertyMock) @patch.object(Job, "create") - @patch("ads.aqua.evaluation.evaluation.get_container_image") + @patch.object(AquaApp, "get_container_image") @patch.object(DataScienceModel, "create") @patch.object(ModelVersionSet, "create") @patch.object(DataScienceModel, "from_id") @@ -750,13 +753,13 @@ def test_list(self): def test_download_report( self, mock_TemporaryDirectory, mock_dsc_model_from_id, mock_download_artifact ): - """Tests download evaluation report successfully.""" + """Tests downloading evaluation report successfully.""" curr_dir = os.path.dirname(os.path.abspath(__file__)) mock_temp_path = os.path.join(curr_dir, "test_data/valid_eval_artifact") mock_TemporaryDirectory.return_value.__enter__.return_value = mock_temp_path response = self.app.download_report(TestDataset.EVAL_ID) - mock_dsc_model_from_id.assert_called_with(TestDataset.EVAL_ID) + self.print_expected_response(response, "DOWNLOAD REPORT") self.assert_payload(response, AquaEvalReport) read_content = base64.b64decode(response.content).decode() @@ -843,14 +846,13 @@ def test_load_metrics( mock_temp_path = os.path.join(curr_dir, "test_data/valid_eval_artifact") mock_TemporaryDirectory.return_value.__enter__.return_value = mock_temp_path response = self.app.load_metrics(TestDataset.EVAL_ID) - mock_dsc_model_from_id.assert_called_with(TestDataset.EVAL_ID) + self.print_expected_response(response, "LOAD METRICS") self.assert_payload(response, AquaEvalMetrics) assert len(response.metric_results) == 1 assert len(response.metric_summary_result) == 1 assert self.app._metrics_cache.currsize == 1 - response1 = self.app.load_metrics(TestDataset.EVAL_ID) assert response1 == self.app._metrics_cache.get(TestDataset.EVAL_ID) @@ -982,76 +984,32 @@ def test_extract_job_lifecycle_details(self, input, expect_output): msg = self.app._extract_job_lifecycle_details(input) assert msg == expect_output, msg - @patch("ads.aqua.evaluation.evaluation.get_evaluation_service_config") - def test_get_supported_metrics(self, mock_get_evaluation_service_config): + @patch.object(AquaApp, "list_service_containers") + def test_get_supported_metrics(self, mock_list_service_containers): """ Tests getting a list of supported metrics for evaluation. """ - test_evaluation_service_config = EvaluationServiceConfig( - ui_config=UIConfig( - metrics=[ - MetricConfig( - **{ - "args": {}, - "description": "BERT Score.", - "key": "bertscore", - "name": "BERT Score", - "tags": [], - "task": ["text-generation"], - }, - ) - ] - ) - ) - mock_get_evaluation_service_config.return_value = test_evaluation_service_config + supported_metrics = [ + "bertscore", + "rouge", + "bleu", + "text_readability", + "perplexity_score", + ] + mock_list_service_containers.return_value = TestDataset.CONTAINERS_LIST response = self.app.get_supported_metrics() assert isinstance(response, list) - assert len(response) == len(test_evaluation_service_config.ui_config.metrics) - assert response == [ - item.to_dict() for item in test_evaluation_service_config.ui_config.metrics - ] + assert len(response) == len(supported_metrics) + assert [m.key for m in response].sort() == supported_metrics.sort() - @patch("ads.aqua.evaluation.evaluation.get_evaluation_service_config") - def test_load_evaluation_config(self, mock_get_evaluation_service_config): + @patch.object(AquaApp, "list_service_containers") + def test_load_evaluation_config(self, mock_list_service_containers): """ Tests loading default config for evaluation. This method currently hardcoded the return value. """ - - test_evaluation_service_config = EvaluationServiceConfig( - ui_config=UIConfig( - model_params=ModelParamsConfig( - **{ - "default": { - "model": "odsc-llm", - "max_tokens": 500, - "temperature": 0.7, - "top_p": 0.9, - "top_k": 50, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, - "stop": [], - } - } - ), - shapes=[ - ShapeConfig( - **{ - "name": "VM.Standard.E3.Flex", - "ocpu": 8, - "memory_in_gbs": 128, - "block_storage_size": 200, - "filter": { - "evaluation_container": ["odsc-llm-evaluate"], - "evaluation_target": ["datasciencemodeldeployment"], - }, - } - ) - ], - ) - ) - mock_get_evaluation_service_config.return_value = test_evaluation_service_config + mock_list_service_containers.return_value = TestDataset.CONTAINERS_LIST expected_result = { "model_params": { @@ -1065,16 +1023,46 @@ def test_load_evaluation_config(self, mock_get_evaluation_service_config): "stop": [], }, "shape": { + "VM.Optimized3.Flex": { + "block_storage_size": 200, + "filter": { + "evaluation_container": ["odsc-llm-evaluate"], + "evaluation_target": ["datasciencemodeldeployment"], + }, + "memory_in_gbs": 128, + "name": "VM.Optimized3.Flex", + "ocpu": 8, + }, "VM.Standard.E3.Flex": { + "block_storage_size": 200, + "filter": { + "evaluation_container": ["odsc-llm-evaluate"], + "evaluation_target": ["datasciencemodeldeployment"], + }, + "memory_in_gbs": 128, "name": "VM.Standard.E3.Flex", "ocpu": 8, + }, + "VM.Standard.E4.Flex": { + "block_storage_size": 200, + "filter": { + "evaluation_container": ["odsc-llm-evaluate"], + "evaluation_target": ["datasciencemodeldeployment"], + }, "memory_in_gbs": 128, + "name": "VM.Standard.E4.Flex", + "ocpu": 8, + }, + "VM.Standard3.Flex": { "block_storage_size": 200, "filter": { "evaluation_container": ["odsc-llm-evaluate"], "evaluation_target": ["datasciencemodeldeployment"], }, - } + "memory_in_gbs": 128, + "name": "VM.Standard3.Flex", + "ocpu": 8, + }, }, "default": { "name": "VM.Standard.E3.Flex", diff --git a/tests/unitary/with_extras/aqua/test_finetuning.py b/tests/unitary/with_extras/aqua/test_finetuning.py index e082595ca..066ae3298 100644 --- a/tests/unitary/with_extras/aqua/test_finetuning.py +++ b/tests/unitary/with_extras/aqua/test_finetuning.py @@ -21,7 +21,7 @@ from ads.aqua.common.errors import AquaValueError from ads.aqua.finetuning import AquaFineTuningApp from ads.aqua.finetuning.constants import FineTuneCustomMetadata -from ads.aqua.finetuning.entities import AquaFineTuningParams +from ads.aqua.finetuning.entities import AquaFineTuningParams, AquaFineTuningConfig from ads.aqua.model.entities import AquaFineTuneModel from ads.jobs.ads_job import Job from ads.jobs.builders.infrastructure.dsc_job import DataScienceJobRun @@ -61,7 +61,7 @@ def tearDownClass(cls): @patch("ads.jobs.ads_job.Job.name", new_callable=PropertyMock) @patch("ads.jobs.ads_job.Job.id", new_callable=PropertyMock) @patch.object(Job, "create") - @patch("ads.aqua.finetuning.finetuning.get_container_image") + @patch.object(AquaApp, "get_container_image") @patch.object(AquaFineTuningApp, "get_finetuning_config") @patch.object(AquaApp, "create_model_catalog") @patch.object(AquaApp, "create_model_version_set") @@ -113,11 +113,13 @@ def test_create_fine_tuning( ft_model.time_created = "test_time_created" mock_ds_model_create.return_value = ft_model - mock_get_finetuning_config.return_value = { - "shape": { - "VM.GPU.A10.1": {"batch_size": 1, "replica": 1}, + mock_get_finetuning_config.return_value = AquaFineTuningConfig( + **{ + "shape": { + "VM.GPU.A10.1": {"batch_size": 1, "replica": 1}, + } } - } + ) mock_get_container_image.return_value = "test_container_image" mock_job_id.return_value = "test_ft_job_id" @@ -279,9 +281,13 @@ def test_get_finetuning_config(self): config_json = os.path.join(self.curr_dir, "test_data/finetuning/ft_config.json") with open(config_json, "r") as _file: config = json.load(_file) - - self.app.get_config = MagicMock(return_value=ModelConfigResult(config=config)) - result = self.app.get_finetuning_config(model_id="test-model-id") + self.app.get_config_from_metadata = MagicMock( + return_value=ModelConfigResult(config=config, model_details=None) + ) + self.app.get_config = MagicMock( + return_value=ModelConfigResult(config=config, model_details=None) + ) + result = self.app.get_finetuning_config(model_id="test-model-id").to_dict() assert result == config def test_get_finetuning_default_params(self): @@ -304,12 +310,16 @@ def test_get_finetuning_default_params(self): with open(config_json, "r") as _file: config = json.load(_file) - self.app.get_finetuning_config = MagicMock(return_value=config) + self.app.get_finetuning_config = MagicMock( + return_value=AquaFineTuningConfig(**config) + ) result = self.app.get_finetuning_default_params(model_id="test_model_id") assert result == params_dict # check when config json is not available - self.app.get_finetuning_config = MagicMock(return_value={}) + self.app.get_finetuning_config = MagicMock( + return_value=AquaFineTuningConfig(**{}) + ) result = self.app.get_finetuning_default_params(model_id="test_model_id") assert result == {"params": {}} diff --git a/tests/unitary/with_extras/aqua/test_model.py b/tests/unitary/with_extras/aqua/test_model.py index f202228fd..e7bc97bb8 100644 --- a/tests/unitary/with_extras/aqua/test_model.py +++ b/tests/unitary/with_extras/aqua/test_model.py @@ -13,6 +13,9 @@ import oci import pytest + +from ads.aqua.app import AquaApp +from ads.aqua.config.container_config import AquaContainerConfig from huggingface_hub.hf_api import HfApi, ModelInfo from parameterized import parameterized @@ -44,7 +47,8 @@ ModelProvenanceMetadata, ModelTaxonomyMetadata, ) -from ads.model.service.oci_datascience_model import OCIDataScienceModel + +from tests.unitary.with_extras.aqua.utils import ServiceManagedContainers @pytest.fixture(autouse=True, scope="class") @@ -54,22 +58,15 @@ def mock_auth(): def get_container_config(): - with open( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "test_data/ui/container_index.json", - ), - "r", - ) as _file: - container_index_json = json.load(_file) - - return container_index_json + return AquaContainerConfig.from_service_config( + service_containers=TestDataset.CONTAINERS_LIST + ) @pytest.fixture(autouse=True, scope="class") def mock_get_container_config(): - with patch("ads.aqua.model.model.get_container_config") as mock_config: - mock_config.return_value = get_container_config() + with patch.object(AquaApp, "get_container_config") as mock_config: + mock_config.return_value = TestDataset yield mock_config @@ -233,6 +230,8 @@ class TestDataset: COMPARTMENT_ID = "ocid1.compartment.oc1.." SERVICE_MODEL_ID = "ocid1.datasciencemodel.oc1.iad." + CONTAINERS_LIST = ServiceManagedContainers.MOCK_OUTPUT + @patch("ads.config.COMPARTMENT_OCID", "ocid1.compartment.oc1.") @patch("ads.config.PROJECT_OCID", "ocid1.datascienceproject.oc1.iad.") @@ -365,7 +364,7 @@ def test_create_model(self, mock_from_id, mock_validate, mock_create): @patch.object(DataScienceModel, "create_custom_metadata_artifact") @patch.object(DataScienceModel, "create") @patch("ads.model.datascience_model.validate") - @patch("ads.aqua.model.model.get_container_config") + @patch.object(AquaApp, "get_container_config") @patch.object(DataScienceModel, "from_id") def test_create_multimodel( self, @@ -469,7 +468,7 @@ def test_create_multimodel( "verified", ], ) - @patch("ads.aqua.model.model.get_container_config") + @patch.object(AquaApp, "get_container_config") @patch("ads.aqua.model.model.read_file") @patch.object(DataScienceModel, "from_id") @patch( @@ -540,17 +539,6 @@ def test_get_foundation_models( mock_from_id.assert_called_with(model_id) - if foundation_model_type == "verified": - mock_read_file.assert_called_with( - file_path="oci://bucket@namespace/prefix/config/README.md", - auth=mock_auth(), - ) - else: - mock_read_file.assert_called_with( - file_path="oci://bucket@namespace/prefix/README.md", - auth=mock_auth(), - ) - assert asdict(aqua_model) == { "arm_cpu_supported": False, "artifact_location": "oci://bucket@namespace/prefix/", @@ -560,7 +548,6 @@ def test_get_foundation_models( "id": f"{ds_model.id}", "is_fine_tuned_model": False, "license": f'{ds_model.freeform_tags["license"]}', - "model_card": f"{mock_read_file.return_value}", "model_formats": [ModelFormat.SAFETENSORS], "model_file": "", "name": f"{ds_model.display_name}", @@ -585,7 +572,7 @@ def test_get_foundation_models( } @patch("ads.aqua.common.utils.query_resource") - @patch("ads.aqua.model.model.get_container_config") + @patch.object(AquaApp, "get_container_config") @patch("ads.aqua.model.model.read_file") @patch.object(DataScienceModel, "from_id") @patch( @@ -660,7 +647,7 @@ def test_get_model_fine_tuned( ) mock_from_id.return_value = ds_model - mock_read_file.return_value = "test_model_card" + mock_read_file.return_value = "" response = MagicMock() job_run = MagicMock() @@ -693,10 +680,6 @@ def test_get_model_fine_tuned( model = self.app.get(model_id="test_model_id") mock_from_id.assert_called_with("test_model_id") - mock_read_file.assert_called_with( - file_path="oci://bucket@namespace/prefix/README.md", - auth=mock_auth(), - ) mock_query_resource.assert_called() assert asdict(model) == { @@ -739,7 +722,6 @@ def test_get_model_fine_tuned( "scores": [], }, ], - "model_card": f"{mock_read_file.return_value}", "model_formats": [ModelFormat.SAFETENSORS], "model_file": "", "name": f"{ds_model.display_name}", @@ -782,8 +764,12 @@ def test_get_model_fine_tuned( @patch("ads.aqua.common.utils.load_config", return_value={}) @patch("huggingface_hub.snapshot_download") @patch("subprocess.check_call") + @patch.object(AquaApp, "get_container_config") + @patch("ads.common.utils.is_path_exists", return_value=True) def test_import_verified_model( self, + mock_is_path_exists, + mock_get_container_config, mock_subprocess, mock_snapshot_download, mock_load_config, @@ -808,7 +794,7 @@ def test_import_verified_model( obj2.name = f"{artifact_path}/config/ft_config.json" objects = [obj1, obj2] mock_list_objects.return_value = MagicMock(objects=objects) - + mock_get_container_config.return_value = get_container_config() ds_model = DataScienceModel() os_path = "oci://aqua-bkt@aqua-ns/prefix/path" model_name = "oracle/aqua-1t-mega-model" @@ -830,6 +816,7 @@ def test_import_verified_model( .with_version_id("ocid1.blah.blah") ) custom_metadata_list = ModelCustomMetadata() + defined_metadata_list = ModelTaxonomyMetadata() custom_metadata_list.add( **{"key": "deployment-container", "value": "odsc-tgi-serving"} ) @@ -845,6 +832,7 @@ def test_import_verified_model( } ) ds_model.with_custom_metadata_list(custom_metadata_list) + ds_model.with_defined_metadata_list(defined_metadata_list) ds_model.set_spec(ds_model.CONST_MODEL_FILE_DESCRIPTION, {}) ds_model.dsc_model = MagicMock(id="test_model_id") DataScienceModel.from_id = MagicMock(return_value=ds_model) @@ -920,8 +908,12 @@ def test_import_verified_model( @patch("ads.model.datascience_model.DataScienceModel.upload_artifact") @patch.object(AquaModelApp, "_validate_model") @patch("ads.aqua.common.utils.load_config", return_value={}) + @patch.object(AquaApp, "get_container_config") + @patch("ads.common.utils.is_path_exists", return_value=True) def test_import_any_model_no_containers_specified( self, + mock_is_path_exists, + mock_get_container_config, mock_load_config, mock__validate_model, mock_upload_artifact, @@ -938,6 +930,7 @@ def test_import_any_model_no_containers_specified( "organization": "oracle", "task": "text-generation", } + mock_get_container_config.return_value = get_container_config() mock__validate_model.return_value = ModelValidationResult( model_file="model_file.gguf", model_formats=[ModelFormat.SAFETENSORS], @@ -974,8 +967,12 @@ def test_import_any_model_no_containers_specified( @patch("ads.aqua.common.utils.load_config", return_value={}) @patch("huggingface_hub.snapshot_download") @patch("subprocess.check_call") + @patch.object(AquaApp, "get_container_config") + @patch("ads.common.utils.is_path_exists", return_value=True) def test_import_model_with_project_compartment_override( self, + mock_is_path_exists, + mock_get_container_config, mock_subprocess, mock_snapshot_download, mock_load_config, @@ -988,7 +985,7 @@ def test_import_model_with_project_compartment_override( mock_get_hf_model_info, ): ObjectStorageDetails.is_bucket_versioned = MagicMock(return_value=True) - + mock_get_container_config.return_value = get_container_config() mock_list_objects.return_value = MagicMock(objects=[]) ds_model = DataScienceModel() os_path = "oci://aqua-bkt@aqua-ns/prefix/path" @@ -1011,6 +1008,7 @@ def test_import_model_with_project_compartment_override( .with_version_id("ocid1.blah.blah") ) custom_metadata_list = ModelCustomMetadata() + defined_metadata_list = ModelTaxonomyMetadata() custom_metadata_list.add( **{"key": "deployment-container", "value": "odsc-tgi-serving"} ) @@ -1018,6 +1016,7 @@ def test_import_model_with_project_compartment_override( **{"key": "evaluation-container", "value": "odsc-llm-evaluate"} ) ds_model.with_custom_metadata_list(custom_metadata_list) + ds_model.with_defined_metadata_list(defined_metadata_list) ds_model.set_spec(ds_model.CONST_MODEL_FILE_DESCRIPTION, {}) DataScienceModel.from_id = MagicMock(return_value=ds_model) mock__find_matching_aqua_model.return_value = "test_model_id" @@ -1063,8 +1062,12 @@ def test_import_model_with_project_compartment_override( @patch("ads.aqua.common.utils.load_config", side_effect=AquaFileNotFoundError) @patch("huggingface_hub.snapshot_download") @patch("subprocess.check_call") + @patch.object(AquaApp, "get_container_config") + @patch("ads.common.utils.is_path_exists", return_value=True) def test_import_model_with_missing_config( self, + mock_is_path_exists, + mock_get_container_config, mock_subprocess, mock_snapshot_download, mock_load_config, @@ -1077,6 +1080,7 @@ def test_import_model_with_missing_config( mock_get_hf_model_info, mock_init_client, ): + mock_get_container_config.return_value = get_container_config() my_model = "oracle/aqua-1t-mega-model" ObjectStorageDetails.is_bucket_versioned = MagicMock(return_value=True) # set object list from OSS without config.json @@ -1130,8 +1134,12 @@ def test_import_model_with_missing_config( @patch("ads.model.datascience_model.DataScienceModel.upload_artifact") @patch("ads.common.object_storage_details.ObjectStorageDetails.list_objects") @patch("ads.aqua.common.utils.load_config", return_value={}) + @patch.object(AquaApp, "get_container_config") + @patch("ads.common.utils.is_path_exists", return_value=True) def test_import_any_model_smc_container( self, + mock_is_path_exist, + mock_get_container_config, mock_load_config, mock_list_objects, mock_upload_artifact, @@ -1142,7 +1150,7 @@ def test_import_any_model_smc_container( ): my_model = "oracle/aqua-1t-mega-model" ObjectStorageDetails.is_bucket_versioned = MagicMock(return_value=True) - + mock_get_container_config.return_value = get_container_config() os_path = "oci://aqua-bkt@aqua-ns/prefix/path" ds_freeform_tags = { "OCI_AQUA": "active", @@ -1201,8 +1209,12 @@ def test_import_any_model_smc_container( @patch("ads.aqua.common.utils.load_config", return_value={}) @patch("huggingface_hub.snapshot_download") @patch("subprocess.check_call") + @patch.object(AquaApp, "get_container_config") + @patch("ads.common.utils.is_path_exists", return_value=True) def test_import_tei_model_byoc( self, + mock_is_path_exists, + mock_get_container_config, mock_subprocess, mock_snapshot_download, mock_load_config, @@ -1215,6 +1227,7 @@ def test_import_tei_model_byoc( mock_get_hf_model_info, mock_init_client, ): + mock_get_container_config.return_value = get_container_config() ObjectStorageDetails.is_bucket_versioned = MagicMock(return_value=True) artifact_path = "service_models/model-name/commit-id/artifact" @@ -1241,10 +1254,12 @@ def test_import_tei_model_byoc( .with_version_id("ocid1.version.id") ) custom_metadata_list = ModelCustomMetadata() + defined_metadata_list = ModelTaxonomyMetadata() custom_metadata_list.add( **{"key": "deployment-container", "value": "odsc-tei-serving"} ) ds_model.with_custom_metadata_list(custom_metadata_list) + ds_model.with_defined_metadata_list(defined_metadata_list) ds_model.set_spec(ds_model.CONST_MODEL_FILE_DESCRIPTION, {}) DataScienceModel.from_id = MagicMock(return_value=ds_model) mock__find_matching_aqua_model.return_value = None @@ -1279,8 +1294,12 @@ def test_import_tei_model_byoc( @patch("ads.common.object_storage_details.ObjectStorageDetails.list_objects") @patch.object(HfApi, "model_info") @patch("ads.aqua.common.utils.load_config", return_value={}) + @patch.object(AquaApp, "get_container_config") + @patch("ads.common.utils.is_path_exists", return_value=True) def test_import_model_with_input_tags( self, + mock_is_path_exists, + mock_get_container_config, mock_load_config, mock_list_objects, mock_upload_artifact, @@ -1291,7 +1310,7 @@ def test_import_model_with_input_tags( ): my_model = "oracle/aqua-1t-mega-model" ObjectStorageDetails.is_bucket_versioned = MagicMock(return_value=True) - + mock_get_container_config.return_value = get_container_config() os_path = "oci://aqua-bkt@aqua-ns/prefix/path" ds_freeform_tags = { "OCI_AQUA": "active", @@ -1402,23 +1421,30 @@ def test_import_cli(self, data, expected_output): @patch("ads.aqua.model.model.read_file") @patch("ads.aqua.model.model.get_artifact_path") - def test_load_license(self, mock_get_artifact_path, mock_read_file): + @patch("ads.common.utils.is_path_exists") + def test_load_license( + self, mock_is_path_exist, mock_get_artifact_path, mock_read_file + ): self.app.ds_client.get_model = MagicMock() + self.app.ds_client.get_model_defined_metadatum_artifact_content.return_value.data.content.decode.return_value = "test_license" mock_get_artifact_path.return_value = ( "oci://bucket@namespace/prefix/config/LICENSE.txt" ) mock_read_file.return_value = "test_license" - + mock_is_path_exist.return_value = True license = self.app.load_license(model_id="test_model_id") mock_get_artifact_path.assert_called() - mock_read_file.assert_called() - assert asdict(license) == {"id": "test_model_id", "license": "test_license"} + assert asdict(license) == { + "id": "test_model_id", + "license": "test_license", + } - def test_list_service_models(self): + @patch.object(AquaApp, "get_container_config") + def test_list_service_models(self, mock_get_container_config): """Tests listing service models succesfully.""" - + mock_get_container_config.return_value = get_container_config() self.app.list_resource = MagicMock( return_value=[ oci.data_science.models.ModelSummary(**item) @@ -1442,9 +1468,10 @@ def test_list_service_models(self): for attr in attributes: assert rdict.get(attr) is not None - def test_list_custom_models(self): + @patch.object(AquaApp, "get_container_config") + def test_list_custom_models(self, mock_get_container_config): """Tests list custom models succesfully.""" - + mock_get_container_config.return_value = get_container_config() self.app._rqs = MagicMock( return_value=[ oci.resource_search.models.ResourceSummary(**item) diff --git a/tests/unitary/with_extras/aqua/test_model_handler.py b/tests/unitary/with_extras/aqua/test_model_handler.py index 818638cea..abc049812 100644 --- a/tests/unitary/with_extras/aqua/test_model_handler.py +++ b/tests/unitary/with_extras/aqua/test_model_handler.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*-- # Copyright (c) 2024, 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ - +from unicodedata import category from unittest import TestCase from unittest.mock import MagicMock, patch @@ -27,6 +27,7 @@ ) from ads.aqua.model import AquaModelApp from ads.aqua.model.entities import AquaModel, AquaModelSummary, HFModelSummary +from ads.config import USER class ModelHandlerTestCase(TestCase): @@ -133,7 +134,7 @@ def test_list(self, mock_list): mock_finish.side_effect = lambda x: x self.model_handler.list() mock_list.assert_called_with( - compartment_id=None, project_id=None, model_type=None + compartment_id=None, project_id=None, model_type=None, category=USER ) @parameterized.expand( @@ -362,12 +363,11 @@ def test_post_negative(self, mock_uuid, mock_format_hf_custom_error_message): self.mock_handler.finish.assert_called_with( ReplyDetails( status=400, - troubleshooting_tips= f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}", - message= "Invalid format of input data.", - service_payload = {}, - reason = "Invalid format of input data.", - request_id = "###" - + troubleshooting_tips=f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}", + message="Invalid format of input data.", + service_payload={}, + reason="Invalid format of input data.", + request_id="###", ) ) get_hf_model_info.cache_clear() @@ -378,12 +378,11 @@ def test_post_negative(self, mock_uuid, mock_format_hf_custom_error_message): self.mock_handler.finish.assert_called_with( ReplyDetails( status=400, - troubleshooting_tips= f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}", - message= "No input data provided.", - service_payload = {}, - reason = "No input data provided.", - request_id = "###" - + troubleshooting_tips=f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}", + message="No input data provided.", + service_payload={}, + reason="No input data provided.", + request_id="###", ) ) get_hf_model_info.cache_clear() @@ -394,12 +393,11 @@ def test_post_negative(self, mock_uuid, mock_format_hf_custom_error_message): self.mock_handler.finish.assert_called_with( ReplyDetails( status=400, - troubleshooting_tips= f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}", - message= "Missing required parameter: \'model_id\'", - service_payload = {}, - reason = "Missing required parameter: \'model_id\'", - request_id = "###" - + troubleshooting_tips=f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}", + message="Missing required parameter: 'model_id'", + service_payload={}, + reason="Missing required parameter: 'model_id'", + request_id="###", ) ) get_hf_model_info.cache_clear() @@ -418,12 +416,11 @@ def test_post_negative(self, mock_uuid, mock_format_hf_custom_error_message): self.mock_handler.finish.assert_called_with( ReplyDetails( status=400, - troubleshooting_tips= f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}", - message= STATUS_CODE_MESSAGES["400"], - service_payload = {}, - reason = "test error message", - request_id = "###" - + troubleshooting_tips=f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}", + message=STATUS_CODE_MESSAGES["400"], + service_payload={}, + reason="test error message", + request_id="###", ) ) get_hf_model_info.cache_clear() @@ -439,12 +436,11 @@ def test_post_negative(self, mock_uuid, mock_format_hf_custom_error_message): self.mock_handler.finish.assert_called_with( ReplyDetails( status=400, - troubleshooting_tips= f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}", - message= STATUS_CODE_MESSAGES["400"], - service_payload = {}, - reason = "The chosen model \'test_model_id\' is currently disabled and cannot be imported into AQUA. Please verify the model\'s status on the Hugging Face Model Hub or select a different model.", - request_id = "###" - + troubleshooting_tips=f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}", + message=STATUS_CODE_MESSAGES["400"], + service_payload={}, + reason="The chosen model 'test_model_id' is currently disabled and cannot be imported into AQUA. Please verify the model's status on the Hugging Face Model Hub or select a different model.", + request_id="###", ) ) get_hf_model_info.cache_clear() diff --git a/tests/unitary/with_extras/aqua/test_ui.py b/tests/unitary/with_extras/aqua/test_ui.py index bbf91e9f1..3fe2021b4 100644 --- a/tests/unitary/with_extras/aqua/test_ui.py +++ b/tests/unitary/with_extras/aqua/test_ui.py @@ -13,6 +13,14 @@ import oci import pytest +from oci.data_science.models import ( + ContainerSummary, + WorkloadConfigurationDetails, + ModelDeployWorkloadConfigurationDetails, + JobRunWorkloadConfigurationDetails, +) + +from ads.aqua.app import AquaApp from ads.aqua.extension.base_handler import AquaAPIhandler from parameterized import parameterized @@ -20,6 +28,7 @@ from ads.aqua.common.errors import AquaValueError from ads.aqua.common.utils import load_config from ads.aqua.ui import AquaUIApp +from tests.unitary.with_extras.aqua.utils import ServiceManagedContainers class TestDataset: @@ -29,6 +38,36 @@ class TestDataset: VCN_ID = "ocid1.vcn.oc1.iad." DEPLOYMENT_SHAPE_NAMES = ["VM.GPU.A10.1", "BM.GPU4.8", "VM.GPU.A10.2"] LIMIT_NAMES = ["ds-gpu-a10-count", "ds-gpu4-count", "ds-gpu-a10-count"] + CONTAINERS_LIST = ServiceManagedContainers.MOCK_OUTPUT + EVAL_CONTAINER_ITEM = ContainerSummary( + **{ + "container_name": "odsc-llm-evaluate", + "display_name": "Evaluate:0.1.3.4", + "family_name": "odsc-llm-evaluate", + "description": "This container supports evaluation on model deployment", + "is_latest": True, + "target_workloads": ["JOB_RUN"], + "usages": ["EVALUATION"], + "tag": "0.1.3.4", + "lifecycle_state": "ACTIVE", + "workload_configuration_details_list": [ + JobRunWorkloadConfigurationDetails( + **{ + "use_case_configuration": { + "useCaseType": "GENERIC", + "additionalConfigurations": { + "metrics": '[{"task":["text-generation"],"key":"bertscore","name":"BERTScore","description":"BERTScoreisametricforevaluatingthequalityoftextgenerationmodels,suchasmachinetranslationorsummarization.Itutilizespre-trainedBERTcontextualembeddingsforboththegeneratedandreferencetexts,andthencalculatesthecosinesimilaritybetweentheseembeddings.","args":{},"tags":[]},{"task":["text-generation"],"key":"rouge","name":"ROUGEScore","description":"ROUGEscorescompareacandidatedocumenttoacollectionofreferencedocumentstoevaluatethesimilaritybetweenthem.Themetricsrangefrom0to1,withhigherscoresindicatinggreatersimilarity.ROUGEismoresuitableformodelsthatdon\'tincludeparaphrasinganddonotgeneratenewtextunitsthatdon\'tappearinthereferences.","args":{},"tags":[]},{"task":["text-generation"],"key":"bleu","name":"BLEUScore","description":"BLEU(BilingualEvaluationUnderstudy)isanalgorithmforevaluatingthequalityoftextwhichhasbeenmachine-translatedfromonenaturallanguagetoanother.Qualityisconsideredtobethecorrespondencebetweenamachine\'soutputandthatofahuman:\'thecloseramachinetranslationistoaprofessionalhumantranslation,thebetteritis\'.","args":{},"tags":[]},{"task":["text-generation"],"key":"perplexity_score","name":"PerplexityScore","description":"Perplexityisametrictoevaluatethequalityoflanguagemodels,particularlyfor\\"TextGeneration\\"tasktype.PerplexityquantifieshowwellaLLMcanpredictthenextwordinasequenceofwords.AhighperplexityscoreindicatesthattheLLMisnotconfidentinitstextgeneration—thatis,themodelis\\"perplexed\\"—whereasalowperplexityscoreindicatesthattheLLMisconfidentinitsgeneration.","args":{},"tags":[]},{"task":["text-generation"],"key":"text_readability","name":"TextReadability","description":"Textquality/readabilitymetricsoffervaluableinsightsintothequalityandsuitabilityofgeneratedresponses.MonitoringthesemetricshelpsensurethatLanguageModel(LLM)outputsareclear,concise,andappropriateforthetargetaudience.Evaluatingtextcomplexityandgradelevelhelpstailorthegeneratedcontenttotheintendedreaders.Byconsideringaspectssuchassentencestructure,vocabulary,anddomain-specificneeds,wecanmakesuretheLLMproducesresponsesthatmatchthedesiredreadinglevelandprofessionalcontext.Additionally,metricslikesyllablecount,wordcount,andcharactercountallowyoutokeeptrackofthelengthandstructureofthegeneratedtext.","args":{},"tags":[]}]', + "shapes": '[{"name":"VM.Standard.E3.Flex","ocpu":8,"memory_in_gbs":128,"block_storage_size":200,"filter":{"evaluation_container":["odsc-llm-evaluate"],"evaluation_target":["datasciencemodeldeployment"]}},{"name":"VM.Standard.E4.Flex","ocpu":8,"memory_in_gbs":128,"block_storage_size":200,"filter":{"evaluation_container":["odsc-llm-evaluate"],"evaluation_target":["datasciencemodeldeployment"]}},{"name":"VM.Standard3.Flex","ocpu":8,"memory_in_gbs":128,"block_storage_size":200,"filter":{"evaluation_container":["odsc-llm-evaluate"],"evaluation_target":["datasciencemodeldeployment"]}},{"name":"VM.Optimized3.Flex","ocpu":8,"memory_in_gbs":128,"block_storage_size":200,"filter":{"evaluation_container":["odsc-llm-evaluate"],"evaluation_target":["datasciencemodeldeployment"]}}]', + }, + } + } + ) + ], + "tag_configuration_list": [], + "freeform_tags": None, + "defined_tags": None, + } + ) class TestAquaUI(unittest.TestCase): @@ -495,96 +534,45 @@ def test_is_bucket_versioned(self, versioned, mock_from_path): result = self.app.is_bucket_versioned("oci://bucket-name-@namespace/prefix") assert result["is_versioned"] == versioned - @patch("ads.aqua.ui.get_container_config") - def test_list_containers(self, mock_get_container_config): + @patch.object(AquaApp, "list_service_containers") + def test_list_containers(self, mock_list_service_containers): """Test to lists AQUA containers.""" - with open( - os.path.join(self.curr_dir, "test_data/ui/container_index.json"), "r" - ) as _file: - container_index_json = json.load(_file) - - mock_get_container_config.return_value = container_index_json - - test_result = self.app.list_containers().to_dict() + mock_list_service_containers.return_value = TestDataset.CONTAINERS_LIST + test_result = self.app.list_containers() + print("test_result: ", test_result) expected_result = { "evaluate": [ { "name": "dsmc://odsc-llm-evaluate", - "version": "0.1.2.1", - "display_name": "0.1.2.1", + "version": "0.1.3.4", + "display_name": "Evaluate:0.1.3.4", "family": "odsc-llm-evaluate", "platforms": [], "model_formats": [], "spec": None, - "usages": [], + "usages": ["EVALUATION"], } ], "inference": [ - { - "name": "dsmc://odsc-llama-cpp-python-aio-linux_arm64_v8", - "version": "0.2.75.5", - "display_name": "LLAMA-CPP:0.2.75", - "family": "odsc-llama-cpp-serving", - "platforms": ["ARM_CPU"], - "model_formats": ["GGUF"], - "spec": { - "cli_param": "", - "env_vars": [ - {"MODEL_DEPLOY_PREDICT_ENDPOINT": "/v1/completions"}, - {"MODEL_DEPLOY_HEALTH_ENDPOINT": "/v1/models"}, - {"MODEL_DEPLOY_ENABLE_STREAMING": "true"}, - {"PORT": "8080"}, - {"HEALTH_CHECK_PORT": "8080"}, - ], - "health_check_port": "8080", - "restricted_params": [], - "server_port": "8080", - }, - "usages": [], - }, - { - "name": "dsmc://odsc-text-generation-inference", - "version": "2.0.1.4", - "display_name": "TGI:2.0.1", - "family": "odsc-tgi-serving", - "platforms": ["NVIDIA_GPU"], - "model_formats": ["SAFETENSORS"], - "spec": { - "cli_param": "--sharded true --trust-remote-code", - "env_vars": [ - {"MODEL_DEPLOY_PREDICT_ENDPOINT": "/v1/completions"}, - {"MODEL_DEPLOY_ENABLE_STREAMING": "true"}, - {"PORT": "8080"}, - {"HEALTH_CHECK_PORT": "8080"}, - ], - "health_check_port": "8080", - "restricted_params": [ - "--port", - "--hostname", - "--num-shard", - "--sharded", - "--trust-remote-code", - ], - "server_port": "8080", - }, - "usages": [], - }, { "name": "dsmc://odsc-vllm-serving", - "version": "0.4.1.3", - "display_name": "VLLM:0.4.1", + "version": "0.6.4.post1.2", + "display_name": "VLLM:0.6.4.post1.2", "family": "odsc-vllm-serving", "platforms": ["NVIDIA_GPU"], "model_formats": ["SAFETENSORS"], "spec": { - "cli_param": "--served-model-name odsc-llm --seed 42 ", + "cli_param": "--served-model-name odsc-llm --disable-custom-all-reduce --seed 42 ", "env_vars": [ - {"MODEL_DEPLOY_PREDICT_ENDPOINT": "/v1/completions"}, - {"MODEL_DEPLOY_ENABLE_STREAMING": "true"}, - {"PORT": "8080"}, - {"HEALTH_CHECK_PORT": "8080"}, + { + "HEALTH_CHECK_PORT": "8080", + "MODEL_DEPLOY_ENABLE_STREAMING": "true", + "MODEL_DEPLOY_HEALTH_ENDPOINT": "", + "MODEL_DEPLOY_PREDICT_ENDPOINT": "/v1/completions", + "PORT": "8080", + } ], "health_check_port": "8080", "restricted_params": [ @@ -594,25 +582,25 @@ def test_list_containers(self, mock_get_container_config): "--seed", ], "server_port": "8080", + "evaluation_configuration": {}, }, - "usages": ["inference", "batch_inference", "multi_model"], + "usages": ["INFERENCE", "BATCH_INFERENCE", "OTHER"], }, ], "finetune": [ { "name": "dsmc://odsc-llm-fine-tuning", - "version": "1.1.37.37", - "display_name": "1.1.37.37", + "version": "2.2.62.70", + "display_name": "Fine-Tune:2.2.62.70", "family": "odsc-llm-fine-tuning", "platforms": [], "model_formats": [], "spec": None, - "usages": [], + "usages": ["FINE_TUNE"], } ], } test_result = json.loads( json.dumps(test_result, default=AquaAPIhandler.serialize) ) - for key in expected_result: - assert expected_result[key] == test_result[key] + assert expected_result == test_result diff --git a/tests/unitary/with_extras/aqua/utils.py b/tests/unitary/with_extras/aqua/utils.py index bdc72e3b5..ee7ab0241 100644 --- a/tests/unitary/with_extras/aqua/utils.py +++ b/tests/unitary/with_extras/aqua/utils.py @@ -1,13 +1,22 @@ #!/usr/bin/env python # -*- coding: utf-8 -*-- -# Copyright (c) 2024 Oracle and/or its affiliates. +# Copyright (c) 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import typing from dataclasses import dataclass, fields from typing import Dict +from oci.data_science.models import ( + ContainerSummary, + ModelDeployWorkloadConfigurationDetails, + JobRunWorkloadConfigurationDetails, + JobRunUseCaseConfigurationDetails, + WorkloadConfigurationDetails, + GenericJobRunUseCaseConfigurationDetails, +) + @dataclass(repr=False) class MockData: @@ -17,6 +26,90 @@ class MockData: name: str = "" +@dataclass(repr=False) +class ServiceManagedContainers: + use_case_configuration = GenericJobRunUseCaseConfigurationDetails( + additional_configurations={ + "metrics": '[{"task":["text-generation"],"key":"bertscore","name":"BERTScore","description":"BERTScoreisametricforevaluatingthequalityoftextgenerationmodels,suchasmachinetranslationorsummarization.Itutilizespre-trainedBERTcontextualembeddingsforboththegeneratedandreferencetexts,andthencalculatesthecosinesimilaritybetweentheseembeddings.","args":{},"tags":[]},{"task":["text-generation"],"key":"rouge","name":"ROUGEScore","description":"ROUGEscorescompareacandidatedocumenttoacollectionofreferencedocumentstoevaluatethesimilaritybetweenthem.Themetricsrangefrom0to1,withhigherscoresindicatinggreatersimilarity.ROUGEismoresuitableformodelsthatdon\'tincludeparaphrasinganddonotgeneratenewtextunitsthatdon\'tappearinthereferences.","args":{},"tags":[]},{"task":["text-generation"],"key":"bleu","name":"BLEUScore","description":"BLEU(BilingualEvaluationUnderstudy)isanalgorithmforevaluatingthequalityoftextwhichhasbeenmachine-translatedfromonenaturallanguagetoanother.Qualityisconsideredtobethecorrespondencebetweenamachine\'soutputandthatofahuman:\'thecloseramachinetranslationistoaprofessionalhumantranslation,thebetteritis\'.","args":{},"tags":[]},{"task":["text-generation"],"key":"perplexity_score","name":"PerplexityScore","description":"Perplexityisametrictoevaluatethequalityoflanguagemodels,particularlyfor\\"TextGeneration\\"tasktype.PerplexityquantifieshowwellaLLMcanpredictthenextwordinasequenceofwords.AhighperplexityscoreindicatesthattheLLMisnotconfidentinitstextgeneration—thatis,themodelis\\"perplexed\\"—whereasalowperplexityscoreindicatesthattheLLMisconfidentinitsgeneration.","args":{},"tags":[]},{"task":["text-generation"],"key":"text_readability","name":"TextReadability","description":"Textquality/readabilitymetricsoffervaluableinsightsintothequalityandsuitabilityofgeneratedresponses.MonitoringthesemetricshelpsensurethatLanguageModel(LLM)outputsareclear,concise,andappropriateforthetargetaudience.Evaluatingtextcomplexityandgradelevelhelpstailorthegeneratedcontenttotheintendedreaders.Byconsideringaspectssuchassentencestructure,vocabulary,anddomain-specificneeds,wecanmakesuretheLLMproducesresponsesthatmatchthedesiredreadinglevelandprofessionalcontext.Additionally,metricslikesyllablecount,wordcount,andcharactercountallowyoutokeeptrackofthelengthandstructureofthegeneratedtext.","args":{},"tags":[]}]', + "shapes": '[{"name":"VM.Standard.E3.Flex","ocpu":8,"memory_in_gbs":128,"block_storage_size":200,"filter":{"evaluation_container":["odsc-llm-evaluate"],"evaluation_target":["datasciencemodeldeployment"]}},{"name":"VM.Standard.E4.Flex","ocpu":8,"memory_in_gbs":128,"block_storage_size":200,"filter":{"evaluation_container":["odsc-llm-evaluate"],"evaluation_target":["datasciencemodeldeployment"]}},{"name":"VM.Standard3.Flex","ocpu":8,"memory_in_gbs":128,"block_storage_size":200,"filter":{"evaluation_container":["odsc-llm-evaluate"],"evaluation_target":["datasciencemodeldeployment"]}},{"name":"VM.Optimized3.Flex","ocpu":8,"memory_in_gbs":128,"block_storage_size":200,"filter":{"evaluation_container":["odsc-llm-evaluate"],"evaluation_target":["datasciencemodeldeployment"]}}]', + } + ) + + workload_configuration = JobRunWorkloadConfigurationDetails( + use_case_configuration=use_case_configuration + ) + MOCK_OUTPUT = [ + ContainerSummary( + **{ + "container_name": "odsc-vllm-serving", + "display_name": "VLLM:0.6.4.post1.2", + "family_name": "odsc-vllm-serving", + "description": "This container is used for llm inference, batch inference and serving", + "is_latest": True, + "target_workloads": ["MODEL_DEPLOYMENT", "JOB_RUN"], + "usages": ["INFERENCE", "BATCH_INFERENCE", "OTHER"], + "tag": "0.6.4.post1.2", + "lifecycle_state": "ACTIVE", + "workload_configuration_details_list": [ + ModelDeployWorkloadConfigurationDetails( + **{ + "cmd": "--served-model-name odsc-llm --disable-custom-all-reduce --seed 42 ", + "server_port": 8080, + "health_check_port": 8080, + "additional_configurations": { + "HEALTH_CHECK_PORT": "8080", + "MODEL_DEPLOY_ENABLE_STREAMING": "true", + "MODEL_DEPLOY_PREDICT_ENDPOINT": "/v1/completions", + "PORT": "8080", + "modelFormats": "SAFETENSORS", + "platforms": "NVIDIA_GPU", + "restrictedParams": '["--port","--host","--served-model-name","--seed"]', + }, + } + ) + ], + "tag_configuration_list": [], + "freeform_tags": None, + "defined_tags": None, + } + ), + ContainerSummary( + **{ + "container_name": "odsc-llm-fine-tuning", + "display_name": "Fine-Tune:2.2.62.70", + "family_name": "odsc-llm-fine-tuning", + "description": "This container is used to fine tune llm", + "is_latest": True, + "target_workloads": ["JOB_RUN"], + "usages": ["FINE_TUNE"], + "tag": "2.2.62.70", + "lifecycle_state": "ACTIVE", + "workload_configuration_details_list": [], + "tag_configuration_list": [], + "freeform_tags": None, + "defined_tags": None, + } + ), + ContainerSummary( + **{ + "container_name": "odsc-llm-evaluate", + "display_name": "Evaluate:0.1.3.4", + "family_name": "odsc-llm-evaluate", + "description": "This container supports evaluation on model deployment", + "is_latest": True, + "target_workloads": ["JOB_RUN"], + "usages": ["EVALUATION"], + "tag": "0.1.3.4", + "lifecycle_state": "ACTIVE", + "workload_configuration_details_list": [workload_configuration], + "tag_configuration_list": [], + "freeform_tags": None, + "defined_tags": None, + } + ), + ] + + class HandlerTestDataset: MOCK_OCID = "ocid.datasciencemdoel." mock_valid_input = dict(