From e0a6b86d4bca47d9d7a4d1b406848fcf339bc836 Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Tue, 15 Oct 2024 19:46:30 +0530 Subject: [PATCH 01/12] updated: type annotations. --- langtest/langtest.py | 6 ++++-- langtest/types.py | 37 ++++++++++++++++++++++++++++++++++ langtest/utils/config_utils.py | 4 ++-- 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 langtest/types.py diff --git a/langtest/langtest.py b/langtest/langtest.py index 09df1b57d..595f019f7 100644 --- a/langtest/langtest.py +++ b/langtest/langtest.py @@ -13,6 +13,8 @@ from pkg_resources import resource_filename +from langtest.types import DatasetConfig, ModelConfig + from .tasks import TaskManager from .augmentation import AugmentRobustness, TemplaticAugment from .datahandler.datasource import DataFactory @@ -90,8 +92,8 @@ class Harness: def __init__( self, task: Union[str, dict], - model: Optional[Union[list, dict]] = None, - data: Optional[Union[list, dict]] = None, + model: Optional[Union[List[ModelConfig], ModelConfig]] = None, + data: Optional[Union[List[DatasetConfig], DatasetConfig]] = None, config: Optional[Union[str, dict]] = None, benchmarking: dict = None, ): diff --git a/langtest/types.py b/langtest/types.py new file mode 100644 index 000000000..bf164eb1d --- /dev/null +++ b/langtest/types.py @@ -0,0 +1,37 @@ +from typing import Literal, TypedDict, Union, List + + +class ModelConfig(TypedDict): + """ + ModelConfig is a TypedDict that defines the configuration for a model. + + Attributes: + model (str): The name of the model. + type (Literal['chat', 'instruct']): The type of the model, either 'chat' or 'instruct'. + hub (str): The hub where the model is located. + """ + + model: str + type: Literal["chat", "instruct"] + hub: str + + +class DatasetConfig(TypedDict): + """ + DatasetConfig is a TypedDict that defines the configuration for a dataset. + + Attributes: + data_source (str): The source of the data, e.g., a file path. + split (str): The data split, e.g., 'train', 'test', or 'validation'. + subset (str): A specific subset of the data, if applicable. + feature_column (Union[str, List[str]]): The column(s) representing the features in the dataset. + target_column (Union[str, List[str]]): The column(s) representing the target variable(s) in the dataset. + source (str): The original source of the dataset ex: huggingface. + """ + + data_source: str + split: str + subset: str + feature_column: Union[str, List[str]] + target_column: Union[str, List[str]] + source: str diff --git a/langtest/utils/config_utils.py b/langtest/utils/config_utils.py index 9f762ce30..5f81e9337 100644 --- a/langtest/utils/config_utils.py +++ b/langtest/utils/config_utils.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Any, Dict, List from pkg_resources import resource_filename @@ -12,7 +12,7 @@ "default": resource_filename("langtest", "data/config/QA_summarization_config.yml"), } -DEFAULTS_CONFIG = { +DEFAULTS_CONFIG: Dict[str, Any] = { "question-answering": LLM_DEFAULTS_CONFIG, "summarization": LLM_DEFAULTS_CONFIG, "ideology": resource_filename("langtest", "data/config/political_config.yml"), From c759246382ecda42225e551e77b4ec8dc3672fb1 Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Tue, 15 Oct 2024 21:34:02 +0530 Subject: [PATCH 02/12] updated model_type parameter in Harness and TaskManager classes --- langtest/langtest.py | 7 ++++--- langtest/tasks/task.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/langtest/langtest.py b/langtest/langtest.py index 595f019f7..d4ca64a4a 100644 --- a/langtest/langtest.py +++ b/langtest/langtest.py @@ -158,11 +158,12 @@ def __init__( raise ValueError(Errors.E003()) if isinstance(model, dict): - hub, model = model["hub"], model["model"] + hub, model, model_type = model["hub"], model["model"], model.get("type") self.hub = hub self._actual_model = model else: hub = None + model_type = None # loading task @@ -217,14 +218,14 @@ def __init__( hub = i["hub"] model_dict[model] = self.task.model( - model, hub, **self._config.get("model_parameters", {}) + model, hub, model_type, **self._config.get("model_parameters", {}) ) self.model = model_dict else: self.model = self.task.model( - model, hub, **self._config.get("model_parameters", {}) + model, hub, model_type, **self._config.get("model_parameters", {}) ) # end model selection formatted_config = json.dumps(self._config, indent=1) diff --git a/langtest/tasks/task.py b/langtest/tasks/task.py index 0e5134eae..a289a7dac 100644 --- a/langtest/tasks/task.py +++ b/langtest/tasks/task.py @@ -23,7 +23,9 @@ def create_sample(cls, *args, **kwargs) -> samples.Sample: pass @classmethod - def load_model(cls, model_path: str, model_hub: str, *args, **kwargs): + def load_model( + cls, model_path: str, model_hub: str, model_type: str, *args, **kwargs + ): """Load the model.""" models = ModelAPI.model_registry @@ -54,6 +56,10 @@ def load_model(cls, model_path: str, model_hub: str, *args, **kwargs): if "server_prompt" in kwargs: cls.server_prompt = kwargs.get("server_prompt") kwargs.pop("server_prompt") + + if model_type: + kwargs["model_type"] = model_type + try: if model_hub in LANGCHAIN_HUBS: # LLM models @@ -145,9 +151,9 @@ def create_sample(self, *args, **kwargs): return self.__task.create_sample(*args, **kwargs) - def model(self, *args, **kwargs) -> "ModelAPI": + def model(self, model_name, hub, model_type, **kwargs) -> "ModelAPI": """Add a task to the task manager.""" - return self.__task.load_model(*args, **kwargs) + return self.__task.load_model(model_name, hub, model_type, **kwargs) def __eq__(self, __value: str) -> bool: """Check if the task is equal to another task.""" From a3d1caadaaa0504ee08dc3ed0ee38e47b4951ab7 Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Wed, 16 Oct 2024 14:36:10 +0530 Subject: [PATCH 03/12] updated: chat models integration from langchain module --- langtest/modelhandler/llm_modelhandler.py | 26 ++++++++++--- langtest/modelhandler/utils.py | 47 +++++++++++++++++++++++ 2 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 langtest/modelhandler/utils.py diff --git a/langtest/modelhandler/llm_modelhandler.py b/langtest/modelhandler/llm_modelhandler.py index 3e64609a4..d050b41b5 100644 --- a/langtest/modelhandler/llm_modelhandler.py +++ b/langtest/modelhandler/llm_modelhandler.py @@ -1,6 +1,8 @@ import inspect + from typing import Any, List, Union import langchain.llms as lc +import langchain.chat_models as chat_models from langchain.chains import LLMChain from langchain_core.prompts import PromptTemplate from langchain_core.exceptions import OutputParserException @@ -71,7 +73,9 @@ def load_model(cls, hub: str, path: str, *args, **kwargs) -> "PretrainedModelFor ValueError: If the model is not found online or locally. ConfigError: If there is an error in the model configuration. """ - exclude_args = ["task", "device", "stream"] + exclude_args = ["task", "device", "stream", "model_type"] + + model_type = kwargs.get("model_type", None) filtered_kwargs = kwargs.copy() @@ -104,13 +108,23 @@ def load_model(cls, hub: str, path: str, *args, **kwargs) -> "PretrainedModelFor model = AzureChatOpenAI(model=path, *args, **filtered_kwargs) return cls(hub, model, *args, **filtered_kwargs) - elif hub == "ollama": - from langchain.chat_models.ollama import ChatOllama + # elif hub == "ollama": + # from langchain.chat_models.ollama import ChatOllama - model = ChatOllama(model=path, *args, **filtered_kwargs) - return cls(hub, model, *args, **filtered_kwargs) + # model = ChatOllama(model=path, *args, **filtered_kwargs) + # return cls(hub, model, *args, **filtered_kwargs) else: - model = getattr(lc, LANGCHAIN_HUBS[hub]) + from .utils import CHAT_MODEL_CLASSES + + if model_type and hub in CHAT_MODEL_CLASSES: + hub_module = getattr(chat_models, hub) + model = getattr(hub_module, CHAT_MODEL_CLASSES[hub]) + elif model_type in [None, "instruct"]: + model = getattr(lc, LANGCHAIN_HUBS[hub]) + else: + raise ValueError( + f"{hub} hub is not supported for the given model type" + ) default_args = inspect.getfullargspec(model).kwonlyargs if "model" in default_args: cls.model = model(model=path, *args, **filtered_kwargs) diff --git a/langtest/modelhandler/utils.py b/langtest/modelhandler/utils.py new file mode 100644 index 000000000..8b32b9b14 --- /dev/null +++ b/langtest/modelhandler/utils.py @@ -0,0 +1,47 @@ +# This file contains the model classes that are used in the model handler. +# from langchain + +CHAT_MODEL_CLASSES = { + "anthropic": "ChatAnthropic", + "anyscale": "ChatAnyscale", + "azure_openai": "AzureChatOpenAI", + "baichuan": "ChatBaichuan", + "baidu_qianfan_endpoint": "QianfanChatEndpoint", + "bedrock": "BedrockChat", + "cohere": "ChatCohere", + "databricks": "ChatDatabricks", + "deepinfra": "ChatDeepInfra", + "ernie": "ErnieBotChat", + "everlyai": "ChatEverlyAI", + "fake": "FakeListChatModel", + "fireworks": "ChatFireworks", + "gigachat": "GigaChat", + "google_palm": "ChatGooglePalm", + "gpt_router": "GPTRouter", + "huggingface": "ChatHuggingFace", + "human": "HumanInputChatModel", + "hunyuan": "ChatHunyuan", + "javelin_ai_gateway": "ChatJavelinAIGateway", + "jinachat": "JinaChat", + "kinetica": "ChatKinetica", + "konko": "ChatKonko", + "litellm": "ChatLiteLLM", + "litellm_router": "ChatLiteLLMRouter", + "llama_edge": "LlamaEdgeChatService", + "maritalk": "ChatMaritalk", + "minimax": "MiniMaxChat", + "mlflow": "ChatMlflow", + "mlflow_ai_gateway": "ChatMLflowAIGateway", + "ollama": "ChatOllama", + "openai": "ChatOpenAI", + "pai_eas_endpoint": "PaiEasChatEndpoint", + "perplexity": "ChatPerplexity", + "promptlayer_openai": "PromptLayerChatOpenAI", + "sparkllm": "ChatSparkLLM", + "tongyi": "ChatTongyi", + "vertexai": "ChatVertexAI", + "volcengine_maas": "VolcEngineMaasChat", + "yandex": "ChatYandexGPT", + "yuan2": "ChatYuan2", + "zhipuai": "ChatZhipuAI", +} From 36b1916d377025f0a33c2b1924bd9137f7268a72 Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Wed, 16 Oct 2024 14:47:49 +0530 Subject: [PATCH 04/12] updated: handling openai and azureopenai hubs in proper way. --- langtest/modelhandler/llm_modelhandler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/langtest/modelhandler/llm_modelhandler.py b/langtest/modelhandler/llm_modelhandler.py index d050b41b5..f0a21271b 100644 --- a/langtest/modelhandler/llm_modelhandler.py +++ b/langtest/modelhandler/llm_modelhandler.py @@ -97,13 +97,15 @@ def load_model(cls, hub: str, path: str, *args, **kwargs) -> "PretrainedModelFor "gpt-3.5-turbo-1106", "gpt-4o-2024-05-13", "gpt-4o", - ): + "o1-preview", + "o1-mini", + ) and hub in ["openai", "azure-openai"]: if hub == "openai": from langchain_openai.chat_models import ChatOpenAI model = ChatOpenAI(model=path, *args, **filtered_kwargs) elif hub == "azure-openai": - from langchain.chat_models.azure_openai import AzureChatOpenAI + from langchain_openai.chat_models import AzureChatOpenAI model = AzureChatOpenAI(model=path, *args, **filtered_kwargs) From f4920a1baf5247de056115031092207848df69d4 Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Wed, 16 Oct 2024 19:41:34 +0530 Subject: [PATCH 05/12] Updated: `PretrainedModelForQA` to support `model_type` parameter, enhance prediction handling for chat models, and update prompt management for flexibility. --- .../modelhandler/transformers_modelhandler.py | 36 +++++++++++++++---- langtest/prompts.py | 2 +- langtest/tasks/task.py | 2 +- langtest/utils/hf_utils.py | 4 +++ 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/langtest/modelhandler/transformers_modelhandler.py b/langtest/modelhandler/transformers_modelhandler.py index cd77f6496..341dab8ca 100644 --- a/langtest/modelhandler/transformers_modelhandler.py +++ b/langtest/modelhandler/transformers_modelhandler.py @@ -685,6 +685,7 @@ def __init__(self, model, **kwargs): ) self.model = model + self.model_type = kwargs.get("model_type", None) @classmethod def _try_initialize_model(cls, path, device, tasks, **kwargs): @@ -729,6 +730,10 @@ def load_model(cls, path: str, **kwargs): - PretrainedModelForQA: An instance of the PretrainedModelForQA class. """ try: + # set the model_type from kwargs + model_type = kwargs.get("model_type", None) + kwargs.pop("model_type", None) + # Setup and pop specific kwargs new_tokens_key = "max_new_tokens" @@ -758,7 +763,7 @@ def load_model(cls, path: str, **kwargs): ) else: model = HuggingFacePipeline(pipeline=path) - return cls(model) + return cls(model, model_type=model_type) else: if isinstance(path, str): model = cls._try_initialize_model( @@ -773,7 +778,7 @@ def load_model(cls, path: str, **kwargs): else: model = HuggingFacePipeline(pipeline=path) - return cls(model) + return cls(model, model_type=model_type) except Exception as e: raise ValueError(Errors.E090(error_message=e)) @@ -792,10 +797,29 @@ def predict(self, text: Union[str, dict], prompt: dict, **kwargs) -> str: - str: The generated prediction. """ try: - prompt_template = SimplePromptTemplate(**prompt) - text = prompt_template.format(**text) - output = self.model._generate([text]) - return output[0] + if self.model_type == "chat": + from langtest.prompts import PromptManager + + prompt_manager = PromptManager() + examples = prompt_manager.get_prompt(hub="transformers") + + if examples: + prompt["template"] = "".join( + f"{k.title()}: {{{k}}}" for k in text.keys() + ) + prompt_template = SimplePromptTemplate(**prompt) + text = prompt_template.format(**text) + messages = [*examples, {"role": "user", "content": text}] + else: + messages = [{"role": "user", "content": text}] + output = self.model._generate([messages]) + return output[0].get("content", "") + + else: + prompt_template = SimplePromptTemplate(**prompt) + text = prompt_template.format(**text) + output = self.model._generate([text]) + return output[0] except Exception as e: raise ValueError(Errors.E089(error_message=e)) diff --git a/langtest/prompts.py b/langtest/prompts.py index d9cc03857..25441ec0e 100644 --- a/langtest/prompts.py +++ b/langtest/prompts.py @@ -169,7 +169,7 @@ def prompt_style(self): return final_prompt def get_prompt(self, hub=None): - if hub == "lm-studio": + if hub in ("lm-studio", "transformers"): return self.lm_studio_prompt() return self.prompt_style() diff --git a/langtest/tasks/task.py b/langtest/tasks/task.py index a289a7dac..6841fa682 100644 --- a/langtest/tasks/task.py +++ b/langtest/tasks/task.py @@ -151,7 +151,7 @@ def create_sample(self, *args, **kwargs): return self.__task.create_sample(*args, **kwargs) - def model(self, model_name, hub, model_type, **kwargs) -> "ModelAPI": + def model(self, model_name, hub, model_type=None, **kwargs) -> "ModelAPI": """Add a task to the task manager.""" return self.__task.load_model(model_name, hub, model_type, **kwargs) diff --git a/langtest/utils/hf_utils.py b/langtest/utils/hf_utils.py index 01b8be4da..7fa80bdc4 100644 --- a/langtest/utils/hf_utils.py +++ b/langtest/utils/hf_utils.py @@ -168,8 +168,12 @@ def __init__( **kwargs: Any, ): """Construct the pipeline object from model_id and task.""" + from transformers import Pipeline + login_with_token() + self.model_id = model_id + self.pipeline: Pipeline = None if pipeline: self.pipeline = pipeline else: From c3cedb0985d882ce5b850ba57a2ba685b9af412f Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Wed, 16 Oct 2024 20:55:44 +0530 Subject: [PATCH 06/12] Updated type annotations in TaskManager class --- langtest/tasks/task.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/langtest/tasks/task.py b/langtest/tasks/task.py index 6841fa682..9e39d56d9 100644 --- a/langtest/tasks/task.py +++ b/langtest/tasks/task.py @@ -1,7 +1,7 @@ import ast import re from abc import ABC, abstractmethod -from typing import Union +from typing import Literal, Union from langtest.modelhandler import ModelAPI, LANGCHAIN_HUBS, INSTALLED_HUBS from langtest.errors import Errors, ColumnNameError @@ -24,7 +24,12 @@ def create_sample(cls, *args, **kwargs) -> samples.Sample: @classmethod def load_model( - cls, model_path: str, model_hub: str, model_type: str, *args, **kwargs + cls, + model_path: str, + model_hub: str, + model_type: Literal["chat", "completion"] = None, + *args, + **kwargs, ): """Load the model.""" @@ -151,9 +156,9 @@ def create_sample(self, *args, **kwargs): return self.__task.create_sample(*args, **kwargs) - def model(self, model_name, hub, model_type=None, **kwargs) -> "ModelAPI": + def model(self, *args, **kwargs) -> "ModelAPI": """Add a task to the task manager.""" - return self.__task.load_model(model_name, hub, model_type, **kwargs) + return self.__task.load_model(*args, **kwargs) def __eq__(self, __value: str) -> bool: """Check if the task is equal to another task.""" From d456fbac040921243b587d0a363a852a07c8af04 Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Thu, 17 Oct 2024 19:26:34 +0530 Subject: [PATCH 07/12] updated: code to remove unnecessary kwargs in PretrainedModelForQA and HuggingFacePipeline classes --- langtest/modelhandler/transformers_modelhandler.py | 3 +-- langtest/utils/hf_utils.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/langtest/modelhandler/transformers_modelhandler.py b/langtest/modelhandler/transformers_modelhandler.py index 341dab8ca..245fdbec9 100644 --- a/langtest/modelhandler/transformers_modelhandler.py +++ b/langtest/modelhandler/transformers_modelhandler.py @@ -732,7 +732,6 @@ def load_model(cls, path: str, **kwargs): try: # set the model_type from kwargs model_type = kwargs.get("model_type", None) - kwargs.pop("model_type", None) # Setup and pop specific kwargs new_tokens_key = "max_new_tokens" @@ -813,7 +812,7 @@ def predict(self, text: Union[str, dict], prompt: dict, **kwargs) -> str: else: messages = [{"role": "user", "content": text}] output = self.model._generate([messages]) - return output[0].get("content", "") + return output[0][0].get("content", "") else: prompt_template = SimplePromptTemplate(**prompt) diff --git a/langtest/utils/hf_utils.py b/langtest/utils/hf_utils.py index 7fa80bdc4..1297c56d6 100644 --- a/langtest/utils/hf_utils.py +++ b/langtest/utils/hf_utils.py @@ -174,6 +174,8 @@ def __init__( self.model_id = model_id self.pipeline: Pipeline = None + self.model_type = kwargs.get("model_type", None) + self.chat_template = kwargs.get("chat_template", None) if pipeline: self.pipeline = pipeline else: @@ -206,6 +208,9 @@ def _initialize_pipeline( tokenizer = AutoTokenizer.from_pretrained(model_id) + # remove the unnecessary kwargs + kwargs.pop("model_type", None) + # Set the pad_token_id for the tokenizer tokenizer.pad_token_id = tokenizer.eos_token_id @@ -237,6 +242,9 @@ def _initialize_pipeline( if device < 0 and cuda_device_count > 0: logging.warning(Warnings.W016(cuda_device_count=cuda_device_count)) + # renove the unnecessary kwargs + kwargs.pop("chat_template", None) + return hf_pipeline( task=task, model=model, @@ -258,6 +266,12 @@ def _generate(self, prompts: List[str]) -> List[str]: text_generations: List[str] = [] for prompt in prompts: + if ( + self.pipeline.tokenizer.chat_template is None + and self.model_type == "chat" + ): + self.pipeline.tokenizer.chat_template = self.chat_template + # response = self.pipeline(prompt, return_full_text=False) response = self.pipeline(prompt) if isinstance(response, list): From 7d07c6af6af9ae9e3c9530fc40e0e74b8a0ba1df Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Thu, 17 Oct 2024 20:33:17 +0530 Subject: [PATCH 08/12] Update model type annotation to support 'completion' type in ModelConfig class and remove unnecessary code in HuggingFacePipeline class --- langtest/types.py | 4 +-- langtest/utils/hf_utils.py | 53 +++++++++++++++++++++----------------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/langtest/types.py b/langtest/types.py index bf164eb1d..d17262597 100644 --- a/langtest/types.py +++ b/langtest/types.py @@ -7,12 +7,12 @@ class ModelConfig(TypedDict): Attributes: model (str): The name of the model. - type (Literal['chat', 'instruct']): The type of the model, either 'chat' or 'instruct'. + type (Literal['chat', 'completion']): The type of the model, either 'chat' or 'completion'. hub (str): The hub where the model is located. """ model: str - type: Literal["chat", "instruct"] + type: Literal["chat", "completion"] hub: str diff --git a/langtest/utils/hf_utils.py b/langtest/utils/hf_utils.py index 1297c56d6..f749f161e 100644 --- a/langtest/utils/hf_utils.py +++ b/langtest/utils/hf_utils.py @@ -271,33 +271,38 @@ def _generate(self, prompts: List[str]) -> List[str]: and self.model_type == "chat" ): self.pipeline.tokenizer.chat_template = self.chat_template - # response = self.pipeline(prompt, return_full_text=False) - response = self.pipeline(prompt) + + # return_full_text = False is available in transformers>=4.11.0 + response = self.pipeline(prompt, return_full_text=False) + + # response = self.pipeline(prompt) if isinstance(response, list): response = response[0] - - if self.pipeline.task == "text-generation": - try: - from transformers.pipelines.text_generation import ReturnType - - remove_prompt = ( - self.pipeline._postprocess_params.get("return_type") - != ReturnType.NEW_TEXT - ) - except Exception as e: - logging.warning(Warnings.W017(e=e)) - remove_prompt = True - if remove_prompt: - text = response["generated_text"][len(prompt) :] - else: - text = response["generated_text"] - elif self.pipeline.task == "text2text-generation": - text = response["generated_text"] - elif self.pipeline.task == "summarization": - text = response["summary_text"] - else: - raise ValueError(Errors.E086(task=self.pipeline.task)) + + output_key = f"{self.pipeline.return_name}_text" if hasattr(self.pipeline, 'return_name') else "generated_text" + text = response[output_key] + # if self.pipeline.task == "text-generation": + # try: + # from transformers.pipelines.text_generation import ReturnType + + # remove_prompt = ( + # self.pipeline._postprocess_params.get("return_type") + # != ReturnType.NEW_TEXT + # ) + # except Exception as e: + # logging.warning(Warnings.W017(e=e)) + # remove_prompt = True + # if remove_prompt: + # text = response["generated_text"][len(prompt) :] + # else: + # text = response["generated_text"] + # elif self.pipeline.task == "text2text-generation": + # text = response["generated_text"] + # elif self.pipeline.task == "summarization": + # text = response["summary_text"] + # else: + # raise ValueError(Errors.E086(task=self.pipeline.task)) text_generations.append(text) From e13b779da8c106b48a33225e61fee7cb0c8fdd5f Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Thu, 17 Oct 2024 20:41:03 +0530 Subject: [PATCH 09/12] updated: formatting issues. --- langtest/utils/hf_utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/langtest/utils/hf_utils.py b/langtest/utils/hf_utils.py index f749f161e..4b35b0182 100644 --- a/langtest/utils/hf_utils.py +++ b/langtest/utils/hf_utils.py @@ -271,7 +271,7 @@ def _generate(self, prompts: List[str]) -> List[str]: and self.model_type == "chat" ): self.pipeline.tokenizer.chat_template = self.chat_template - + # return_full_text = False is available in transformers>=4.11.0 response = self.pipeline(prompt, return_full_text=False) @@ -279,8 +279,12 @@ def _generate(self, prompts: List[str]) -> List[str]: if isinstance(response, list): response = response[0] - - output_key = f"{self.pipeline.return_name}_text" if hasattr(self.pipeline, 'return_name') else "generated_text" + + output_key = ( + f"{self.pipeline.return_name}_text" + if hasattr(self.pipeline, "return_name") + else "generated_text" + ) text = response[output_key] # if self.pipeline.task == "text-generation": # try: From 9b6862c71157309c82faafb51d9d24c4e1d4a37f Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Thu, 17 Oct 2024 21:41:15 +0530 Subject: [PATCH 10/12] fixed temporary issues: in prompt manager --- langtest/modelhandler/llm_modelhandler.py | 2 +- .../modelhandler/transformers_modelhandler.py | 4 ++-- langtest/prompts.py | 15 +++++++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/langtest/modelhandler/llm_modelhandler.py b/langtest/modelhandler/llm_modelhandler.py index f0a21271b..a970c7f79 100644 --- a/langtest/modelhandler/llm_modelhandler.py +++ b/langtest/modelhandler/llm_modelhandler.py @@ -73,7 +73,7 @@ def load_model(cls, hub: str, path: str, *args, **kwargs) -> "PretrainedModelFor ValueError: If the model is not found online or locally. ConfigError: If there is an error in the model configuration. """ - exclude_args = ["task", "device", "stream", "model_type"] + exclude_args = ["task", "device", "stream", "model_type", "chat_template"] model_type = kwargs.get("model_type", None) diff --git a/langtest/modelhandler/transformers_modelhandler.py b/langtest/modelhandler/transformers_modelhandler.py index 245fdbec9..e78bd9810 100644 --- a/langtest/modelhandler/transformers_modelhandler.py +++ b/langtest/modelhandler/transformers_modelhandler.py @@ -804,7 +804,7 @@ def predict(self, text: Union[str, dict], prompt: dict, **kwargs) -> str: if examples: prompt["template"] = "".join( - f"{k.title()}: {{{k}}}" for k in text.keys() + f"{k.title()}:\n{{{k}}}\n" for k in text.keys() ) prompt_template = SimplePromptTemplate(**prompt) text = prompt_template.format(**text) @@ -812,7 +812,7 @@ def predict(self, text: Union[str, dict], prompt: dict, **kwargs) -> str: else: messages = [{"role": "user", "content": text}] output = self.model._generate([messages]) - return output[0][0].get("content", "") + return output[0].strip() else: prompt_template = SimplePromptTemplate(**prompt) diff --git a/langtest/prompts.py b/langtest/prompts.py index 25441ec0e..9b9269ff5 100644 --- a/langtest/prompts.py +++ b/langtest/prompts.py @@ -33,11 +33,17 @@ def get_template(self): temp = [] order_less = [] - for field in self.__dict__: + + sorted_fields = sorted( + self.__dict__.keys(), key=lambda x: self.__field_order.index(x.lower()) + ) + + for field in sorted_fields: if field in self.__field_order: temp.append(f"{field.title()}: {{{field}}}") else: order_less.append(f"{field.title()}: {{{field}}}") + if order_less: temp.extend(order_less) return "\n" + "\n".join(temp) @@ -194,7 +200,12 @@ def lm_studio_prompt(self): # assistant role temp_ai["role"] = "assistant" - temp_ai["content"] = example.ai.get_template.format(**example.ai.get_example) + temp_ai["content"] = ( + example.ai.get_template.format(**example.ai.get_example) + .replace("Answer:", "") + .strip() + + "\n\n" + ) messages.append(temp_user) messages.append(temp_ai) From db57f31ebc56623f4159c362ea9f0f1cf95e1fd6 Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Thu, 17 Oct 2024 21:52:49 +0530 Subject: [PATCH 11/12] fixed error: `model_kwargs` are not used by the model: ['return_full_text'] --- langtest/utils/hf_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/langtest/utils/hf_utils.py b/langtest/utils/hf_utils.py index 4b35b0182..d930e47fc 100644 --- a/langtest/utils/hf_utils.py +++ b/langtest/utils/hf_utils.py @@ -273,9 +273,10 @@ def _generate(self, prompts: List[str]) -> List[str]: self.pipeline.tokenizer.chat_template = self.chat_template # return_full_text = False is available in transformers>=4.11.0 - response = self.pipeline(prompt, return_full_text=False) - - # response = self.pipeline(prompt) + if self.pipeline.task == "text-generation": + response = self.pipeline(prompt, return_full_text=False) + else: + response = self.pipeline(prompt) if isinstance(response, list): response = response[0] @@ -286,7 +287,6 @@ def _generate(self, prompts: List[str]) -> List[str]: else "generated_text" ) text = response[output_key] - # if self.pipeline.task == "text-generation": # try: # from transformers.pipelines.text_generation import ReturnType From 572bbe4d96a2c2bbf14414585bda012839d1d602 Mon Sep 17 00:00:00 2001 From: Kalyan Chakravarthy Date: Sat, 19 Oct 2024 13:24:09 +0530 Subject: [PATCH 12/12] removed unneccesary comments --- langtest/utils/hf_utils.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/langtest/utils/hf_utils.py b/langtest/utils/hf_utils.py index d930e47fc..b7e051ffc 100644 --- a/langtest/utils/hf_utils.py +++ b/langtest/utils/hf_utils.py @@ -287,26 +287,6 @@ def _generate(self, prompts: List[str]) -> List[str]: else "generated_text" ) text = response[output_key] - # try: - # from transformers.pipelines.text_generation import ReturnType - - # remove_prompt = ( - # self.pipeline._postprocess_params.get("return_type") - # != ReturnType.NEW_TEXT - # ) - # except Exception as e: - # logging.warning(Warnings.W017(e=e)) - # remove_prompt = True - # if remove_prompt: - # text = response["generated_text"][len(prompt) :] - # else: - # text = response["generated_text"] - # elif self.pipeline.task == "text2text-generation": - # text = response["generated_text"] - # elif self.pipeline.task == "summarization": - # text = response["summary_text"] - # else: - # raise ValueError(Errors.E086(task=self.pipeline.task)) text_generations.append(text)