From 8e40f57b65d5520d0bbdae662b8b6531837dd042 Mon Sep 17 00:00:00 2001 From: Nupur Khare Date: Wed, 17 Jan 2024 21:11:30 +0530 Subject: [PATCH] 1. Gave prompt specific hyperparameters, collection top_results and similarity_threshold. 2. Added unit and integration test cases. 3. Fixed test cases. --- kairon/actions/definitions/prompt.py | 27 +- kairon/api/models.py | 57 +- kairon/importer/validator/file_validator.py | 22 +- kairon/shared/actions/data_objects.py | 32 +- kairon/shared/cognition/processor.py | 2 +- kairon/shared/data/processor.py | 5 - kairon/shared/llm/gpt3.py | 24 +- kairon/shared/utils.py | 2 + template/use-cases/Hi-Hello-GPT/actions.yml | 14 - tests/integration_test/action_service_test.py | 106 ++-- tests/integration_test/services_test.py | 222 ++++--- tests/testing_data/actions/actions.yml | 10 +- .../actions/validation_action_data.json | 103 ++-- tests/unit_test/action/action_test.py | 55 +- .../data_processor/data_processor_test.py | 540 ++++++++++-------- tests/unit_test/llm_test.py | 89 +-- .../validator/training_data_validator_test.py | 38 +- 17 files changed, 715 insertions(+), 633 deletions(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 222e427e4..b6902546e 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -120,11 +120,10 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti system_prompt = None context_prompt = '' query_prompt = '' + query_prompt_dict = {} history_prompt = None is_query_prompt_enabled = False - similarity_prompt_name = None - similarity_prompt_instructions = None - use_similarity_prompt = False + similarity_prompt = {} params = {} num_bot_responses = k_faq_action_config['num_bot_responses'] for prompt in k_faq_action_config['llm_prompts']: @@ -134,9 +133,11 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti if prompt['source'] == LlmPromptSource.history.value: history_prompt = ActionUtility.prepare_bot_responses(tracker, num_bot_responses) elif prompt['source'] == LlmPromptSource.bot_content.value: - similarity_prompt_name = prompt['name'] - similarity_prompt_instructions = prompt['instructions'] - use_similarity_prompt = True + similarity_prompt.update({'similarity_prompt_name': prompt['name'], + 'similarity_prompt_instructions': prompt['instructions'], + 'collection': prompt['collection'], + 'use_similarity_prompt': True, 'top_results': prompt.get('top_results'), + 'similarity_threshold': prompt.get('similarity_threshold')}) elif prompt['source'] == LlmPromptSource.slot.value: slot_data = tracker.get_slot(prompt['data']) context_prompt += f"{prompt['name']}:\n{slot_data}\n" @@ -159,21 +160,15 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti if prompt['instructions']: query_prompt += f"Instructions on how to use {prompt['name']}:\n{prompt['instructions']}\n\n" is_query_prompt_enabled = True + query_prompt_dict.update({'query_prompt': query_prompt, 'use_query_prompt': is_query_prompt_enabled, + 'hyperparameters': prompt.get('hyperparameters', Utility.get_llm_hyperparameters())}) - params["top_results"] = k_faq_action_config.get('top_results', 10) - params["similarity_threshold"] = k_faq_action_config.get('similarity_threshold', 0.70) - params["hyperparameters"] = k_faq_action_config.get('hyperparameters', Utility.get_llm_hyperparameters()) - params['enable_response_cache'] = k_faq_action_config.get('enable_response_cache', False) params["system_prompt"] = system_prompt params["context_prompt"] = context_prompt - params["query_prompt"] = query_prompt - params["use_query_prompt"] = is_query_prompt_enabled + params["query_prompt"] = query_prompt_dict params["previous_bot_responses"] = history_prompt - params['use_similarity_prompt'] = use_similarity_prompt - params['similarity_prompt_name'] = similarity_prompt_name - params['similarity_prompt_instructions'] = similarity_prompt_instructions + params["similarity_prompt"] = similarity_prompt params['instructions'] = k_faq_action_config.get('instructions', []) - params['collection'] = k_faq_action_config.get('collection') return params @staticmethod diff --git a/kairon/api/models.py b/kairon/api/models.py index 6d3a5100c..f655bbacf 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -874,12 +874,40 @@ def check(cls, values): class LlmPromptRequest(BaseModel): name: str + top_results: int = 10 + similarity_threshold: float = 0.70 + hyperparameters: dict = None data: str = None + collection: str = None instructions: str = None type: LlmPromptType source: LlmPromptSource is_enabled: bool = True + @validator("similarity_threshold") + def validate_similarity_threshold(cls, v, values, **kwargs): + if not 0.3 <= v <= 1: + raise ValueError("similarity_threshold should be within 0.3 and 1") + return v + + @validator("top_results") + def validate_top_results(cls, v, values, **kwargs): + if v > 30: + raise ValueError("top_results should not be greater than 30") + return v + + @root_validator + def check(cls, values): + from kairon.shared.utils import Utility + + if not values.get('hyperparameters'): + values['hyperparameters'] = {} + + for key, value in Utility.get_llm_hyperparameters().items(): + if key not in values['hyperparameters']: + values['hyperparameters'][key] = value + return values + class UserQuestionModel(BaseModel): type: UserMessageType = UserMessageType.from_user_message.value @@ -891,22 +919,11 @@ class PromptActionConfigRequest(BaseModel): num_bot_responses: int = 5 failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE user_question: UserQuestionModel = UserQuestionModel() - top_results: int = 10 - similarity_threshold: float = 0.70 - enable_response_cache: bool = False - hyperparameters: dict = None llm_prompts: List[LlmPromptRequest] instructions: List[str] = [] - collection: str = None set_slots: List[SetSlotsUsingActionResponse] = [] dispatch_response: bool = True - @validator("similarity_threshold") - def validate_similarity_threshold(cls, v, values, **kwargs): - if not 0.3 <= v <= 1: - raise ValueError("similarity_threshold should be within 0.3 and 1") - return v - @validator("llm_prompts") def validate_llm_prompts(cls, v, values, **kwargs): from kairon.shared.utils import Utility @@ -914,30 +931,12 @@ def validate_llm_prompts(cls, v, values, **kwargs): Utility.validate_kairon_faq_llm_prompts([vars(value) for value in v], ValueError) return v - @validator("top_results") - def validate_top_results(cls, v, values, **kwargs): - if v > 30: - raise ValueError("top_results should not be greater than 30") - return v - @validator("num_bot_responses") def validate_num_bot_responses(cls, v, values, **kwargs): if v > 5: raise ValueError("num_bot_responses should not be greater than 5") return v - @root_validator - def check(cls, values): - from kairon.shared.utils import Utility - - if not values.get('hyperparameters'): - values['hyperparameters'] = {} - - for key, value in Utility.get_llm_hyperparameters().items(): - if key not in values['hyperparameters']: - values['hyperparameters'][key] = value - return values - class ColumnMetadata(BaseModel): column_name: str diff --git a/kairon/importer/validator/file_validator.py b/kairon/importer/validator/file_validator.py index 1494dd813..b3d74369d 100644 --- a/kairon/importer/validator/file_validator.py +++ b/kairon/importer/validator/file_validator.py @@ -672,15 +672,7 @@ def __validate_prompt_actions(prompt_actions: list): continue if action.get('num_bot_responses') and (action['num_bot_responses'] > 5 or not isinstance(action['num_bot_responses'], int)): data_error.append(f'num_bot_responses should not be greater than 5 and of type int: {action.get("name")}') - if action.get('top_results') and (action['top_results'] > 30 or not isinstance(action['top_results'], int)): - data_error.append(f'top_results should not be greater than 30 and of type int: {action.get("name")}') - if action.get('similarity_threshold'): - if not (0.3 <= action['similarity_threshold'] <= 1) or not (isinstance(action['similarity_threshold'], float) or isinstance(action['similarity_threshold'], int)): - data_error.append(f'similarity_threshold should be within 0.3 and 1 and of type int or float: {action.get("name")}') llm_prompts_errors = TrainingDataValidator.__validate_llm_prompts(action['llm_prompts']) - if action.get('hyperparameters') is not None: - llm_hyperparameters_errors = TrainingDataValidator.__validate_llm_prompts_hyperparamters(action.get('hyperparameters')) - data_error.extend(llm_hyperparameters_errors) data_error.extend(llm_prompts_errors) if action['name'] in actions_present: data_error.append(f'Duplicate action found: {action["name"]}') @@ -696,6 +688,18 @@ def __validate_llm_prompts(llm_prompts: dict): history_prompt_count = 0 bot_content_prompt_count = 0 for prompt in llm_prompts: + if prompt.get('top_results') and (prompt['top_results'] > 30 or not isinstance(prompt['top_results'], int)): + error_list.append(f'top_results should not be greater than 30 and of type int: {prompt.get("name")}') + if prompt.get('similarity_threshold'): + if not (0.3 <= prompt['similarity_threshold'] <= 1) or not ( + isinstance(prompt['similarity_threshold'], float) or isinstance(prompt['similarity_threshold'], + int)): + error_list.append( + f'similarity_threshold should be within 0.3 and 1 and of type int or float: {prompt.get("name")}') + if prompt.get('hyperparameters') is not None: + llm_hyperparameters_errors = TrainingDataValidator.__validate_llm_prompts_hyperparamters( + prompt.get('hyperparameters')) + error_list.extend(llm_hyperparameters_errors) if prompt.get('type') == 'system': system_prompt_count += 1 elif prompt.get('source') == 'history': @@ -726,6 +730,8 @@ def __validate_llm_prompts(llm_prompts: dict): error_list.append('data field in prompts should of type string.') if not prompt.get('data') and prompt.get('source') == 'static': error_list.append('data is required for static prompts') + if Utility.check_empty_string(prompt.get('collection')) and prompt.get('source') == 'bot_content': + error_list.append("Collection is required for bot content prompts!") if system_prompt_count > 1: error_list.append('Only one system prompt can be present') if system_prompt_count == 0: diff --git a/kairon/shared/actions/data_objects.py b/kairon/shared/actions/data_objects.py index c148fe4c9..6bee1c43e 100644 --- a/kairon/shared/actions/data_objects.py +++ b/kairon/shared/actions/data_objects.py @@ -627,7 +627,11 @@ def clean(self): class LlmPrompt(EmbeddedDocument): name = StringField(required=True) + top_results = IntField(default=10) + similarity_threshold = FloatField(default=0.70) + hyperparameters = DictField(default=Utility.get_llm_hyperparameters) data = StringField() + collection = StringField(default=None) instructions = StringField() type = StringField(required=True, choices=[LlmPromptType.user.value, LlmPromptType.system.value, LlmPromptType.query.value]) source = StringField(choices=[LlmPromptSource.static.value, LlmPromptSource.history.value, LlmPromptSource.bot_content.value, @@ -635,9 +639,20 @@ class LlmPrompt(EmbeddedDocument): default=LlmPromptSource.static.value) is_enabled = BooleanField(default=True) + def clean(self): + for key, value in Utility.get_llm_hyperparameters().items(): + if key not in self.hyperparameters: + self.hyperparameters.update({key: value}) + def validate(self, clean=True): + if not 0.3 <= self.similarity_threshold <= 1: + raise ValidationError("similarity_threshold should be within 0.3 and 1") + if self.top_results > 30: + raise ValidationError("top_results should not be greater than 30") if self.type == LlmPromptType.system.value and self.source != LlmPromptSource.static.value: raise ValidationError("System prompt must have static source!") + dict_data = self.to_mongo().to_dict() + Utility.validate_llm_hyperparameters(dict_data['hyperparameters'], ValidationError) class UserQuestion(EmbeddedDocument): @@ -651,43 +666,30 @@ class UserQuestion(EmbeddedDocument): class PromptAction(Auditlog): name = StringField(required=True) num_bot_responses = IntField(default=5) - top_results = IntField(default=10) - similarity_threshold = FloatField(default=0.70) - enable_response_cache = BooleanField(default=False) failure_message = StringField(default=DEFAULT_NLU_FALLBACK_RESPONSE) user_question = EmbeddedDocumentField(UserQuestion, default=UserQuestion()) bot = StringField(required=True) user = StringField(required=True) timestamp = DateTimeField(default=datetime.utcnow) - hyperparameters = DictField(default=Utility.get_llm_hyperparameters) llm_prompts = EmbeddedDocumentListField(LlmPrompt, required=True) instructions = ListField(StringField()) - collection = StringField(default=None) set_slots = EmbeddedDocumentListField(SetSlotsFromResponse) dispatch_response = BooleanField(default=True) status = BooleanField(default=True) meta = {"indexes": [{"fields": ["bot", ("bot", "name", "status")]}]} - def clean(self): - for key, value in Utility.get_llm_hyperparameters().items(): - if key not in self.hyperparameters: - self.hyperparameters.update({key: value}) - def validate(self, clean=True): if clean: self.clean() if self.num_bot_responses > 5: raise ValidationError("num_bot_responses should not be greater than 5") - if not 0.3 <= self.similarity_threshold <= 1: - raise ValidationError("similarity_threshold should be within 0.3 and 1") - if self.top_results > 30: - raise ValidationError("top_results should not be greater than 30") if not self.llm_prompts: raise ValidationError("llm_prompts are required!") + for prompts in self.llm_prompts: + prompts.validate() dict_data = self.to_mongo().to_dict() Utility.validate_kairon_faq_llm_prompts(dict_data['llm_prompts'], ValidationError) - Utility.validate_llm_hyperparameters(dict_data['hyperparameters'], ValidationError) @auditlogger.log diff --git a/kairon/shared/cognition/processor.py b/kairon/shared/cognition/processor.py index 6789841c8..1d07d79b5 100644 --- a/kairon/shared/cognition/processor.py +++ b/kairon/shared/cognition/processor.py @@ -246,7 +246,7 @@ def find_matching_metadata(bot: Text, data: Any, collection: Text = None): @staticmethod def validate_collection_name(bot: Text, collection: Text): - prompt_action = list(PromptAction.objects(bot=bot, collection__iexact=collection)) + prompt_action = list(PromptAction.objects(bot=bot, llm_prompts__collection__iexact=collection)) database_action = list(DatabaseAction.objects(bot=bot, collection__iexact=collection)) if prompt_action: raise AppException(f'Cannot remove collection {collection} linked to action "{prompt_action[0].name}"!') diff --git a/kairon/shared/data/processor.py b/kairon/shared/data/processor.py index 68221c294..e3b7e7c35 100644 --- a/kairon/shared/data/processor.py +++ b/kairon/shared/data/processor.py @@ -5510,14 +5510,9 @@ def edit_prompt_action(self, prompt_action_id: str, request_data: dict, bot: Tex action.name = request_data.get("name") action.failure_message = request_data.get("failure_message") action.user_question = UserQuestion(**request_data.get("user_question")) - action.top_results = request_data.get("top_results") - action.enable_response_cache = request_data.get("enable_response_cache", False) - action.similarity_threshold = request_data.get("similarity_threshold") action.num_bot_responses = request_data.get('num_bot_responses', 5) - action.hyperparameters = request_data.get('hyperparameters', Utility.get_llm_hyperparameters()) action.llm_prompts = [LlmPrompt(**prompt) for prompt in request_data.get('llm_prompts', [])] action.instructions = request_data.get('instructions', []) - action.collection = request_data.get('collection') action.set_slots = request_data.get('set_slots', []) action.dispatch_response = request_data.get('dispatch_response', True) action.timestamp = datetime.utcnow() diff --git a/kairon/shared/llm/gpt3.py b/kairon/shared/llm/gpt3.py index 9135d9983..854b5b5af 100644 --- a/kairon/shared/llm/gpt3.py +++ b/kairon/shared/llm/gpt3.py @@ -104,10 +104,15 @@ async def __get_embedding(self, text: Text) -> List[float]: return result async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs): - query_prompt = kwargs.get('query_prompt') - use_query_prompt = kwargs.get('use_query_prompt') + use_query_prompt = False + query_prompt = '' + hyperparameters = Utility.get_llm_hyperparameters() + if kwargs.get('query_prompt', {}): + query_prompt_dict = kwargs.pop('query_prompt') + query_prompt = query_prompt_dict.get('query_prompt', '') + use_query_prompt = query_prompt_dict.get('use_query_prompt') + hyperparameters = query_prompt_dict.get('hyperparameters', Utility.get_llm_hyperparameters()) previous_bot_responses = kwargs.get('previous_bot_responses') - hyperparameters = kwargs.get('hyperparameters', Utility.get_llm_hyperparameters()) instructions = kwargs.get('instructions', []) instructions = '\n'.join(instructions) @@ -196,13 +201,14 @@ def logs(self): return self.__logs async def __attach_similarity_prompt_if_enabled(self, query_embedding, context_prompt, **kwargs): - use_similarity_prompt = kwargs.pop('use_similarity_prompt') - similarity_prompt_name = kwargs.pop('similarity_prompt_name') - similarity_prompt_instructions = kwargs.pop('similarity_prompt_instructions') - limit = kwargs.pop('top_results', 10) - score_threshold = kwargs.pop('similarity_threshold', 0.70) + similarity_prompt = kwargs.pop('similarity_prompt') + use_similarity_prompt = similarity_prompt.get('use_similarity_prompt') + similarity_prompt_name = similarity_prompt.get('similarity_prompt_name') + similarity_prompt_instructions = similarity_prompt.get('similarity_prompt_instructions') + limit = similarity_prompt.get('top_results', 10) + score_threshold = similarity_prompt.get('similarity_threshold', 0.70) if use_similarity_prompt: - collection_name = f"{self.bot}_{kwargs.get('collection')}{self.suffix}" if kwargs.get('collection') else f"{self.bot}{self.suffix}" + collection_name = f"{self.bot}_{similarity_prompt.get('collection')}{self.suffix}" if similarity_prompt.get('collection') else f"{self.bot}{self.suffix}" search_result = await self.__collection_search__(collection_name, vector=query_embedding, limit=limit, score_threshold=score_threshold) similarity_context = "\n".join([item['payload']['content'] for item in search_result['result']]) diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index 75d718e9c..d9d0313a2 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -1781,6 +1781,8 @@ def validate_kairon_faq_llm_prompts(llm_prompts: List, exception_class): raise exception_class("Name cannot be empty!") if Utility.check_empty_string(prompt.get('data')) and prompt['source'] == LlmPromptSource.static.value: raise exception_class("data is required for static prompts!") + if Utility.check_empty_string(prompt.get('collection')) and prompt['source'] == LlmPromptSource.bot_content.value: + raise exception_class("Collection is required for bot content prompts!") if prompt['type'] == LlmPromptType.query.value and prompt['source'] != LlmPromptSource.static.value: raise exception_class("Query prompt must have static source!") if prompt.get('type') == LlmPromptType.system.value: diff --git a/template/use-cases/Hi-Hello-GPT/actions.yml b/template/use-cases/Hi-Hello-GPT/actions.yml index 9953ca552..a708670b9 100644 --- a/template/use-cases/Hi-Hello-GPT/actions.yml +++ b/template/use-cases/Hi-Hello-GPT/actions.yml @@ -13,19 +13,7 @@ jira_action: [] pipedrive_leads_action: [] prompt_action: - dispatch_response: true - enable_response_cache: false failure_message: Kindly share more details so I can assist you effectively. - hyperparameters: - frequency_penalty: 0 - logit_bias: {} - max_tokens: 300 - model: gpt-3.5-turbo - n: 1 - presence_penalty: 0 - stop: null - stream: false - temperature: 0 - top_p: 0 instructions: [] llm_prompts: - data: 'You are a helpful personal assistant. Answer the question based on the @@ -47,9 +35,7 @@ prompt_action: name: kairon_faq_action num_bot_responses: 5 set_slots: [] - similarity_threshold: 0.7 status: true - top_results: 10 slot_set_action: [] two_stage_fallback: [] zendesk_action: [] diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index 3f60d5469..8504eaf77 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -9709,7 +9709,7 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc 'type': 'query', 'source': 'static', 'is_enabled': False}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', + 'type': 'user', 'source': 'bot_content', 'collection': 'python', 'is_enabled': True} ] @@ -9750,21 +9750,15 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc assert mock_completion.call_args.args[ 3] == """\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer question based on the context above, if answer is not in the context go check previous logs.\n""" print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == {'top_results': 10, 'similarity_threshold': 0.7, - 'enable_response_cache': False, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, - 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, - 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, - 'query_prompt': '', 'use_query_prompt': False, + assert mock_completion.call_args.kwargs == {'query_prompt': {}, 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}], - 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'instructions': [], 'collection': None} + 'similarity_prompt': {'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', + 'collection': 'python', + 'use_similarity_prompt': True, 'top_results': 10, + 'similarity_threshold': 0.7}, 'instructions': []} @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) @@ -9790,7 +9784,7 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe 'type': 'query', 'source': 'static', 'is_enabled': False}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', + 'type': 'user', 'source': 'bot_content', 'collection': 'python', 'is_enabled': True} ] @@ -9831,21 +9825,15 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe assert mock_completion.call_args.args[ 3] == """\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer question based on the context above, if answer is not in the context go check previous logs.\n""" print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == {'top_results': 10, 'similarity_threshold': 0.7, - 'enable_response_cache': False, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, - 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, - 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, - 'query_prompt': '', 'use_query_prompt': False, + assert mock_completion.call_args.kwargs == {'query_prompt': {}, 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}], - 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'instructions': [], 'collection': None} + 'similarity_prompt': {'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', + 'collection': 'python', + 'use_similarity_prompt': True, 'top_results': 10, + 'similarity_threshold': 0.7}, 'instructions': []} @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) @@ -9873,7 +9861,7 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock 'type': 'query', 'source': 'static', 'is_enabled': False}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', + 'type': 'user', 'source': 'bot_content', 'collection': 'python', 'is_enabled': True} ] @@ -9915,22 +9903,16 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock assert mock_completion.call_args.args[ 3] == """\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer question based on the context above, if answer is not in the context go check previous logs.\n""" print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == {'top_results': 10, 'similarity_threshold': 0.7, - 'enable_response_cache': False, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, - 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, - 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, - 'query_prompt': '', 'use_query_prompt': False, + assert mock_completion.call_args.kwargs == {'query_prompt': {}, 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}], - 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'instructions': ['Answer in a short way.', 'Keep it simple.'], - 'collection': None} + 'similarity_prompt': {'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', + 'collection': 'python', + 'use_similarity_prompt': True, 'top_results': 10, + 'similarity_threshold': 0.7}, + 'instructions': ['Answer in a short way.', 'Keep it simple.']} @mock.patch.object(GPT3Resources, "invoke", autospec=True) @@ -9954,7 +9936,7 @@ def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embed 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'collection': 'python', 'is_enabled': True}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', @@ -9995,6 +9977,7 @@ def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embed response = client.post("/webhook", json=request_object) response_json = response.json() + print(response_json) assert response_json['events'] == [ {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': generated_text}] assert response_json['responses'] == [ @@ -10027,7 +10010,7 @@ def test_prompt_response_action(mock_search, mock_embedding, mock_completion): {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', 'source': 'static'}, {'name': 'Similarity Prompt', - 'instructions': 'Answer question based on the context above.', 'type': 'user', 'source': 'bot_content'}, + 'instructions': 'Answer question based on the context above.', 'type': 'user', 'source': 'bot_content', 'collection': 'python'}, ] embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) mock_embedding.return_value = embedding @@ -10074,7 +10057,7 @@ def test_prompt_response_action_with_instructions(mock_search, mock_embedding, m {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', 'source': 'static'}, {'name': 'Similarity Prompt', - 'instructions': 'Answer question based on the context above.', 'type': 'user', 'source': 'bot_content'}, + 'instructions': 'Answer question based on the context above.', 'type': 'user', 'source': 'bot_content', 'collection': 'python'}, ] embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) mock_embedding.return_value = embedding @@ -10120,7 +10103,13 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', 'source': 'static'}, {'name': 'Similarity Prompt', - 'instructions': 'Answer question based on the context above.', 'type': 'user', 'source': 'bot_content'}, + 'instructions': 'Answer question based on the context above.', 'type': 'user', 'source': 'bot_content', 'collection': 'python', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': True, 'stop': None, + 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }, ] embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) @@ -10130,10 +10119,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() - hyperparameters = Utility.get_llm_hyperparameters().copy() - hyperparameters['stream'] = True - PromptAction(name=action_name, bot=bot, user=user, hyperparameters=hyperparameters, - llm_prompts=llm_prompts).save() + PromptAction(name=action_name, bot=bot, user=user, llm_prompts=llm_prompts).save() BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() request_object = json.load(open("tests/testing_data/actions/action-request.json")) @@ -10150,13 +10136,13 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - assert mock_completion.call_args.kwargs == {'messages': [ - {'role': 'system', 'content': 'You are a personal assistant.\n'}, - {'role': 'user', - 'content': '\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer question based on the context above.\n \nQ: What kind of language is python? \nA:'}], - 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': True, 'stop': None, - 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + print(mock_completion.call_args.kwargs) + assert mock_completion.call_args.kwargs == { + 'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, + {'role': 'user', + 'content': '\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer question based on the context above.\n \nQ: What kind of language is python? \nA:'}], + 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert mock_completion.call_args.args[1] == 'chat/completions' @@ -10271,7 +10257,7 @@ def test_prompt_action_response_action_with_static_user_prompt(mock_search, mock 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, 'collection': 'python'}, {'name': 'Python Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'static', @@ -10384,7 +10370,7 @@ def test_prompt_action_response_action_with_action_prompt(mock_search, mock_embe 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'collection': 'python', 'is_enabled': True}, {'name': 'Python Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'static', @@ -10578,7 +10564,7 @@ def test_prompt_action_dispatch_response_disabled(mock_search, mock_embedding, m 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'collection': 'python', 'is_enabled': True}, {'name': 'Language Prompt', 'data': 'type', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'slot', @@ -10727,7 +10713,7 @@ def test_prompt_action_response_action_slot_prompt(mock_search, mock_embedding, 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'collection': 'python', 'is_enabled': True}, {'name': 'Language Prompt', 'data': 'type', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'slot', @@ -10803,7 +10789,7 @@ def test_prompt_action_user_message_in_slot(mock_search, mock_embedding, mock_co 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'collection': 'python', 'is_enabled': True}, ] def mock_completion_for_answer(*args, **kwargs): diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 5b9d550ff..9ab76b81d 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -1375,14 +1375,16 @@ def _mock_get_bot_settings(*args, **kwargs): assert not actual_three["data"] assert actual_three["error_code"] == 422 - response = client.delete( + response_five = client.delete( url=f"/api/bot/{pytest.bot}/data/cognition/schema/{pytest.schema_id_one}", headers={"Authorization": pytest.token_type + " " + pytest.access_token} ) - response = client.delete( + actual_five = response_five.json() + response_six = client.delete( url=f"/api/bot/{pytest.bot}/data/cognition/schema/{pytest.schema_id_two}", headers={"Authorization": pytest.token_type + " " + pytest.access_token} ) + actual_six = response_six.json() response_four = client.post( url=f"/api/bot/{pytest.bot}/data/cognition/schema", @@ -1569,7 +1571,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, 'collection': 'python'}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -1579,9 +1581,8 @@ def _mock_get_bot_settings(*args, **kwargs): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], - 'collection': 'python', 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2262,7 +2263,8 @@ def test_add_prompt_action_with_invalid_similarity_threshold(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "top_results": 10, "similarity_threshold": 1.70, + "collection": "Science"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -2271,7 +2273,7 @@ def test_add_prompt_action_with_invalid_similarity_threshold(): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 1.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2279,8 +2281,7 @@ def test_add_prompt_action_with_invalid_similarity_threshold(): ) actual = response.json() assert actual["message"] == [ - {'loc': ['body', 'similarity_threshold'], 'msg': 'similarity_threshold should be within 0.3 and 1', - 'type': 'value_error'}] + {'loc': ['body', 'llm_prompts', 1, 'similarity_threshold'], 'msg': 'similarity_threshold should be within 0.3 and 1', 'type': 'value_error'}] assert not actual["data"] assert not actual["success"] assert actual["error_code"] == 422 @@ -2292,7 +2293,8 @@ def test_add_prompt_action_with_invalid_top_results(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "top_results": 40, "similarity_threshold": 0.70, + "collection": "Science"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -2301,7 +2303,7 @@ def test_add_prompt_action_with_invalid_top_results(): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 40, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2309,8 +2311,7 @@ def test_add_prompt_action_with_invalid_top_results(): ) actual = response.json() print(actual["message"]) - assert actual["message"] == [{'loc': ['body', 'top_results'], - 'msg': 'top_results should not be greater than 30', 'type': 'value_error'}] + assert actual["message"] == [{'loc': ['body', 'llm_prompts', 1, 'top_results'], 'msg': 'top_results should not be greater than 30', 'type': 'value_error'}] assert not actual["data"] assert not actual["success"] assert actual["error_code"] == 422 @@ -2322,12 +2323,12 @@ def test_add_prompt_action_with_invalid_query_prompt(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Science"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'history', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2347,7 +2348,7 @@ def test_add_prompt_action_with_invalid_num_bot_responses(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Science"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -2356,7 +2357,7 @@ def test_add_prompt_action_with_invalid_num_bot_responses(): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 10} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -2379,7 +2380,7 @@ def test_add_prompt_action_with_invalid_system_prompt_source(): 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Science"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -2388,7 +2389,7 @@ def test_add_prompt_action_with_invalid_system_prompt_source(): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2413,7 +2414,7 @@ def test_add_prompt_action_with_multiple_system_prompt(): 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Science"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -2422,7 +2423,7 @@ def test_add_prompt_action_with_multiple_system_prompt(): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2443,7 +2444,7 @@ def test_add_prompt_action_with_empty_llm_prompt_name(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Science"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -2452,7 +2453,7 @@ def test_add_prompt_action_with_empty_llm_prompt_name(): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2467,13 +2468,43 @@ def test_add_prompt_action_with_empty_llm_prompt_name(): assert actual["error_code"] == 422 +def test_add_prompt_action_with_collection_for_bot_content(): + action = {'name': 'test_add_prompt_action_with_empty_llm_prompt_name', + 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', + 'source': 'static', 'is_enabled': True}, + {'name': 'Similarity Prompt', + 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', + 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + {'name': 'Query Prompt', + 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True}, + {'name': 'Query Prompt', + 'data': 'If there is no specific query, assume that user is aking about java programming.', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + response = client.post( + f"/api/bot/{pytest.bot}/action/prompt", + json=action, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + print(actual["message"]) + assert actual["message"] == [{'loc': ['body', 'llm_prompts'], + 'msg': 'Collection is required for bot content prompts!', 'type': 'value_error'}] + assert not actual["data"] + assert not actual["success"] + assert actual["error_code"] == 422 + + def test_add_prompt_action_with_empty_data_for_static_prompt(): action = {'name': 'test_add_prompt_action_with_empty_data_for_static_prompt', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Science"}, {'name': 'Query Prompt', 'data': '', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, @@ -2481,7 +2512,7 @@ def test_add_prompt_action_with_empty_data_for_static_prompt(): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2504,12 +2535,12 @@ def test_add_prompt_action_with_multiple_history_source_prompts(): {'name': 'Analytical Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Science"}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2531,16 +2562,16 @@ def test_add_prompt_action_with_multiple_bot_content_source_prompts(): {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Science"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, {'name': 'Another Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True} + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Data"} ], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2561,7 +2592,7 @@ def test_add_prompt_action_with_gpt_feature_disabled(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Science"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -2570,7 +2601,7 @@ def test_add_prompt_action_with_gpt_feature_disabled(): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2593,18 +2624,17 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', - 'source': 'static', 'is_enabled': True}, + 'source': 'static', 'is_enabled': True, "top_results": 10, "similarity_threshold": 0.70}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], - 'collection': 'Bot_collection', 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2624,21 +2654,22 @@ def _mock_get_bot_settings(*args, **kwargs): return BotSettings(bot=pytest.bot, user="integration@demo.ai", llm_settings=LLMSettings(enable_faq=True)) monkeypatch.setattr(MongoProcessor, 'get_bot_settings', _mock_get_bot_settings) - action = {'name': 'test_add_prompt_action', - 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', - 'source': 'static', 'is_enabled': True}, - {'name': 'Similarity Prompt', - 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, - {'name': 'Query Prompt', - 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', - 'instructions': 'Answer according to the context', 'type': 'query', - 'source': 'static', 'is_enabled': True}, - {'name': 'Query Prompt', - 'data': 'If there is no specific query, assume that user is aking about java programming.', - 'instructions': 'Answer according to the context', 'type': 'query', - 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + action = {'name': 'test_add_prompt_action', 'user_question': {'type': 'from_user_message'}, + 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', + 'source': 'static', 'is_enabled': True}, + {'name': 'Similarity Prompt', + 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, + {'name': 'Query Prompt', + 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True, "top_results": 10, "similarity_threshold": 0.70}, + {'name': 'Query Prompt', + 'data': 'If there is no specific query, assume that user is aking about java programming.', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], + 'num_bot_responses': 5, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -2658,7 +2689,7 @@ def test_update_prompt_action_does_not_exist(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -2667,7 +2698,7 @@ def test_update_prompt_action_does_not_exist(): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} response = client.put( f"/api/bot/{pytest.bot}/action/prompt/61512cc2c6219f0aae7bba3d", json=action, @@ -2687,7 +2718,8 @@ def test_update_prompt_action_with_invalid_similarity_threshold(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection", + "top_results": 9, "similarity_threshold": 1.50}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -2696,7 +2728,7 @@ def test_update_prompt_action_with_invalid_similarity_threshold(): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": "updated_failure_message", "top_results": 9, "similarity_threshold": 1.50} + "failure_message": "updated_failure_message"} response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", json=action, @@ -2704,8 +2736,7 @@ def test_update_prompt_action_with_invalid_similarity_threshold(): ) actual = response.json() print(actual["message"]) - assert actual["message"] == [{'loc': ['body', 'similarity_threshold'], - 'msg': 'similarity_threshold should be within 0.3 and 1', 'type': 'value_error'}] + assert actual["message"] == [ {'loc': ['body', 'llm_prompts', 1, 'similarity_threshold'], 'msg': 'similarity_threshold should be within 0.3 and 1', 'type': 'value_error'}] assert not actual["data"] assert not actual["success"] assert actual["error_code"] == 422 @@ -2717,12 +2748,13 @@ def test_update_prompt_action_with_invalid_top_results(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection" + , "top_results": 39, "similarity_threshold": 0.50}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming here.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 5, - "failure_message": "updated_failure_message", "top_results": 39, "similarity_threshold": 0.50} + "failure_message": "updated_failure_message"} response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", json=action, @@ -2730,8 +2762,7 @@ def test_update_prompt_action_with_invalid_top_results(): ) actual = response.json() print(actual["message"]) - assert actual["message"] == [{'loc': ['body', 'top_results'], - 'msg': 'top_results should not be greater than 30', 'type': 'value_error'}] + assert actual["message"] == [{'loc': ['body', 'llm_prompts', 1, 'top_results'], 'msg': 'top_results should not be greater than 30', 'type': 'value_error'}] assert not actual["data"] assert not actual["success"] assert actual["error_code"] == 422 @@ -2743,12 +2774,12 @@ def test_update_prompt_action_with_invalid_num_bot_responses(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'num_bot_responses': 50, - "failure_message": "updated_failure_message", "top_results": 39, "similarity_threshold": 0.50} + "failure_message": "updated_failure_message"} response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", json=action, @@ -2758,8 +2789,7 @@ def test_update_prompt_action_with_invalid_num_bot_responses(): print(actual["message"]) assert actual["message"] == [ {'loc': ['body', 'num_bot_responses'], 'msg': 'num_bot_responses should not be greater than 5', - 'type': 'value_error'}, - {'loc': ['body', 'top_results'], 'msg': 'top_results should not be greater than 30', 'type': 'value_error'}] + 'type': 'value_error'}] assert not actual["data"] assert not actual["success"] assert actual["error_code"] == 422 @@ -2771,13 +2801,13 @@ def test_update_prompt_action_with_invalid_query_prompt(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'history', 'is_enabled': True}, ], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, 'num_bot_responses': 5, "use_query_prompt": True, "query_prompt": ""} response = client.put( @@ -2797,13 +2827,13 @@ def test_update_prompt_action_with_query_prompt_with_false(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', - 'source': 'bot_content', 'is_enabled': False}, + 'source': 'bot_content', 'is_enabled': False, "collection": "Bot_collection"}, ], - "failure_message": "updated_failure_message", "top_results": 9, "similarity_threshold": 0.50, + "failure_message": "updated_failure_message", 'num_bot_responses': 5, "set_slots": [{"name": "gpt_result", "value": "${data}", "evaluation_type": "expression"}, {"name": "gpt_result_type", "value": "${data.type}", "evaluation_type": "script"}], @@ -2825,7 +2855,7 @@ def test_update_prompt_action(): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity_analytical Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -2835,7 +2865,7 @@ def test_update_prompt_action(): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": "updated_failure_message", "top_results": 9, "similarity_threshold": 0.50} + "failure_message": "updated_failure_message"} response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", json=action, @@ -2861,25 +2891,35 @@ def test_get_prompt_action(): actual['data'][0].pop("_id") print(actual["data"]) assert actual["data"] == [ - {'name': 'test_update_prompt_action', 'num_bot_responses': 5, 'top_results': 9, 'similarity_threshold': 0.5, - 'enable_response_cache': False, 'failure_message': 'updated_failure_message', - 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, - 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', - 'source': 'static', 'is_enabled': True}, - {'name': 'Similarity_analytical Prompt', - 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, - {'name': 'Query Prompt', - 'data': 'A programming language is a system of notation for writing computer programs.Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', - 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, - {'name': 'Query Prompt', - 'data': 'If there is no specific query, assume that user is aking about java programming language,', - 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'instructions': ['Answer in a short manner.', 'Keep it simple.'], - 'set_slots': [], 'dispatch_response': True, 'status': True}] + {'name': 'test_update_prompt_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', + 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [ + {'name': 'System Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'logit_bias': {}}, 'data': 'You are a personal assistant.', 'type': 'system', + 'source': 'static', 'is_enabled': True}, + {'name': 'Similarity_analytical Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'logit_bias': {}}, 'collection': 'Bot_collection', + 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', + 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + {'name': 'Query Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'logit_bias': {}}, + 'data': 'A programming language is a system of notation for writing computer programs.Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', + 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', + 'is_enabled': True}, {'name': 'Query Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', + 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'logit_bias': {}}, + 'data': 'If there is no specific query, assume that user is aking about java programming language,', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True}], + 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], 'dispatch_response': True, + 'status': True}] def test_delete_prompt_action_not_exists(): diff --git a/tests/testing_data/actions/actions.yml b/tests/testing_data/actions/actions.yml index 23782da09..c73b91128 100644 --- a/tests/testing_data/actions/actions.yml +++ b/tests/testing_data/actions/actions.yml @@ -240,9 +240,6 @@ two_stage_fallback: prompt_action: - name: prompt_action_1 num_bot_responses: 5 - top_results: 5 - similarity_threshold: 1 - enable_response_cache: True llm_prompts: - name: System Prompt data: You are a personal assistant. @@ -251,15 +248,14 @@ prompt_action: is_enabled: True - name: Query Prompt data: A programming language is a system of notation for writing computer programs. + top_results: 5 + similarity_threshold: 1 instructions: Answer according to the context. type: query source: static is_enabled: True - name: prompt_action_2 num_bot_responses: 5 - top_results: 5 - similarity_threshold: 1 - enable_response_cache: True llm_prompts: - name: System Prompt data: You are a personal assistant. @@ -268,6 +264,8 @@ prompt_action: is_enabled: True - name: Similarity Prompt data: Data Science is an emerging field. + top_results: 5 + similarity_threshold: 1 instructions: Answer according to the context. type: user source: static diff --git a/tests/testing_data/actions/validation_action_data.json b/tests/testing_data/actions/validation_action_data.json index 779735a64..c211c8b72 100644 --- a/tests/testing_data/actions/validation_action_data.json +++ b/tests/testing_data/actions/validation_action_data.json @@ -513,6 +513,8 @@ "llm_prompts":[ { "name":"Similarity Prompt", + "top_results":40, + "similarity_threshold":2, "instructions":"Answer question based on the context above, if answer is not in the context go check previous logs.", "type":"user", "source":"bot_content", @@ -528,8 +530,6 @@ } ], "failure_message":"DEFAULT_NLU_FALLBACK_RESPONSE", - "top_results":40, - "similarity_threshold":2, "num_bot_responses":5 }, { @@ -567,8 +567,6 @@ } ], "failure_message":"DEFAULT_NLU_FALLBACK_RESPONSE", - "top_results":10, - "similarity_threshold":0.70, "num_bot_responses":15 }, { @@ -586,7 +584,21 @@ "instructions":"Answer question based on the context above, if answer is not in the context go check previous logs.", "type":"user", "source":"bot_content", - "is_enabled":true + "is_enabled":true, + "hyperparameters": { + "temperature": 3.0, + "max_tokens": 5000, + "model": "gpt - 3.5 - turbo", + "top_p": 4, + "n": 10, + "stream": false, + "stop": { + }, + "presence_penalty": 5, + "frequency_penalty": 5, + "logit_bias": [ + ] + } }, { "name":"Similarity Prompt two", @@ -597,46 +609,12 @@ } ], "failure_message":"DEFAULT_NLU_FALLBACK_RESPONSE", - "top_results":10, - "similarity_threshold":0.70, - "num_bot_responses":5, - "hyperparameters":{ - "temperature":3.0, - "max_tokens":5000, - "model":"gpt - 3.5 - turbo", - "top_p":4, - "n":10, - "stream":false, - "stop":{ - - }, - "presence_penalty":5, - "frequency_penalty":5, - "logit_bias":[ - - ] - } + "num_bot_responses":5 }, { "name":"prompt_action_with_no_llm_prompts", "failure_message":"DEFAULT_NLU_FALLBACK_RESPONSE", - "top_results":10, - "similarity_threshold":0.70, - "num_bot_responses":5, - "hyperparameters":{ - "temperature":3.0, - "max_tokens":300, - "model":"gpt - 3.5 - turbo", - "top_p":0.0, - "n":1, - "stream":false, - "stop":"None", - "presence_penalty":0.0, - "frequency_penalty":0.0, - "logit_bias":{ - - } - } + "num_bot_responses":5 }, { "name":"test_add_prompt_action_one", @@ -725,7 +703,26 @@ "instructions":50, "type":1, "source":2, - "is_enabled":true + "is_enabled":true, + "hyperparameters": { + "temperature": 3.0, + "max_tokens": 5000, + "model": "gpt - 3.5 - turbo", + "top_p": 4, + "n": 10, + "stream": false, + "stop": [ + "a", + "b", + "c", + "d", + "e" + ], + "presence_penalty": 5, + "frequency_penalty": 5, + "logit_bias": [ + ] + } }, { "name":"Http action Prompt", @@ -756,27 +753,7 @@ "is_enabled":true } ], - "dispatch_response":false, - "hyperparameters":{ - "temperature":3.0, - "max_tokens":5000, - "model":"gpt - 3.5 - turbo", - "top_p":4, - "n":10, - "stream":false, - "stop":[ - "a", - "b", - "c", - "d", - "e" - ], - "presence_penalty":5, - "frequency_penalty":5, - "logit_bias":[ - - ] - } + "dispatch_response":false } ], "razorpay_action":[ diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index 5b101b913..3923a107e 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -2671,17 +2671,24 @@ def test_get_prompt_action_config(self): actual_config.pop("timestamp") actual_config.pop("status") actual_config.pop("user") - assert actual_config == {'name': 'kairon_faq_action', 'num_bot_responses': 5, 'top_results': 10, - 'user_question': {'type': 'from_user_message'}, - 'similarity_threshold': 0.7, + assert actual_config == {'name': 'kairon_faq_action', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", - 'bot': 'test_action_server', 'enable_response_cache': False, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, - 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {} - }, - 'dispatch_response': True, 'set_slots': [], 'llm_prompts': llm_prompts, - 'instructions': []} + 'user_question': {'type': 'from_user_message'}, 'bot': 'test_action_server', + 'llm_prompts': [ + {'name': 'System Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', + 'is_enabled': True}, + {'name': 'History Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, 'type': 'user', + 'source': 'history', 'is_enabled': True}], 'instructions': [], 'set_slots': [], + 'dispatch_response': True} bot_settings.pop("_id") bot_settings.pop("timestamp") bot_settings.pop("status") @@ -3958,21 +3965,21 @@ def test_get_prompt_action_config_2(self): PromptAction(name='kairon_faq_action', bot=bot, user=user, llm_prompts=llm_prompts).save() k_faq_action_config = ActionUtility.get_faq_action_config(bot, "kairon_faq_action") k_faq_action_config.pop('timestamp') - assert k_faq_action_config == {'name': 'kairon_faq_action', 'num_bot_responses': 5, 'top_results': 10, - 'similarity_threshold': 0.7, - 'enable_response_cache': False, - 'user_question': {'type': 'from_user_message'}, + print(k_faq_action_config) + assert k_faq_action_config == {'name': 'kairon_faq_action', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", - 'bot': 'test_bot_action_test', 'user': 'test_user_action_test', - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, - 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'dispatch_response': True, 'set_slots': [], - 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', - 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', - 'is_enabled': True}], 'instructions': [], - 'status': True} + 'user_question': {'type': 'from_user_message'}, 'bot': 'test_bot_action_test', + 'user': 'test_user_action_test', 'llm_prompts': [ + {'name': 'System Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, + {'name': 'History Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, 'type': 'user', 'source': 'history', + 'is_enabled': True}], 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True} def test_retrieve_config_two_stage_fallback_not_found(self): with pytest.raises(ActionFailure, match="Two stage fallback action config not found"): diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index f9c4a0ab7..3185a7e9c 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -156,7 +156,7 @@ def test_add_prompt_action_with_gpt_feature_disabled(self): bot = 'test' user = 'test_user' request = {"system_prompt": DEFAULT_SYSTEM_PROMPT, "context_prompt": DEFAULT_CONTEXT_PROMPT, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 5} with pytest.raises(AppException, match="Faq feature is disabled for the bot! Please contact support."): processor.add_prompt_action(request, bot, user) @@ -168,18 +168,19 @@ def test_add_prompt_action_with_invalid_slots(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_slots', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 10, - 'similarity_threshold': 1.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection", + 'top_results': 10, + 'similarity_threshold': 1.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }, {'name': 'Identification Prompt', 'data': 'info', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'slot', @@ -194,18 +195,19 @@ def test_add_prompt_action_with_invalid_http_action(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_http_action', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 10, - 'similarity_threshold': 1.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection", + 'top_results': 10, + 'similarity_threshold': 1.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }, {'name': 'Http action Prompt', 'data': 'test_http_action', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'action', @@ -220,17 +222,19 @@ def test_add_prompt_action_with_invalid_similarity_threshold(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_prompt_action_similarity', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 10, - 'similarity_threshold': 1.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection", + 'top_results': 10, + 'similarity_threshold': 1.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -248,17 +252,19 @@ def test_add_prompt_action_with_invalid_top_results(self): user = 'test_user' request = {'name': 'test_prompt_action_invalid_top_results', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 40, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection", + 'top_results': 40, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -270,6 +276,28 @@ def test_add_prompt_action_with_invalid_top_results(self): with pytest.raises(ValidationError, match="top_results should not be greater than 30"): processor.add_prompt_action(request, bot, user) + def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): + processor = MongoProcessor() + bot = 'test_bot' + user = 'test_user' + request = {'name': 'test_add_prompt_action_with_empty_collection_for_bot_content_prompt', 'num_bot_responses': 5, + 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, + 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', + 'source': 'static', 'is_enabled': True}, + {'name': 'Similarity Prompt', + 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', + 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + {'name': 'Query Prompt', + 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True}, + {'name': 'Query Prompt', + 'data': 'If there is no specific query, assume that user is aking about java programming.', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True}]} + with pytest.raises(ValidationError, match="Collection is required for bot content prompts!"): + processor.add_prompt_action(request, bot, user) + def test_add_prompt_action_with_invalid_query_prompt(self): processor = MongoProcessor() bot = 'test_bot' @@ -279,13 +307,12 @@ def test_add_prompt_action_with_invalid_query_prompt(self): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'history', 'is_enabled': True}], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70, - "num_bot_responses": 5} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 5} with pytest.raises(ValidationError, match="Query prompt must have static source!"): processor.add_prompt_action(request, bot, user) @@ -298,7 +325,7 @@ def test_add_prompt_action_with_invalid_num_bot_responses(self): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -307,7 +334,7 @@ def test_add_prompt_action_with_invalid_num_bot_responses(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 15} with pytest.raises(ValidationError, match="num_bot_responses should not be greater than 5"): processor.add_prompt_action(request, bot, user) @@ -322,7 +349,7 @@ def test_add_prompt_action_with_invalid_system_prompt_source(self): 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -331,8 +358,7 @@ def test_add_prompt_action_with_invalid_system_prompt_source(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, - "similarity_threshold": 0.70, "num_bot_responses": 5} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 5} with pytest.raises(ValidationError, match="System prompt must have static source!"): processor.add_prompt_action(request, bot, user) @@ -349,7 +375,7 @@ def test_add_prompt_action_with_multiple_system_prompt(self): 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -358,8 +384,7 @@ def test_add_prompt_action_with_multiple_system_prompt(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, - "similarity_threshold": 0.70, "num_bot_responses": 5} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 5} with pytest.raises(ValidationError, match="Only one system prompt can be present!"): processor.add_prompt_action(request, bot, user) @@ -372,7 +397,7 @@ def test_add_prompt_action_with_empty_llm_prompt_name(self): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -381,8 +406,7 @@ def test_add_prompt_action_with_empty_llm_prompt_name(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, - "similarity_threshold": 0.70, "num_bot_responses": 5} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 5} with pytest.raises(ValidationError, match="Name cannot be empty!"): processor.add_prompt_action(request, bot, user) @@ -395,7 +419,7 @@ def test_add_prompt_action_with_empty_data_for_static_prompt(self): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': '', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, @@ -403,12 +427,10 @@ def test_add_prompt_action_with_empty_data_for_static_prompt(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, - "similarity_threshold": 0.70, "num_bot_responses": 5} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 5} with pytest.raises(ValidationError, match="data is required for static prompts!"): processor.add_prompt_action(request, bot, user) - def test_add_prompt_action_with_multiple_history_source_prompts(self): processor = MongoProcessor() bot = 'test_bot' @@ -420,7 +442,7 @@ def test_add_prompt_action_with_multiple_history_source_prompts(self): {'name': 'Analytical Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -429,8 +451,7 @@ def test_add_prompt_action_with_multiple_history_source_prompts(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, - "similarity_threshold": 0.70, "num_bot_responses": 5} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 5} with pytest.raises(ValidationError, match="Only one history source can be present!"): processor.add_prompt_action(request, bot, user) @@ -444,17 +465,16 @@ def test_add_prompt_action_with_multiple_bot_content_source_prompts(self): {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, {'name': 'Another Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True} + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"} ], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, - "similarity_threshold": 0.70, "num_bot_responses": 5} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 5} with pytest.raises(ValidationError, match="Only one bot_content source can be present!"): processor.add_prompt_action(request, bot, user) @@ -467,13 +487,12 @@ def test_add_prompt_action_with_no_system_prompts(self): {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"}, {'name': 'Another Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True} + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, "collection": "Bot_collection"} ], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, - "similarity_threshold": 0.70, "num_bot_responses": 5} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 5} with pytest.raises(ValidationError, match="System prompt is required!"): processor.add_prompt_action(request, bot, user) @@ -483,12 +502,6 @@ def test_add_prompt_action_with_empty_llm_prompts(self): user = 'test_user' request = {'name': 'test_add_prompt_action_with_empty_llm_prompts', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': []} with pytest.raises(ValidationError, match="llm_prompts are required!"): processor.add_prompt_action(request, bot, user) @@ -510,22 +523,23 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) pytest.action_id = processor.add_prompt_action(request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - assert action == [ - {'name': 'test_add_prompt_action_faq_action_with_default_values', - 'num_bot_responses': 5, 'top_results': 10, 'similarity_threshold': 0.7, - 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", - 'enable_response_cache': False, 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, - 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', - 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}], - 'instructions': ['Answer in a short manner.', 'Keep it simple.'], - 'status': True, "set_slots": [ - {"name": "gpt_result", "value": "${data}", "evaluation_type": "expression"}, - {"name": "gpt_result_type", "value": "${data.type}", "evaluation_type": "script"}], - "dispatch_response": False}] + print(action) + assert action == [{'name': 'test_add_prompt_action_faq_action_with_default_values', 'num_bot_responses': 5, + 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", + 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [ + {'name': 'System Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, + {'name': 'History Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, 'type': 'user', 'source': 'history', + 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], + 'set_slots': [{'name': 'gpt_result', 'value': '${data}', 'evaluation_type': 'expression'}, + {'name': 'gpt_result_type', 'value': '${data.type}', + 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}] def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): processor = MongoProcessor() @@ -534,15 +548,16 @@ def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_temperature_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True, + 'top_results': 20, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }]} with pytest.raises(ValidationError, match="Temperature must be between 0.0 and 2.0!"): processor.add_prompt_action(request, bot, user) @@ -553,15 +568,17 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_stop_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True, + 'top_results': 20, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], + 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }]} with pytest.raises(ValidationError, match="Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers."): processor.add_prompt_action(request, bot, user) @@ -572,15 +589,16 @@ def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_presence_penalty_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True, + 'top_results': 20, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }]} with pytest.raises(ValidationError, match="Presence penalty must be between -2.0 and 2.0!"): processor.add_prompt_action(request, bot, user) @@ -591,15 +609,16 @@ def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, - 'frequency_penalty': 3.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True, + 'top_results': 20, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'frequency_penalty': 3.0, 'logit_bias': {}}, + }]} with pytest.raises(ValidationError, match="Frequency penalty must be between -2.0 and 2.0!"): processor.add_prompt_action(request, bot, user) @@ -610,15 +629,16 @@ def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_max_tokens_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True, + 'top_results': 20, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }]} with pytest.raises(ValidationError, match="max_tokens must be between 5 and 4096 and should not be 0!"): processor.add_prompt_action(request, bot, user) @@ -629,15 +649,16 @@ def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_zero_max_tokens_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True, + 'top_results': 20, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }]} with pytest.raises(ValidationError, match="max_tokens must be between 5 and 4096 and should not be 0!"): processor.add_prompt_action(request, bot, user) @@ -648,15 +669,16 @@ def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_top_p_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt - 3.5 - turbo', - 'top_p': 3.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True, + 'top_results': 20, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt - 3.5 - turbo', + 'top_p': 3.0, + 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }]} with pytest.raises(ValidationError, match="top_p must be between 0.0 and 1.0!"): processor.add_prompt_action(request, bot, user) @@ -667,15 +689,16 @@ def test_add_prompt_action_with_invalid_n_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_n_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True, + 'top_results': 20, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }]} with pytest.raises(ValidationError, match="n must be between 1 and 5 and should not be 0!"): processor.add_prompt_action(request, bot, user) @@ -686,15 +709,16 @@ def test_add_prompt_action_with_zero_n_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_zero_n_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True, + 'top_results': 20, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + }]} with pytest.raises(ValidationError, match="n must be between 1 and 5 and should not be 0!"): processor.add_prompt_action(request, bot, user) @@ -705,15 +729,16 @@ def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_logit_bias_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'top_results': 20, - 'similarity_threshold': 0.70, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', - 'top_p': 0.0, - 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': 'a'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True, + 'top_results': 20, + 'similarity_threshold': 0.70, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'top_p': 0.0, + 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': 'a'}, + }]} with pytest.raises(ValidationError, match="logit_bias must be a dictionary!"): processor.add_prompt_action(request, bot, user) @@ -722,9 +747,15 @@ def test_add_prompt_action_faq_action_already_exist(self): bot = 'test_bot' user = 'test_user' request = {'name': 'test_add_prompt_action_faq_action_with_default_values', + 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, - {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} + {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}], + 'instructions': ['Answer in a short manner.', 'Keep it simple.'], + "set_slots": [{"name": "gpt_result", "value": "${data}", "evaluation_type": "expression"}, + {"name": "gpt_result_type", "value": "${data.type}", "evaluation_type": "script"}], + "dispatch_response": False + } with pytest.raises(AppException, match='Action exists!'): processor.add_prompt_action(request, bot, user) @@ -738,7 +769,7 @@ def test_edit_prompt_action_does_not_exist(self): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True, 'collection': 'Bot_collection'}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -747,8 +778,7 @@ def test_edit_prompt_action_does_not_exist(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70, - "num_bot_responses": 5} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 5} with pytest.raises(AppException, match="Action not found"): processor.edit_prompt_action(action_id, request, bot, user) @@ -762,7 +792,7 @@ def test_edit_prompt_action_faq_action(self): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True,'collection': 'Bot_collection'}, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', @@ -771,9 +801,9 @@ def test_edit_prompt_action_faq_action(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - "failure_message": "updated_failure_message", "top_results": 10, "similarity_threshold": 0.70, + "failure_message": "updated_failure_message", "use_query_prompt": True, "use_bot_responses": True, "query_prompt": "updated_query_prompt", - "num_bot_responses": 5, "hyperparameters": Utility.get_llm_hyperparameters(), + "num_bot_responses": 5, "set_slots": [{"name": "gpt_result", "value": "${data}", "evaluation_type": "expression"}, {"name": "gpt_result_type", "value": "${data.type}", "evaluation_type": "script"}], "dispatch_response": False @@ -781,27 +811,43 @@ def test_edit_prompt_action_faq_action(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - assert action == [ - {'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, 'top_results': 10, - 'similarity_threshold': 0.7, 'failure_message': 'updated_failure_message', 'enable_response_cache': False, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, - 'user_question': {'type': 'from_user_message'}, - 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', - 'source': 'static', 'is_enabled': True}, - {'name': 'Similarity Prompt', - 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, - {'name': 'Query Prompt', - 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', - 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, - {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', - 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'status': True, 'instructions': [], - "set_slots": [{"name": "gpt_result", "value": "${data}", "evaluation_type": "expression"}, - {"name": "gpt_result_type", "value": "${data.type}", "evaluation_type": "script"}], - "dispatch_response": False}] + print(action) + assert action == [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, + 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_user_message'}, + 'llm_prompts': [{'name': 'System Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'You are a personal assistant.', 'type': 'system', + 'source': 'static', 'is_enabled': True}, + {'name': 'Similarity Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'collection': 'Bot_collection', + 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', + 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + {'name': 'Query Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True}, + {'name': 'Query Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'If there is no specific query, assume that user is aking about java programming.', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True}], 'instructions': [], + 'set_slots': [{'name': 'gpt_result', 'value': '${data}', 'evaluation_type': 'expression'}, + {'name': 'gpt_result_type', 'value': '${data.type}', + 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}] request = {'name': 'test_edit_prompt_action_faq_action_again', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -809,19 +855,17 @@ def test_edit_prompt_action_faq_action(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - assert action == [ - {'name': 'test_edit_prompt_action_faq_action_again', 'num_bot_responses': 5, 'top_results': 10, - 'similarity_threshold': 0.7, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", - 'enable_response_cache': False, - 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, - 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', - 'source': 'static', 'is_enabled': True}], 'status': True, - "set_slots": [], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], - "dispatch_response": True - }] + print(action) + assert action == [{'name': 'test_edit_prompt_action_faq_action_again', 'num_bot_responses': 5, + 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", + 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [ + {'name': 'System Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}], + 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], + 'dispatch_response': True, 'status': True}] def test_edit_prompt_action_with_less_hyperparameters(self): processor = MongoProcessor() @@ -834,7 +878,14 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + 'type': 'user', 'source': 'bot_content', 'is_enabled': True,'collection': 'Bot_collection', + "top_results": 10, "similarity_threshold": 0.70, + "hyperparameters": {"temperature": 0.0, + "max_tokens": 300, + "model": "gpt-3.5-turbo", + "top_p": 0.0, + "n": 1} + }, {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', @@ -843,33 +894,42 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - "failure_message": "updated_failure_message", "top_results": 10, "similarity_threshold": 0.70, + "failure_message": "updated_failure_message", "use_query_prompt": True, "use_bot_responses": True, "query_prompt": "updated_query_prompt", - "num_bot_responses": 5, "hyperparameters": {"temperature": 0.0, - "max_tokens": 300, - "model": "gpt-3.5-turbo", - "top_p": 0.0, - "n": 1}} + "num_bot_responses": 5} processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - assert action == [ - {'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, 'top_results': 10, - 'similarity_threshold': 0.7, 'failure_message': 'updated_failure_message', 'enable_response_cache': False, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, - 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, - 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', - 'source': 'static', 'is_enabled': True}, - {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, - {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', - 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, - {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', - 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'status': True, 'instructions': [], "set_slots": [], "dispatch_response": True}] + print(action) + assert action == [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, + 'failure_message': 'updated_failure_message', + 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [ + {'name': 'System Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, + {'name': 'Similarity Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1}, 'collection': 'Bot_collection', + 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', + 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + {'name': 'Query Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', + 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', + 'is_enabled': True}, {'name': 'Query Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'If there is no specific query, assume that user is aking about java programming.', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True}], 'instructions': [], 'set_slots': [], + 'dispatch_response': True, 'status': True}] def test_get_prompt_action_does_not_exist(self): processor = MongoProcessor() @@ -882,22 +942,35 @@ def test_get_prompt_faq_action(self): bot = 'test_bot' action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - assert action == [ - {'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, 'top_results': 10, - 'similarity_threshold': 0.7, 'failure_message': 'updated_failure_message', 'enable_response_cache': False, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, - 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, - 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', - 'source': 'static', 'is_enabled': True}, - {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, - {'name': 'Query Prompt', 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', - 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, - {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', - 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'status': True, 'instructions': [], "set_slots": [], "dispatch_response": True}] + print(action) + assert action == [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, + 'failure_message': 'updated_failure_message', + 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [ + {'name': 'System Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, + {'name': 'Similarity Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1}, 'collection': 'Bot_collection', + 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', + 'type': 'user', 'source': 'bot_content', 'is_enabled': True}, + {'name': 'Query Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.', + 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', + 'is_enabled': True}, {'name': 'Query Prompt', 'top_results': 10, 'similarity_threshold': 0.7, + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'data': 'If there is no specific query, assume that user is aking about java programming.', + 'instructions': 'Answer according to the context', 'type': 'query', + 'source': 'static', 'is_enabled': True}], 'instructions': [], 'set_slots': [], + 'dispatch_response': True, 'status': True}] def test_delete_prompt_action(self): processor = MongoProcessor() @@ -2915,7 +2988,9 @@ def _mock_bot_info(*args, **kwargs): assert zip_file.getinfo('actions.yml') file_info_actions = zip_file.getinfo('actions.yml') file_content_actions = zip_file.read(file_info_actions) - expected_content = b"name: System Prompt\n source: static\n type: system\n - is_enabled: true" + print(file_content_actions) + expected_content = b"email_action: []\nform_validation_action: []\ngoogle_search_action: []\nhttp_action: []\njira_action: []\npipedrive_leads_action: []\nprompt_action:\n- dispatch_response: false\n failure_message: I'm sorry, I didn't quite understand that. Could you rephrase?\n instructions: []\n llm_prompts:\n - data: You are a personal assistant.\n hyperparameters:\n frequency_penalty: 0.0\n logit_bias: {}\n max_tokens: 300\n model: gpt-3.5-turbo\n n: 1\n presence_penalty: 0.0\n stop: null\n stream: false\n temperature: 0.0\n top_p: 0.0\n is_enabled: true\n name: System Prompt\n similarity_threshold: 0.7\n source: static\n top_results: 10\n type: system\n - hyperparameters:\n frequency_penalty: 0.0\n logit_bias: {}\n max_tokens: 300\n model: gpt-3.5-turbo\n n: 1\n presence_penalty: 0.0\n stop: null\n stream: false\n temperature: 0.0\n top_p: 0.0\n is_enabled: true\n name: History Prompt\n similarity_threshold: 0.7\n source: history\n top_results: 10\n type: user\n - data: A programming language is a system of notation for writing computer programs.[1]\n Most programming languages are text-based formal languages, but they may also\n be graphical. They are a kind of computer language.\n hyperparameters:\n frequency_penalty: 0.0\n logit_bias: {}\n max_tokens: 300\n model: gpt-3.5-turbo\n n: 1\n presence_penalty: 0.0\n stop: null\n stream: false\n temperature: 0.0\n top_p: 0.0\n instructions: Answer according to the context\n is_enabled: true\n name: Query Prompt\n similarity_threshold: 0.7\n source: static\n top_results: 10\n type: query\n name: prompt_action_with_default_values\n num_bot_responses: 5\n set_slots:\n - evaluation_type: expression\n name: gpt_result\n value: ${data}\n - evaluation_type: script\n name: gpt_result_type\n value: ${data.type}\n status: true\n user_question:\n type: from_user_message\npyscript_action: []\nrazorpay_action: []\nslot_set_action: []\ntwo_stage_fallback: []\nzendesk_action: []\n" + assert file_content_actions == expected_content assert file_content_actions.__contains__(expected_content) zip_file.close() @@ -14678,10 +14753,9 @@ def test_delete_schema_attached_to_prompt_action(self): {'name': 'Similarity Prompt', 'instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', 'type': 'user', 'source': 'bot_content', - 'is_enabled': True} + 'is_enabled': True, 'collection': 'python'} ], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], - 'collection': 'python', "set_slots": [{"name": "gpt_result", "value": "${data}", "evaluation_type": "expression"}, {"name": "gpt_result_type", "value": "${data.type}", "evaluation_type": "script"}], "dispatch_response": False diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 62527a3ae..b053309ab 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -508,9 +508,10 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): bot = "test_embed_faq_predict" user = "test" value = "knupur" + collection = 'python' test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - bot=bot, user=user).save() + collection=collection, bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." @@ -519,9 +520,10 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", - "top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.'} + "similarity_prompt": {"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}} hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ {'role': 'system', @@ -550,7 +552,7 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} @@ -576,16 +578,17 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - bot="test_embed_faq_predict", user="test").save() + collection='python', bot="test_embed_faq_predict", user="test").save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", - "top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.'} + "similarity_prompt": {"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}} hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ @@ -615,7 +618,7 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} @@ -651,16 +654,17 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, ai test_content = CognitionData( data="Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ", - bot="test_embed_faq_predict", user="test").save() + collection='java', bot="test_embed_faq_predict", user="test").save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", - "top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', + "similarity_prompt": {"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": "java"}, 'instructions': ['Answer in a short way.', 'Keep it simple.']} hyperparameters = Utility.get_llm_hyperparameters() @@ -691,7 +695,7 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, ai gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} @@ -734,16 +738,17 @@ async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - bot="test_embed_faq_predict", user="test").save() + collection='python', bot="test_embed_faq_predict", user="test").save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", - "top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', "enable_response_cache": True} + "similarity_prompt": {"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}} def __mock_connection_error(*args, **kwargs): import openai @@ -757,7 +762,7 @@ def __mock_connection_error(*args, **kwargs): gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} @@ -777,10 +782,11 @@ def __mock_connection_error(*args, **kwargs): Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected. Instructions on how to use Similarity Prompt: Answer according to this context. """ - assert mock_completion.call_args.kwargs == {'top_results': 10, 'similarity_threshold': 0.7, - 'use_similarity_prompt': True, 'enable_response_cache': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.'} + assert mock_completion.call_args.kwargs == { + 'similarity_prompt': {'top_results': 10, 'similarity_threshold': 0.7, 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}} assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} @@ -793,15 +799,16 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - bot="test_embed_faq_predict", user="test").save() + collection='python', bot="test_embed_faq_predict", user="test").save() query = "What kind of language is python?" k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", - "top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', "enable_response_cache": True} + "similarity_prompt": {"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}} mock_embedding.return_value = embedding mock_llm_request.side_effect = ClientConnectionError() @@ -830,8 +837,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ query = "What kind of language is python?" k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", - "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", - "top_results": 10, "similarity_threshold": 0.70, "enable_response_cache": True} + "context_prompt": "Based on below context answer question, if answer not in context check previous logs."} mock_embedding.side_effect = [openai.error.APIConnectionError("Connection reset by peer!"), embedding] @@ -852,7 +858,7 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior user = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - bot=bot, user=user).save() + collection='python', bot=bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" @@ -860,8 +866,10 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior "previous_bot_responses": [ {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, - ], 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.'} + ], + "similarity_prompt": {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}} hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below'}, @@ -890,7 +898,7 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} @@ -918,22 +926,23 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): user = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - bot=bot, user=user).save() + collection='python', bot=bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" rephrased_query = "Explain python is called high level programming language in laymen terms?" k_faq_action_config = { - "query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", - "use_query_prompt": True, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.' + "query_prompt": {"query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", + "use_query_prompt": True}, + "similarity_prompt": {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', "collection": 'python'} } hyperparameters = Utility.get_llm_hyperparameters() mock_rephrase_request = {"messages": [ {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, {"role": "user", - "content": f"{k_faq_action_config['query_prompt']}\n\n Q: {query}\n A:"} + "content": f"{k_faq_action_config.get('query_prompt')['query_prompt']}\n\n Q: {query}\n A:"} ]} mock_completion_request = {"messages": [ @@ -971,7 +980,7 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} diff --git a/tests/unit_test/validator/training_data_validator_test.py b/tests/unit_test/validator/training_data_validator_test.py index 08195d919..cf2f493f9 100644 --- a/tests/unit_test/validator/training_data_validator_test.py +++ b/tests/unit_test/validator/training_data_validator_test.py @@ -472,37 +472,37 @@ def test_validate_custom_actions_with_errors(self): assert len(error_summary['google_search_actions']) == 2 assert len(error_summary['zendesk_actions']) == 2 assert len(error_summary['pipedrive_leads_actions']) == 3 - assert len(error_summary['prompt_actions']) == 46 + assert len(error_summary['prompt_actions']) == 50 assert len(error_summary['razorpay_actions']) == 3 assert len(error_summary['pyscript_actions']) == 3 - required_fields_error = error_summary["prompt_actions"][18] + required_fields_error = error_summary["prompt_actions"][22] assert re.match(r"Required fields .* not found in action: prompt_action_with_no_llm_prompts", required_fields_error) - del error_summary["prompt_actions"][18] + del error_summary["prompt_actions"][22] + print(error_summary['prompt_actions']) assert error_summary['prompt_actions'] == [ - 'top_results should not be greater than 30 and of type int: prompt_action_invalid_query_prompt', - 'similarity_threshold should be within 0.3 and 1 and of type int or float: prompt_action_invalid_query_prompt', - 'System prompt is required', 'Query prompt must have static source', - 'Name cannot be empty', 'System prompt is required', + 'top_results should not be greater than 30 and of type int: Similarity Prompt', + 'similarity_threshold should be within 0.3 and 1 and of type int or float: Similarity Prompt', + 'Collection is required for bot content prompts!', 'System prompt is required', + 'Query prompt must have static source', 'Name cannot be empty', 'System prompt is required', 'num_bot_responses should not be greater than 5 and of type int: prompt_action_invalid_num_bot_responses', - 'data field in prompts should of type string.', 'data is required for static prompts', + 'Collection is required for bot content prompts!', 'data field in prompts should of type string.', + 'data is required for static prompts', 'System prompt must have static source', 'Temperature must be between 0.0 and 2.0!', 'max_tokens must be between 5 and 4096!', 'top_p must be between 0.0 and 1.0!', 'n must be between 1 and 5!', 'presence_penality must be between -2.0 and 2.0!', 'frequency_penalty must be between -2.0 and 2.0!', - 'logit_bias must be a dictionary!', 'System prompt must have static source', - 'Only one bot_content source can be present', + 'logit_bias must be a dictionary!', 'Collection is required for bot content prompts!', + 'Collection is required for bot content prompts!', 'Only one bot_content source can be present', 'Duplicate action found: test_add_prompt_action_one', - 'Invalid action configuration format. Dictionary expected.', + 'Invalid action configuration format. Dictionary expected.', 'Only one system prompt can be present', + 'Invalid prompt type', 'Invalid prompt source', 'Only one system prompt can be present', 'Temperature must be between 0.0 and 2.0!', 'max_tokens must be between 5 and 4096!', 'top_p must be between 0.0 and 1.0!', 'n must be between 1 and 5!', 'Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers.', - 'presence_penality must be between -2.0 and 2.0!', - 'frequency_penalty must be between -2.0 and 2.0!', - 'logit_bias must be a dictionary!', 'Only one system prompt can be present', - 'Invalid prompt type', 'Invalid prompt source', 'Only one system prompt can be present', - 'Invalid prompt type', 'Invalid prompt source', 'type in LLM Prompts should be of type string.', - 'source in LLM Prompts should be of type string.', 'Instructions in LLM Prompts should be of type string.', - 'Only one system prompt can be present', 'Data must contain action name', - 'Only one system prompt can be present', 'Data must contain slot name', + 'presence_penality must be between -2.0 and 2.0!', 'frequency_penalty must be between -2.0 and 2.0!', + 'logit_bias must be a dictionary!', 'Invalid prompt type', 'Invalid prompt source', + 'type in LLM Prompts should be of type string.', 'source in LLM Prompts should be of type string.', + 'Instructions in LLM Prompts should be of type string.', 'Only one system prompt can be present', + 'Data must contain action name', 'Only one system prompt can be present', 'Data must contain slot name', 'Only one system prompt can be present', 'Only one system prompt can be present', 'Only one system prompt can be present', 'Only one history source can be present'] assert component_count == {'http_actions': 7, 'slot_set_actions': 10, 'form_validation_actions': 9,