From 45c16d442e88984d95cb538ca3ed24031fff457e Mon Sep 17 00:00:00 2001 From: Spandan Mondal Date: Thu, 17 Oct 2024 18:59:20 +0530 Subject: [PATCH] Action serialization (#1570) * temporary commit action serialization * added unit tests for action_serializer.py and data_validation.py * added unit tests for action_serializer.py and data_validation.py * fix * fixed more test cases and logical errors * fixed test cases * temp push * temp push * temp push * added more testcases, removed some unused code * removed print statements --------- Co-authored-by: spandan.mondal --- kairon/api/app/routers/bot/bot.py | 1 - kairon/api/models.py | 1328 +--------------- kairon/importer/data_importer.py | 1 + kairon/importer/validator/file_validator.py | 11 +- kairon/shared/callback/data_objects.py | 14 + kairon/shared/data/action_serializer.py | 457 ++++++ kairon/shared/data/data_models.py | 1343 +++++++++++++++++ kairon/shared/data/data_validation.py | 184 +++ kairon/shared/data/processor.py | 80 +- kairon/shared/importer/processor.py | 4 +- kairon/shared/utils.py | 11 + tests/integration_test/services_test.py | 269 ++-- tests/testing_data/actions/actions.yml | 22 + tests/testing_data/error/actions.yml | 4 +- .../valid_with_multiflow/actions.yml | 25 + tests/testing_data/valid_yml/actions.yml | 48 + .../testing_data/validator/valid/actions.yml | 25 + .../yml_training_files/actions.yml | 267 +++- .../data_processor/action_serializer_test.py | 688 +++++++++ .../data_processor/data_processor_test.py | 320 ++-- tests/unit_test/events/events_test.py | 9 +- tests/unit_test/utility_test.py | 2 +- .../unit_test/validator/data_importer_test.py | 3 +- 23 files changed, 3448 insertions(+), 1668 deletions(-) create mode 100644 kairon/shared/data/action_serializer.py create mode 100644 kairon/shared/data/data_models.py create mode 100644 kairon/shared/data/data_validation.py create mode 100644 tests/testing_data/validator/valid/actions.yml create mode 100644 tests/unit_test/data_processor/action_serializer_test.py diff --git a/kairon/api/app/routers/bot/bot.py b/kairon/api/app/routers/bot/bot.py index 0fee25946..94858f3e0 100644 --- a/kairon/api/app/routers/bot/bot.py +++ b/kairon/api/app/routers/bot/bot.py @@ -598,7 +598,6 @@ def upload_files( """ Uploads training data nlu.yml, domain.yml, stories.yml, config.yml, rules.yml and actions.yml files. """ - print(training_files) event = TrainingDataImporterEvent( current_user.get_bot(), current_user.get_user(), import_data=import_data, overwrite=overwrite ) diff --git a/kairon/api/models.py b/kairon/api/models.py index 25c9ed677..674698fa9 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -1,1328 +1,2 @@ -from typing import List, Any, Dict, Optional, Text +from kairon.shared.data.data_models import * -from validators import url -from validators.utils import ValidationError as ValidationFailure -from fastapi.param_functions import Form -from fastapi.security import OAuth2PasswordRequestForm -from rasa.shared.constants import DEFAULT_NLU_FALLBACK_INTENT_NAME - -from kairon.exceptions import AppException -from kairon.shared.data.constant import ( - EVENT_STATUS, - SLOT_MAPPING_TYPE, - SLOT_TYPE, - ACCESS_ROLES, - ACTIVITY_STATUS, - INTEGRATION_STATUS, - FALLBACK_MESSAGE, - DEFAULT_NLU_FALLBACK_RESPONSE -) -from ..shared.actions.models import ( - ActionParameterType, - EvaluationType, - DispatchType, - DbQueryValueType, - DbActionOperationType, UserMessageType -) -from ..shared.callback.data_objects import CallbackExecutionMode -from ..shared.constants import SLOT_SET_TYPE, FORM_SLOT_SET_TYPE - -from pydantic import BaseModel, validator, SecretStr, root_validator, constr -from ..shared.models import ( - StoryStepType, - StoryType, - TemplateType, - HttpContentType, - LlmPromptSource, - LlmPromptType, - CognitionDataType, - CognitionMetadataType, -) -from kairon.shared.utils import Utility - - -class RecaptchaVerifiedRequest(BaseModel): - recaptcha_response: str = None - remote_ip: str = None - - @root_validator - def validate_recaptcha(cls, values): - from kairon.shared.utils import Utility - - secret = Utility.environment["security"].get("recaptcha_secret", None) - if Utility.environment["security"][ - "validate_recaptcha" - ] and not Utility.check_empty_string(secret): - Utility.validate_recaptcha( - values.get("recaptcha_response"), values.get("remote_ip") - ) - return values - - -class RecaptchaVerifiedOAuth2PasswordRequestForm(OAuth2PasswordRequestForm): - """ - Dependency class overridden from OAuth2PasswordRequestForm. - """ - - def __init__( - self, - grant_type: str = Form(None, pattern="password"), - username: str = Form(...), - password: str = Form(...), - scope: str = Form(""), - client_id: Optional[str] = Form(None), - client_secret: Optional[str] = Form(None), - recaptcha_response: str = Form(None), - remote_ip: str = Form(None), - fingerprint: str = Form(None), - ): - """ - @param grant_type: the OAuth2 spec says it is required and MUST be the fixed string "password". - Nevertheless, this dependency class is permissive and allows not passing it. If you want to enforce it, - use instead the OAuth2PasswordRequestFormStrict dependency. - @param username: username string. The OAuth2 spec requires the exact field name "username". - @param password: password string. The OAuth2 spec requires the exact field name "password". - @param scope: Optional string. Several scopes (each one a string) separated by spaces. - E.g. "items:read items:write users:read profile openid" - @param client_id: optional string. OAuth2 recommends sending the client_id and client_secret (if any) - using HTTP Basic auth, as: client_id:client_secret - @param client_secret: optional string. OAuth2 recommends sending the client_id and client_secret (if any) - using HTTP Basic auth, as: client_id:client_secret - @param recaptcha_response: optional string. recaptcha response. - @param remote_ip: optional string. remote ip address. - @param fingerprint: optional string. device fingerprint. - """ - from kairon.shared.utils import Utility - - secret = Utility.environment["security"].get("recaptcha_secret", None) - if Utility.environment["security"][ - "validate_recaptcha" - ] and not Utility.check_empty_string(secret): - Utility.validate_recaptcha(recaptcha_response, remote_ip) - OAuth2PasswordRequestForm.__init__( - self, - grant_type=grant_type, - username=username, - password=password, - scope=scope, - client_id=client_id, - client_secret=client_secret - ) - self.recaptcha_response = recaptcha_response - self.remote_ip = remote_ip - if Utility.environment["user"]["validate_trusted_device"] and Utility.check_empty_string(fingerprint): - raise AppException("fingerprint is required") - self.fingerprint = fingerprint - - -class Token(BaseModel): - access_token: str - token_type: str - - -class TokenData(BaseModel): - username: str - - -class Response(BaseModel): - success: bool = True - message: Any = None - data: Any - error_code: int = 0 - -class ActionResponse(BaseModel): - success: bool = True - error: str = None - action_name: str = None - events: List[Dict[Text, Any]] = None - responses: List[Dict[Text, Any]] = None - error_code: int = 200 - - -class RequestData(BaseModel): - data: Any - - -class TextData(BaseModel): - data: str - - -class RecaptchaVerifiedTextData(RecaptchaVerifiedRequest): - data: str - - -class TextDataLowerCase(BaseModel): - data: constr(to_lower=True, strip_whitespace=True) - - -class ListData(BaseModel): - data: List[str] - - -class RegisterAccount(RecaptchaVerifiedRequest): - email: constr(to_lower=True, strip_whitespace=True) - first_name: str - last_name: str - password: SecretStr - confirm_password: SecretStr - account: str - fingerprint: str = None - - @validator("email") - def validate_email(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - try: - Utility.verify_email(v) - except AppException as e: - raise ValueError(str(e)) - return v - - @validator("password") - def validate_password(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - try: - Utility.valid_password(v.get_secret_value()) - except AppException as e: - raise ValueError(str(e)) - return v - - @validator("confirm_password") - def validate_confirm_password(cls, v, values, **kwargs): - if ( - "password" in values - and v.get_secret_value() != values["password"].get_secret_value() - ): - raise ValueError("Password and Confirm Password does not match") - return v - - @root_validator - def validate_fingerprint(cls, values): - from kairon.shared.utils import Utility - - if Utility.environment["user"]["validate_trusted_device"] and Utility.check_empty_string(values.get("fingerprint")): - raise ValueError("fingerprint is required") - return values - - -class BotAccessRequest(RecaptchaVerifiedRequest): - email: constr(to_lower=True, strip_whitespace=True) - role: ACCESS_ROLES = ACCESS_ROLES.TESTER.value - activity_status: ACTIVITY_STATUS = ACTIVITY_STATUS.INACTIVE.value - - @validator("email") - def validate_email(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - try: - Utility.verify_email(v) - except AppException as e: - raise ValueError(str(e)) - return v - - @validator("role") - def validate_role(cls, v, values, **kwargs): - if v == ACCESS_ROLES.OWNER.value: - raise ValueError("There can be only 1 owner per bot") - return v - - -class EndPointBot(BaseModel): - url: str - token: str = None - token_type: str = None - - -class EndPointAction(BaseModel): - url: str - - -class EndPointHistory(BaseModel): - url: str - token: str = None - - -class Endpoint(BaseModel): - bot_endpoint: EndPointBot = None - action_endpoint: EndPointAction = None - history_endpoint: EndPointHistory = None - - -class RasaConfig(BaseModel): - language: str = "en" - pipeline: List[Dict] - policies: List[Dict] - - -class ComponentConfig(BaseModel): - nlu_epochs: int = None - response_epochs: int = None - ted_epochs: int = None - nlu_confidence_threshold: float = None - action_fallback: str = None - action_fallback_threshold: float = None - - @validator("nlu_epochs", "response_epochs", "ted_epochs") - def validate_epochs(cls, v): - from kairon.shared.utils import Utility - - if v is not None and v < 1: - raise ValueError("Choose a positive number as epochs") - elif v > Utility.environment["model"]["config_properties"]["epoch_max_limit"]: - epoch_max_limit = Utility.environment["model"]["config_properties"][ - "epoch_max_limit" - ] - raise ValueError(f"Please choose a epoch between 1 and {epoch_max_limit}") - return v - - @validator("nlu_confidence_threshold", "action_fallback_threshold") - def validate_confidence_threshold(cls, v): - if v is not None and (v < 0.3 or v > 0.9): - raise ValueError("Please choose a threshold between 0.3 and 0.9") - return v - - -class Password(RecaptchaVerifiedRequest): - data: str - password: SecretStr - confirm_password: SecretStr - - @validator("password") - def validate_password(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - try: - Utility.valid_password(v.get_secret_value()) - except AppException as e: - raise ValueError(str(e)) - return v - - @validator("confirm_password") - def validate_confirm_password(cls, v, values, **kwargs): - if ( - "password" in values - and v.get_secret_value() != values["password"].get_secret_value() - ): - raise ValueError("Password and Confirm Password does not match") - return v - - -class HttpActionParameters(BaseModel): - key: str - value: str = None - parameter_type: ActionParameterType - encrypt: bool = False - - @root_validator - def check(cls, values): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(values.get("key")): - raise ValueError("key cannot be empty") - - if values.get( - "parameter_type" - ) == ActionParameterType.slot and Utility.check_empty_string( - values.get("value") - ): - raise ValueError("Provide name of the slot as value") - - if values.get( - "parameter_type" - ) == ActionParameterType.key_vault and Utility.check_empty_string( - values.get("value") - ): - raise ValueError("Provide key from key vault as value") - - if values.get("parameter_type") == ActionParameterType.key_vault: - values["encrypt"] = True - - return values - - -class SetSlotsUsingActionResponse(BaseModel, use_enum_values=True): - name: str - value: str - evaluation_type: EvaluationType = EvaluationType.expression - - @validator("name") - def validate_name(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(v): - raise ValueError("slot name is required") - return v - - @validator("value") - def validate_expression(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(v): - raise ValueError("expression is required to evaluate value of slot") - return v - - -class ActionResponseEvaluation(BaseModel): - value: str = None - dispatch: bool = True - evaluation_type: EvaluationType = EvaluationType.expression - dispatch_type: DispatchType = DispatchType.text.value - - @root_validator - def check(cls, values): - from kairon.shared.utils import Utility - - if values.get("dispatch") is True and Utility.check_empty_string( - values.get("value") - ): - raise ValueError("response is required for dispatch") - - return values - - -class HttpActionConfigRequest(BaseModel): - action_name: constr(to_lower=True, strip_whitespace=True) - content_type: HttpContentType = HttpContentType.application_json - response: ActionResponseEvaluation = None - http_url: str - request_method: str - params_list: List[HttpActionParameters] = [] - dynamic_params: str = None - headers: List[HttpActionParameters] = [] - set_slots: List[SetSlotsUsingActionResponse] = [] - - @validator("action_name") - def validate_action_name(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(v): - raise ValueError("action_name is required") - return v - - @validator("http_url") - def validate_http_url(cls, v, values, **kwargs): - if isinstance(url(v), ValidationFailure): - raise ValueError("URL is malformed") - return v - - @validator("request_method") - def validate_request_method(cls, v, values, **kwargs): - if v.upper() not in ("GET", "POST", "PUT", "DELETE"): - raise ValueError("Invalid HTTP method") - return v.upper() - - -class PayloadConfig(BaseModel): - type: DbQueryValueType - value: Any - query_type: DbActionOperationType - - @root_validator - def check(cls, values): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(values.get("type")): - raise ValueError("type is required") - - return values - - -class PyscriptActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - source_code: str - dispatch_response: bool = True - - @validator("name") - def validate_action_name(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(v): - raise ValueError("name is required") - return v - - @validator("source_code") - def validate_source_code(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(v): - raise ValueError("source_code is required") - return v - - -class DatabaseActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - collection: str - payload: List[PayloadConfig] - response: ActionResponseEvaluation = None - set_slots: List[SetSlotsUsingActionResponse] = [] - - @validator("name") - def validate_action_name(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(v): - raise ValueError("name is required") - return v - - @validator("collection") - def validate_collection_name(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(v): - raise ValueError("collection is required") - return v - - @validator("payload") - def validate_payload(cls, v, values, **kwargs): - count_payload_search = 0 - if v: - for item in v: - if item.query_type == DbActionOperationType.payload_search: - count_payload_search += 1 - if count_payload_search > 1: - raise ValueError(f"Only One {DbActionOperationType.payload_search} is allowed!") - else: - raise ValueError("payload is required") - return v - - -class LiveAgentActionRequest(BaseModel): - bot_response: str = 'Connecting to live agent' - agent_connect_response: str = 'Connected to live agent' - agent_disconnect_response: str = 'Agent has closed the conversation' - agent_not_available_response: str = 'No agents available at this moment. An agent will reply to you shortly.' - dispatch_bot_response: bool = True - dispatch_agent_connect_response: bool = True - dispatch_agent_disconnect_response: bool = True - dispatch_agent_not_available_response: bool = True - - -class TrainingData(BaseModel): - intent: constr(to_lower=True, strip_whitespace=True) - training_examples: List[str] - response: str - - -class BulkTrainingDataAddRequest(BaseModel): - history_id: str - training_data: List[TrainingData] - - -class TrainingDataGeneratorResponseModel(BaseModel): - intent: str - training_examples: List[str] - response: str - - -class TrainingDataGeneratorStatusModel(BaseModel): - status: EVENT_STATUS - response: List[TrainingData] = None - exception: str = None - - -class StoryStepRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) = None - type: StoryStepType - value: Any = None - - -class MultiStoryStepRequest(StoryStepRequest): - node_id: str - component_id: str - - -class StoryStepData(BaseModel): - step: MultiStoryStepRequest - connections: List[MultiStoryStepRequest] = None - - -class StoryMetadata(BaseModel): - node_id: str - flow_type: StoryType = StoryType.story.value - - -class MultiFlowStoryRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - steps: List[StoryStepData] - metadata: List[StoryMetadata] = None - - @validator("steps") - def validate_request_method(cls, v, values, **kwargs): - if not v: - raise ValueError("Steps are required to form Flow") - return v - - -class StoryRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - type: StoryType - steps: List[StoryStepRequest] - template_type: TemplateType = None - - class Config: - use_enum_values = True - - def get_steps(self): - return [step.dict() for step in self.steps] - - @validator("steps") - def validate_request_method(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if not v: - raise ValueError("Steps are required to form Flow") - - if v[0].type != StoryStepType.intent: - raise ValueError("First step should be an intent") - - if v[len(v) - 1].type == StoryStepType.intent: - raise ValueError("Intent should be followed by utterance or action") - - intents = 0 - for i, j in enumerate(range(1, len(v))): - if v[i].type == StoryStepType.intent: - intents = intents + 1 - if v[i].type == StoryStepType.intent and v[j].type == StoryStepType.intent: - raise ValueError("Found 2 consecutive intents") - if ( - Utility.check_empty_string(v[i].name) - and v[i].type != StoryStepType.form_end - ): - raise ValueError( - f"Only {StoryStepType.form_end} step type can have empty name" - ) - if v[i].type == StoryStepType.stop_flow_action and i != len(v) - 1: - raise ValueError("Stop Flow Action should only be at the end of the flow") - if v[i].type == StoryStepType.intent and v[j].type == StoryStepType.stop_flow_action: - raise ValueError("Stop Flow Action should not be after intent") - - if "type" in values: - if values["type"] == StoryType.rule and intents > 1: - raise ValueError( - f"""Found rules '{values['name']}' that contain more than intent.\nPlease use stories for this case""" - ) - return v - - -class AnalyticsModel(BaseModel): - fallback_intent: str = DEFAULT_NLU_FALLBACK_INTENT_NAME - - @validator('fallback_intent') - def validate_fallback_intent(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - if Utility.check_empty_string(v): - raise ValueError("fallback_intent field cannot be empty") - return v - - -class BotSettingsRequest(BaseModel): - analytics: AnalyticsModel = AnalyticsModel() - - -class FeedbackRequest(BaseModel): - rating: float - scale: float = 5 - feedback: str = None - - -class GPTRequest(BaseModel): - api_key: str - data: List[str] - engine: str = "davinci" - temperature: float = 0.75 - max_tokens: int = 100 - num_responses: int = 10 - - @validator("data") - def validate_gpt_questions(cls, v, values, **kwargs): - if len(v) <= 0: - raise ValueError("Question Please!") - elif len(v) > 5: - raise ValueError("Max 5 Questions are allowed!") - return v - - -class ParaphrasesRequest(BaseModel): - data: List[str] - - @validator("data") - def validate_paraphrases_questions(cls, v, values, **kwargs): - if len(v) <= 0: - raise ValueError("Question Please!") - elif len(v) > 5: - raise ValueError("Max 5 Questions are allowed!") - return v - - -class SlotRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - type: SLOT_TYPE - initial_value: Any = None - values: List[str] = None - max_value: float = None - min_value: float = None - influence_conversation: bool = False - - class Config: - use_enum_values = True - - -class SynonymRequest(BaseModel): - value: List[str] - - @validator("value") - def validate_value(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if len(v) <= 0: - raise ValueError("value field cannot be empty") - for ele in v: - if Utility.check_empty_string(ele): - raise ValueError("value cannot be an empty string") - return v - - -class AddBotRequest(BaseModel): - name: str - from_template: str = None - - -class DictData(BaseModel): - data: dict - - -class RecaptchaVerifiedDictData(DictData): - recaptcha_response: str = None - - -class RegexRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - pattern: str - - @validator("name") - def validate_name(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(v): - raise ValueError("Regex name cannot be empty or a blank space") - return v - - @validator("pattern") - def validate_pattern(cls, f, values, **kwargs): - from kairon.shared.utils import Utility - import re - - if Utility.check_empty_string(f): - raise ValueError("Regex pattern cannot be empty or a blank space") - try: - re.compile(f) - except Exception: - raise AppException("invalid regular expression") - return f - - -class LookupTablesRequest(BaseModel): - value: List[str] - - @validator("value") - def validate_value(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if len(v) <= 0: - raise ValueError("value field cannot be empty") - for ele in v: - if Utility.check_empty_string(ele): - raise ValueError("lookup value cannot be empty or a blank space") - return v - - -class MappingCondition(BaseModel): - active_loop: str = None - requested_slot: str = None - - @root_validator - def validate(cls, values): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(values.get("active_loop")) and not Utility.check_empty_string(values.get("requested_slot")): - raise ValueError("active_loop is required to add requested_slot as condition!") - - return values - - -class SlotMapping(BaseModel): - entity: constr(to_lower=True, strip_whitespace=True) = None - type: SLOT_MAPPING_TYPE - value: Any = None - intent: List[constr(to_lower=True, strip_whitespace=True)] = None - not_intent: List[constr(to_lower=True, strip_whitespace=True)] = None - conditions: List[MappingCondition] = None - - class Config: - use_enum_values = True - - -class SlotMappingRequest(BaseModel): - slot: constr(to_lower=True, strip_whitespace=True) - mapping: SlotMapping - - class Config: - use_enum_values = True - - @validator("mapping") - def validate_mapping(cls, v, values, **kwargs): - if not v or v == [{}]: - raise ValueError("At least one mapping is required") - return v - - -class FormSlotSetModel(BaseModel): - type: FORM_SLOT_SET_TYPE = FORM_SLOT_SET_TYPE.current.value - value: Any = None - class Config: - use_enum_values = True - - - -class FormSettings(BaseModel): - ask_questions: List[str] - slot: str - is_required: bool = True - validation_semantic: str = None - valid_response: str = None - invalid_response: str = None - slot_set: FormSlotSetModel = FormSlotSetModel() - - @validator("ask_questions") - def validate_responses(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - err_msg = "Questions cannot be empty or contain spaces" - if not v: - raise ValueError(err_msg) - - for response in v: - if Utility.check_empty_string(response): - raise ValueError(err_msg) - return v - - @validator("slot") - def validate_slot(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(v): - raise ValueError("Slot is required") - return v - - -class Forms(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - settings: List[FormSettings] - - -class SetSlots(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - type: SLOT_SET_TYPE - value: Any = None - - class Config: - use_enum_values = True - - -class SlotSetActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - set_slots: List[SetSlots] - - -class CustomActionParameter(BaseModel): - value: str = None - parameter_type: ActionParameterType = ActionParameterType.value - class Config: - use_enum_values = True - - @validator("parameter_type") - def validate_parameter_type(cls, v, values, **kwargs): - allowed_values = { - ActionParameterType.value, - ActionParameterType.slot, - ActionParameterType.key_vault, - ActionParameterType.sender_id, - } - if v not in allowed_values: - raise ValueError( - f"Invalid parameter type. Allowed values: {allowed_values}" - ) - return v - - @root_validator - def check(cls, values): - from kairon.shared.utils import Utility - - if values.get( - "parameter_type" - ) == ActionParameterType.slot and Utility.check_empty_string( - values.get("value") - ): - raise ValueError("Provide name of the slot as value") - - if values.get( - "parameter_type" - ) == ActionParameterType.key_vault and Utility.check_empty_string( - values.get("value") - ): - raise ValueError("Provide key from key vault as value") - - return values - - -class GoogleSearchActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - api_key: CustomActionParameter = None - search_engine_id: str = None - website: str = None - failure_response: str = "I have failed to process your request." - num_results: int = 1 - dispatch_response: bool = True - set_slot: str = None - - @validator("num_results") - def validate_num_results(cls, v, values, **kwargs): - if not v or v < 1: - raise ValueError("num_results must be greater than or equal to 1!") - return v - - -class WebSearchActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - website: str = None - failure_response: str = 'I have failed to process your request.' - topn: int = 1 - dispatch_response: bool = True - set_slot: str = None - - @validator("name") - def validate_action_name(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if Utility.check_empty_string(v): - raise ValueError("name is required") - return v - - @validator("topn") - def validate_top_n(cls, v, values, **kwargs): - if not v or v < 1: - raise ValueError("topn must be greater than or equal to 1!") - return v - - -class CustomActionParameterModel(BaseModel): - value: Any = None - parameter_type: ActionParameterType = ActionParameterType.value - - @validator("parameter_type") - def validate_parameter_type(cls, v, values, **kwargs): - allowed_values = {ActionParameterType.value, ActionParameterType.slot} - if v not in allowed_values: - raise ValueError(f"Invalid parameter type. Allowed values: {allowed_values}") - return v - - @root_validator - def check(cls, values): - if values.get('parameter_type') == ActionParameterType.slot and not values.get('value'): - raise ValueError("Provide name of the slot as value") - - if values.get('parameter_type') == ActionParameterType.value and not isinstance(values.get('value'), list): - raise ValueError("Provide list of emails as value") - - return values - - -class CustomActionDynamicParameterModel(BaseModel): - value: Any = None - parameter_type: ActionParameterType = ActionParameterType.value - - @validator("parameter_type") - def validate_parameter_type(cls, v, values, **kwargs): - allowed_values = {ActionParameterType.value, ActionParameterType.slot} - if v not in allowed_values: - raise ValueError(f"Invalid parameter type. Allowed values: {allowed_values}") - return v - - @root_validator - def check(cls, values): - if values.get('parameter_type') == ActionParameterType.slot and not values.get('value'): - raise ValueError("Provide name of the slot as value") - - if values.get('parameter_type') == ActionParameterType.value and Utility.check_empty_string(values.get("value")): - raise ValueError("Value can not be blank") - - return values - - -class EmailActionRequest(BaseModel): - action_name: constr(to_lower=True, strip_whitespace=True) - smtp_url: str - smtp_port: int - smtp_userid: CustomActionParameter = None - smtp_password: CustomActionParameter - from_email: CustomActionParameter - subject: str - custom_text: CustomActionParameter = None - to_email: CustomActionParameterModel - response: str - tls: bool = False - - -class JiraActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - url: str - user_name: str - api_token: CustomActionParameter - project_key: str - issue_type: str - parent_key: str = None - summary: str - response: str - - -class ZendeskActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - subdomain: str - user_name: str - api_token: CustomActionParameter - subject: str - response: str - - -class PipedriveActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - domain: str - api_token: CustomActionParameter - title: str - response: str - metadata: dict - - @validator("metadata") - def validate_metadata(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if not v or Utility.check_empty_string(v.get("name")): - raise ValueError("name is required") - return v - - -class HubspotFormsActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - portal_id: str - form_guid: str - fields: List[HttpActionParameters] - response: str - - -class QuickReplies(BaseModel): - text: str - payload: str - message: str = None - is_dynamic_msg: bool = False - - -class TwoStageFallbackTextualRecommendations(BaseModel): - count: int = 0 - use_intent_ranking: bool = False - - -class TwoStageFallbackConfigRequest(BaseModel): - fallback_message: str = FALLBACK_MESSAGE - text_recommendations: TwoStageFallbackTextualRecommendations = None - trigger_rules: List[QuickReplies] = None - - @root_validator - def check(cls, values): - if not values.get("text_recommendations") and not values["trigger_rules"]: - raise ValueError( - "One of text_recommendations or trigger_rules should be defined" - ) - if ( - values.get("text_recommendations") - and values["text_recommendations"].count < 0 - ): - raise ValueError("count cannot be negative") - return values - - -class PromptHyperparameters(BaseModel): - top_results: int = 10 - similarity_threshold: float = 0.70 - - @root_validator - def check(cls, values): - if not 0.3 <= values.get('similarity_threshold') <= 1: - raise ValueError("similarity_threshold should be within 0.3 and 1") - if values.get('top_results') > 30: - raise ValueError("top_results should not be greater than 30") - return values - - -class LlmPromptRequest(BaseModel, use_enum_values=True): - name: str - hyperparameters: PromptHyperparameters = None - data: str = None - instructions: str = None - type: LlmPromptType - source: LlmPromptSource - is_enabled: bool = True - - @root_validator - def check(cls, values): - from kairon.shared.utils import Utility - - if (values.get('source') == LlmPromptSource.bot_content.value and - Utility.check_empty_string(values.get('data'))): - values['data'] = "default" - return values - - -class UserQuestionModel(BaseModel): - type: UserMessageType = UserMessageType.from_user_message.value - value: str = None - - -class PromptActionConfigRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - num_bot_responses: int = 5 - failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE - user_question: UserQuestionModel = UserQuestionModel() - llm_type: str - hyperparameters: dict - llm_prompts: List[LlmPromptRequest] - instructions: List[str] = [] - set_slots: List[SetSlotsUsingActionResponse] = [] - dispatch_response: bool = True - bot: str - - @validator("llm_type", pre=True, always=True) - def validate_llm_type(cls, v, values, **kwargs): - if v not in Utility.get_llms(): - raise ValueError("Invalid llm type") - return v - - @validator("llm_prompts") - def validate_llm_prompts(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - Utility.validate_kairon_faq_llm_prompts( - [vars(value) for value in v], ValueError - ) - return v - - @validator("num_bot_responses") - def validate_num_bot_responses(cls, v, values, **kwargs): - if v > 5: - raise ValueError("num_bot_responses should not be greater than 5") - return v - - @validator("hyperparameters") - def validate_hyperparameters(cls, v, values, **kwargs): - bot = values.get('bot') - llm_type = values.get('llm_type') - if llm_type and v: - Utility.validate_llm_hyperparameters(v, llm_type, bot, ValueError) - return v - - @root_validator(pre=True) - def validate_required_fields(cls, values): - bot = values.get('bot') - if not bot: - raise ValueError("bot field is missing") - return values - - -class ColumnMetadata(BaseModel): - column_name: str - data_type: CognitionMetadataType - enable_search: bool = True - create_embeddings: bool = True - - @root_validator - def check(cls, values): - from kairon.shared.utils import Utility - - if values.get('data_type') not in [CognitionMetadataType.str.value, CognitionMetadataType.int.value, CognitionMetadataType.float.value]: - raise ValueError("Only str, int and float data types are supported") - if Utility.check_empty_string(values.get('column_name')): - raise ValueError("Column name cannot be empty") - return values - - -class CognitionSchemaRequest(BaseModel): - metadata: List[ColumnMetadata] = None - collection_name: constr(to_lower=True, strip_whitespace=True) - - -class CollectionDataRequest(BaseModel): - data: dict - is_secure: list = [] - collection_name: constr(to_lower=True, strip_whitespace=True) - - @root_validator - def check(cls, values): - from kairon.shared.utils import Utility - - data = values.get("data") - is_secure = values.get("is_secure") - collection_name = values.get("collection_name") - if Utility.check_empty_string(collection_name): - raise ValueError("collection_name should not be empty!") - - if not isinstance(is_secure, list): - raise ValueError("is_secure should be list of keys!") - - if is_secure: - if not data or not isinstance(data, dict): - raise ValueError("data cannot be empty and should be of type dict!") - data_keys = set(data.keys()) - is_secure_set = set(is_secure) - - if not is_secure_set.issubset(data_keys): - raise ValueError("is_secure contains keys that are not present in data") - return values - - -class CognitiveDataRequest(BaseModel): - data: Any - content_type: CognitionDataType = CognitionDataType.text.value - collection: constr(to_lower=True, strip_whitespace=True) = None - - @root_validator - def check(cls, values): - from kairon.shared.utils import Utility - - data = values.get("data") - content_type = values.get("content_type") - if isinstance(data, dict) and content_type != CognitionDataType.json.value: - raise ValueError("content type and type of data do not match!") - if not data or (isinstance(data, str) and Utility.check_empty_string(data)): - raise ValueError("data cannot be empty") - return values - - -class RazorpayActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - api_key: CustomActionParameter - api_secret: CustomActionParameter - amount: CustomActionParameter - currency: CustomActionParameter - username: CustomActionParameter = None - email: CustomActionParameter = None - contact: CustomActionParameter = None - notes: Optional[List[HttpActionParameters]] - - -class IntegrationRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - expiry_minutes: int = 0 - access_list: list = None - role: ACCESS_ROLES = ACCESS_ROLES.CHAT.value - status: INTEGRATION_STATUS = INTEGRATION_STATUS.ACTIVE.value - - -class KeyVaultRequest(BaseModel): - key: str - value: str - - @validator("key") - def validate_key(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if not v or Utility.check_empty_string(v): - raise ValueError("key is required") - return v - - @validator("value") - def validate_value(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if not v or Utility.check_empty_string(v): - raise ValueError("value is required") - return v - - -class EventConfig(BaseModel): - ws_url: str - headers: dict - method: str - - @validator("ws_url") - def validate_ws_url(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if not v or Utility.check_empty_string(v): - raise ValueError("url can not be empty") - return v - - @validator("headers") - def validate_headers(cls, v, values, **kwargs): - if not v or len(v) < 1: - v = {} - return v - - -class IDPConfig(BaseModel): - config: dict - organization: str - - @validator("config") - def validate_config(cls, v, values, **kwargs): - if not v or len(v) == 0: - v = {} - return v - - @validator("organization") - def validate_organization(cls, v, values, **kwargs): - from kairon.shared.utils import Utility - - if not v or Utility.check_empty_string(v): - raise ValueError("Organization can not be empty") - return v - - -class CallbackConfigRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - pyscript_code: str - execution_mode: str = CallbackExecutionMode.ASYNC.value - standalone: bool = False - shorten_token: bool = False - standalone_id_path: Optional[str] = None - expire_in: int = 0 - - -class CallbackActionConfigRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - callback_name: str - dynamic_url_slot_name: Optional[str] - metadata_list: list[HttpActionParameters] = [] - bot_response: Optional[str] - dispatch_bot_response: bool = True - - -class ScheduleActionRequest(BaseModel): - name: constr(to_lower=True, strip_whitespace=True) - schedule_time: CustomActionDynamicParameterModel - timezone: str = None - schedule_action: str - response_text: Optional[str] - params_list: Optional[List[HttpActionParameters]] - dispatch_bot_response: bool = True - - @root_validator - def validate_name(cls, values): - from kairon.shared.utils import Utility - - if not values.get("name") or Utility.check_empty_string(values.get("name")): - raise ValueError("Schedule action name can not be empty") - - if not values.get("schedule_action") or Utility.check_empty_string(values.get("schedule_action")): - raise ValueError("Schedule action can not be empty, it is needed to execute on schedule time") - - return values diff --git a/kairon/importer/data_importer.py b/kairon/importer/data_importer.py index d864d8d7e..919d8bbda 100644 --- a/kairon/importer/data_importer.py +++ b/kairon/importer/data_importer.py @@ -55,4 +55,5 @@ def import_data(self): self.validator.multiflow_stories, self.validator.bot_content, self.validator.chat_client_config.get('config'), + self.validator.other_collections, self.overwrite, self.files_to_save) diff --git a/kairon/importer/validator/file_validator.py b/kairon/importer/validator/file_validator.py index 7d1052cb9..9cdeed4c4 100644 --- a/kairon/importer/validator/file_validator.py +++ b/kairon/importer/validator/file_validator.py @@ -24,6 +24,7 @@ from kairon.shared.actions.models import ActionType, ActionParameterType, DbActionOperationType from kairon.shared.cognition.data_objects import CognitionSchema from kairon.shared.constants import DEFAULT_ACTIONS, DEFAULT_INTENTS, SYSTEM_TRIGGERED_UTTERANCES, SLOT_SET_TYPE +from kairon.shared.data.action_serializer import ActionSerializer from kairon.shared.data.constant import KAIRON_TWO_STAGE_FALLBACK from kairon.shared.data.data_objects import MultiflowStories from kairon.shared.data.processor import MongoProcessor @@ -32,6 +33,9 @@ from kairon.shared.utils import Utility, StoryValidator +DEFAULT_OTHER_COLLECTIONS_PATH = 'other_collections.yml' + + class TrainingDataValidator(Validator): """ Tool to verify usage of intents, utterances, @@ -84,6 +88,8 @@ async def from_training_files(cls, training_data_paths: str, domain_path: str, c cls.multiflow_stories_graph = StoryValidator.create_multiflow_story_graphs(multiflow_stories) bot_content = Utility.read_yaml(os.path.join(root_dir, 'bot_content.yml')) cls.bot_content = bot_content if bot_content else {} + other_collections = Utility.read_yaml(os.path.join(root_dir, DEFAULT_OTHER_COLLECTIONS_PATH)) + cls.other_collections = other_collections if other_collections else {} return await TrainingDataValidator.from_importer(file_importer) except YamlValidationException as e: @@ -503,6 +509,7 @@ def validate_multiflow(self, raise_exception: bool = True): if errors and raise_exception: raise AppException("Invalid multiflow_stories.yml. Check logs!") + #TODO: Depricated not needed anymore @staticmethod def validate_custom_actions(actions: Dict, bot: Text = None): """ @@ -1032,10 +1039,10 @@ def validate_actions(self, bot: Text = None, raise_exception: bool = True): @param raise_exception: Set this flag to false to prevent raising exceptions. @return: """ - is_data_invalid, summary, component_count = TrainingDataValidator.validate_custom_actions(self.actions, bot) + is_data_valid, summary, component_count = ActionSerializer.validate(bot, self.actions, self.other_collections) self.component_count.update(component_count) self.summary.update(summary) - if not is_data_invalid and raise_exception: + if not is_data_valid and raise_exception: raise AppException("Invalid actions.yml. Check logs!") @staticmethod diff --git a/kairon/shared/callback/data_objects.py b/kairon/shared/callback/data_objects.py index ac36fa842..d2232cec9 100644 --- a/kairon/shared/callback/data_objects.py +++ b/kairon/shared/callback/data_objects.py @@ -36,6 +36,10 @@ def decrypt_secret(encrypted_secret: str) -> str: def xor_encrypt_secret(secret: str) -> str: + """ + AES small length text encryption + TODO: change function name + """ key = Utility.environment['async_callback_action']['short_secret']['aes_key'] iv = Utility.environment['async_callback_action']['short_secret']['aes_iv'] key = bytes.fromhex(key) @@ -49,6 +53,10 @@ def xor_encrypt_secret(secret: str) -> str: def xor_decrypt_secret(encoded_secret: str) -> str: + """ + AES encripted text decription function + TODO: change function name + """ key = Utility.environment['async_callback_action']['short_secret']['aes_key'] iv = Utility.environment['async_callback_action']['short_secret']['aes_iv'] key = bytes.fromhex(key) @@ -225,6 +233,9 @@ class CallbackRecordStatusType(Enum): @push_notification.apply class CallbackData(Document): + """ + this represents a record of every callback execution generated by action trigger + """ action_name = StringField() callback_name = StringField(required=True) bot = StringField(required=True) @@ -322,6 +333,9 @@ def update_state(bot: str, identifier: str, state: dict, invalidate: bool): @push_notification.apply class CallbackLog(Document): + """ + this represents the record of actual execution record of callback after the callback url is triggered + """ callback_name = StringField(required=True) bot = StringField(required=True) channel = StringField(default='unsupported') diff --git a/kairon/shared/data/action_serializer.py b/kairon/shared/data/action_serializer.py new file mode 100644 index 000000000..2d265d91d --- /dev/null +++ b/kairon/shared/data/action_serializer.py @@ -0,0 +1,457 @@ +from enum import Enum +from typing import Optional + +from mongoengine import Document + +from kairon import Utility +from kairon.exceptions import AppException +from kairon.shared.actions.data_objects import HttpActionConfig, KaironTwoStageFallbackAction, EmailActionConfig, \ + ZendeskAction, JiraAction, FormValidationAction, SlotSetAction, GoogleSearchAction, PipedriveLeadsAction, \ + PromptAction, WebSearchAction, RazorpayAction, PyscriptActionConfig, DatabaseAction, LiveAgentActionConfig, \ + CallbackActionConfig, ScheduleAction, Actions +from kairon.shared.actions.models import ActionType +from kairon.shared.callback.data_objects import CallbackConfig +from kairon.shared.data.data_models import HttpActionConfigRequest, TwoStageFallbackConfigRequest, EmailActionRequest, \ + JiraActionRequest, ZendeskActionRequest, SlotSetActionRequest, GoogleSearchActionRequest, PipedriveActionRequest, \ + RazorpayActionRequest, PyscriptActionRequest, DatabaseActionRequest, \ + LiveAgentActionRequest, CallbackActionConfigRequest, ScheduleActionRequest, WebSearchActionRequest, \ + CallbackConfigRequest, PromptActionConfigUploadValidation +from kairon.shared.data.data_objects import Forms +from kairon.shared.data.data_validation import DataValidation +from pydantic import ValidationError as PValidationError + + +class ReconfigarableProperty(Enum): + bot = "bot" + user = "user" + status = "status" + + +class ActionSerializer: + """ + action_lookup: dict = { + export_name_str : { + "db_model": Document, + "validation_model": Document, + "custom_validation": Optional[list[function]] = [], // [(bot: str, data: dict) -> list[str]] + "modify": Optional[function] = None, // (bot: str, data: dict) -> dict + "single_instance": Optional[bool] = False + "group": Optional[str] = 'action' // action -> action.yml, anything_else -> other_collections.yml + "reconfigure": Optional[list[ReconfigarableProperty]] = default_reconfigurable + } + """ + action_lookup = { + ActionType.http_action.value: { + "db_model": HttpActionConfig, + "validation_model": HttpActionConfigRequest, + "custom_validation": [DataValidation.validate_http_action], + }, + ActionType.two_stage_fallback.value: { + "db_model": KaironTwoStageFallbackAction, + "validation_model": TwoStageFallbackConfigRequest, + "single_instance": True, + }, + ActionType.email_action.value: { + "db_model": EmailActionConfig, + "validation_model": EmailActionRequest, + }, + ActionType.zendesk_action.value: { + "db_model": ZendeskAction, + "validation_model": ZendeskActionRequest, + }, + ActionType.jira_action.value: { + "db_model": JiraAction, + "validation_model": JiraActionRequest, + }, + ActionType.form_validation_action.value: { + "db_model": FormValidationAction, + "validation_model": None, + "custom_validation": [DataValidation.validate_form_validation_action], + }, + ActionType.slot_set_action.value: { + "db_model": SlotSetAction, + "validation_model": SlotSetActionRequest, + }, + ActionType.google_search_action.value: { + "db_model": GoogleSearchAction, + "validation_model": GoogleSearchActionRequest, + }, + ActionType.pipedrive_leads_action.value: { + "db_model": PipedriveLeadsAction, + "validation_model": PipedriveActionRequest, + }, + ActionType.prompt_action.value: { + "db_model": PromptAction, + "validation_model": PromptActionConfigUploadValidation, + "custom_validation": [DataValidation.validate_prompt_action], + }, + ActionType.web_search_action.value: { + "db_model": WebSearchAction, + "validation_model": WebSearchActionRequest + }, + ActionType.razorpay_action.value: { + "db_model": RazorpayAction, + "validation_model": RazorpayActionRequest, + }, + ActionType.pyscript_action.value: { + "db_model": PyscriptActionConfig, + "validation_model": PyscriptActionRequest, + "custom_validation": [DataValidation.validate_pyscript_action], + }, + ActionType.database_action.value: { + "db_model": DatabaseAction, + "validation_model": DatabaseActionRequest, + "custom_validation": [DataValidation.validate_database_action], + }, + ActionType.live_agent_action.value: { + "db_model": LiveAgentActionConfig, + "validation_model": LiveAgentActionRequest, + "single_instance": True, + }, + ActionType.callback_action.value: { + "db_model": CallbackActionConfig, + "validation_model": CallbackActionConfigRequest, + "entry_check": "callback", + }, + ActionType.schedule_action.value: { + "db_model": ScheduleAction, + "validation_model": ScheduleActionRequest, + }, + str(CallbackConfig.__name__).lower(): { + "db_model": CallbackConfig, + "validation_model": CallbackConfigRequest, + "group": "callback", + "custom_validation": [DataValidation.validate_callback_config], + "modify": DataValidation.modify_callback_config, + "reconfigure": [ReconfigarableProperty.bot.value] + } + + } + + default_reconfigurable = [ReconfigarableProperty.bot.value, + ReconfigarableProperty.user.value, + ReconfigarableProperty.status.value] + + @staticmethod + def get_item_name(data: dict, raise_exception: bool = True): + """ + Gets the name of the action + :param data: action data + :param raise_exception: bool + :return: action name + """ + action_name = data.get("action_name") or data.get("name") + if Utility.check_empty_string(action_name): + if raise_exception: + raise AppException("Action name cannot be empty or blank spaces!") + action_name = None + return action_name + + @staticmethod + def validate(bot: str, actions: dict, other_collections: dict): + """ + Validates the action configuration data, first return parameter is true if validation is successful + :param bot: bot id + :param actions: action configuration data + :param other_collections: other collection configuration data + :return: is_data_valid: bool, error_summary: dict, component_count: dict + """ + is_data_invalid = False + component_count = dict.fromkeys(ActionSerializer.action_lookup.keys(), 0) + error_summary = {key: [] for key in ActionSerializer.action_lookup.keys()} + encountered_action_names = set() + if not actions: + return True, error_summary, component_count + if not isinstance(actions, dict): + error_summary = {'action.yml': ['Expected dictionary with action types as keys']} + return True, error_summary, component_count + + for action_type, actions_list in actions.items(): + if action_type not in ActionSerializer.action_lookup: + error_summary[action_type] = [f"Invalid action type: {action_type}."] + is_data_invalid = True + continue + + if not isinstance(actions_list, list): + error_summary[action_type] = [f"Expected list of actions for {action_type}."] + is_data_invalid = True + continue + + component_count[action_type] = len(actions_list) + + if error_list := ActionSerializer.collection_config_validation(bot, action_type, actions_list, encountered_action_names): + error_summary[action_type].extend(error_list) + is_data_invalid = True + continue + + if other_collections: + for collection_name, collection_data in other_collections.items(): + if collection_name not in ActionSerializer.action_lookup: + error_summary[collection_name] = [f"Invalid collection type: {collection_name}."] + is_data_invalid = True + continue + if not isinstance(collection_data, list): + error_summary[collection_name] = [f"Expected list of data for {collection_name}."] + is_data_invalid = True + continue + + component_count[collection_name] = len(collection_data) + + if error_list := ActionSerializer.collection_config_validation(bot, collection_name, collection_data, set()): + error_summary[collection_name].extend(error_list) + is_data_invalid = True + + return not is_data_invalid, error_summary, component_count + + @staticmethod + def collection_config_validation(bot: str, action_type: str, actions_list: list[dict], encountered_action_names: set): + """ + Validates the action configuration data for an action type + :param bot: bot id + :param action_type: action type + :param actions_list: list of action configuration data + :param encountered_action_names: set of action names encountered + :return: error_summary: list + """ + action_info = ActionSerializer.action_lookup.get(action_type) + if not action_info: + return [f"Action type not found: {action_type}."] + validation_model = action_info.get("validation_model") # pydantic model + collection_model = action_info.get("db_model") # mongoengine model + + err_summary = [] + custom_validation = action_info.get("custom_validation") + required_fields = {k for k, v in collection_model._fields.items() if + v.required and k not in {'bot', 'user', 'timestamp', 'status'}} + + for action in actions_list: + if not isinstance(action, dict): + err_summary.append(f"Expected dictionary for [{action_type}] ") + continue + action_name = ActionSerializer.get_item_name(action, raise_exception=False) + if not action_name: + err_summary.append(f"No name found for [{action_type}].") + continue + if action_name in encountered_action_names: + if action_type != ActionType.form_validation_action.value and not action_name.startswith("utter_"): + err_summary.append({action_name: "Duplicate Name found for other action."}) + continue + encountered_action_names.add(action_name) + if not action: + err_summary.append("Action configuration cannot be empty.") + continue + not_present_fields = required_fields.difference(set(action.keys())) + if len(not_present_fields) > 0: + err_summary.append({ + action_name: f' Required fields {not_present_fields} not found.' + }) + continue + if custom_validation: + for cv in custom_validation: + if validation_result := cv(bot, action): + err_summary.extend(validation_result) + if err_summary: + continue + try: + if validation_model: + validation_model(**action) + except PValidationError as pe: + err_summary.append({action_name: f"{str(pe)}"}) + except Exception as e: + err_summary.append({action_name: f"{str(e)}"}) + + return err_summary + + @staticmethod + def get_collection_infos(): + """ + Get action and other collection information as seperate dicts + :return: actions_collections: dict, other_collections: dict + """ + actions_collections = {k: v for k, v in ActionSerializer.action_lookup.items() + if not v.get("group") or v.get("group") == "action"} + other_collections = {k: v for k, v in ActionSerializer.action_lookup.items() + if v.get("group") and v.get("group") != "action"} + + return actions_collections, other_collections + + @staticmethod + def serialize(bot: str): + """ + Serialize / export all the actions and configuration collection data + :param bot: bot id + :return: action_config: dict, other_config: dict + """ + action_config = {} + other_config = {} + + actions_collections, other_collections = ActionSerializer.get_collection_infos() + + for action_type, action_info in actions_collections.items(): + action_model = action_info.get("db_model") + actions = ActionSerializer.get_action_config_data_list(bot, action_model, query={'status': True}) + if actions: + action_config[action_type] = actions + + for other_type, other_info in other_collections.items(): + other_model = other_info.get("db_model") + other_collections = ActionSerializer.get_action_config_data_list(bot, other_model) + if other_collections: + other_config[other_type] = other_collections + return action_config, other_config + + @staticmethod + def deserialize(bot: str, user: str, actions: Optional[dict] = None, other_collections_data: Optional[dict] = None, overwrite: bool = False): + """ + Deserialize / import the actions and configuration collection data + :param bot: bot id + :param user: user id + :param actions: action configuration data + :param other_collections_data: other collection configuration data + :param overwrite: bool + """ + actions_collections, _ = ActionSerializer.get_collection_infos() + + if overwrite: + for _, info in ActionSerializer.action_lookup.items(): + model = info.get("db_model") + if model: + model.objects(bot=bot).delete() + + saved_actions = set( + Actions.objects(bot=bot, status=True, type__ne=None).values_list("name") + ) + form_names = set( + Forms.objects(bot=bot, status=True).values_list("name") + ) + + filtered_actions = {} + if actions: + for action_type, action_info in actions_collections.items(): + if action_type in actions: + # Skip if no actions are present + if len(actions[action_type]) == 0: + continue + + if action_type == ActionType.form_validation_action.value: + filtered_actions[action_type] = actions[action_type] + elif action_info.get('single_instance'): + if overwrite: + filtered_actions[action_type] = actions[action_type] + else: + new_actions = [] + action_names = [] + for a in actions[action_type]: + action_name = ActionSerializer.get_item_name(a) + if action_name in form_names: + raise AppException(f"Form with name {action_name} already exists!") + if (action_name not in saved_actions + and not action_name.startswith("utter_") + and action_name not in action_names): + action_names.append(action_name) + new_actions.append(a) + filtered_actions[action_type] = new_actions + + for action_type, data in filtered_actions.items(): + if data: + ActionSerializer.save_collection_data_list(action_type, bot, user, data) + if other_collections_data: + ActionSerializer.save_other_collections(other_collections_data, bot, user, overwrite) + + @staticmethod + def get_action_config_data_list(bot: str, action_model: Document, with_doc_id: bool = False, query: dict = {}) -> list[dict]: + """ + Get the action configuration data list + :param bot: bot id + :param action_model: mongoengine model + :param with_doc_id: bool + :param query: dict + :return: list[dict] + """ + query['bot'] = bot + key_to_remove = {"_id", "user", "bot", "status", "timestamp"} + query_result = action_model.objects(**query).as_pymongo() + actions = [] + if query_result: + actions = [ + { + **({"_id": str(action["_id"])} if with_doc_id else {}), + **{k: v for k, v in action.items() if k not in key_to_remove} + } + for action in query_result + ] + return actions + + + @staticmethod + def is_action(action_type: str): + if not action_type or action_type not in ActionSerializer.action_lookup: + return False + return not ActionSerializer.action_lookup[action_type].get("group") or ActionSerializer.action_lookup[action_type].get("group") == "action" + + @staticmethod + def save_collection_data_list(action_type: str, bot: str, user: str, configs: list[dict]): + """ + Save the collection data list for action or any collection. Mongoengine model must be available in the lookup + """ + if not configs: # Early exit if no configs are present + return + + model = ActionSerializer.action_lookup.get(action_type, {}).get("db_model") + modify = ActionSerializer.action_lookup.get(action_type, {}).get("modify") + is_action = ActionSerializer.is_action(action_type) + if not model: + raise AppException(f"Action type not found: [{action_type}]!") + + try: + model_entries = [] + action_entries = [] + action_names = set() + reconfig_props = (ActionSerializer.action_lookup.get(action_type, {}) + .get("reconfigure", ActionSerializer.default_reconfigurable)) + for config in configs: + if ReconfigarableProperty.bot.value in reconfig_props: + config["bot"] = bot + if ReconfigarableProperty.user.value in reconfig_props: + config["user"] = user + if ReconfigarableProperty.status.value in reconfig_props: + config["status"] = True + if modify: + config = modify(bot, config) + model_entries.append(model(**config)) + action_name = ActionSerializer.get_item_name(config) + + if is_action and action_name not in action_names: + action_entries.append(Actions( + name=action_name, + type=action_type, + bot=bot, + user=user, + status=True + )) + action_names.add(action_name) + if action_entries: + Actions.objects.insert(action_entries) + if model_entries: + model.objects.insert(model_entries) + except Exception as e: + raise AppException(f"Error saving action config data: {str(e)}") from e + + @staticmethod + def save_other_collections(other_collections_data: dict, bot: str, user: str, overwrite: bool = False): + _, other_collections = ActionSerializer.get_collection_infos() + for collection_name, collection_info in other_collections.items(): + collection_data = other_collections_data.get(collection_name) + if overwrite: + collection_info.get("db_model").objects(bot=bot).delete() + else: + prev_data = collection_info.get("db_model").objects(bot=bot) + names = set(prev_data.values_list("name")) + collection_data = [data for data in collection_data if data.get("name") not in names] + + if collection_name and collection_data: + collection_model = collection_info.get("db_model") + if collection_model: + ActionSerializer.save_collection_data_list(collection_name, bot, user, collection_data) + else: + raise AppException(f"Collection model not found for [{collection_name}]!") diff --git a/kairon/shared/data/data_models.py b/kairon/shared/data/data_models.py new file mode 100644 index 000000000..346e0707f --- /dev/null +++ b/kairon/shared/data/data_models.py @@ -0,0 +1,1343 @@ +from typing import List, Any, Dict, Optional, Text, Union + +from validators import url +from validators.utils import ValidationError as ValidationFailure +from fastapi.param_functions import Form +from fastapi.security import OAuth2PasswordRequestForm +from rasa.shared.constants import DEFAULT_NLU_FALLBACK_INTENT_NAME + +from kairon.exceptions import AppException +from kairon.shared.data.constant import ( + EVENT_STATUS, + SLOT_MAPPING_TYPE, + SLOT_TYPE, + ACCESS_ROLES, + ACTIVITY_STATUS, + INTEGRATION_STATUS, + FALLBACK_MESSAGE, + DEFAULT_NLU_FALLBACK_RESPONSE +) +from kairon.shared.actions.models import ( + ActionParameterType, + EvaluationType, + DispatchType, + DbQueryValueType, + DbActionOperationType, UserMessageType, HttpRequestContentType +) +from kairon.shared.callback.data_objects import CallbackExecutionMode +from kairon.shared.constants import SLOT_SET_TYPE, FORM_SLOT_SET_TYPE + +from pydantic import BaseModel, validator, SecretStr, root_validator, constr +from kairon.shared.models import ( + StoryStepType, + StoryType, + TemplateType, + HttpContentType, + LlmPromptSource, + LlmPromptType, + CognitionDataType, + CognitionMetadataType, +) + + +class RecaptchaVerifiedRequest(BaseModel): + recaptcha_response: str = None + remote_ip: str = None + + @root_validator + def validate_recaptcha(cls, values): + from kairon.shared.utils import Utility + + secret = Utility.environment["security"].get("recaptcha_secret", None) + if Utility.environment["security"][ + "validate_recaptcha" + ] and not Utility.check_empty_string(secret): + Utility.validate_recaptcha( + values.get("recaptcha_response"), values.get("remote_ip") + ) + return values + + +class RecaptchaVerifiedOAuth2PasswordRequestForm(OAuth2PasswordRequestForm): + """ + Dependency class overridden from OAuth2PasswordRequestForm. + """ + + def __init__( + self, + grant_type: str = Form(None, pattern="password"), + username: str = Form(...), + password: str = Form(...), + scope: str = Form(""), + client_id: Optional[str] = Form(None), + client_secret: Optional[str] = Form(None), + recaptcha_response: str = Form(None), + remote_ip: str = Form(None), + fingerprint: str = Form(None), + ): + """ + @param grant_type: the OAuth2 spec says it is required and MUST be the fixed string "password". + Nevertheless, this dependency class is permissive and allows not passing it. If you want to enforce it, + use instead the OAuth2PasswordRequestFormStrict dependency. + @param username: username string. The OAuth2 spec requires the exact field name "username". + @param password: password string. The OAuth2 spec requires the exact field name "password". + @param scope: Optional string. Several scopes (each one a string) separated by spaces. + E.g. "items:read items:write users:read profile openid" + @param client_id: optional string. OAuth2 recommends sending the client_id and client_secret (if any) + using HTTP Basic auth, as: client_id:client_secret + @param client_secret: optional string. OAuth2 recommends sending the client_id and client_secret (if any) + using HTTP Basic auth, as: client_id:client_secret + @param recaptcha_response: optional string. recaptcha response. + @param remote_ip: optional string. remote ip address. + @param fingerprint: optional string. device fingerprint. + """ + from kairon.shared.utils import Utility + + secret = Utility.environment["security"].get("recaptcha_secret", None) + if Utility.environment["security"][ + "validate_recaptcha" + ] and not Utility.check_empty_string(secret): + Utility.validate_recaptcha(recaptcha_response, remote_ip) + OAuth2PasswordRequestForm.__init__( + self, + grant_type=grant_type, + username=username, + password=password, + scope=scope, + client_id=client_id, + client_secret=client_secret + ) + self.recaptcha_response = recaptcha_response + self.remote_ip = remote_ip + if Utility.environment["user"]["validate_trusted_device"] and Utility.check_empty_string(fingerprint): + raise AppException("fingerprint is required") + self.fingerprint = fingerprint + + +class Token(BaseModel): + access_token: str + token_type: str + + +class TokenData(BaseModel): + username: str + + +class Response(BaseModel): + success: bool = True + message: Any = None + data: Any + error_code: int = 0 + +class ActionResponse(BaseModel): + success: bool = True + error: str = None + action_name: str = None + events: List[Dict[Text, Any]] = None + responses: List[Dict[Text, Any]] = None + error_code: int = 200 + + +class RequestData(BaseModel): + data: Any + + +class TextData(BaseModel): + data: str + + +class RecaptchaVerifiedTextData(RecaptchaVerifiedRequest): + data: str + + +class TextDataLowerCase(BaseModel): + data: constr(to_lower=True, strip_whitespace=True) + + +class ListData(BaseModel): + data: List[str] + + +class RegisterAccount(RecaptchaVerifiedRequest): + email: constr(to_lower=True, strip_whitespace=True) + first_name: str + last_name: str + password: SecretStr + confirm_password: SecretStr + account: str + fingerprint: str = None + + @validator("email") + def validate_email(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + try: + Utility.verify_email(v) + except AppException as e: + raise ValueError(str(e)) + return v + + @validator("password") + def validate_password(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + try: + Utility.valid_password(v.get_secret_value()) + except AppException as e: + raise ValueError(str(e)) + return v + + @validator("confirm_password") + def validate_confirm_password(cls, v, values, **kwargs): + if ( + "password" in values + and v.get_secret_value() != values["password"].get_secret_value() + ): + raise ValueError("Password and Confirm Password does not match") + return v + + @root_validator + def validate_fingerprint(cls, values): + from kairon.shared.utils import Utility + + if Utility.environment["user"]["validate_trusted_device"] and Utility.check_empty_string(values.get("fingerprint")): + raise ValueError("fingerprint is required") + return values + + +class BotAccessRequest(RecaptchaVerifiedRequest): + email: constr(to_lower=True, strip_whitespace=True) + role: ACCESS_ROLES = ACCESS_ROLES.TESTER.value + activity_status: ACTIVITY_STATUS = ACTIVITY_STATUS.INACTIVE.value + + @validator("email") + def validate_email(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + try: + Utility.verify_email(v) + except AppException as e: + raise ValueError(str(e)) + return v + + @validator("role") + def validate_role(cls, v, values, **kwargs): + if v == ACCESS_ROLES.OWNER.value: + raise ValueError("There can be only 1 owner per bot") + return v + + +class EndPointBot(BaseModel): + url: str + token: str = None + token_type: str = None + + +class EndPointAction(BaseModel): + url: str + + +class EndPointHistory(BaseModel): + url: str + token: str = None + + +class Endpoint(BaseModel): + bot_endpoint: EndPointBot = None + action_endpoint: EndPointAction = None + history_endpoint: EndPointHistory = None + + +class RasaConfig(BaseModel): + language: str = "en" + pipeline: List[Dict] + policies: List[Dict] + + +class ComponentConfig(BaseModel): + nlu_epochs: int = None + response_epochs: int = None + ted_epochs: int = None + nlu_confidence_threshold: float = None + action_fallback: str = None + action_fallback_threshold: float = None + + @validator("nlu_epochs", "response_epochs", "ted_epochs") + def validate_epochs(cls, v): + from kairon.shared.utils import Utility + + if v is not None and v < 1: + raise ValueError("Choose a positive number as epochs") + elif v > Utility.environment["model"]["config_properties"]["epoch_max_limit"]: + epoch_max_limit = Utility.environment["model"]["config_properties"][ + "epoch_max_limit" + ] + raise ValueError(f"Please choose a epoch between 1 and {epoch_max_limit}") + return v + + @validator("nlu_confidence_threshold", "action_fallback_threshold") + def validate_confidence_threshold(cls, v): + if v is not None and (v < 0.3 or v > 0.9): + raise ValueError("Please choose a threshold between 0.3 and 0.9") + return v + + +class Password(RecaptchaVerifiedRequest): + data: str + password: SecretStr + confirm_password: SecretStr + + @validator("password") + def validate_password(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + try: + Utility.valid_password(v.get_secret_value()) + except AppException as e: + raise ValueError(str(e)) + return v + + @validator("confirm_password") + def validate_confirm_password(cls, v, values, **kwargs): + if ( + "password" in values + and v.get_secret_value() != values["password"].get_secret_value() + ): + raise ValueError("Password and Confirm Password does not match") + return v + + +class HttpActionParameters(BaseModel): + key: str + value: str = None + parameter_type: ActionParameterType + encrypt: bool = False + + @root_validator + def check(cls, values): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(values.get("key")): + raise ValueError("key cannot be empty") + + if values.get( + "parameter_type" + ) == ActionParameterType.slot and Utility.check_empty_string( + values.get("value") + ): + raise ValueError("Provide name of the slot as value") + + if values.get( + "parameter_type" + ) == ActionParameterType.key_vault and Utility.check_empty_string( + values.get("value") + ): + raise ValueError("Provide key from key vault as value") + + if values.get("parameter_type") == ActionParameterType.key_vault: + values["encrypt"] = True + + return values + + +class SetSlotsUsingActionResponse(BaseModel, use_enum_values=True): + name: str + value: str + evaluation_type: EvaluationType = EvaluationType.expression + + @validator("name") + def validate_name(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(v): + raise ValueError("slot name is required") + return v + + @validator("value") + def validate_expression(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(v): + raise ValueError("expression is required to evaluate value of slot") + return v + + +class ActionResponseEvaluation(BaseModel): + value: str = None + dispatch: bool = True + evaluation_type: EvaluationType = EvaluationType.expression + dispatch_type: DispatchType = DispatchType.text.value + + @root_validator + def check(cls, values): + from kairon.shared.utils import Utility + + if values.get("dispatch") is True and Utility.check_empty_string( + values.get("value") + ): + raise ValueError("response is required for dispatch") + + return values + + +class HttpActionConfigRequest(BaseModel): + action_name: constr(to_lower=True, strip_whitespace=True) + content_type: Union[HttpContentType, HttpRequestContentType] = HttpContentType.application_json + response: ActionResponseEvaluation = None + http_url: str + request_method: str + params_list: List[HttpActionParameters] = [] + dynamic_params: str = None + headers: List[HttpActionParameters] = [] + set_slots: List[SetSlotsUsingActionResponse] = [] + + @validator("action_name") + def validate_action_name(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(v): + raise ValueError("action_name is required") + return v + + @validator("http_url") + def validate_http_url(cls, v, values, **kwargs): + if isinstance(url(v), ValidationFailure): + raise ValueError("URL is malformed") + return v + + @validator("request_method") + def validate_request_method(cls, v, values, **kwargs): + if v.upper() not in ("GET", "POST", "PUT", "DELETE"): + raise ValueError("Invalid HTTP method") + return v.upper() + + +class PayloadConfig(BaseModel): + type: DbQueryValueType + value: Any + query_type: DbActionOperationType + + @root_validator + def check(cls, values): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(values.get("type")): + raise ValueError("type is required") + + return values + + +class PyscriptActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + source_code: str + dispatch_response: bool = True + + @validator("name") + def validate_action_name(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(v): + raise ValueError("name is required") + return v + + @validator("source_code") + def validate_source_code(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(v): + raise ValueError("source_code is required") + return v + + +class DatabaseActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + collection: str + payload: List[PayloadConfig] + response: ActionResponseEvaluation = None + set_slots: List[SetSlotsUsingActionResponse] = [] + + @validator("name") + def validate_action_name(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(v): + raise ValueError("name is required") + return v + + @validator("collection") + def validate_collection_name(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(v): + raise ValueError("collection is required") + return v + + @validator("payload") + def validate_payload(cls, v, values, **kwargs): + count_payload_search = 0 + if v: + for item in v: + if item.query_type == DbActionOperationType.payload_search: + count_payload_search += 1 + if count_payload_search > 1: + raise ValueError(f"Only One {DbActionOperationType.payload_search} is allowed!") + else: + raise ValueError("payload is required") + return v + + +class LiveAgentActionRequest(BaseModel): + bot_response: str = 'Connecting to live agent' + agent_connect_response: str = 'Connected to live agent' + agent_disconnect_response: str = 'Agent has closed the conversation' + agent_not_available_response: str = 'No agents available at this moment. An agent will reply to you shortly.' + dispatch_bot_response: bool = True + dispatch_agent_connect_response: bool = True + dispatch_agent_disconnect_response: bool = True + dispatch_agent_not_available_response: bool = True + + +class TrainingData(BaseModel): + intent: constr(to_lower=True, strip_whitespace=True) + training_examples: List[str] + response: str + + +class BulkTrainingDataAddRequest(BaseModel): + history_id: str + training_data: List[TrainingData] + + +class TrainingDataGeneratorResponseModel(BaseModel): + intent: str + training_examples: List[str] + response: str + + +class TrainingDataGeneratorStatusModel(BaseModel): + status: EVENT_STATUS + response: List[TrainingData] = None + exception: str = None + + +class StoryStepRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) = None + type: StoryStepType + value: Any = None + + +class MultiStoryStepRequest(StoryStepRequest): + node_id: str + component_id: str + + +class StoryStepData(BaseModel): + step: MultiStoryStepRequest + connections: List[MultiStoryStepRequest] = None + + +class StoryMetadata(BaseModel): + node_id: str + flow_type: StoryType = StoryType.story.value + + +class MultiFlowStoryRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + steps: List[StoryStepData] + metadata: List[StoryMetadata] = None + + @validator("steps") + def validate_request_method(cls, v, values, **kwargs): + if not v: + raise ValueError("Steps are required to form Flow") + return v + + +class StoryRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + type: StoryType + steps: List[StoryStepRequest] + template_type: TemplateType = None + + class Config: + use_enum_values = True + + def get_steps(self): + return [step.dict() for step in self.steps] + + @validator("steps") + def validate_request_method(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if not v: + raise ValueError("Steps are required to form Flow") + + if v[0].type != StoryStepType.intent: + raise ValueError("First step should be an intent") + + if v[len(v) - 1].type == StoryStepType.intent: + raise ValueError("Intent should be followed by utterance or action") + + intents = 0 + for i, j in enumerate(range(1, len(v))): + if v[i].type == StoryStepType.intent: + intents = intents + 1 + if v[i].type == StoryStepType.intent and v[j].type == StoryStepType.intent: + raise ValueError("Found 2 consecutive intents") + if ( + Utility.check_empty_string(v[i].name) + and v[i].type != StoryStepType.form_end + ): + raise ValueError( + f"Only {StoryStepType.form_end} step type can have empty name" + ) + if v[i].type == StoryStepType.stop_flow_action and i != len(v) - 1: + raise ValueError("Stop Flow Action should only be at the end of the flow") + if v[i].type == StoryStepType.intent and v[j].type == StoryStepType.stop_flow_action: + raise ValueError("Stop Flow Action should not be after intent") + + if "type" in values: + if values["type"] == StoryType.rule and intents > 1: + raise ValueError( + f"""Found rules '{values['name']}' that contain more than intent.\nPlease use stories for this case""" + ) + return v + + +class AnalyticsModel(BaseModel): + fallback_intent: str = DEFAULT_NLU_FALLBACK_INTENT_NAME + + @validator('fallback_intent') + def validate_fallback_intent(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + if Utility.check_empty_string(v): + raise ValueError("fallback_intent field cannot be empty") + return v + + +class BotSettingsRequest(BaseModel): + analytics: AnalyticsModel = AnalyticsModel() + + +class FeedbackRequest(BaseModel): + rating: float + scale: float = 5 + feedback: str = None + + +class GPTRequest(BaseModel): + api_key: str + data: List[str] + engine: str = "davinci" + temperature: float = 0.75 + max_tokens: int = 100 + num_responses: int = 10 + + @validator("data") + def validate_gpt_questions(cls, v, values, **kwargs): + if len(v) <= 0: + raise ValueError("Question Please!") + elif len(v) > 5: + raise ValueError("Max 5 Questions are allowed!") + return v + + +class ParaphrasesRequest(BaseModel): + data: List[str] + + @validator("data") + def validate_paraphrases_questions(cls, v, values, **kwargs): + if len(v) <= 0: + raise ValueError("Question Please!") + elif len(v) > 5: + raise ValueError("Max 5 Questions are allowed!") + return v + + +class SlotRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + type: SLOT_TYPE + initial_value: Any = None + values: List[str] = None + max_value: float = None + min_value: float = None + influence_conversation: bool = False + + class Config: + use_enum_values = True + + +class SynonymRequest(BaseModel): + value: List[str] + + @validator("value") + def validate_value(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if len(v) <= 0: + raise ValueError("value field cannot be empty") + for ele in v: + if Utility.check_empty_string(ele): + raise ValueError("value cannot be an empty string") + return v + + +class AddBotRequest(BaseModel): + name: str + from_template: str = None + + +class DictData(BaseModel): + data: dict + + +class RecaptchaVerifiedDictData(DictData): + recaptcha_response: str = None + + +class RegexRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + pattern: str + + @validator("name") + def validate_name(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(v): + raise ValueError("Regex name cannot be empty or a blank space") + return v + + @validator("pattern") + def validate_pattern(cls, f, values, **kwargs): + from kairon.shared.utils import Utility + import re + + if Utility.check_empty_string(f): + raise ValueError("Regex pattern cannot be empty or a blank space") + try: + re.compile(f) + except Exception: + raise AppException("invalid regular expression") + return f + + +class LookupTablesRequest(BaseModel): + value: List[str] + + @validator("value") + def validate_value(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if len(v) <= 0: + raise ValueError("value field cannot be empty") + for ele in v: + if Utility.check_empty_string(ele): + raise ValueError("lookup value cannot be empty or a blank space") + return v + + +class MappingCondition(BaseModel): + active_loop: str = None + requested_slot: str = None + + @root_validator + def validate(cls, values): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(values.get("active_loop")) and not Utility.check_empty_string(values.get("requested_slot")): + raise ValueError("active_loop is required to add requested_slot as condition!") + + return values + + +class SlotMapping(BaseModel): + entity: constr(to_lower=True, strip_whitespace=True) = None + type: SLOT_MAPPING_TYPE + value: Any = None + intent: List[constr(to_lower=True, strip_whitespace=True)] = None + not_intent: List[constr(to_lower=True, strip_whitespace=True)] = None + conditions: List[MappingCondition] = None + + class Config: + use_enum_values = True + + +class SlotMappingRequest(BaseModel): + slot: constr(to_lower=True, strip_whitespace=True) + mapping: SlotMapping + + class Config: + use_enum_values = True + + @validator("mapping") + def validate_mapping(cls, v, values, **kwargs): + if not v or v == [{}]: + raise ValueError("At least one mapping is required") + return v + + +class FormSlotSetModel(BaseModel): + type: FORM_SLOT_SET_TYPE = FORM_SLOT_SET_TYPE.current.value + value: Any = None + class Config: + use_enum_values = True + + + +class FormSettings(BaseModel): + ask_questions: List[str] + slot: str + is_required: bool = True + validation_semantic: str = None + valid_response: str = None + invalid_response: str = None + slot_set: FormSlotSetModel = FormSlotSetModel() + + @validator("ask_questions") + def validate_responses(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + err_msg = "Questions cannot be empty or contain spaces" + if not v: + raise ValueError(err_msg) + + for response in v: + if Utility.check_empty_string(response): + raise ValueError(err_msg) + return v + + @validator("slot") + def validate_slot(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(v): + raise ValueError("Slot is required") + return v + + +class Forms(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + settings: List[FormSettings] + + +class SetSlots(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + type: SLOT_SET_TYPE + value: Any = None + + class Config: + use_enum_values = True + + +class SlotSetActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + set_slots: List[SetSlots] + + +class CustomActionParameter(BaseModel): + value: str = None + parameter_type: ActionParameterType = ActionParameterType.value + class Config: + use_enum_values = True + + @validator("parameter_type") + def validate_parameter_type(cls, v, values, **kwargs): + allowed_values = { + ActionParameterType.value, + ActionParameterType.slot, + ActionParameterType.key_vault, + ActionParameterType.sender_id, + } + if v not in allowed_values: + raise ValueError( + f"Invalid parameter type. Allowed values: {allowed_values}" + ) + return v + + @root_validator + def check(cls, values): + from kairon.shared.utils import Utility + + if values.get( + "parameter_type" + ) == ActionParameterType.slot and Utility.check_empty_string( + values.get("value") + ): + raise ValueError("Provide name of the slot as value") + + if values.get( + "parameter_type" + ) == ActionParameterType.key_vault and Utility.check_empty_string( + values.get("value") + ): + raise ValueError("Provide key from key vault as value") + + return values + + +class GoogleSearchActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + api_key: CustomActionParameter = None + search_engine_id: str = None + website: str = None + failure_response: str = "I have failed to process your request." + num_results: int = 1 + dispatch_response: bool = True + set_slot: str = None + + @validator("num_results") + def validate_num_results(cls, v, values, **kwargs): + if not v or v < 1: + raise ValueError("num_results must be greater than or equal to 1!") + return v + + +class WebSearchActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + website: str = None + failure_response: str = 'I have failed to process your request.' + topn: int = 1 + dispatch_response: bool = True + set_slot: str = None + + @validator("name") + def validate_action_name(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if Utility.check_empty_string(v): + raise ValueError("name is required") + return v + + @validator("topn") + def validate_top_n(cls, v, values, **kwargs): + if not v or v < 1: + raise ValueError("topn must be greater than or equal to 1!") + return v + + +class CustomActionParameterModel(BaseModel): + value: Any = None + parameter_type: ActionParameterType = ActionParameterType.value + + @validator("parameter_type") + def validate_parameter_type(cls, v, values, **kwargs): + allowed_values = {ActionParameterType.value, ActionParameterType.slot} + if v not in allowed_values: + raise ValueError(f"Invalid parameter type. Allowed values: {allowed_values}") + return v + + @root_validator + def check(cls, values): + if values.get('parameter_type') == ActionParameterType.slot and not values.get('value'): + raise ValueError("Provide name of the slot as value") + + if values.get('parameter_type') == ActionParameterType.value and not isinstance(values.get('value'), list): + raise ValueError("Provide list of emails as value") + + return values + + +class CustomActionDynamicParameterModel(BaseModel): + value: Any = None + parameter_type: ActionParameterType = ActionParameterType.value + + @validator("parameter_type") + def validate_parameter_type(cls, v, values, **kwargs): + allowed_values = {ActionParameterType.value, ActionParameterType.slot} + if v not in allowed_values: + raise ValueError(f"Invalid parameter type. Allowed values: {allowed_values}") + return v + + @root_validator + def check(cls, values): + from kairon.shared.utils import Utility + if values.get('parameter_type') == ActionParameterType.slot and not values.get('value'): + raise ValueError("Provide name of the slot as value") + + if values.get('parameter_type') == ActionParameterType.value and Utility.check_empty_string(values.get("value")): + raise ValueError("Value can not be blank") + + return values + + +class EmailActionRequest(BaseModel): + action_name: constr(to_lower=True, strip_whitespace=True) + smtp_url: str + smtp_port: int + smtp_userid: CustomActionParameter = None + smtp_password: CustomActionParameter + from_email: CustomActionParameter + subject: str + custom_text: CustomActionParameter = None + to_email: CustomActionParameterModel + response: str + tls: bool = False + + +class JiraActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + url: str + user_name: str + api_token: CustomActionParameter + project_key: str + issue_type: str + parent_key: str = None + summary: str + response: str + + +class ZendeskActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + subdomain: str + user_name: str + api_token: CustomActionParameter + subject: str + response: str + + +class PipedriveActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + domain: str + api_token: CustomActionParameter + title: str + response: str + metadata: dict + + @validator("metadata") + def validate_metadata(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if not v or Utility.check_empty_string(v.get("name")): + raise ValueError("name is required") + return v + + +class HubspotFormsActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + portal_id: str + form_guid: str + fields: List[HttpActionParameters] + response: str + + +class QuickReplies(BaseModel): + text: str + payload: str + message: str = None + is_dynamic_msg: bool = False + + +class TwoStageFallbackTextualRecommendations(BaseModel): + count: int = 0 + use_intent_ranking: bool = False + + +class TwoStageFallbackConfigRequest(BaseModel): + fallback_message: str = FALLBACK_MESSAGE + text_recommendations: TwoStageFallbackTextualRecommendations = None + trigger_rules: List[QuickReplies] = None + + @root_validator + def check(cls, values): + if not values.get("text_recommendations") and not values["trigger_rules"]: + raise ValueError( + "One of text_recommendations or trigger_rules should be defined" + ) + if ( + values.get("text_recommendations") + and values["text_recommendations"].count < 0 + ): + raise ValueError("count cannot be negative") + return values + + +class PromptHyperparameters(BaseModel): + top_results: int = 10 + similarity_threshold: float = 0.70 + + @root_validator + def check(cls, values): + if not 0.3 <= values.get('similarity_threshold') <= 1: + raise ValueError("similarity_threshold should be within 0.3 and 1") + if values.get('top_results') > 30: + raise ValueError("top_results should not be greater than 30") + return values + + +class LlmPromptRequest(BaseModel, use_enum_values=True): + name: str + hyperparameters: PromptHyperparameters = None + data: str = None + instructions: str = None + type: LlmPromptType + source: LlmPromptSource + is_enabled: bool = True + + @root_validator + def check(cls, values): + from kairon.shared.utils import Utility + + if (values.get('source') == LlmPromptSource.bot_content.value and + Utility.check_empty_string(values.get('data'))): + values['data'] = "default" + return values + + +class UserQuestionModel(BaseModel): + type: UserMessageType = UserMessageType.from_user_message.value + value: str = None + + +class PromptActionConfigUploadValidation(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + num_bot_responses: int = 5 + failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE + user_question: UserQuestionModel = UserQuestionModel() + llm_type: str + hyperparameters: dict + llm_prompts: List[LlmPromptRequest] + instructions: List[str] = [] + set_slots: List[SetSlotsUsingActionResponse] = [] + dispatch_response: bool = True + + +class PromptActionConfigRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + num_bot_responses: int = 5 + failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE + user_question: UserQuestionModel = UserQuestionModel() + llm_type: str + hyperparameters: dict + llm_prompts: List[LlmPromptRequest] + instructions: List[str] = [] + set_slots: List[SetSlotsUsingActionResponse] = [] + dispatch_response: bool = True + bot: str + + @validator("llm_type", pre=True, always=True) + def validate_llm_type(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + if v not in Utility.get_llms(): + raise ValueError("Invalid llm type") + return v + + @validator("llm_prompts") + def validate_llm_prompts(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + Utility.validate_kairon_faq_llm_prompts( + [vars(value) for value in v], ValueError + ) + return v + + @validator("num_bot_responses") + def validate_num_bot_responses(cls, v, values, **kwargs): + if v > 5: + raise ValueError("num_bot_responses should not be greater than 5") + return v + + @validator("hyperparameters") + def validate_hyperparameters(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + bot = values.get('bot') + llm_type = values.get('llm_type') + if llm_type and v: + Utility.validate_llm_hyperparameters(v, llm_type, bot, ValueError) + return v + + @root_validator(pre=True) + def validate_required_fields(cls, values): + bot = values.get('bot') + if not bot: + raise ValueError("bot field is missing") + return values + + +class ColumnMetadata(BaseModel): + column_name: str + data_type: CognitionMetadataType + enable_search: bool = True + create_embeddings: bool = True + + @root_validator + def check(cls, values): + from kairon.shared.utils import Utility + + if values.get('data_type') not in [CognitionMetadataType.str.value, CognitionMetadataType.int.value, CognitionMetadataType.float.value]: + raise ValueError("Only str, int and float data types are supported") + if Utility.check_empty_string(values.get('column_name')): + raise ValueError("Column name cannot be empty") + return values + + +class CognitionSchemaRequest(BaseModel): + metadata: List[ColumnMetadata] = None + collection_name: constr(to_lower=True, strip_whitespace=True) + + +class CollectionDataRequest(BaseModel): + data: dict + is_secure: list = [] + collection_name: constr(to_lower=True, strip_whitespace=True) + + @root_validator + def check(cls, values): + from kairon.shared.utils import Utility + + data = values.get("data") + is_secure = values.get("is_secure") + collection_name = values.get("collection_name") + if Utility.check_empty_string(collection_name): + raise ValueError("collection_name should not be empty!") + + if not isinstance(is_secure, list): + raise ValueError("is_secure should be list of keys!") + + if is_secure: + if not data or not isinstance(data, dict): + raise ValueError("data cannot be empty and should be of type dict!") + data_keys = set(data.keys()) + is_secure_set = set(is_secure) + + if not is_secure_set.issubset(data_keys): + raise ValueError("is_secure contains keys that are not present in data") + return values + + +class CognitiveDataRequest(BaseModel): + data: Any + content_type: CognitionDataType = CognitionDataType.text.value + collection: constr(to_lower=True, strip_whitespace=True) = None + + @root_validator + def check(cls, values): + from kairon.shared.utils import Utility + + data = values.get("data") + content_type = values.get("content_type") + if isinstance(data, dict) and content_type != CognitionDataType.json.value: + raise ValueError("content type and type of data do not match!") + if not data or (isinstance(data, str) and Utility.check_empty_string(data)): + raise ValueError("data cannot be empty") + return values + + +class RazorpayActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + api_key: CustomActionParameter + api_secret: CustomActionParameter + amount: CustomActionParameter + currency: CustomActionParameter + username: CustomActionParameter = None + email: CustomActionParameter = None + contact: CustomActionParameter = None + notes: Optional[List[HttpActionParameters]] + + +class IntegrationRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + expiry_minutes: int = 0 + access_list: list = None + role: ACCESS_ROLES = ACCESS_ROLES.CHAT.value + status: INTEGRATION_STATUS = INTEGRATION_STATUS.ACTIVE.value + + +class KeyVaultRequest(BaseModel): + key: str + value: str + + @validator("key") + def validate_key(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if not v or Utility.check_empty_string(v): + raise ValueError("key is required") + return v + + @validator("value") + def validate_value(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if not v or Utility.check_empty_string(v): + raise ValueError("value is required") + return v + + +class EventConfig(BaseModel): + ws_url: str + headers: dict + method: str + + @validator("ws_url") + def validate_ws_url(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if not v or Utility.check_empty_string(v): + raise ValueError("url can not be empty") + return v + + @validator("headers") + def validate_headers(cls, v, values, **kwargs): + if not v or len(v) < 1: + v = {} + return v + + +class IDPConfig(BaseModel): + config: dict + organization: str + + @validator("config") + def validate_config(cls, v, values, **kwargs): + if not v or len(v) == 0: + v = {} + return v + + @validator("organization") + def validate_organization(cls, v, values, **kwargs): + from kairon.shared.utils import Utility + + if not v or Utility.check_empty_string(v): + raise ValueError("Organization can not be empty") + return v + + +class CallbackConfigRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + pyscript_code: str + execution_mode: str = CallbackExecutionMode.ASYNC.value + standalone: bool = False + shorten_token: bool = False + standalone_id_path: Optional[str] = None + expire_in: int = 0 + + +class CallbackActionConfigRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + callback_name: str + dynamic_url_slot_name: Optional[str] + metadata_list: list[HttpActionParameters] = [] + bot_response: Optional[str] + dispatch_bot_response: bool = True + + +class ScheduleActionRequest(BaseModel): + name: constr(to_lower=True, strip_whitespace=True) + schedule_time: CustomActionDynamicParameterModel + timezone: str = None + schedule_action: str + response_text: Optional[str] + params_list: Optional[List[HttpActionParameters]] + dispatch_bot_response: bool = True + + @root_validator + def validate_name(cls, values): + from kairon.shared.utils import Utility + + if not values.get("name") or Utility.check_empty_string(values.get("name")): + raise ValueError("Schedule action name can not be empty") + + if not values.get("schedule_action") or Utility.check_empty_string(values.get("schedule_action")): + raise ValueError("Schedule action can not be empty, it is needed to execute on schedule time") + + return values diff --git a/kairon/shared/data/data_validation.py b/kairon/shared/data/data_validation.py new file mode 100644 index 000000000..936d9c9ad --- /dev/null +++ b/kairon/shared/data/data_validation.py @@ -0,0 +1,184 @@ +from uuid6 import uuid7 + +from kairon import Utility +from kairon.exceptions import AppException +from kairon.shared.actions.models import ActionParameterType, DbActionOperationType +import ast + +from kairon.shared.callback.data_objects import encrypt_secret + + +class DataValidation: + @staticmethod + def validate_http_action(bot: str, data: dict): + action_param_types = {param.value for param in ActionParameterType} + data_error = [] + if data.get('params_list'): + for param in data.get('params_list'): + if not param.get('key'): + data_error.append('Invalid params_list for http action: ' + data['action_name']) + continue + if param.get('parameter_type') not in action_param_types: + data_error.append('Invalid params_list for http action: ' + data['action_name']) + continue + if param.get('parameter_type') == 'slot' and not param.get('value'): + param['value'] = param.get('key') + + if data.get('headers'): + for param in data.get('headers'): + if not param.get('key'): + data_error.append('Invalid headers for http action: ' + data['action_name']) + continue + if param.get('parameter_type') not in action_param_types: + data_error.append('Invalid headers for http action: ' + data['action_name']) + continue + if param.get('parameter_type') == 'slot' and not param.get('value'): + param['value'] = param.get('key') + + return data_error + + @staticmethod + def validate_form_validation_action(bot: str, data: dict): + data_error = [] + if data.get('validation_semantic') and not isinstance(data['validation_semantic'], str): + data_error.append(f'Invalid validation semantic: {data["name"]}') + if data.get('slot_set'): + if Utility.check_empty_string(data['slot_set'].get('type')): + data_error.append('slot_set should have type current as default!') + if data['slot_set'].get('type') == 'current' and not Utility.check_empty_string( + data['slot_set'].get('value')): + data_error.append('slot_set with type current should not have any value!') + if data['slot_set'].get('type') == 'slot' and Utility.check_empty_string( + data['slot_set'].get('value')): + data_error.append('slot_set with type slot should have a valid slot value!') + if data['slot_set'].get('type') not in ['current', 'custom', 'slot']: + data_error.append('Invalid slot_set type!') + return data_error + + @staticmethod + def validate_database_action(bot: str, data: dict): + data_error = [] + for idx, item in enumerate(data.get('payload', [])): + if not item.get('query_type') or not item.get('type') or not item.get('value'): + data_error.append(f"Payload {idx} must contain fields 'query_type', 'type' and 'value'!") + if item.get('query_type') not in [qtype.value for qtype in DbActionOperationType]: + data_error.append(f"Unknown query_type found: {item['query_type']} in payload {idx}") + return data_error + + @staticmethod + def validate_prompt_action(bot: str, data: dict): + data_error = [] + if data.get('num_bot_responses') and ( + data['num_bot_responses'] > 5 or not isinstance(data['num_bot_responses'], int)): + data_error.append( + f'num_bot_responses should not be greater than 5 and of type int: {data.get("name")}') + llm_prompts_errors = DataValidation.validate_llm_prompts(data['llm_prompts']) + if data.get('hyperparameters'): + llm_hyperparameters_errors = DataValidation.validate_llm_prompts_hyperparameters( + data.get('hyperparameters'), data.get("llm_type", "openai"), bot) + data_error.extend(llm_hyperparameters_errors) + data_error.extend(llm_prompts_errors) + + return data_error + + @staticmethod + def validate_pyscript_action(bot, data: dict): + data_error = [] + if not data.get('source_code'): + data_error.append('Script is required for pyscript action!') + return data_error + + compile_time_error = DataValidation.validate_python_script_compile_time(data['source_code']) + if compile_time_error: + data_error.append(f"Error in python script: {compile_time_error}") + return data_error + + @staticmethod + def validate_python_script_compile_time(script: str): + try: + ast.parse(script) + except SyntaxError as e: + return e.msg + return None + + @staticmethod + def validate_llm_prompts_hyperparameters(hyperparameters: dict, llm_type: str, bot: str = None): + error_list = [] + try: + Utility.validate_llm_hyperparameters(hyperparameters, llm_type, bot, AppException) + except AppException as e: + error_list.append(e.__str__()) + return error_list + + @staticmethod + def validate_llm_prompts(llm_prompts: list): + error_list = [] + system_prompt_count = 0 + history_prompt_count = 0 + for prompt in llm_prompts: + if prompt.get('hyperparameters') is not None: + hyperparameters = prompt.get('hyperparameters') + for key, value in hyperparameters.items(): + if key == 'similarity_threshold': + if not (0.3 <= value <= 1.0) or not ( + isinstance(value, float) or isinstance(value, int)): + error_list.append( + "similarity_threshold should be within 0.3 and 1.0 and of type int or float!") + if key == 'top_results' and (value > 30 or not isinstance(value, int)): + error_list.append("top_results should not be greater than 30 and of type int!") + + if prompt.get('type') == 'system': + system_prompt_count += 1 + elif prompt.get('source') == 'history': + history_prompt_count += 1 + if prompt.get('type') not in ['user', 'system', 'query']: + error_list.append('Invalid prompt type') + if prompt.get('source') not in ['static', 'slot', 'action', 'history', 'bot_content']: + error_list.append('Invalid prompt source') + if prompt.get('type') and not isinstance(prompt.get('type'), str): + error_list.append('type in LLM Prompts should be of type string.') + if prompt.get('source') and not isinstance(prompt.get('source'), str): + error_list.append('source in LLM Prompts should be of type string.') + if prompt.get('instructions') and not isinstance(prompt.get('instructions'), str): + error_list.append('Instructions in LLM Prompts should be of type string.') + if prompt.get('type') == 'system' and prompt.get('source') != 'static': + error_list.append('System prompt must have static source') + if prompt.get('type') == 'query' and prompt.get('source') != 'static': + error_list.append('Query prompt must have static source') + if not prompt.get('data') and prompt.get('source') == 'action': + error_list.append('Data must contain action name') + if not prompt.get('data') and prompt.get('source') == 'slot': + error_list.append('Data must contain slot name') + if Utility.check_empty_string(prompt.get('name')): + error_list.append('Name cannot be empty') + if prompt.get('data') and not isinstance(prompt.get('data'), str): + error_list.append('data field in prompts should of type string.') + if not prompt.get('data') and prompt.get('source') == 'static': + error_list.append('data is required for static prompts') + if prompt.get('source') == 'bot_content' and Utility.check_empty_string(prompt.get('data')): + error_list.append("Collection is required for bot content prompts!") + if system_prompt_count > 1: + error_list.append('Only one system prompt can be present') + if system_prompt_count == 0: + error_list.append('System prompt is required') + if history_prompt_count > 1: + error_list.append('Only one history source can be present') + return error_list + + @staticmethod + def validate_callback_config(bot: str, data: dict): + data_error = [] + if not data.get('pyscript_code'): + data_error.append('pyscript_code is required') + return data_error + + compile_time_error = DataValidation.validate_python_script_compile_time(data['pyscript_code']) + if compile_time_error: + data_error.append(f"Error in python script: {compile_time_error}") + return data_error + + @staticmethod + def modify_callback_config(bot: str, data: dict) -> dict: + data['token_hash'] = uuid7().hex + data['validation_secret'] = encrypt_secret(uuid7().hex) + return data diff --git a/kairon/shared/data/processor.py b/kairon/shared/data/processor.py index c73a39702..541d4a46c 100644 --- a/kairon/shared/data/processor.py +++ b/kairon/shared/data/processor.py @@ -161,6 +161,8 @@ MultiflowStories, MultiflowStoryEvents, MultiFlowStoryMetadata, Synonyms, Lookup, Analytics, ModelTraining, ConversationsHistoryDeleteLogs, DemoRequestLogs ) +from .action_serializer import ActionSerializer +from .data_validation import DataValidation from .utils import DataUtility from ..callback.data_objects import CallbackConfig, CallbackLog from ..chat.broadcast.data_objects import MessageBroadcastLogs @@ -235,8 +237,9 @@ def download_files(self, bot: Text, user: Text, download_multiflow: bool = False stories = stories.merge(multiflow_stories[0]) rules = rules.merge(multiflow_stories[1]) multiflow_stories = self.load_multiflow_stories_yaml(bot) - actions = self.load_action_configurations(bot) + #actions = self.load_action_configurations(bot) bot_content = self.load_bot_content(bot) + actions, other_collections = ActionSerializer.serialize(bot) return Utility.create_zip_file( nlu, domain, @@ -247,7 +250,8 @@ def download_files(self, bot: Text, user: Text, download_multiflow: bool = False actions, multiflow_stories, chat_client_config, - bot_content + bot_content, + other_collections, ) async def apply_template(self, template: Text, bot: Text, user: Text): @@ -289,6 +293,7 @@ async def save_from_path( actions_yml = os.path.join(path, "actions.yml") multiflow_stories_yml = os.path.join(path, "multiflow_stories.yml") bot_content_yml = os.path.join(path, "bot_content.yml") + other_collections_yml = os.path.join(path, "other_collections.yml") importer = RasaFileImporter.load_from_config( config_path=config_path, domain_path=domain_path, @@ -309,7 +314,14 @@ async def save_from_path( if bot_content_yml else None ) - TrainingDataValidator.validate_custom_actions(actions) + other_collections = ( + Utility.read_yaml(other_collections_yml) + if other_collections_yml + else None + ) + + ActionSerializer.validate(bot, actions, other_collections) + self.save_training_data( bot, @@ -321,6 +333,7 @@ async def save_from_path( actions, multiflow_stories, bot_content, + other_collections=other_collections, overwrite=overwrite, what=REQUIREMENTS.copy() - {"chat_client_config"}, ) @@ -340,6 +353,7 @@ def save_training_data( multiflow_stories: dict = None, bot_content: list = None, chat_client_config: dict = None, + other_collections: dict = None, overwrite: bool = False, what: set = REQUIREMENTS.copy(), ): @@ -347,7 +361,8 @@ def save_training_data( self.delete_bot_data(bot, user, what) if "actions" in what: - self.save_integrated_actions(actions, bot, user) + #self.save_integrated_actions(actions, bot, user) + ActionSerializer.deserialize(bot, user, actions, other_collections, overwrite) if "domain" in what: self.save_domain(domain, bot, user) if "stories" in what: @@ -551,7 +566,7 @@ def save_domain(self, domain: Domain, bot: Text, user: Text): lambda actions: not actions.startswith("utter_"), domain.user_actions ) ) - self.__save_actions(actions, bot, user) + # self.verify_actions_presence(actions, bot, user) self.__save_responses(domain.responses, bot, user) self.save_utterances(domain.responses.keys(), bot, user) self.__save_slots(domain.slots, bot, user) @@ -1070,11 +1085,16 @@ def __save_form_logic(self, name, slots, bot, user): if Utility.is_exist( Actions, raise_error=False, name=f"validate_{name}", bot=bot, status=True ): - form_validation_action = Actions.objects( - name=f"validate_{name}", bot=bot, status=True - ).get() - form_validation_action.type = ActionType.form_validation_action.value - form_validation_action.save() + try: + form_validation_action = Actions.objects( + name__iexact=f"validate_{name}", bot=bot, status=True + ).get() + form_validation_action.type = ActionType.form_validation_action.value + form_validation_action.save() + except Exception as e: + print(e) + + self.__check_for_form_and_action_existance(bot, name) form = Forms(name=name, required_slots=slots, bot=bot, user=user) form.clean() @@ -1147,6 +1167,13 @@ def __check_for_form_and_action_existance(self, bot: Text, name: Text, action_ty exp_message=f"Form with the name '{name}' already exists", name=name, bot=bot, status=True) + # def verify_actions_presence(self, actions: list[str], bot: str, user: str): + # if actions: + # found_names = Actions.objects(name__in=actions, bot=bot, user=user).values_list('name') + # for action in actions: + # if action not in found_names: + # raise AppException(f"Action [{action}] not present in actions.yml") + def __save_actions(self, actions, bot: Text, user: Text): if actions: new_actions = list(self.__extract_actions(actions, bot, user)) @@ -3955,7 +3982,9 @@ def get_http_action_config(self, bot: str, action_name: str): http_config_dict["content_type"] = { HttpRequestContentType.json.value: HttpContentType.application_json.value, + HttpContentType.application_json.value: HttpContentType.application_json.value, HttpRequestContentType.data.value: HttpContentType.urlencoded_form_data.value, + HttpContentType.urlencoded_form_data.value: HttpContentType.urlencoded_form_data.value, }[http_config_dict["content_type"]] return http_config_dict except DoesNotExist as ex: @@ -3985,6 +4014,8 @@ def add_pyscript_action(self, pyscript_config: Dict, user: str, bot: str): Utility.is_valid_action_name( pyscript_config.get("name"), bot, PyscriptActionConfig ) + if compile_error := DataValidation.validate_python_script_compile_time(pyscript_config["source_code"]): + raise AppException(f"source code syntax error: {compile_error}") action_id = ( PyscriptActionConfig( name=pyscript_config["name"], @@ -4024,9 +4055,12 @@ def update_pyscript_action(self, request_data: Dict, user: str, bot: str): raise AppException( f'Action with name "{request_data.get("name")}" not found' ) + if compile_error := DataValidation.validate_python_script_compile_time(request_data["source_code"]): + raise AppException(f"source code syntax error: {compile_error}") action = PyscriptActionConfig.objects( name=request_data.get("name"), bot=bot, status=True ).get() + action.source_code = request_data["source_code"] action.dispatch_response = request_data["dispatch_response"] action.user = user @@ -4710,6 +4744,7 @@ def save_integrated_actions(self, actions: dict, bot: Text, user: Text): :param user: user id :return: None """ + if not actions: return document_types = { @@ -5039,10 +5074,11 @@ def save_data_without_event( if os.path.exists(actions_path): actions = Utility.read_yaml(actions_path) ( - validation_failed, + is_successful, error_summary, actions_count, - ) = TrainingDataValidator.validate_custom_actions(actions) + ) = ActionSerializer.validate(bot, actions, {}) + validation_failed = not is_successful component_count.update(actions_count) if os.path.exists(config_path): config = Utility.read_yaml(config_path) @@ -6356,17 +6392,6 @@ def update_slot_mapping(self, mapping: dict, slot_mapping_id: str): except Exception as e: raise AppException(e) - def delete_single_slot_mapping(self, slot_mapping_id: str, user: str = None): - """ - Delete slot mapping. - - :param slot_mapping_id: document id of the mapping - """ - try: - slot_mapping = SlotMapping.objects(id=slot_mapping_id, status=True).get() - Utility.delete_documents(slot_mapping, user) - except Exception as e: - raise AppException(e) def __prepare_slot_mappings(self, bot: Text): """ @@ -7963,6 +7988,8 @@ def add_callback(self, request_data: dict, bot: Text): standalone_id_path = request_data.get("standalone_id_path") if standalone and not standalone_id_path: raise AppException("Standalone id path is required!") + if compile_error := DataValidation.validate_python_script_compile_time(pyscript_code): + raise AppException(f"source code syntax error: {compile_error}") config = CallbackConfig.create_entry(bot, name, pyscript_code, @@ -7983,6 +8010,9 @@ def edit_callback(self, request_data: dict, bot: Text): """ name = request_data.get("name") request_data.pop('name') + if pyscript_code := request_data.get('pyscript_code'): + if compile_error := DataValidation.validate_python_script_compile_time(pyscript_code): + raise AppException(f"source code syntax error: {compile_error}") config = CallbackConfig.edit(bot, name, **request_data) config.pop('_id') return config @@ -8352,8 +8382,8 @@ def get_column_datatype_dict(self, bot, table_name): schemas = list(cognition_processor.list_cognition_schema(bot)) table_metadata = next((schema['metadata'] for schema in schemas if schema['collection_name'] == table_name.lower()), None) - if table_metadata is None: - print(f"Schema for table '{table_name}' not found.") + if not table_metadata: + logger.info(f"Schema for table '{table_name}' not found.") column_datatype_dict = {column['column_name']: column['data_type'] for column in table_metadata} diff --git a/kairon/shared/importer/processor.py b/kairon/shared/importer/processor.py index df014f1fd..02e9c33bf 100644 --- a/kairon/shared/importer/processor.py +++ b/kairon/shared/importer/processor.py @@ -80,8 +80,8 @@ def update_summary(bot: str, user: str, component_count: dict, summary: dict, st doc.training_examples = TrainingComponentLog(count=component_count['training_examples'], data=summary.get('training_examples')) doc.config = TrainingComponentLog(data=summary.get('config')) doc.rules = TrainingComponentLog(count=component_count['rules'], data=summary.get('rules')) - action_summary = [{'type': s, 'count': component_count.get(s), 'data': summary.get(s)} for s in - summary.keys() if s in {f'{a_type.value}s' for a_type in ActionType}] + action_summary = [{'type': f"{s}s", 'count': component_count.get(s), 'data': summary.get(s)} for s in + summary.keys() if s in {f'{a_type.value}' for a_type in ActionType}] doc.multiflow_stories = TrainingComponentLog(count=component_count.get('multiflow_stories'), data=summary.get('multiflow_stories')) doc.bot_content = TrainingComponentLog(data=summary.get('bot_content')) diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index 683013bc1..0407c1248 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -1138,6 +1138,7 @@ def write_training_data( chat_client_config: dict = None, multiflow_stories: dict = None, bot_content: list = None, + other_collections: dict = None, ): """ convert mongo data to individual files @@ -1151,6 +1152,7 @@ def write_training_data( :param actions: action configuration data :param multiflow_stories: multiflow_stories configurations :param bot_content: bot content + :param other_collections: other collections used by the bot :return: files path """ from rasa.shared.core.training_data.story_writer.yaml_story_writer import ( @@ -1175,6 +1177,7 @@ def write_training_data( chat_client_config_path = os.path.join(temp_path, "chat_client_config.yml") multiflow_stories_config_path = os.path.join(temp_path, "multiflow_stories.yml") bot_content_path = os.path.join(temp_path, "bot_content.yml") + other_collections_path = os.path.join(temp_path, "other_collections.yml") nlu_as_str = nlu.nlu_as_yaml().encode() config_as_str = yaml.dump(config).encode() @@ -1204,6 +1207,11 @@ def write_training_data( Utility.write_to_file( bot_content_path, bot_content_as_str ) + if other_collections: + other_collections_as_str = yaml.dump(other_collections).encode() + Utility.write_to_file( + other_collections_path, other_collections_as_str + ) return temp_path @staticmethod @@ -1218,6 +1226,7 @@ def create_zip_file( multiflow_stories: Dict = None, chat_client_config: Dict = None, bot_content: List = None, + other_collections: Dict = None, ): """ adds training files to zip @@ -1232,6 +1241,7 @@ def create_zip_file( :param actions: action configurations :param multiflow_stories: multiflow_stories configurations :param bot_content: bot_content + :param other_collections: other collections used by the bot :return: None """ directory = Utility.write_training_data( @@ -1244,6 +1254,7 @@ def create_zip_file( chat_client_config, multiflow_stories, bot_content, + other_collections ) zip_path = os.path.join(tempfile.gettempdir(), bot) zip_file = shutil.make_archive(zip_path, format="zip", root_dir=directory) diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index ffcb82431..dca4792c9 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -1438,7 +1438,7 @@ def test_upload_doc_content(): ) actual = response.json() - assert actual["success"] == True + assert actual["success"] assert actual["message"] == "Document content upload in progress! Check logs." assert actual["error_code"] == 0 @@ -1451,14 +1451,14 @@ def test_upload_doc_content(): headers={"Authorization": pytest.token_type + " " + pytest.access_token}, ) actual = response.json() - assert actual["success"] == True + assert actual["success"] assert actual["error_code"] == 0 logs = actual['data']['logs'] assert len(logs) == 1 assert logs[0]['file_received'] == 'Salesstore.csv' assert logs[0]['status'] == 'Success' assert logs[0]['event_status'] == 'Completed' - assert logs[0]['is_data_uploaded'] == True + assert logs[0]['is_data_uploaded'] assert logs[0]['start_timestamp'] is not None assert logs[0]['end_timestamp'] is not None assert logs[0]['validation_errors'] == {} @@ -1559,7 +1559,7 @@ def test_upload_doc_content_append(): assert logs[0]['file_received'] == 'Salesstore.csv' assert logs[0]['status'] == 'Success' assert logs[0]['event_status'] == 'Completed' - assert logs[0]['is_data_uploaded'] == True + assert logs[0]['is_data_uploaded'] assert logs[0]['start_timestamp'] is not None assert logs[0]['end_timestamp'] is not None assert logs[0]['validation_errors'] == {} @@ -1626,7 +1626,7 @@ def test_upload_doc_content_basic_validation_failure(): ) actual = response.json() - assert actual["success"] == True + assert actual["success"] assert actual["message"] == "Document content upload in progress! Check logs." assert actual["error_code"] == 0 @@ -1636,7 +1636,7 @@ def test_upload_doc_content_basic_validation_failure(): ) actual = response.json() - assert actual["success"] == True + assert actual["success"] assert actual["error_code"] == 0 logs = actual['data']['logs'] assert len(logs) == 3 @@ -1695,7 +1695,7 @@ def test_download_error_csv_error_report_not_found(): ) actual = response.json() - assert actual["success"] == True + assert actual["success"] assert actual["message"] == "Document content upload in progress! Check logs." assert actual["error_code"] == 0 @@ -1708,7 +1708,7 @@ def test_download_error_csv_error_report_not_found(): headers={"Authorization": pytest.token_type + " " + pytest.access_token}, ) actual = response.json() - assert actual["success"] == True + assert actual["success"] assert actual["error_code"] == 0 event_id = ContentImporterLogProcessor.get_event_id_for_latest_event(pytest.bot) @@ -1765,7 +1765,7 @@ def test_upload_doc_content_datatype_validation_failure(): ) actual = response.json() - assert actual["success"] == True + assert actual["success"] assert actual["message"] == "Document content upload in progress! Check logs." assert actual["error_code"] == 0 @@ -1780,7 +1780,7 @@ def test_upload_doc_content_datatype_validation_failure(): ) actual = response.json() - assert actual["success"] == True + assert actual["success"] assert actual["error_code"] == 0 logs = actual['data']['logs'] assert len(logs) == 5 @@ -3240,6 +3240,25 @@ def test_get_live_agent_after_disabled(): assert actual["success"] +def test_callback_config_add_syntax_error(): + request_body = { + "name": "callback_1", + "pyscript_code": "bot_response = Hello World!'", + "validation_secret": "string", + "execution_mode": "async" + } + + response = client.post( + url=f"/api/bot/{pytest.bot}/action/callback", + json=request_body, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + + actual = response.json() + print(actual) + assert not actual['success'] + assert actual['error_code'] == 422 + assert actual['message'] == 'source code syntax error: unterminated string literal (detected at line 1)' def test_callback_config_add(): request_body = { @@ -3305,7 +3324,7 @@ def test_callback_get_standalone_url(): assert response.status_code == 200 actual = response.json() print(actual) - assert actual['success'] == True + assert actual['success'] assert actual['error_code'] == 0 assert isinstance(actual['data'], str) assert '/callback/s/' in actual['data'] @@ -3330,6 +3349,27 @@ def test_callback_config_add_standalone_fail_no_path(): assert actual == {'success': False, 'message': 'Standalone id path is required!', 'data': None, 'error_code': 422} +def test_callback_config_edit_syntex_error(): + request_body = { + "name": "callback_1", + "pyscript_code": "bot_response is if = 'Hello World2!'", + "validation_secret": "string", + "execution_mode": "async" + } + + response = client.put( + url=f"/api/bot/{pytest.bot}/action/callback", + json=request_body, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + + actual = response.json() + print(actual) + + assert not actual['success'] + assert actual['error_code'] == 422 + assert actual['message'] == 'source code syntax error: invalid syntax' + def test_callback_config_edit(): request_body = { "name": "callback_1", @@ -3364,7 +3404,7 @@ def test_callback_config_get(): ) actual = response.json() - assert actual['success'] == True + assert actual['success'] assert actual['error_code'] == 0 assert len(actual['data']) == 2 assert 'callback_1' in actual['data'] @@ -3377,7 +3417,7 @@ def test_callback_single_config_get(): ) actual = response.json() - assert actual['success'] == True + assert actual['success'] assert actual['error_code'] == 0 assert actual['data']['name'] == 'callback_1' assert actual['data']['pyscript_code'] == "bot_response = 'Hello World2!'" @@ -3443,7 +3483,7 @@ def test_callback_action_add(): actual = response.json() print(actual) - assert actual['success'] == True + assert actual['success'] assert actual['message'] == 'Callback action added successfully!' data = actual['data'] assert data['name'] == 'callback_action1' @@ -3451,8 +3491,6 @@ def test_callback_action_add(): def test_callback_action_update(): - - request_body = { "name": "callback_action1", "callback_name": "callback_1", @@ -3470,7 +3508,7 @@ def test_callback_action_update(): actual = response.json() print(actual) - assert actual['success'] == True + assert actual['success'] assert actual['message'] == 'Callback action updated successfully!' @@ -3481,7 +3519,7 @@ def test_callback_action_get(): ) actual = response.json() - assert actual['success'] == True + assert actual['success'] assert actual['error_code'] == 0 assert actual['data']['name'] == 'callback_action1' assert actual['data']['callback_name'] == 'callback_1' @@ -3496,7 +3534,7 @@ def test_callback_action_get_all(): actual = response.json() print(actual) - assert actual['success'] == True + assert actual['success'] assert actual['error_code'] == 0 assert isinstance(actual['data'], list) assert len(actual['data']) == 1 @@ -3525,7 +3563,7 @@ def test_callback_get_logs(): actual = response.json() - assert actual['success'] == True + assert actual['success'] assert actual['error_code'] == 0 assert isinstance(actual['data']['logs'], list) assert len(actual['data']['logs']) == 1 @@ -3543,13 +3581,7 @@ def test_callback_action_delete(): def test_add_pyscript_action_empty_name(): - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" request_body = { "name": "", "source_code": script, @@ -3594,13 +3626,7 @@ def test_add_pyscript_action_empty_source_code(): def test_add_pyscript_action_with_utter(): - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" request_body = { "name": "utter_pyscript_action", "source_code": script, @@ -3618,7 +3644,7 @@ def test_add_pyscript_action_with_utter(): assert not actual["success"] -def test_add_pyscript_action(): +def test_add_pyscript_action_syntex_error(): script = """ data = [1, 2, 3, 4, 5] total = 0 @@ -3638,19 +3664,32 @@ def test_add_pyscript_action(): ) actual = response.json() + print(actual) + assert actual["error_code"] == 422 + assert actual["message"] == "source code syntax error: unexpected indent" + assert not actual["success"] + +def test_add_pyscript_action(): + script= "bot_response='hello world'" + request_body = { + "name": "test_add_pyscript_action", + "source_code": script, + "dispatch_response": False, + } + response = client.post( + url=f"/api/bot/{pytest.bot}/action/pyscript", + json=request_body, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + + actual = response.json() + print(actual) assert actual["error_code"] == 0 assert actual["message"] == "Action added!" assert actual["success"] - def test_add_pyscript_action_name_already_exist(): - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script= "bot_response='hello world'" request_body = { "name": "test_add_pyscript_action", "source_code": script, @@ -3669,13 +3708,7 @@ def test_add_pyscript_action_name_already_exist(): def test_add_pyscript_action_case_insensitivity(): - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script= "bot_response='hello world'" request_body = { "name": "TEST_ADD_PYSCRIPT_ACTION_CASE_INSENSITIVITY", "source_code": script, @@ -3697,13 +3730,7 @@ def test_add_pyscript_action_case_insensitivity(): def test_update_pyscript_action_does_not_exist(): - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script= "bot_response='hello world'" request_body = { "name": "test_update_pyscript_action", "source_code": script, @@ -3723,7 +3750,7 @@ def test_update_pyscript_action_does_not_exist(): assert not actual["success"] -def test_update_pyscript_action(): +def test_update_pyscript_action_syntex_error(): script = """ data = [1, 2, 3, 4, 5, 6, 7] total = 0 @@ -3742,27 +3769,33 @@ def test_update_pyscript_action(): headers={"Authorization": pytest.token_type + " " + pytest.access_token}, ) + actual = response.json() + assert actual["error_code"] == 422 + assert actual["message"] == "source code syntax error: unexpected indent" + assert not actual["success"] + + +def test_update_pyscript_action(): + script = "bot_response='hello world'" + request_body = { + "name": "test_add_pyscript_action", + "source_code": script, + "dispatch_response": True, + } + response = client.put( + url=f"/api/bot/{pytest.bot}/action/pyscript", + json=request_body, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() assert actual["error_code"] == 0 assert actual["message"] == "Action updated!" assert actual["success"] - def test_list_pyscript_actions(): - script1 = """ - data = [1, 2, 3, 4, 5, 6, 7] - total = 0 - for i in data: - total += i - print(total) - """ - script2 = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script1 = "bot_response='hello world'" + script2 = "bot_response='hello world'" response = client.get( url=f"/api/bot/{pytest.bot}/action/pyscript", headers={"Authorization": pytest.token_type + " " + pytest.access_token}, @@ -3805,13 +3838,7 @@ def test_delete_pyscript_action(): def test_list_pyscript_actions_after_action_deleted(): - script2 = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script2 = "bot_response='hello world'" response = client.get( url=f"/api/bot/{pytest.bot}/action/pyscript", headers={"Authorization": pytest.token_type + " " + pytest.access_token}, @@ -4395,13 +4422,7 @@ def test_get_broadcast_logs_with_resend_broadcasts(): def test_list_available_actions(): - script = """ - data = [1, 2, 3, 4, 5, 6] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" request_body = { "name": "test_list_available_actions_pyscript_action", "source_code": script, @@ -7629,7 +7650,7 @@ def __mock_qna(*args, **kwargs): assert actual["data"] == {"qna": data, "total": 0} -def test_model_testing_not_trained(): +def test_model_testing_exhosted_daily_limit(): bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.test_limit_per_day = 0 bot_settings.save() @@ -7681,17 +7702,23 @@ def test_get_data_importer_logs(): 'utterances_count': 14, 'forms_count': 2, 'entities_count': 8, 'data': []}, 'config': {'count': 0, 'data': []}, 'rules': {'count': 1, 'data': []}, - 'actions': [{'type': 'http_actions', 'count': 5, 'data': []}, - {'type': 'slot_set_actions', 'count': 0, 'data': []}, - {'type': 'form_validation_actions', 'count': 0, 'data': []}, + 'actions': [{'type': 'http_actions', 'count': 14, 'data': []}, + {'type': 'two_stage_fallbacks', 'count': 0, 'data': []}, {'type': 'email_actions', 'count': 0, 'data': []}, - {'type': 'google_search_actions', 'count': 0, 'data': []}, - {'type': 'jira_actions', 'count': 0, 'data': []}, {'type': 'zendesk_actions', 'count': 0, 'data': []}, + {'type': 'jira_actions', 'count': 0, 'data': []}, + {'type': 'form_validation_actions', 'count': 0, 'data': []}, + {'type': 'slot_set_actions', 'count': 0, 'data': []}, + {'type': 'google_search_actions', 'count': 0, 'data': []}, {'type': 'pipedrive_leads_actions', 'count': 0, 'data': []}, {'type': 'prompt_actions', 'count': 0, 'data': []}, + {'type': 'web_search_actions', 'count': 0, 'data': []}, {'type': 'razorpay_actions', 'count': 0, 'data': []}, - {'type': 'pyscript_actions', 'count': 0, 'data': []}], + {'type': 'pyscript_actions', 'count': 0, 'data': []}, + {'type': 'database_actions', 'count': 0, 'data': []}, + {'type': 'live_agent_actions', 'count': 0, 'data': []}, + {'type': 'callback_actions', 'count': 0, 'data': []}, + {'type': 'schedule_actions', 'count': 0, 'data': []}], 'multiflow_stories': {'count': 0, 'data': []}, 'bot_content': {'count': 0, 'data': []}, 'user_actions': {'count': 9, 'data': []}, @@ -7728,18 +7755,23 @@ def test_get_data_importer_logs(): 'utterances_count': 27, 'forms_count': 2, 'entities_count': 9, 'data': []} assert actual['data']["logs"][3]['config'] == {'count': 0, 'data': []} - assert actual['data']["logs"][3]['actions'] == [{'type': 'http_actions', 'count': 5, 'data': []}, - {'type': 'slot_set_actions', 'count': 0, 'data': []}, - {'type': 'form_validation_actions', 'count': 0, 'data': []}, + assert actual['data']["logs"][3]['actions'] == [{'type': 'http_actions', 'count': 17, 'data': []}, + {'type': 'two_stage_fallbacks', 'count': 0, 'data': []}, {'type': 'email_actions', 'count': 0, 'data': []}, - {'type': 'google_search_actions', 'count': 1, 'data': []}, - {'type': 'jira_actions', 'count': 0, 'data': []}, {'type': 'zendesk_actions', 'count': 0, 'data': []}, + {'type': 'jira_actions', 'count': 0, 'data': []}, + {'type': 'form_validation_actions', 'count': 1, 'data': []}, + {'type': 'slot_set_actions', 'count': 0, 'data': []}, + {'type': 'google_search_actions', 'count': 1, 'data': []}, {'type': 'pipedrive_leads_actions', 'count': 0, 'data': []}, {'type': 'prompt_actions', 'count': 0, 'data': []}, + {'type': 'web_search_actions', 'count': 0, 'data': []}, {'type': 'razorpay_actions', 'count': 0, 'data': []}, - {'type': 'pyscript_actions', 'count': 0, 'data': []} - ] + {'type': 'pyscript_actions', 'count': 0, 'data': []}, + {'type': 'database_actions', 'count': 0, 'data': []}, + {'type': 'live_agent_actions', 'count': 0, 'data': []}, + {'type': 'callback_actions', 'count': 0, 'data': []}, + {'type': 'schedule_actions', 'count': 0, 'data': []}] assert actual['data']["logs"][3]['is_data_uploaded'] assert set(actual['data']["logs"][3]['files_received']) == {'rules', 'stories', 'nlu', 'config', 'domain', 'actions', 'chat_client_config', 'multiflow_stories', @@ -8186,7 +8218,7 @@ def test_training_example_does_not_exist(): ) actual = response.json() assert actual["data"] == {"is_exists": False, "intent": None} - assert actual["success"] == True + assert actual["success"] assert actual["error_code"] == 0 assert Utility.check_empty_string(actual["message"]) @@ -16619,8 +16651,8 @@ def test_upload_with_http_error(): assert actual["data"]["logs"][0]["start_timestamp"] assert actual["data"]["logs"][0]["start_timestamp"] assert ( - "Required http action fields" - in actual["data"]["logs"][0]["actions"][0]["data"][0] + "Required fields {'request_method'} not found" + in str(actual["data"]["logs"][0]["actions"][0]["data"][0]) ) assert actual["data"]["logs"][0]["config"]["data"] == ["Invalid component XYZ"] @@ -16649,6 +16681,7 @@ def test_upload_actions_and_config(): files=files, ) actual = response.json() + print(actual) assert actual["message"] == "Upload in progress! Check logs." assert actual["error_code"] == 0 assert actual["data"] is None @@ -16659,6 +16692,7 @@ def test_upload_actions_and_config(): headers={"Authorization": pytest.token_type + " " + pytest.access_token}, ) actual = response.json() + print(actual) assert actual["success"] assert actual["error_code"] == 0 assert len(actual["data"]["logs"]) == 5 @@ -16668,17 +16702,24 @@ def test_upload_actions_and_config(): assert actual['data']["logs"][0]['is_data_uploaded'] assert actual['data']["logs"][0]['start_timestamp'] assert actual['data']["logs"][0]['end_timestamp'] - assert actual['data']["logs"][0]['actions'] == [{'type': 'http_actions', 'count': 5, 'data': []}, - {'type': 'slot_set_actions', 'count': 0, 'data': []}, - {'type': 'form_validation_actions', 'count': 0, 'data': []}, + print(actual['data']["logs"][0]['actions']) + assert actual['data']["logs"][0]['actions'] == [{'type': 'http_actions', 'count': 17, 'data': []}, + {'type': 'two_stage_fallbacks', 'count': 0, 'data': []}, {'type': 'email_actions', 'count': 0, 'data': []}, - {'type': 'google_search_actions', 'count': 1, 'data': []}, - {'type': 'jira_actions', 'count': 0, 'data': []}, {'type': 'zendesk_actions', 'count': 0, 'data': []}, - {'type': 'pipedrive_leads_actions', 'data': [], 'count': 0}, - {'type': 'prompt_actions', 'data': [], 'count': 0}, - {'type': 'razorpay_actions', 'data': [], 'count': 0}, - {'type': 'pyscript_actions', 'data': [], 'count': 0}] + {'type': 'jira_actions', 'count': 0, 'data': []}, + {'type': 'form_validation_actions', 'count': 1, 'data': []}, + {'type': 'slot_set_actions', 'count': 0, 'data': []}, + {'type': 'google_search_actions', 'count': 1, 'data': []}, + {'type': 'pipedrive_leads_actions', 'count': 0, 'data': []}, + {'type': 'prompt_actions', 'count': 0, 'data': []}, + {'type': 'web_search_actions', 'count': 0, 'data': []}, + {'type': 'razorpay_actions', 'count': 0, 'data': []}, + {'type': 'pyscript_actions', 'count': 0, 'data': []}, + {'type': 'database_actions', 'count': 0, 'data': []}, + {'type': 'live_agent_actions', 'count': 0, 'data': []}, + {'type': 'callback_actions', 'count': 0, 'data': []}, + {'type': 'schedule_actions', 'count': 0, 'data': []}] assert not actual['data']["logs"][0]['config']['data'] response = client.get( @@ -16688,7 +16729,7 @@ def test_upload_actions_and_config(): actual = response.json() assert actual["success"] assert actual["error_code"] == 0 - assert len(actual["data"]) == 5 + assert len(actual["data"]) == 17 @patch( diff --git a/tests/testing_data/actions/actions.yml b/tests/testing_data/actions/actions.yml index 364dbccce..baec608b9 100644 --- a/tests/testing_data/actions/actions.yml +++ b/tests/testing_data/actions/actions.yml @@ -252,6 +252,17 @@ two_stage_fallback: prompt_action: - name: prompt_action_1 num_bot_responses: 5 + llm_type: openai + hyperparameters: + temperature: 0.5 + max_tokens: 100 + top_p: 1 + frequency_penalty: 0 + presence_penalty: 0 + n: 1 + stop: null + stream: false + model: gpt-4o-mini llm_prompts: - name: System Prompt data: You are a personal assistant. @@ -266,6 +277,17 @@ prompt_action: is_enabled: True - name: prompt_action_2 num_bot_responses: 5 + llm_type: openai + hyperparameters: + temperature: 0.5 + max_tokens: 100 + top_p: 1 + frequency_penalty: 0 + presence_penalty: 0 + n: 1 + stop: null + stream: false + model: gpt-4o-mini llm_prompts: - name: System Prompt data: You are a personal assistant. diff --git a/tests/testing_data/error/actions.yml b/tests/testing_data/error/actions.yml index 2b3ffb87e..d36bad885 100644 --- a/tests/testing_data/error/actions.yml +++ b/tests/testing_data/error/actions.yml @@ -1,5 +1,5 @@ http_action: -- action_name: action_performanceUser1001@digite.com +- action_name: action_performanceUser1001@digitecom http_url: http://www.alphabet.com headers: - key: auth_token @@ -15,7 +15,7 @@ http_action: request_method: GET response: value: json -- action_name: action_performanceUser1000@digite.com +- action_name: action_performanceUser1000@digitecom http_url: http://www.alphabet.com params_list: - key: testParam1 diff --git a/tests/testing_data/multiflow_stories/valid_with_multiflow/actions.yml b/tests/testing_data/multiflow_stories/valid_with_multiflow/actions.yml index 37a7e3030..b1f9c5790 100644 --- a/tests/testing_data/multiflow_stories/valid_with_multiflow/actions.yml +++ b/tests/testing_data/multiflow_stories/valid_with_multiflow/actions.yml @@ -1,3 +1,28 @@ +http_action: +- action_name: action_say_hello + content_type: json + headers: [] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + set_slots: [] +- action_name: action_say_goodbye + content_type: json + headers: [] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + set_slots: [] form_validation_action: - invalid_response: Please ensure that you attach a valid image file. This upload only supports image file. diff --git a/tests/testing_data/valid_yml/actions.yml b/tests/testing_data/valid_yml/actions.yml index b57afc0b4..a4ad2bb56 100644 --- a/tests/testing_data/valid_yml/actions.yml +++ b/tests/testing_data/valid_yml/actions.yml @@ -52,3 +52,51 @@ http_action: request_method: GET response: value: json +- action_name: action_small_talk + http_url: http://www.alphabet.com + request_method: GET + response: + value: json +- action_name: action_identify_ticket_attributes + http_url: http://www.alphabet.com + request_method: GET + response: + value: json +- action_name: action_log_ticket + http_url: http://www.alphabet.com + request_method: GET + response: + value: json +- action_name: action_get_ticket_status + http_url: http://www.alphabet.com + request_method: GET + response: + value: json +- action_name: action_clear_file + http_url: http://www.alphabet.com + request_method: GET + response: + value: json +- action_name: action_clear_priority + http_url: http://www.alphabet.com + request_method: GET + response: + value: json +- action_name: ticket_attributes_form_action + http_url: http://www.alphabet.com + request_method: GET + response: + value: json +- action_name: ticket_file_form_action + http_url: http://www.alphabet.com + request_method: GET + response: + value: json +- action_name: action_validate_ticket_for_status + http_url: http://www.alphabet.com + request_method: GET + response: + value: json + + + diff --git a/tests/testing_data/validator/valid/actions.yml b/tests/testing_data/validator/valid/actions.yml new file mode 100644 index 000000000..2cd53cbfd --- /dev/null +++ b/tests/testing_data/validator/valid/actions.yml @@ -0,0 +1,25 @@ +http_action: +- action_name: action_say_hello + content_type: json + headers: [] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + set_slots: [] +- action_name: action_say_goodbye + content_type: json + headers: [] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + set_slots: [] \ No newline at end of file diff --git a/tests/testing_data/yml_training_files/actions.yml b/tests/testing_data/yml_training_files/actions.yml index cd58dde5a..7e00a2046 100644 --- a/tests/testing_data/yml_training_files/actions.yml +++ b/tests/testing_data/yml_training_files/actions.yml @@ -1,65 +1,228 @@ http_action: -- action_name: action_performanceUser1001@digite.com - http_url: http://www.alphabet.com - params_list: - - key: testParam1 - parameter_type: value - value: testValue1 - - key: testParam2 - parameter_type: slot - value: testValue1 - headers: - - key: authorization - parameter_type: value - value: bearer hjklfsdjsjkfbjsbfjsvhfjksvfjksvfjksvf - - key: sender - parameter_type: sender_id - request_method: GET - response: - value: json -- action_name: action_performanceUser1000@digite.com - http_url: http://www.alphabet.com - params_list: - - key: authorization - parameter_type: slot - value: authorization - request_method: GET - response: - value: json -- action_name: action_performanceUser999@digite.com - http_url: http://www.alphabet.com - params_list: - - key: testParam1 - parameter_type: value - value: testValue1 - - key: testParam2 - parameter_type: slot - value: testValue1 +- action_name: action_performanceUser1001@digitecom + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] request_method: GET response: - value: json -- action_name: action_performanceUser998@digite.com - http_url: http://www.alphabet.com - params_list: - - key: testParam1 - parameter_type: value - value: testValue1 - - key: testParam2 - parameter_type: slot - value: testValue1 + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + + +- action_name: action_performanceUser1000@digitecom + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + + +- action_name: action_performanceUser999@digitecom + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + + +- action_name: action_performanceUser998@digitecom + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_performanceUser997@digitecom + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + + +- action_name: action_identify_ticket_attributes + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_log_ticket + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_get_ticket_status + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_reset_slots + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_validate_ticket_for_fileUpload + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_validate_ticket_for_status + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] request_method: GET response: - value: json -- action_name: action_performanceUser997@digite.com - http_url: http://www.alphabet.com + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_clear_file + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] request_method: GET response: - value: json + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_clear_priority + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_log_message + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_small_talk + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: ticket_file_form_action + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +- action_name: action_clear_memory + content_type: json + headers: [ ] + http_url: https://jsonplaceholder.typicode.com/posts/1 + params_list: [ ] + request_method: GET + response: + dispatch: true + dispatch_type: text + evaluation_type: expression + value: ${data} + +form_validation_action: +- invalid_response: '' + is_required: true + name: validate_ticket_attributes_form + slot: customer_name + slot_set: + type: current + value: '' + valid_response: '' + validation_semantic: '' + google_search_action: - name: google_search_action_mf api_key: - key: api_key value: asdfghjklrtyu parameter_type: value - search_engine_id: sdfg34567dfgh \ No newline at end of file + search_engine_id: sdfg34567dfgh + diff --git a/tests/unit_test/data_processor/action_serializer_test.py b/tests/unit_test/data_processor/action_serializer_test.py new file mode 100644 index 000000000..86991883a --- /dev/null +++ b/tests/unit_test/data_processor/action_serializer_test.py @@ -0,0 +1,688 @@ +import os + +from mongoengine import connect + +from kairon import Utility +from kairon.shared.actions.data_objects import Actions, HttpActionConfig, PyscriptActionConfig +from kairon.shared.actions.models import ActionType, ActionParameterType, DbActionOperationType +from kairon.shared.callback.data_objects import CallbackConfig +from kairon.shared.data.data_validation import DataValidation + +os.environ["system_file"] = "./tests/testing_data/system.yaml" +Utility.load_environment() +Utility.load_system_metadata() + +import pytest + +from kairon.shared.data.action_serializer import ActionSerializer +from kairon.shared.data.data_objects import * + + +bot_id_download = "66fb95f3629a37edd68e0cbc" +user_download = "spandan.mondal@nimblework.com" + +valid_http_action_config = { + "action_name": "a_api_action", + "http_url": "https://jsonplaceholder.typicode.com/posts/1", + "request_method": "GET", + "content_type": "json", + "params_list": [], + "headers": [], + "response": { + "value": "${data}", + "dispatch": True, + "evaluation_type": "expression", + "dispatch_type": "text" + }, + "set_slots": [], + "bot": bot_id_download, + "user": user_download, + "status": True +} + +valid_http_action_config2 = { + "action_name": "a_api_action2", + "http_url": "https://jsonplaceholder.typicode.com/posts/2", + "request_method": "GET", + "content_type": "json", + "params_list": [], + "headers": [], + "response": { + "value": "${data}", + "dispatch": True, + "evaluation_type": "expression", + "dispatch_type": "text" + }, + "set_slots": [], + "bot": bot_id_download, + "user": user_download, + "status": True +} + +invalid_http_action_config_field_missing = { + "action_name": "a_api_action_field_missing", + "request_method": "GET", + "params_list": [], + "headers": [], + "response": { + "value": "${data}", + "dispatch": True, + "evaluation_type": "expression", + "dispatch_type": "text" + }, + "set_slots": [], + "bot": bot_id_download, + "user": user_download, + "status": True +} + +valid_pyscript_action_config = { + "name": "a_pyscript_action", + "source_code": "bot_response = \"hello world form pyscript!!\"", + "dispatch_response": True, + "bot": bot_id_download, + "user": user_download, + "status": True +} + +invalid_pyscript_action_config_field_missing = { + "name": "a_pyscript_action_field_missing", + "dispatch_response": True, + "bot": bot_id_download, + "user": user_download, + "status": True +} + +invalid_pyscript_action_config_compiler_error = { + "name": "a_pyscript_action", + "source_code": "if for:\n bot_response = \"hello world form pyscript!!\"", + "dispatch_response": True, + "bot": bot_id_download, + "user": user_download, + "status": True +} + +action_no_name = { + "source_code": "bot_response = \"hello world form pyscript!!\"", + "dispatch_response": True, + "bot": bot_id_download, + "user": user_download +} + +valid_callback_config = { + "name": "cb1", + "pyscript_code": "bot_response = f\"callback data: {metadata}, {req}\"", + "validation_secret": "gAAAAABm-mHUvuS1_vBsGRd5RKLX4Vek5kG05Y8iIUPB788yC75Y15HPaxIdDOnlE4i_HlLV046f2owJSb8CR2YoXXBc8hrnxoft3qdJ7qcdDH_Br5QJIi8ABp1KETumBbYgjKCWTdXr", + "execution_mode": "async", + "expire_in": 0, + "shorten_token": False, + "token_hash": "019247a8285a77d4bb6581d78379d6c6", + "standalone": False, + "standalone_id_path": "", + "bot": bot_id_download +} + +invalid_callback_config_missing_field = { + "name": "cb1", + "standalone": False, + "standalone_id_path": "", + "bot": bot_id_download +} + + +@pytest.fixture(autouse=True, scope='class') +def setup(): + connect(**Utility.mongoengine_connection(Utility.environment['database']["url"])) + + +def test_get_item_name(): + assert ActionSerializer.get_item_name(valid_http_action_config) == "a_api_action" + assert ActionSerializer.get_item_name(valid_pyscript_action_config) == "a_pyscript_action" + with pytest.raises(Exception): + ActionSerializer.get_item_name(action_no_name) + assert ActionSerializer.get_item_name(valid_http_action_config, False) == "a_api_action" + assert not ActionSerializer.get_item_name(action_no_name, False) + + +def test_get_collection_infos(): + action_info, other_collection_info = ActionSerializer.get_collection_infos() + + assert isinstance(action_info, dict) + assert isinstance(other_collection_info, dict) + + action_info_keys = action_info.keys() + other_collection_info_keys = other_collection_info.keys() + test_action_keys = ['http_action', 'two_stage_fallback', 'email_action', 'zendesk_action', 'jira_action', 'form_validation_action', 'slot_set_action', 'google_search_action', 'pipedrive_leads_action', 'prompt_action', 'web_search_action', 'razorpay_action', 'pyscript_action', 'database_action', 'live_agent_action', 'callback_action', 'schedule_action'] + test_other_collection_keys = ['callbackconfig'] + + for k in test_action_keys: + assert k in action_info_keys + assert action_info[k].get('db_model') is not None + + for k in test_other_collection_keys: + assert k in other_collection_info_keys + assert other_collection_info[k].get('db_model') is not None + + assert len(action_info_keys) == 17 + assert len(other_collection_info_keys) == 1 + + +def test_is_action(): + assert ActionSerializer.is_action(ActionType.http_action.value) == True + assert ActionSerializer.is_action(ActionType.two_stage_fallback.value) == True + assert ActionSerializer.is_action("callbackconfig") == False + assert ActionSerializer.is_action(None) == False + + +def test_data_validator_validate_http_action(): + bot = "my_test_bot" + action_param_types = {param.value for param in ActionParameterType} + + # Test case 1: Valid params_list and headers + data = { + "params_list": [{"key": "param1", "parameter_type": list(action_param_types)[0], "value": "value1"}], + "headers": [{"key": "header1", "parameter_type": list(action_param_types)[0], "value": "value1"}], + "action_name": "http_action" + } + assert not DataValidation.validate_http_action(bot, data) + + # Test case 2: Invalid params_list (missing key) + data["params_list"][0]["key"] = None + assert DataValidation.validate_http_action(bot, data) == ['Invalid params_list for http action: http_action'] + + # Test case 3: Invalid headers (missing key) + data["params_list"][0]["key"] = "param1" + data["headers"][0]["key"] = None + assert DataValidation.validate_http_action(bot, data) == ['Invalid headers for http action: http_action'] + + # Test case 4: Invalid params_list (invalid parameter_type) + data["headers"][0]["key"] = "header1" + data["params_list"][0]["parameter_type"] = "invalid_type" + assert DataValidation.validate_http_action(bot, data) == ['Invalid params_list for http action: http_action'] + + # Test case 5: Invalid headers (invalid parameter_type) + data["params_list"][0]["parameter_type"] = list(action_param_types)[0] + data["headers"][0]["parameter_type"] = "invalid_type" + assert DataValidation.validate_http_action(bot, data) == ['Invalid headers for http action: http_action'] + + +def test_data_validator_validate_form_validation_action(): + bot = "my_test_bot" + + # Test case 1: Valid validation_semantic and slot_set + data = { + "name" : "test_action", + "validation_semantic": "valid_semantic", + "slot_set": {"type": "current", "value": ""} + } + assert not DataValidation.validate_form_validation_action(bot, data) + + # Test case 2: Invalid validation_semantic (not a string) + data["validation_semantic"] = 123 + assert DataValidation.validate_form_validation_action(bot, data) == ['Invalid validation semantic: test_action'] + + # Test case 3: Invalid slot_set (missing type) + data["validation_semantic"] = "valid_semantic" + data["slot_set"]["type"] = None + assert DataValidation.validate_form_validation_action(bot, data) == ['slot_set should have type current as default!', 'Invalid slot_set type!'] + + # Test case 4: Invalid slot_set (type current with value) + data["slot_set"]["type"] = "current" + data["slot_set"]["value"] = "value" + assert DataValidation.validate_form_validation_action(bot, data) == ['slot_set with type current should not have any value!'] + + # Test case 5: Invalid slot_set (type slot without value) + data["slot_set"]["type"] = "slot" + data["slot_set"]["value"] = None + assert DataValidation.validate_form_validation_action(bot, data) == ['slot_set with type slot should have a valid slot value!'] + + # Test case 6: Invalid slot_set (invalid type) + data["slot_set"]["type"] = "invalid_type" + data["slot_set"]["value"] = "value" + assert DataValidation.validate_form_validation_action(bot, data) == ['Invalid slot_set type!'] + + +def test_data_validator_validate_database_action(): + bot = "my_test_bot" + db_action_operation_types = {qtype.value for qtype in DbActionOperationType} + + # Test case 1: Valid payload + data = { + "payload": [{"query_type": list(db_action_operation_types)[0], "type": "type1", "value": "value1"}] + } + assert not DataValidation.validate_database_action(bot, data) + + # Test case 2: Invalid payload (missing query_type) + data["payload"][0]["query_type"] = None + assert DataValidation.validate_database_action(bot, data) == ["Payload 0 must contain fields 'query_type', 'type' and 'value'!", 'Unknown query_type found: None in payload 0'] + + # Test case 3: Invalid payload (missing type) + data["payload"][0]["query_type"] = list(db_action_operation_types)[0] + data["payload"][0]["type"] = None + assert DataValidation.validate_database_action(bot, data) == ["Payload 0 must contain fields 'query_type', 'type' and 'value'!"] + + # Test case 4: Invalid payload (missing value) + data["payload"][0]["type"] = "type1" + data["payload"][0]["value"] = None + assert DataValidation.validate_database_action(bot, data) == ["Payload 0 must contain fields 'query_type', 'type' and 'value'!"] + + # Test case 5: Invalid payload (invalid query_type) + data["payload"][0]["value"] = "value1" + data["payload"][0]["query_type"] = "invalid_query_type" + assert DataValidation.validate_database_action(bot, data) == ["Unknown query_type found: invalid_query_type in payload 0", ] + + +def test_data_validator_validate_python_script_compile_time(): + # Test case 1: Valid Python script + script = "print('Hello, World!')" + assert DataValidation.validate_python_script_compile_time(script) is None + + # Test case 2: Invalid Python script (SyntaxError) + script = "print('Hello, World!" + assert DataValidation.validate_python_script_compile_time(script) == "unterminated string literal (detected at line 1)" + + +def test_data_validator_validate_pyscript_action(): + bot = "my_test_bot" + + # Test case 1: Valid Python script + data = {"source_code": "print('Hello, World!')"} + assert not DataValidation.validate_pyscript_action(bot, data) + + # Test case 2: Invalid Python script (SyntaxError) + data["source_code"] = "print('Hello, World!" + assert DataValidation.validate_pyscript_action(bot, data) == ["Error in python script: unterminated string literal (detected at line 1)"] + + # Test case 3: Missing source_code + data = {} + assert DataValidation.validate_pyscript_action(bot, data) == ['Script is required for pyscript action!'] + + +def test_data_validator_validate_callback_config(): + bot = "my_test_bot" + + # Test case 1: Valid Python script + data = {"pyscript_code": "print('Hello, World!')"} + assert not DataValidation.validate_callback_config(bot, data) + + # Test case 2: Invalid Python script (SyntaxError) + data["pyscript_code"] = "print('Hello, World!" + assert DataValidation.validate_callback_config(bot, data) == ["Error in python script: unterminated string literal (detected at line 1)"] + + # Test case 3: Missing pyscript_code + data = {} + assert DataValidation.validate_callback_config(bot, data) == ['pyscript_code is required'] + + +@pytest.mark.parametrize( + "llm_prompts, expected_errors", + [ + # Test: Valid prompts (no errors expected) + ( + [ + {"type": "system", "source": "static", "data": "name", "name": "Prompt1", "hyperparameters": {"similarity_threshold": 0.5, "top_results": 5}}, + {"type": "user", "source": "slot", "data": "name", "name": "Prompt1"} + ], + [] + ), + # Test: Invalid similarity_threshold (out of range) + ( + [ + {"type": "system", "source": "static", "data": "name", "name": "Prompt1", "hyperparameters": {"similarity_threshold": 1.5}}, + {"type": "user", "source": "slot", "data": "name", "name": "Prompt2"} + ], + ["similarity_threshold should be within 0.3 and 1.0 and of type int or float!"] + ), + # Test: Missing data for action source + ( + [ + {"type": "system", "source": "static", "data": "name", "name": "Prompt1"}, + {"type": "user", "source": "action", "name": "Prompt2"} + ], + ["Data must contain action name"] + ), + # Test: Invalid type value + ( + [ + {"type": "invalid", "source": "static", "data": "name", "name": "Prompt3"} + ], + ["Invalid prompt type", "System prompt is required"] + ), + # Test: Multiple system prompts + ( + [ + {"type": "system", "source": "static", "data": "name", "name": "Prompt1"}, + {"type": "system", "source": "static", "data": "name", "name": "Prompt2"} + ], + ["Only one system prompt can be present"] + ), + # Test: Missing system prompt + ( + [ + {"type": "user", "source": "slot", "data": "name", "name": "Prompt4"} + ], + ["System prompt is required"] + ), + # Test: Invalid source value + ( + [ + {"type": "system", "source": "invalid_source", "data": "name", "name": "Prompt5"} + ], + ["Invalid prompt source", 'System prompt must have static source'] + ), + # Test: Invalid top_results value (greater than 30) + ( + [ + {"type": "system", "source": "static", "data": "name", "name": "Prompt1", "hyperparameters": {"top_results": 50}} + ], + ["top_results should not be greater than 30 and of type int!"] + ), + # Test: Empty name field + ( + [ + {"type": "system", "source": "static", "data": "name", "name": ""} + ], + ["Name cannot be empty"] + ) + ] +) +def test_data_validation_llm_prompt(llm_prompts, expected_errors): + assert DataValidation.validate_llm_prompts(llm_prompts) == expected_errors + + +def test_modify_callback_config(): + bot = 'test_bot' + data = {} + + result = DataValidation.modify_callback_config(bot, data) + + assert 'token_hash' in result + assert 'validation_secret' in result + + +def test_validate_prompt_action(): + bot = "my_test_bot" + + # Test case 1: Valid prompt action + data = { + "num_bot_responses": 3, + "llm_prompts": [ + { + "type": "system", + "source": "static", + "data": "Hello, World!", + "name": "Prompt1", + "hyperparameters": { + "similarity_threshold": 0.5, + "top_results": 5 + } + } + ], + "hyperparameters": { + "similarity_threshold": 0.5, + "top_results": 5 + }, + "llm_type": "openai" + } + assert not DataValidation.validate_prompt_action(bot, data) + + # Test case 2: Invalid num_bot_responses (greater than 5) + data["num_bot_responses"] = 6 + assert DataValidation.validate_prompt_action(bot, data) == ['num_bot_responses should not be greater than 5 and of type int: None'] + + # Test case 3: Invalid llm_prompts (invalid type) + data["num_bot_responses"] = 3 + data["llm_prompts"][0]["type"] = "invalid_type" + assert DataValidation.validate_prompt_action(bot, data) == ['Invalid prompt type', 'System prompt is required'] + + # Test case 4: Invalid hyperparameters (similarity_threshold out of range) + data["llm_prompts"][0]["type"] = "system" + data["llm_prompts"][0]["hyperparameters"]["similarity_threshold"] = 1.5 + data["hyperparameters"]["similarity_threshold"] = 0.2 + assert DataValidation.validate_prompt_action(bot, data) == ["similarity_threshold should be within 0.3 and 1.0 and of type int or float!"] + + +def test_action_serializer_validate(): + bot = "my_test_bot" + + # Test case 1: Valid actions and other_collections + actions = { + "http_action": [ + valid_http_action_config + ], + "pyscript_action": [ + valid_pyscript_action_config + ] + } + other_collections = { + str(CallbackConfig.__name__).lower(): [ + valid_callback_config + ] + } + val = ActionSerializer.validate(bot, actions, other_collections) + assert val[0] + for k in val[1].keys(): + assert not val[1][k] + + assert val[2][str(CallbackConfig.__name__).lower()] == 1 + assert val[2]['http_action'] == 1 + assert val[2]['pyscript_action'] == 1 + + # Test case 2: Invalid actions (not a dictionary) + actions = ["http_action"] + val = ActionSerializer.validate(bot, actions, other_collections) + assert val[0] + assert val[1] == {'action.yml': ['Expected dictionary with action types as keys']} + + # Test case 3: Invalid actions (invalid action type) + actions = { + "invalid_action_type": [] + } + val = ActionSerializer.validate(bot, actions, other_collections) + assert not val[0] + assert val[1]['invalid_action_type'] == ['Invalid action type: invalid_action_type.'] + + # Test case 4: not a list + actions = { + "http_action": {} + } + val = ActionSerializer.validate(bot, actions, other_collections) + assert not val[0] + assert val[1]['http_action'] == ['Expected list of actions for http_action.'] + + # Test case 5: Invalid action data + actions = { + "http_action": [ + invalid_http_action_config_field_missing + ], + "pyscript_action": [ + invalid_pyscript_action_config_field_missing + ] + } + val = ActionSerializer.validate(bot, actions, other_collections) + assert not val[0] + assert val[1]['http_action'] == [{ 'a_api_action_field_missing': " Required fields {'http_url'} not found."}] + assert val[1]['pyscript_action'] == [{'a_pyscript_action_field_missing': " Required fields {'source_code'} not found."}] + + # Test case 6: unknown other collection type + oc2 = { + "unknown": [ + valid_callback_config + ] + } + val = ActionSerializer.validate(bot, actions, oc2) + assert not val[0] + assert val[1]['unknown'] == ['Invalid collection type: unknown.'] + + # Test case 7: other collection entry type not list + oc2 = { + str(CallbackConfig.__name__).lower(): { + 'data': valid_callback_config + } + } + + val = ActionSerializer.validate(bot, actions, oc2) + assert not val[0] + assert val[1][str(CallbackConfig.__name__).lower()] == ['Expected list of data for callbackconfig.'] + + # test case 8: duplicate entries for action + actions = { + "http_action": [ + valid_http_action_config, + valid_http_action_config + ] + } + + val = ActionSerializer.validate(bot, actions, other_collections) + assert not val[0] + assert val[1]['http_action'] == [{'a_api_action': 'Duplicate Name found for other action.'}] + + # test case 9: invalid other collection + actions = { + "http_action": [ + valid_http_action_config + ], + "pyscript_action": [ + valid_pyscript_action_config + ] + } + other_collections = { + str(CallbackConfig.__name__).lower(): [ + invalid_callback_config_missing_field + ] + } + + val = ActionSerializer.validate(bot, actions, other_collections) + assert not val[0] + assert "Required fields" in val[1][str(CallbackConfig.__name__).lower()][0]['cb1'] + + +def test_action_serializer_deserialize(): + bot = "my_test_bot" + user = "test_user@test_user.com" + + # Test case 1: Valid actions and other_collections + actions = { + "http_action": [ + valid_http_action_config, + ], + "pyscript_action": [ + valid_pyscript_action_config + ] + } + + other_collections = { + str(CallbackConfig.__name__).lower(): [ + valid_callback_config + ] + } + + ActionSerializer.deserialize(bot, user, actions, other_collections) + actions_added = Actions.objects(bot=bot, user=user) + assert len(list(actions_added)) == 2 + names = [action.name for action in actions_added] + assert "a_api_action" in names + assert "a_pyscript_action" in names + + http_action = HttpActionConfig.objects(bot=bot, user=user).get() + assert http_action.action_name == "a_api_action" + + callback = CallbackConfig.objects(bot=bot).get() + assert callback.name == "cb1" + + +def test_action_serializer_deserialize_overwrite(): + bot = "my_test_bot" + user = "test_user@test_user.com" + + # Test case 1: Valid actions and other_collections + actions = { + "http_action": [ + valid_http_action_config, + ], + "pyscript_action": [ + valid_pyscript_action_config + ] + } + + other_collections = { + str(CallbackConfig.__name__).lower(): [ + valid_callback_config + ] + } + + Actions.objects(bot=bot, user=user).delete() + HttpActionConfig.objects(bot=bot, user=user).delete() + PyscriptActionConfig.objects(bot=bot, user=user).delete() + CallbackConfig.objects(bot=bot).delete() + + ActionSerializer.deserialize(bot, user, actions, other_collections,True) + actions_added = Actions.objects(bot=bot, user=user) + assert len(list(actions_added)) == 2 + names = [action.name for action in actions_added] + assert "a_api_action" in names + assert "a_pyscript_action" in names + + http_action = HttpActionConfig.objects(bot=bot, user=user).get() + assert http_action.action_name == "a_api_action" + + callback = CallbackConfig.objects(bot=bot).get() + assert callback.name == "cb1" + + +def test_action_serialize_duplicate_data(): + bot = "my_test_bot" + user = "test_user@test_user.com" + + actions = { + "http_action": [ + valid_http_action_config, + valid_http_action_config2 + ], + "pyscript_action": [ + valid_pyscript_action_config + ] + } + + other_collections = { + str(CallbackConfig.__name__).lower(): [ + valid_callback_config + ] + } + + actions_added = Actions.objects(bot=bot, user=user) + assert len(list(actions_added)) == 2 + ActionSerializer.deserialize(bot, user, actions, other_collections) + actions_added = Actions.objects(bot=bot, user=user) + assert len(list(actions_added)) == 3 + names = [action.name for action in actions_added] + assert "a_api_action" in names + assert "a_api_action2" in names + assert "a_pyscript_action" in names + + http_action = HttpActionConfig.objects(bot=bot, user=user) + assert len(list(http_action)) == 2 + + +def test_action_serializer_serialize(): + bot = "my_test_bot" + user = "test_user@test_user.com" + + actions, others = ActionSerializer.serialize(bot) + assert actions + assert others + + assert len(actions['http_action']) == 2 + assert len(actions['pyscript_action']) == 1 + + assert len(others[str(CallbackConfig.__name__).lower()]) == 1 + + +def test_action_save_collection_data_list_unknown_data(): + bot = "my_test_bot" + user = "test_user@test_user.com" + + with pytest.raises(AppException, match="Action type not found"): + ActionSerializer.save_collection_data_list('unknown1', bot, user, [{'data1': 'value1'}]) + + diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 56b872b21..980715cb2 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -9,6 +9,8 @@ from io import BytesIO from typing import List +import yaml + from kairon.api.app.routers.bot.data import processor from kairon.shared.content_importer.data_objects import ContentValidationLogs from kairon.shared.utils import Utility @@ -1160,6 +1162,15 @@ def test_list_live_agent_actions(self): assert list_data[0].get('_id') assert len(list_data[0]['_id']) == 24 + def test_enable_live_agent_service_not_available(self): + bot = "test_bot" + user = "test_user" + request_data = {"key": "value"} + processor = MongoProcessor() + with patch.object(processor, 'is_live_agent_enabled', return_value=False): + with pytest.raises(AppException, match="Live agent service is not available for the bot"): + processor.enable_live_agent(request_data, bot, user) + def test_disable_live_agent(self): processor = MongoProcessor() bot = 'test_bot' @@ -1184,6 +1195,53 @@ def test_edit_live_agent_does_not_exist(self): with pytest.raises(AppException, match=f'Live agent not enabled for the bot'): result = processor.edit_live_agent(request_data=request_data, bot=bot, user=user) + + def test_get_callback_service_log_with_conditions(self): + bot = "test_bot" + service = MongoProcessor() + + with patch('kairon.shared.callback.data_objects.CallbackLog.get_logs', return_value=(["log1", "log2"], 2)) as mock_get_logs: + name = "test_callback" + sender_id = "test_sender" + channel = "test_channel" + identifier = "test_identifier" + start = 0 + limit = 100 + + logs, total_count = service.get_callback_service_log( + bot=bot, + name=name, + sender_id=sender_id, + channel=channel, + identifier=identifier, + start=start, + limit=limit + ) + + expected_query = { + "bot": bot, + "callback_name": name, + "sender_id": sender_id, + "channel": channel, + "identifier": identifier + } + mock_get_logs.assert_called_once_with(expected_query, start, limit) + assert logs == ["log1", "log2"] + assert total_count == 2 + + def test_get_callback_service_log_without_conditions(self): + bot = "test_bot" + service = MongoProcessor() + + with patch('kairon.shared.callback.data_objects.CallbackLog.get_logs', return_value=(["log1", "log2"], 2)) as mock_get_logs: + + logs, total_count = service.get_callback_service_log(bot=bot) + + expected_query = {"bot": bot} + mock_get_logs.assert_called_once_with(expected_query, 0, 100) + assert logs == ["log1", "log2"] + assert total_count == 2 + def test_auditlog_event_config_does_not_exist(self): result = MongoProcessor.get_auditlog_event_config("nobot") assert result == {} @@ -1997,13 +2055,7 @@ def test_add_pyscript_action_empty_name(self): bot = 'test_bot' user = 'test_user' action = "test_add_pyscript_action_empty_name" - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" processor = MongoProcessor() pyscript_config = PyscriptActionRequest( name=action, @@ -2019,13 +2071,7 @@ def test_add_pyscript_action_empty_source_code(self): bot = 'test_bot' user = 'test_user' action = "test_add_pyscript_action_empty_source_code" - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" processor = MongoProcessor() pyscript_config = PyscriptActionRequest( name=action, @@ -2041,13 +2087,7 @@ def test_add_pyscript_action_with_utter(self): bot = 'test_bot' user = 'test_user' action = "test_add_pyscript_action_empty_name" - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" processor = MongoProcessor() pyscript_config = PyscriptActionRequest( name=action, @@ -2063,13 +2103,7 @@ def test_add_pyscript_action(self): bot = 'test_bot' user = 'test_user' action = "test_add_pyscript_action" - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" processor = MongoProcessor() pyscript_config = PyscriptActionRequest( name=action, @@ -2088,13 +2122,7 @@ def test_add_pyscript_action_with_name_already_exist(self): bot = 'test_bot' user = 'test_user' action = "test_add_pyscript_action" - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" processor = MongoProcessor() pyscript_config = PyscriptActionRequest( name=action, @@ -2108,13 +2136,7 @@ def test_add_pyscript_action_case_insensitivity(self): bot = 'test_bot' user = 'test_user' action = "TEST_ADD_PYSCRIPT_ACTION_CASE_INSENSITIVITY" - script = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" processor = MongoProcessor() pyscript_config = PyscriptActionRequest( name=action, @@ -2134,13 +2156,7 @@ def test_update_pyscript_action_doesnot_exist(self): bot = 'test_bot' user = 'test_user' action = "test_update_pyscript_action" - script = """ - data = [1, 2, 3, 4, 5, 6, 7] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" processor = MongoProcessor() pyscript_config = PyscriptActionRequest( name=action, @@ -2154,13 +2170,7 @@ def test_update_pyscript_action(self): bot = 'test_bot' user = 'test_user' action = "test_add_pyscript_action" - script = """ - data = [1, 2, 3, 4, 5, 6, 7] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" processor = MongoProcessor() pyscript_config = PyscriptActionRequest( name=action, @@ -2177,20 +2187,8 @@ def test_update_pyscript_action(self): def test_list_pyscript_actions(self): bot = 'test_bot' user = 'test_user' - script1 = """ - data = [1, 2, 3, 4, 5, 6, 7] - total = 0 - for i in data: - total += i - print(total) - """ - script2 = """ - data = [1, 2, 3, 4, 5] - total = 0 - for i in data: - total += i - print(total) - """ + script1 = "bot_response='hello world'" + script2 = "bot_response='hello world'" processor = MongoProcessor() actions = list(processor.list_pyscript_actions(bot, True)) assert len(actions) == 2 @@ -2430,10 +2428,10 @@ async def test_upload_case_insensitivity(self): 'longitude', 'latitude', 'flow_reply', 'quick_reply'], ignore_order=True) assert domain.forms == {'ask_user': {'required_slots': ['user', 'email_id']}, 'ask_location': {'required_slots': ['location', 'application_name']}} - assert domain.user_actions == ['action_get_google_application', 'action_get_microsoft_application', + assert domain.user_actions == ['ACTION_GET_GOOGLE_APPLICATION', 'ACTION_GET_MICROSOFT_APPLICATION', 'utter_default', 'utter_goodbye', 'utter_greet', 'utter_please_rephrase'] - assert processor.fetch_actions('test_upload_case_insensitivity') == ['action_get_google_application', - 'action_get_microsoft_application'] + assert processor.fetch_actions('test_upload_case_insensitivity') == ['ACTION_GET_GOOGLE_APPLICATION', + 'ACTION_GET_MICROSOFT_APPLICATION'] assert domain.intents == ['back', 'deny', 'greet', 'nlu_fallback', 'out_of_scope', 'restart', 'session_start'] assert domain.responses == { 'utter_please_rephrase': [{'text': "I'm sorry, I didn't quite understand that. Could you rephrase?"}], @@ -2444,7 +2442,7 @@ async def test_upload_case_insensitivity(self): 'ask the user to rephrase whenever they send a message with low nlu confidence'] actions = processor.load_http_action("test_upload_case_insensitivity") assert actions == {'http_action': [ - {'action_name': 'action_get_google_application', 'http_url': 'http://www.alphabet.com', + {'action_name': 'ACTION_GET_GOOGLE_APPLICATION', 'http_url': 'http://www.alphabet.com', 'content_type': 'json', 'response': {'value': 'json', 'dispatch': True, 'evaluation_type': 'expression', 'dispatch_type': 'text'}, 'request_method': 'GET', 'headers': [ @@ -2458,11 +2456,11 @@ async def test_upload_case_insensitivity(self): 'encrypt': False}, {'_cls': 'HttpActionRequestBody', 'key': 'testParam5', 'value': '', 'parameter_type': 'sender_id', 'encrypt': False}, - {'_cls': 'HttpActionRequestBody', 'key': 'testParam4', 'value': 'testvalue1', + {'_cls': 'HttpActionRequestBody', 'key': 'testParam4', 'value': 'testValue1', 'parameter_type': 'slot', 'encrypt': False}], 'params_list': [{'_cls': 'HttpActionRequestBody', 'key': 'testParam1', 'value': 'testValue1', 'parameter_type': 'value', 'encrypt': False}, - {'_cls': 'HttpActionRequestBody', 'key': 'testParam2', 'value': 'testvalue1', + {'_cls': 'HttpActionRequestBody', 'key': 'testParam2', 'value': 'testValue1', 'parameter_type': 'slot', 'encrypt': False}], 'dynamic_params': { "farmid": '120d37d6-6159-45f1-a3d0-edfead442971', @@ -2473,12 +2471,12 @@ async def test_upload_case_insensitivity(self): } ] }}, - {'action_name': 'action_get_microsoft_application', + {'action_name': 'ACTION_GET_MICROSOFT_APPLICATION', 'response': {'value': 'json', 'dispatch': True, 'evaluation_type': 'expression', 'dispatch_type': 'text'}, 'http_url': 'http://www.alphabet.com', 'request_method': 'GET', 'content_type': 'json', 'params_list': [{'_cls': 'HttpActionRequestBody', 'key': 'testParam1', 'value': 'testValue1', 'parameter_type': 'value', 'encrypt': False}, - {'_cls': 'HttpActionRequestBody', 'key': 'testParam2', 'value': 'testvalue1', + {'_cls': 'HttpActionRequestBody', 'key': 'testParam2', 'value': 'testValue1', 'parameter_type': 'slot', 'encrypt': False}, {'_cls': 'HttpActionRequestBody', 'key': 'testParam1', 'value': '', 'parameter_type': 'chat_log', 'encrypt': False}, @@ -2490,7 +2488,7 @@ async def test_upload_case_insensitivity(self): 'parameter_type': 'intent', 'encrypt': False}, {'_cls': 'HttpActionRequestBody', 'key': 'testParam5', 'value': '', 'parameter_type': 'sender_id', 'encrypt': False}, - {'_cls': 'HttpActionRequestBody', 'key': 'testParam4', 'value': 'testvalue1', + {'_cls': 'HttpActionRequestBody', 'key': 'testParam4', 'value': 'testValue1', 'parameter_type': 'slot', 'encrypt': False}]}]} assert set(Utterances.objects(bot='test_upload_case_insensitivity').values_list('name')) == {'utter_goodbye', 'utter_greet', @@ -2545,7 +2543,9 @@ async def test_load_from_path_yml_training_files(self): 'required_slots': ['file']} assert isinstance(domain.forms, dict) assert domain.user_actions.__len__() == 48 - assert processor.list_actions('test_load_from_path_yml_training_files')["actions"].__len__() == 12 + + assert processor.list_actions('test_load_from_path_yml_training_files')["http_action"].__len__() == 17 + assert processor.list_actions('test_load_from_path_yml_training_files')["utterances"].__len__() == 29 assert processor.list_actions('test_load_from_path_yml_training_files')["form_validation_action"].__len__() == 1 assert domain.intents.__len__() == 32 assert not Utility.check_empty_string( @@ -2558,7 +2558,7 @@ async def test_load_from_path_yml_training_files(self): actions = processor.load_http_action("test_load_from_path_yml_training_files") actions_google = processor.load_google_search_action("test_load_from_path_yml_training_files") assert isinstance(actions, dict) is True - assert len(actions['http_action']) == 5 + assert len(actions['http_action']) == 17 assert len(actions_google['google_search_action']) == 1 assert Utterances.objects(bot='test_load_from_path_yml_training_files').count() == 29 @@ -2603,8 +2603,7 @@ async def test_load_from_path_all_scenario(self): assert domain.forms.__len__() == 2 assert domain.forms['ticket_attributes_form'] == {'required_slots': {}} assert isinstance(domain.forms, dict) - assert domain.user_actions.__len__() == 38 - assert processor.list_actions('all')["actions"].__len__() == 11 + assert domain.user_actions.__len__() == 27 assert domain.intents.__len__() == 29 assert not Utility.check_empty_string( domain.responses["utter_cheer_up"][0]["image"] @@ -2644,9 +2643,8 @@ async def test_load_from_path_all_scenario_append(self): assert domain.entities.__len__() == 23 assert domain.forms.__len__() == 2 assert isinstance(domain.forms, dict) - assert domain.user_actions.__len__() == 38 + assert domain.user_actions.__len__() == 27 assert domain.intents.__len__() == 29 - assert processor.list_actions('all')["actions"].__len__() == 11 assert not Utility.check_empty_string( domain.responses["utter_cheer_up"][0]["image"] ) @@ -3883,7 +3881,7 @@ def _mock_bot_info(*args, **kwargs): file = processor.download_files("tests_download_empty_data", "user@integration.com") assert file.endswith(".zip") zip_file = ZipFile(file, mode='r') - assert zip_file.filelist.__len__() == 10 + assert zip_file.filelist.__len__() == 9 assert zip_file.getinfo('data/stories.yml') assert zip_file.getinfo('data/rules.yml') file_info_stories = zip_file.getinfo('data/stories.yml') @@ -3958,18 +3956,36 @@ def _mock_bot_info(*args, **kwargs): def test_download_data_files_with_actions(self, monkeypatch): from zipfile import ZipFile - expected_actions = b'database_action: []\nemail_action: []\nform_validation_action: []\ngoogle_search_action: []\nhttp_action: []\njira_action: []\nlive_agent_action: []\npipedrive_leads_action: []\nprompt_action: []\npyscript_action: []\nrazorpay_action: []\nslot_set_action: []\ntwo_stage_fallback: []\nzendesk_action: []\n'.decode( - encoding='utf-8') def _mock_bot_info(*args, **kwargs): return { - "_id": "9876543210", 'name': 'test_bot', 'account': 2, 'user': 'user@integration.com', 'status': True, + "_id": "9876543210", 'name': 'tests', 'account': 2, 'user': 'user@integration.com', 'status': True, "metadata": {"source_bot_id": None} } + #add a http action to the bot + act_config = HttpActionConfig() + act_config.action_name = "my_http_action" + act_config.bot = 'tests' + act_config.user = 'user@integration.com' + act_config.status = True + act_config.http_url = "https://jsonplaceholder.typicode.com/posts/1" + act_config.request_method = "GET" + act_config.content_type = "json" + act_config.response = HttpActionResponse(value='zxcvb') + act_config.save() + + action = Actions() + action.name = 'my_http_action' + action.bot = 'tests' + action.user = 'user@integration.com' + action.status = True + action.type = ActionType.http_action.value + action.save() monkeypatch.setattr(AccountProcessor, 'get_bot', _mock_bot_info) processor = MongoProcessor() file_path = processor.download_files("tests", "user@integration.com") + assert file_path.endswith(".zip") zip_file = ZipFile(file_path, mode='r') assert zip_file.filelist.__len__() == 10 @@ -3985,14 +4001,42 @@ def _mock_bot_info(*args, **kwargs): file_info = zip_file.getinfo('actions.yml') file_content = zip_file.read(file_info) actual_actions = file_content.decode(encoding='utf-8') - print(actual_actions) - assert actual_actions == expected_actions + actual_actions_dict = yaml.safe_load(actual_actions) + assert actual_actions_dict == {'http_action': [ + { + 'action_name': 'my_http_action', + 'content_type': 'json', + 'http_url': 'https://jsonplaceholder.typicode.com/posts/1', + 'params_list': [], + 'request_method': 'GET', + 'response': { + 'dispatch': True, + 'dispatch_type': 'text', + 'evaluation_type': 'expression', + 'value': 'zxcvb', + }, + 'headers': [], + 'set_slots': [] + }]} zip_file.close() def test_load_action_configurations(self): processor = MongoProcessor() action_config = processor.load_action_configurations("tests") - assert action_config == {'http_action': [], 'jira_action': [], 'email_action': [], 'zendesk_action': [], + assert action_config == {'http_action': [ + { + 'action_name': 'my_http_action', + 'content_type': 'json', + 'http_url': 'https://jsonplaceholder.typicode.com/posts/1', + 'request_method': 'GET', + 'response': { + 'dispatch': True, + 'dispatch_type': 'text', + 'evaluation_type': 'expression', + 'value': 'zxcvb', + }, + } + ], 'jira_action': [], 'email_action': [], 'zendesk_action': [], 'form_validation_action': [], 'slot_set_action': [], 'google_search_action': [], 'pipedrive_leads_action': [], 'two_stage_fallback': [], 'prompt_action': [], 'razorpay_action': [], 'pyscript_action': [], 'database_action': [], 'live_agent_action': []} @@ -5602,7 +5646,7 @@ def _mock_bot_info(*args, **kwargs): mongo_processor = MongoProcessor() mongo_processor.save_training_data(bot, user, config, domain, story_graph, nlu, http_actions, multiflow_stories, bot_content, - chat_client_config, True) + chat_client_config, overwrite=True) training_data = mongo_processor.load_nlu(bot) assert isinstance(training_data, TrainingData) @@ -5647,8 +5691,8 @@ def _mock_bot_info(*args, **kwargs): assert len(rules) == 4 actions = mongo_processor.load_http_action(bot) assert isinstance(actions, dict) is True - assert len(actions['http_action']) == 5 - assert len(Actions.objects(type='http_action', bot=bot)) == 5 + assert len(actions['http_action']) == 17 + assert len(Actions.objects(type='http_action', bot=bot)) == 17 multiflow_stories = mongo_processor.load_multiflow_stories_yaml(bot) assert isinstance(multiflow_stories, dict) is True bot_content = mongo_processor.load_bot_content(bot) @@ -5671,7 +5715,7 @@ def _mock_bot_info(*args, **kwargs): mongo_processor = MongoProcessor() mongo_processor.save_training_data(bot, user, config, domain, story_graph, nlu, http_actions, multiflow_stories, bot_content, - chat_client_config, True) + chat_client_config, overwrite=True) training_data = mongo_processor.load_nlu(bot) assert isinstance(training_data, TrainingData) @@ -5694,7 +5738,7 @@ def _mock_bot_info(*args, **kwargs): assert domain.responses.keys().__len__() == 27 assert domain.entities.__len__() == 23 assert domain.form_names.__len__() == 2 - assert domain.user_actions.__len__() == 38 + assert domain.user_actions.__len__() == 27 assert domain.intents.__len__() == 29 assert not Utility.check_empty_string( domain.responses["utter_cheer_up"][0]["image"] @@ -5725,7 +5769,7 @@ def _mock_bot_info(*args, **kwargs): domain.slots[0].mappings[0]['conditions'] = [{"active_loop": "ticket_attributes_form", "requested_slot": "date_time"}] mongo_processor = MongoProcessor() mongo_processor.save_training_data(bot, user, config, domain, story_graph, nlu, http_actions, multiflow_stories, bot_content, - chat_client_config, True) + chat_client_config, overwrite=True) slot_mapping = SlotMapping.objects(form_name="ticket_attributes_form").get() assert slot_mapping.slot == "date_time" @@ -5747,7 +5791,7 @@ def _mock_bot_info(*args, **kwargs): mongo_processor = MongoProcessor() mongo_processor.save_training_data(bot, user, config, domain, story_graph, nlu, http_actions, multiflow_stories, bot_content, - chat_client_config, True) + chat_client_config, overwrite=True) training_data = mongo_processor.load_nlu(bot) assert isinstance(training_data, TrainingData) @@ -5792,7 +5836,7 @@ def _mock_bot_info(*args, **kwargs): assert len(rules) == 4 actions = mongo_processor.load_http_action(bot) assert isinstance(actions, dict) is True - assert len(actions['http_action']) == 5 + assert len(actions['http_action']) == 17 @pytest.mark.asyncio async def test_save_training_data_all_append(self, get_training_data, monkeypatch): @@ -5811,7 +5855,8 @@ def _mock_bot_info(*args, **kwargs): mongo_processor = MongoProcessor() mongo_processor.save_training_data(bot, user, config, domain, story_graph, nlu, http_actions, multiflow_stories, bot_content, - chat_client_config, False, REQUIREMENTS.copy() - {"chat_client_config"}) + chat_client_config, overwrite=False, + what=REQUIREMENTS.copy() - {"chat_client_config"}) training_data = mongo_processor.load_nlu(bot) assert isinstance(training_data, TrainingData) @@ -5856,7 +5901,7 @@ def _mock_bot_info(*args, **kwargs): assert len(rules) == 4 actions = mongo_processor.load_http_action(bot) assert isinstance(actions, dict) is True - assert len(actions['http_action']) == 5 + assert len(actions['http_action']) == 17 def test_delete_nlu_only(self): bot = 'test' @@ -5906,7 +5951,7 @@ def test_delete_nlu_only(self): assert len(rules) == 4 actions = mongo_processor.load_http_action(bot) assert isinstance(actions, dict) is True - assert len(actions['http_action']) == 5 + assert len(actions['http_action']) == 17 @pytest.mark.asyncio async def test_save_nlu_only(self, get_training_data): @@ -5964,7 +6009,7 @@ def test_delete_stories_only(self): assert len(rules) == 4 actions = mongo_processor.load_http_action(bot) assert isinstance(actions, dict) is True - assert len(actions['http_action']) == 5 + assert len(actions['http_action']) == 17 def test_delete_multiflow_stories_only(self): bot = 'test' @@ -6047,7 +6092,7 @@ def test_delete_config_and_actions_only(self): assert domain.responses.keys().__len__() == 31 assert domain.entities.__len__() == 24 assert domain.form_names.__len__() == 2 - assert domain.user_actions.__len__() == 43 + assert domain.user_actions.__len__() == 31 assert domain.intents.__len__() == 33 rules = mongo_processor.fetch_rule_block_names(bot) assert len(rules) == 4 @@ -6069,7 +6114,7 @@ async def test_save_actions_and_config_only(self, get_training_data): mongo_processor.save_training_data(bot, user, config=config, actions=http_actions, overwrite=True, what={'actions', 'config'}) - assert len(mongo_processor.load_http_action(bot)['http_action']) == 5 + assert len(mongo_processor.load_http_action(bot)['http_action']) == 17 config = mongo_processor.load_config(bot) assert config['language'] == 'fr' assert config['pipeline'] @@ -6096,13 +6141,13 @@ def test_delete_rules_and_domain_only(self): assert domain.responses.keys().__len__() == 0 assert domain.entities.__len__() == 0 assert domain.form_names.__len__() == 0 - assert domain.user_actions.__len__() == 6 + assert domain.user_actions.__len__() == 19 assert domain.intents.__len__() == 5 rules = mongo_processor.fetch_rule_block_names(bot) assert len(rules) == 0 actions = mongo_processor.load_http_action(bot) assert isinstance(actions, dict) is True - assert len(actions['http_action']) == 5 + assert len(actions['http_action']) == 17 @pytest.mark.asyncio async def test_save_rules_and_domain_only(self, get_training_data): @@ -6319,7 +6364,7 @@ def _mock_bot_info(*args, **kwargs): file_path = processor.download_files(pytest.bot, "user@integration.com") assert file_path.endswith(".zip") zip_file = ZipFile(file_path, mode='r') - assert zip_file.filelist.__len__() == 10 + assert zip_file.filelist.__len__() == 9 assert zip_file.getinfo('chat_client_config.yml') @pytest.fixture() @@ -6458,11 +6503,12 @@ async def test_validate_and_prepare_data_save_actions(self, resource_save_and_va training_file = [pytest.http_actions] files_received, is_event_data, non_event_validation_summary = await processor.validate_and_prepare_data( pytest.bot, 'test', training_file, True) + print(non_event_validation_summary) assert {'actions'} == files_received assert not is_event_data assert not non_event_validation_summary.get("http_actions") assert not non_event_validation_summary.get("config") - assert processor.list_http_actions(pytest.bot).__len__() == 5 + assert processor.list_http_actions(pytest.bot).__len__() == 17 @pytest.mark.asyncio async def test_validate_and_prepare_data_save_domain(self, resource_save_and_validate_training_files): @@ -6488,11 +6534,12 @@ async def test_validate_and_prepare_data_save_actions_and_config_overwrite(self, training_file = [pytest.http_actions, pytest.config] files_received, is_event_data, non_event_validation_summary = await processor.validate_and_prepare_data( pytest.bot, 'test', training_file, True) + print(non_event_validation_summary) assert {'actions', 'config'} == files_received assert not is_event_data assert not non_event_validation_summary.get("http_actions") assert not non_event_validation_summary.get("config") - assert processor.list_http_actions(pytest.bot).__len__() == 5 + assert processor.list_http_actions(pytest.bot).__len__() == 17 config = processor.load_config(pytest.bot) assert config['pipeline'] assert config['policies'] @@ -6509,7 +6556,7 @@ async def test_validate_and_prepare_data_save_actions_and_config_append(self, assert not is_event_data assert not non_event_validation_summary.get("http_actions") assert not non_event_validation_summary.get("config") - assert processor.list_http_actions(pytest.bot).__len__() == 6 + assert processor.list_http_actions(pytest.bot).__len__() == 18 config = processor.load_config(pytest.bot) assert config['pipeline'] assert config['policies'] @@ -6557,7 +6604,7 @@ async def test_validate_and_prepare_data_zip_actions_config(self, assert not is_event_data assert not non_event_validation_summary.get("http_actions") assert not non_event_validation_summary.get("config") - assert processor.list_http_actions(pytest.bot).__len__() == 5 + assert processor.list_http_actions(pytest.bot).__len__() == 17 config = processor.load_config(pytest.bot) assert config['pipeline'] assert config['policies'] @@ -6586,7 +6633,8 @@ async def test_validate_and_prepare_data_invalid_zip_actions_config(self, processor = MongoProcessor() files_received, is_event_data, non_event_validation_summary = await processor.validate_and_prepare_data( pytest.bot, 'test', [pytest.zip], True) - assert 'Required http action' in non_event_validation_summary['summary']['http_actions'][0] + print(non_event_validation_summary) + assert 'No name found for [http_action]' in non_event_validation_summary['summary']['http_action'][0] assert files_received == {'actions', 'config'} assert not is_event_data @@ -6604,21 +6652,23 @@ async def test_validate_and_prepare_data_all_actions(self): files_received, is_event_data, non_event_validation_summary = await processor.validate_and_prepare_data( 'test_validate_and_prepare_data_all_actions', 'test', [actions], True) assert non_event_validation_summary['summary'] == { - 'http_actions': [], 'slot_set_actions': [], 'form_validation_actions': [], - 'email_actions': [], - 'google_search_actions': [], 'jira_actions': [], 'zendesk_actions': [], - 'pipedrive_leads_actions': [], 'prompt_actions': [], 'razorpay_actions': [], - 'pyscript_actions': [] + 'http_action': [], 'slot_set_action': [], 'form_validation_action': [], + 'email_action': [], + 'google_search_action': [], 'jira_action': [], 'zendesk_action': [], + 'pipedrive_leads_action': [], 'prompt_action': [], 'razorpay_action': [], + 'pyscript_action': [], 'database_action': [], 'callback_action': [], 'callbackconfig': [], + 'two_stage_fallback': [], 'schedule_action': [], 'web_search_action': [], 'live_agent_action': [] } - assert non_event_validation_summary['component_count']['http_actions'] == 4 - assert non_event_validation_summary['component_count']['jira_actions'] == 2 - assert non_event_validation_summary['component_count']['google_search_actions'] == 2 - assert non_event_validation_summary['component_count']['zendesk_actions'] == 2 - assert non_event_validation_summary['component_count']['email_actions'] == 2 - assert non_event_validation_summary['component_count']['slot_set_actions'] == 3 - assert non_event_validation_summary['component_count']['form_validation_actions'] == 4 - assert non_event_validation_summary['component_count']['pipedrive_leads_actions'] == 2 - assert non_event_validation_summary['component_count']['prompt_actions'] == 2 + print(non_event_validation_summary) + assert non_event_validation_summary['component_count']['http_action'] == 4 + assert non_event_validation_summary['component_count']['jira_action'] == 2 + assert non_event_validation_summary['component_count']['google_search_action'] == 2 + assert non_event_validation_summary['component_count']['zendesk_action'] == 2 + assert non_event_validation_summary['component_count']['email_action'] == 2 + assert non_event_validation_summary['component_count']['slot_set_action'] == 3 + assert non_event_validation_summary['component_count']['form_validation_action'] == 4 + assert non_event_validation_summary['component_count']['pipedrive_leads_action'] == 2 + assert non_event_validation_summary['component_count']['prompt_action'] == 2 assert non_event_validation_summary['validation_failed'] is False assert files_received == {'actions'} assert not is_event_data @@ -12061,7 +12111,7 @@ def test_add_complex_story(self): actions = processor.list_actions("tests") assert not DeepDiff(actions, {'actions': [], 'zendesk_action': [], 'pipedrive_leads_action': [], 'hubspot_forms_action': [], - 'http_action': [], 'google_search_action': [], 'jira_action': [], + 'http_action': ['my_http_action'], 'google_search_action': [], 'jira_action': [], 'two_stage_fallback': [], 'slot_set_action': [], 'email_action': [], 'form_validation_action': [], 'kairon_bot_response': [], @@ -13900,7 +13950,7 @@ def test_list_actions(self): actions = processor.list_actions("test_upload_and_save") assert not DeepDiff(actions, { 'actions': ['reset_slot'], 'google_search_action': [], 'jira_action': [], 'pipedrive_leads_action': [], - 'http_action': ['action_performanceuser1000@digite.com'], 'zendesk_action': [], 'slot_set_action': [], + 'http_action': ['action_performanceUser1000@digite.com'], 'zendesk_action': [], 'slot_set_action': [], 'hubspot_forms_action': [], 'two_stage_fallback': [], 'kairon_bot_response': [], 'razorpay_action': [], 'email_action': [], 'form_validation_action': [], 'prompt_action': [], 'database_action': [], 'pyscript_action': [], 'web_search_action': [], 'live_agent_action': [], 'callback_action': [], 'schedule_action': [], @@ -14009,7 +14059,7 @@ def test_add_rule(self): actions = processor.list_actions("tests") assert not DeepDiff(actions, { 'actions': [], 'zendesk_action': [], 'hubspot_forms_action': [], 'two_stage_fallback': [], - 'http_action': [], 'google_search_action': [], 'pipedrive_leads_action': [], 'kairon_bot_response': [], + 'http_action': ['my_http_action'], 'google_search_action': [], 'pipedrive_leads_action': [], 'kairon_bot_response': [], 'razorpay_action': [], 'prompt_action': ['gpt_llm_faq'], 'slot_set_action': [], 'email_action': [], 'form_validation_action': [], 'jira_action': [], 'database_action': [], 'pyscript_action': [], 'web_search_action': [], 'live_agent_action': [], @@ -15052,13 +15102,7 @@ def test_list_all_actions(self): bot = 'test_bot' user = 'test_user' action = "test_list_all_action_pyscript_action" - script = """ - data = [1, 2, 3, 4, 5, 6] - total = 0 - for i in data: - total += i - print(total) - """ + script = "bot_response='hello world'" processor = MongoProcessor() pyscript_config = PyscriptActionRequest( name=action, diff --git a/tests/unit_test/events/events_test.py b/tests/unit_test/events/events_test.py index 85f4f2695..3ea582297 100644 --- a/tests/unit_test/events/events_test.py +++ b/tests/unit_test/events/events_test.py @@ -17,6 +17,8 @@ from rasa.shared.constants import DEFAULT_DOMAIN_PATH, DEFAULT_DATA_PATH, DEFAULT_CONFIG_PATH from rasa.shared.importers.rasa import RasaFileImporter from responses import matchers + +from kairon.shared.actions.data_objects import Actions from kairon.shared.utils import Utility Utility.load_system_metadata() @@ -58,7 +60,8 @@ def init(self): pytest.tmp_dir = tmp_dir yield None shutil.rmtree(tmp_dir) - shutil.rmtree('models/test_events_bot') + if os.path.exists('models/test_events_bot'): + shutil.rmtree('models/test_events_bot') @pytest.fixture() def get_training_data(self): @@ -812,7 +815,7 @@ def _path(*args, **kwargs): monkeypatch.setattr(Utility, "get_latest_file", _path) BotSettings(force_import=True, bot=bot, user=user).save() - + Actions.objects(bot=bot).delete() DataImporterLogProcessor.add_log(bot, user, files_received=['nlu', 'stories', 'domain', 'config']) TrainingDataImporterEvent(bot, user, import_data=True, overwrite=True).execute() logs = list(DataImporterLogProcessor.get_logs(bot)) @@ -834,7 +837,7 @@ def _path(*args, **kwargs): assert len(mongo_processor.fetch_stories(bot)) == 3 assert len(list(mongo_processor.fetch_training_examples(bot))) == 21 assert len(list(mongo_processor.fetch_responses(bot))) == 14 - assert len(mongo_processor.fetch_actions(bot)) == 4 + assert len(mongo_processor.fetch_actions(bot)) == 0 assert len(mongo_processor.fetch_rule_block_names(bot)) == 1 def test_trigger_faq_importer_validate_only(self, monkeypatch): diff --git a/tests/unit_test/utility_test.py b/tests/unit_test/utility_test.py index 3b27d6759..7b7bf4c47 100644 --- a/tests/unit_test/utility_test.py +++ b/tests/unit_test/utility_test.py @@ -406,7 +406,7 @@ def test_write_training_data_with_rules(self): def test_read_yaml(self): path = "tests/testing_data/yml_training_files/actions.yml" content = Utility.read_yaml(path) - assert len(content["http_action"]) == 5 + assert len(content["http_action"]) == 17 def test_read_yaml_multiflow_story(self): path = "tests/testing_data/yml_training_files/multiflow_stories.yml" diff --git a/tests/unit_test/validator/data_importer_test.py b/tests/unit_test/validator/data_importer_test.py index c1773661c..3a7e63729 100644 --- a/tests/unit_test/validator/data_importer_test.py +++ b/tests/unit_test/validator/data_importer_test.py @@ -65,7 +65,8 @@ async def test_validate_all_including_http_actions(self): assert not summary.get('intents') assert not summary.get('stories') assert not summary.get('utterances') - assert 'Required http action' in summary.get('http_actions')[0] + assert len(summary.get('http_action')) == 3 + summary.get('http_action')[0] = {'action_performanceUser1000@digite.com': " Required fields {'request_method'} not found."} assert not summary.get('training_examples') assert not summary.get('domain') assert not summary.get('config')