Database Action - Fixed requested changes. (#1065)

* Fixed requested changes.

* Fixed requested changes.

* Fixed requested changes.

* Fixed requested changes.

---------

Co-authored-by: Nupur Khare <[email protected]>

kairon/actions/definitions/vector_action.py

@@ -70,13 +70,13 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma
             collection_name = vector_action_config['collection']
             db_type = vector_action_config['db_type']
             vector_db = VectorEmbeddingsDbFactory.get_instance(db_type)(collection_name)
-            operation_type = vector_action_config['query']
+            operation_type = vector_action_config['query_type']
             payload_type = vector_action_config['payload']
             request_body = tracker.get_slot(payload_type.get('value')) if payload_type.get('type') == DbQueryValueType.from_slot.value \
                 else payload_type.get('value')
             msg_logger.append(request_body)
             tracker_data = ActionUtility.build_context(tracker, True)
-            response = vector_db.perform_operation(operation_type.get('value'), request_body)
+            response = vector_db.perform_operation(operation_type, request_body)
             logger.info("response: " + str(response))
             response_context = self.__add_user_context_to_http_response(response, tracker_data)
             bot_response, bot_resp_log = ActionUtility.compose_response(vector_action_config['response'], response_context)

kairon/api/app/routers/bot/data.py

@@ -4,16 +4,18 @@
 from starlette.requests import Request
 from starlette.responses import FileResponse
 
-from kairon.api.models import Response, TextData, CognitiveDataRequest
+from kairon.api.models import Response, CognitiveDataRequest, CognitionSchemaRequest
 from kairon.events.definitions.faq_importer import FaqDataImporterEvent
 from kairon.shared.auth import Authentication
+from kairon.shared.cognition.processor import CognitionDataProcessor
 from kairon.shared.constants import DESIGNER_ACCESS
 from kairon.shared.data.processor import MongoProcessor
 from kairon.shared.models import User
 from kairon.shared.utils import Utility
 
 router = APIRouter()
 processor = MongoProcessor()
+cognition_processor = CognitionDataProcessor()
 
 
 @router.post("/faq/upload", response_model=Response)
@@ -51,86 +53,48 @@ async def download_faq_files(
     return response
 
 
-@router.post("/text/faq", response_model=Response)
-async def save_bot_text(
-        text: TextData,
+@router.post("/cognition/schema", response_model=Response)
+async def save_cognition_schema(
+        schema: CognitionSchemaRequest,
         current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
-        collection: str = None
 ):
     """
-    Saves text content into the bot
+    Saves and updates cognition metadata into the bot
     """
     return {
-        "message": "Text saved!",
+        "message": "Schema saved!",
         "data": {
-            "_id": processor.save_content(
-                    text.data,
+            "_id": cognition_processor.save_cognition_schema(
+                    schema.dict(),
                     current_user.get_user(),
                     current_user.get_bot(),
-                    collection
             )
         }
     }
 
 
-@router.put("/text/faq/{text_id}", response_model=Response)
-async def update_bot_text(
-        text_id: str,
-        text: TextData,
+@router.delete("/cognition/schema/{schema_id}", response_model=Response)
+async def delete_cognition_schema(
+        schema_id: str,
         current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
-        collection: str = None,
 ):
     """
-    Updates text content into the bot
-    """
-    return {
-        "message": "Text updated!",
-        "data": {
-            "_id": processor.update_content(
-                text_id,
-                text.data,
-                current_user.get_user(),
-                current_user.get_bot(),
-                collection
-            )
-        }
-    }
-
-
-@router.delete("/text/faq/{text_id}", response_model=Response)
-async def delete_bot_text(
-        text_id: str,
-        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
-):
-    """
-    Deletes text content of the bot
+    Deletes cognition content of the bot
     """
-    processor.delete_content(text_id, current_user.get_user(), current_user.get_bot())
+    cognition_processor.delete_cognition_schema(schema_id, current_user.get_bot())
     return {
-        "message": "Text deleted!"
+        "message": "Schema deleted!"
     }
 
 
-@router.get("/text/faq", response_model=Response)
-async def get_text(
-        request: Request,
-        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
-):
-    """
-    Fetches text content of the bot
-    """
-    kwargs = request.query_params._dict.copy()
-    return {"data": list(processor.get_content(current_user.get_bot(), **kwargs))}
-
-
-@router.get("/text/faq/collection", response_model=Response)
-async def list_collection(
+@router.get("/cognition/schema", response_model=Response)
+async def list_cognition_schema(
         current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
 ):
     """
-    Fetches text content of the bot
+    Fetches cognition content of the bot
     """
-    return {"data": processor.list_collection(current_user.get_bot())}
+    return {"data": list(cognition_processor.list_cognition_schema(current_user.get_bot()))}
 
 
 @router.post("/cognition", response_model=Response)
@@ -144,7 +108,7 @@ async def save_cognition_data(
     return {
         "message": "Record saved!",
         "data": {
-            "_id": processor.save_cognition_data(
+            "_id": cognition_processor.save_cognition_data(
                     cognition.dict(),
                     current_user.get_user(),
                     current_user.get_bot(),
@@ -165,7 +129,7 @@ async def update_cognition_data(
     return {
         "message": "Record updated!",
         "data": {
-            "_id": processor.update_cognition_data(
+            "_id": cognition_processor.update_cognition_data(
                 cognition_id,
                 cognition.dict(),
                 current_user.get_user(),
@@ -183,17 +147,20 @@ async def delete_cognition_data(
     """
     Deletes cognition content of the bot
     """
-    processor.delete_cognition_data(cognition_id, current_user.get_bot())
+    cognition_processor.delete_cognition_data(cognition_id, current_user.get_bot())
     return {
         "message": "Record deleted!"
     }
 
 
 @router.get("/cognition", response_model=Response)
 async def list_cognition_data(
+        request: Request,
+        start_idx: int = 0, page_size: int = 10,
         current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
 ):
     """
     Fetches cognition content of the bot
     """
-    return {"data": list(processor.list_cognition_data(current_user.get_bot()))}
+    kwargs = request.query_params._dict.copy()
+    return {"data": list(cognition_processor.list_cognition_data(current_user.get_bot(), start_idx, page_size, **kwargs))}

kairon/api/models.py

@@ -3,9 +3,9 @@
 import validators
 from fastapi.param_functions import Form
 from fastapi.security import OAuth2PasswordRequestForm
+from rasa.shared.constants import DEFAULT_NLU_FALLBACK_INTENT_NAME
 
 from kairon.exceptions import AppException
-from rasa.shared.constants import DEFAULT_NLU_FALLBACK_INTENT_NAME
 from kairon.shared.data.constant import EVENT_STATUS, SLOT_MAPPING_TYPE, SLOT_TYPE, ACCESS_ROLES, ACTIVITY_STATUS, \
     INTEGRATION_STATUS, FALLBACK_MESSAGE, DEFAULT_NLU_FALLBACK_RESPONSE
 from ..shared.actions.models import ActionParameterType, EvaluationType, DispatchType, DbQueryValueType, \
@@ -422,7 +422,7 @@ def validate_source_code(cls, v, values, **kwargs):
 
 class DatabaseActionRequest(BaseModel):
     name: constr(to_lower=True, strip_whitespace=True)
-    query: QueryConfig
+    query_type: DbActionOperationType
     payload: PayloadConfig
     response: ActionResponseEvaluation = None
     set_slots: List[SetSlotsUsingActionResponse] = []
@@ -947,28 +947,43 @@ def check(cls, values):
         return values
 
 
-class Metadata(BaseModel):
+class ColumnMetadata(BaseModel):
     column_name: str
     data_type: CognitionMetadataType
     enable_search: bool = True
     create_embeddings: bool = True
 
+    @root_validator
+    def check(cls, values):
+        from kairon.shared.utils import Utility
+
+        if values.get('data_type') not in [CognitionMetadataType.str.value, CognitionMetadataType.int.value]:
+            raise ValueError("Only str and int data types are supported")
+        if Utility.check_empty_string(values.get('column_name')):
+            raise ValueError("Column name cannot be empty")
+        return values
+
+
+class CognitionSchemaRequest(BaseModel):
+    metadata: List[ColumnMetadata] = None
+    collection_name: str
+
 
 class CognitiveDataRequest(BaseModel):
     data: Any
-    content_type: CognitionDataType
-    metadata: List[Metadata] = None
+    content_type: CognitionDataType = CognitionDataType.text.value
     collection: str = None
 
     @root_validator
     def check(cls, values):
         from kairon.shared.utils import Utility
 
         data = values.get("data")
-        metadata = values.get("metadata", [])
-        if metadata:
-            for metadata_item in metadata:
-                Utility.retrieve_data(data, metadata_item.dict())
+        content_type = values.get("content_type")
+        if isinstance(data, dict) and content_type != CognitionDataType.json.value:
+            raise ValueError("content type and type of data do not match!")
+        if not data or (isinstance(data, str) and Utility.check_empty_string(data)):
+            raise ValueError("data cannot be empty")
         return values
 
 

kairon/shared/actions/data_objects.py

@@ -162,17 +162,6 @@ def pre_save_post_validation(cls, sender, document, **kwargs):
                     param.value = Utility.encrypt_message(param.value)
 
 
-class DbOperation(EmbeddedDocument):
-    type = StringField(required=True, choices=[op_type.value for op_type in DbQueryValueType])
-    value = StringField(required=True, choices=[payload.value for payload in DbActionOperationType])
-
-    def validate(self, clean=True):
-        if Utility.check_empty_string(self.type):
-            raise ValidationError("query type is required")
-        if not self.value or self.value is None:
-            raise ValidationError("query value is required")
-
-
 class DbQuery(EmbeddedDocument):
     type = StringField(required=True, choices=[op_type.value for op_type in DbQueryValueType])
     value = DynamicField(required=True)
@@ -189,7 +178,7 @@ def validate(self, clean=True):
 class DatabaseAction(Auditlog):
     name = StringField(required=True)
     collection = StringField(required=True)
-    query = EmbeddedDocumentField(DbOperation, required=True)
+    query_type = StringField(required=True, choices=[payload.value for payload in DbActionOperationType])
     payload = EmbeddedDocumentField(DbQuery, default=DbQuery())
     response = EmbeddedDocumentField(HttpActionResponse, default=HttpActionResponse())
     set_slots = ListField(EmbeddedDocumentField(SetSlotsFromResponse))
@@ -206,9 +195,10 @@ def validate(self, clean=True):
 
         if self.name is None or not self.name.strip():
             raise ValidationError("Action name cannot be empty")
+        if not self.query_type or self.query_type is None:
+            raise ValidationError("query type is required")
         self.response.validate()
         self.payload.validate()
-        self.query.validate()
 
     def clean(self):
         self.name = self.name.strip().lower()

kairon/shared/cognition/data_objects.py

@@ -0,0 +1,78 @@
+from datetime import datetime
+
+from mongoengine import EmbeddedDocument, StringField, BooleanField, ValidationError, ListField, EmbeddedDocumentField, \
+    DateTimeField, SequenceField, DynamicField
+
+from kairon.shared.data.audit.data_objects import Auditlog
+from kairon.shared.data.signals import push_notification, auditlogger
+from kairon.shared.models import CognitionMetadataType, CognitionDataType
+
+
+class ColumnMetadata(EmbeddedDocument):
+    column_name = StringField(required=True)
+    data_type = StringField(required=True, default=CognitionMetadataType.str.value,
+                            choices=[CognitionMetadataType.str.value, CognitionMetadataType.int.value])
+    enable_search = BooleanField(default=True)
+    create_embeddings = BooleanField(default=True)
+
+    def validate(self, clean=True):
+        from kairon import Utility
+
+        if clean:
+            self.clean()
+        if self.data_type not in [CognitionMetadataType.str.value, CognitionMetadataType.int.value]:
+            raise ValidationError("Only str and int data types are supported")
+        if Utility.check_empty_string(self.column_name):
+            raise ValidationError("Column name cannot be empty")
+
+    def clean(self):
+        from kairon import Utility
+
+        if not Utility.check_empty_string(self.column_name):
+            self.column_name = self.column_name.strip().lower()
+
+
+@auditlogger.log
+@push_notification.apply
+class CognitionSchema(Auditlog):
+    metadata = ListField(EmbeddedDocumentField(ColumnMetadata, default=None))
+    collection_name = StringField(required=True)
+    user = StringField(required=True)
+    bot = StringField(required=True)
+    timestamp = DateTimeField(default=datetime.utcnow)
+
+    meta = {"indexes": [{"fields": ["bot"]}]}
+
+    def validate(self, clean=True):
+        if clean:
+            self.clean()
+
+        if self.metadata:
+            for metadata_dict in self.metadata:
+                metadata_dict.validate()
+
+
+@auditlogger.log
+@push_notification.apply
+class CognitionData(Auditlog):
+    vector_id = SequenceField(required=True)
+    data = DynamicField(required=True)
+    content_type = StringField(default=CognitionDataType.text.value, choices=[CognitionDataType.text.value,
+                                                                              CognitionDataType.json.value])
+    collection = StringField(default=None)
+    user = StringField(required=True)
+    bot = StringField(required=True)
+    timestamp = DateTimeField(default=datetime.utcnow)
+
+    meta = {"indexes": [{"fields": ["$data", "bot"]}]}
+
+    def validate(self, clean=True):
+        from kairon import Utility
+
+        if clean:
+            self.clean()
+
+        if isinstance(self.data, dict) and self.content_type != CognitionDataType.json.value:
+            raise ValidationError("content type and type of data do not match!")
+        if not self.data or (isinstance(self.data, str) and Utility.check_empty_string(self.data)):
+            raise ValidationError("data cannot be empty")