Database action changes.

kairon/api/app/routers/bot/data.py

@@ -4,16 +4,18 @@
 from starlette.requests import Request
 from starlette.responses import FileResponse
 
-from kairon.api.models import Response, TextData, CognitiveDataRequest
+from kairon.api.models import Response, TextData, CognitiveDataRequest, CognitionSchemaRequest
 from kairon.events.definitions.faq_importer import FaqDataImporterEvent
 from kairon.shared.auth import Authentication
+from kairon.shared.cognition.processor import CognitionDataProcessor
 from kairon.shared.constants import DESIGNER_ACCESS
 from kairon.shared.data.processor import MongoProcessor
 from kairon.shared.models import User
 from kairon.shared.utils import Utility
 
 router = APIRouter()
 processor = MongoProcessor()
+cognition_processor = CognitionDataProcessor()
 
 
 @router.post("/faq/upload", response_model=Response)
@@ -63,7 +65,7 @@ async def save_bot_text(
     return {
         "message": "Text saved!",
         "data": {
-            "_id": processor.save_content(
+            "_id": cognition_processor.save_content(
                     text.data,
                     current_user.get_user(),
                     current_user.get_bot(),
@@ -86,7 +88,7 @@ async def update_bot_text(
     return {
         "message": "Text updated!",
         "data": {
-            "_id": processor.update_content(
+            "_id": cognition_processor.update_content(
                 text_id,
                 text.data,
                 current_user.get_user(),
@@ -105,7 +107,7 @@ async def delete_bot_text(
     """
     Deletes text content of the bot
     """
-    processor.delete_content(text_id, current_user.get_user(), current_user.get_bot())
+    cognition_processor.delete_content(text_id, current_user.get_user(), current_user.get_bot())
     return {
         "message": "Text deleted!"
     }
@@ -120,7 +122,7 @@ async def get_text(
     Fetches text content of the bot
     """
     kwargs = request.query_params._dict.copy()
-    return {"data": list(processor.get_content(current_user.get_bot(), **kwargs))}
+    return {"data": list(cognition_processor.get_content(current_user.get_bot(), **kwargs))}
 
 
 @router.get("/text/faq/collection", response_model=Response)
@@ -130,7 +132,73 @@ async def list_collection(
     """
     Fetches text content of the bot
     """
-    return {"data": processor.list_collection(current_user.get_bot())}
+    return {"data": cognition_processor.list_cognition_collections(current_user.get_bot())}
+
+
+@router.post("/cognition/schema", response_model=Response)
+async def save_cognition_schema(
+        metadata: CognitionSchemaRequest,
+        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
+):
+    """
+    Saves and updates cognition metadata into the bot
+    """
+    return {
+        "message": "Schema saved!",
+        "data": {
+            "_id": cognition_processor.save_cognition_schema(
+                    metadata.dict(),
+                    current_user.get_user(),
+                    current_user.get_bot(),
+            )
+        }
+    }
+
+
+@router.put("/cognition/schema/{metadata_id}", response_model=Response)
+async def update_cognition_schema(
+        metadata_id: str,
+        metadata: CognitionSchemaRequest,
+        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
+):
+    """
+    Saves and updates cognition metadata into the bot
+    """
+    return {
+        "message": "Schema updated!",
+        "data": {
+            "_id": cognition_processor.update_cognition_schema(
+                    metadata_id,
+                    metadata.dict(),
+                    current_user.get_user(),
+                    current_user.get_bot(),
+            )
+        }
+    }
+
+
+@router.delete("/cognition/schema/{metadata_id}", response_model=Response)
+async def delete_cognition_schema(
+        metadata_id: str,
+        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
+):
+    """
+    Deletes cognition content of the bot
+    """
+    cognition_processor.delete_cognition_schema(metadata_id, current_user.get_bot())
+    return {
+        "message": "Schema deleted!"
+    }
+
+
+@router.get("/cognition/schema", response_model=Response)
+async def list_cognition_schema(
+        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
+):
+    """
+    Fetches cognition content of the bot
+    """
+    return {"data": list(cognition_processor.list_cognition_schema(current_user.get_bot()))}
 
 
 @router.post("/cognition", response_model=Response)
@@ -144,7 +212,7 @@ async def save_cognition_data(
     return {
         "message": "Record saved!",
         "data": {
-            "_id": processor.save_cognition_data(
+            "_id": cognition_processor.save_cognition_data(
                     cognition.dict(),
                     current_user.get_user(),
                     current_user.get_bot(),
@@ -165,7 +233,7 @@ async def update_cognition_data(
     return {
         "message": "Record updated!",
         "data": {
-            "_id": processor.update_cognition_data(
+            "_id": cognition_processor.update_cognition_data(
                 cognition_id,
                 cognition.dict(),
                 current_user.get_user(),
@@ -183,7 +251,7 @@ async def delete_cognition_data(
     """
     Deletes cognition content of the bot
     """
-    processor.delete_cognition_data(cognition_id, current_user.get_bot())
+    cognition_processor.delete_cognition_data(cognition_id, current_user.get_bot())
     return {
         "message": "Record deleted!"
     }
@@ -196,4 +264,4 @@ async def list_cognition_data(
     """
     Fetches cognition content of the bot
     """
-    return {"data": list(processor.list_cognition_data(current_user.get_bot()))}
+    return {"data": list(cognition_processor.list_cognition_data(current_user.get_bot()))}

kairon/api/models.py

@@ -3,9 +3,9 @@
 import validators
 from fastapi.param_functions import Form
 from fastapi.security import OAuth2PasswordRequestForm
+from rasa.shared.constants import DEFAULT_NLU_FALLBACK_INTENT_NAME
 
 from kairon.exceptions import AppException
-from rasa.shared.constants import DEFAULT_NLU_FALLBACK_INTENT_NAME
 from kairon.shared.data.constant import EVENT_STATUS, SLOT_MAPPING_TYPE, SLOT_TYPE, ACCESS_ROLES, ACTIVITY_STATUS, \
     INTEGRATION_STATUS, FALLBACK_MESSAGE, DEFAULT_NLU_FALLBACK_RESPONSE
 from ..shared.actions.models import ActionParameterType, EvaluationType, DispatchType, DbQueryValueType, \
@@ -947,28 +947,43 @@ def check(cls, values):
         return values
 
 
-class Metadata(BaseModel):
+class ColumnMetadata(BaseModel):
     column_name: str
     data_type: CognitionMetadataType
     enable_search: bool = True
     create_embeddings: bool = True
 
+    @root_validator
+    def check(cls, values):
+        from kairon.shared.utils import Utility
+
+        if values.get('data_type') not in [CognitionMetadataType.str.value, CognitionMetadataType.int.value]:
+            raise ValueError("Only str and int data types are supported")
+        if Utility.check_empty_string(values.get('column_name')):
+            raise ValueError("Column name cannot be empty")
+        return values
+
+
+class CognitionSchemaRequest(BaseModel):
+    metadata: List[ColumnMetadata]
+    collection_name: str
+
 
 class CognitiveDataRequest(BaseModel):
     data: Any
     content_type: CognitionDataType
-    metadata: List[Metadata] = None
     collection: str = None
 
     @root_validator
     def check(cls, values):
         from kairon.shared.utils import Utility
 
         data = values.get("data")
-        metadata = values.get("metadata", [])
-        if metadata:
-            for metadata_item in metadata:
-                Utility.retrieve_data(data, metadata_item.dict())
+        content_type = values.get("content_type")
+        if isinstance(data, dict) and content_type != CognitionDataType.json.value:
+            raise ValueError("data of type dict is required if content type is json")
+        if not data or (isinstance(data, str) and Utility.check_empty_string(data)):
+            raise ValueError("data cannot be empty")
         return values
 
 

kairon/shared/cognition/data_objects.py

@@ -0,0 +1,73 @@
+from datetime import datetime
+
+from mongoengine import EmbeddedDocument, StringField, BooleanField, ValidationError, ListField, EmbeddedDocumentField, \
+    DateTimeField, SequenceField, DynamicField
+
+from kairon import Utility
+from kairon.shared.data.audit.data_objects import Auditlog
+from kairon.shared.data.signals import push_notification, auditlogger
+from kairon.shared.models import CognitionMetadataType, CognitionDataType
+
+
+class ColumnMetadata(EmbeddedDocument):
+    column_name = StringField(required=True)
+    data_type = StringField(required=True, default=CognitionMetadataType.str.value,
+                            choices=[CognitionMetadataType.str.value, CognitionMetadataType.int.value])
+    enable_search = BooleanField(default=True)
+    create_embeddings = BooleanField(default=True)
+
+    def validate(self, clean=True):
+        if clean:
+            self.clean()
+        if self.data_type not in [CognitionMetadataType.str.value, CognitionMetadataType.int.value]:
+            raise ValidationError("Only str and int data types are supported")
+        if Utility.check_empty_string(self.column_name):
+            raise ValidationError("Column name cannot be empty")
+
+    def clean(self):
+        if not Utility.check_empty_string(self.column_name):
+            self.column_name = self.column_name.strip().lower()
+
+
+@auditlogger.log
+@push_notification.apply
+class CognitionSchema(Auditlog):
+    metadata = ListField(EmbeddedDocumentField(ColumnMetadata))
+    collection_name = StringField(required=True)
+    user = StringField(required=True)
+    bot = StringField(required=True)
+    timestamp = DateTimeField(default=datetime.utcnow)
+
+    meta = {"indexes": [{"fields": ["bot"]}]}
+
+    def validate(self, clean=True):
+        if clean:
+            self.clean()
+
+        if self.metadata:
+            for metadata_dict in self.metadata:
+                metadata_dict.validate()
+
+
+@auditlogger.log
+@push_notification.apply
+class CognitionData(Auditlog):
+    vector_id = SequenceField(required=True)
+    data = DynamicField(required=True)
+    content_type = StringField(default=CognitionDataType.text.value, choices=[CognitionDataType.text.value,
+                                                            CognitionDataType.json.value])
+    collection = StringField(default=None)
+    user = StringField(required=True)
+    bot = StringField(required=True)
+    timestamp = DateTimeField(default=datetime.utcnow)
+
+    meta = {"indexes": [{"fields": ["$data", "bot"]}]}
+
+    def validate(self, clean=True):
+        if clean:
+            self.clean()
+
+        if isinstance(self.data, dict) and self.content_type != CognitionDataType.json.value:
+            raise ValidationError("data of type dict is required if content type is json")
+        if not self.data or (isinstance(self.data, str) and Utility.check_empty_string(self.data)):
+            raise ValidationError("data cannot be empty")