Collection Data (#1402)

* Added CRUD operations for the CollectionData and added unit and integration tests for the same.

* Added CRUD operations for the CollectionData and added unit and integration tests for the same.

* removed unused import statement

kairon/api/app/routers/bot/data.py

@@ -4,7 +4,7 @@
 from starlette.requests import Request
 from starlette.responses import FileResponse
 
-from kairon.api.models import Response, CognitiveDataRequest, CognitionSchemaRequest
+from kairon.api.models import Response, CognitiveDataRequest, CognitionSchemaRequest, CollectionDataRequest
 from kairon.events.definitions.faq_importer import FaqDataImporterEvent
 from kairon.shared.auth import Authentication
 from kairon.shared.cognition.processor import CognitionDataProcessor
@@ -171,3 +171,83 @@ async def list_cognition_data(
         "total": row_cnt
     }
     return Response(data=data)
+
+
+@router.post("/collection", response_model=Response)
+async def save_collection_data(
+        collection: CollectionDataRequest,
+        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
+):
+    """
+    Saves collection data
+    """
+    return {
+        "message": "Record saved!",
+        "data": {
+            "_id": cognition_processor.save_collection_data(
+                collection.dict(),
+                current_user.get_user(),
+                current_user.get_bot(),
+            )
+        }
+    }
+
+
+@router.put("/collection/{collection_id}", response_model=Response)
+async def update_collection_data(
+        collection_id: str,
+        collection: CollectionDataRequest,
+        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
+):
+    """
+    Updates collection data
+    """
+    return {
+        "message": "Record updated!",
+        "data": {
+            "_id": cognition_processor.update_collection_data(
+                collection_id,
+                collection.dict(),
+                current_user.get_user(),
+                current_user.get_bot(),
+            )
+        }
+    }
+
+
+@router.delete("/collection/{collection_id}", response_model=Response)
+async def delete_collection_data(
+        collection_id: str,
+        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
+):
+    """
+    Deletes collection data
+    """
+    cognition_processor.delete_collection_data(collection_id, current_user.get_bot(), current_user.get_user())
+    return {
+        "message": "Record deleted!"
+    }
+
+
+@router.get("/collection", response_model=Response)
+async def list_collection_data(
+        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
+):
+    """
+    Fetches collection data of the bot
+    """
+    return {"data": list(cognition_processor.list_collection_data(current_user.get_bot()))}
+
+
+@router.get("/collection/{collection_name}", response_model=Response)
+async def get_collection_data(
+        collection_name: str,
+        key: str = None, value: str = None,
+        current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS),
+):
+    """
+    Fetches collection data based on the filters provided
+    """
+    return {"data": list(cognition_processor.get_collection_data(current_user.get_bot(),
+                                                                 collection_name=collection_name,
+                                                                 key=key, value=value))}

kairon/api/models.py

@@ -1138,6 +1138,35 @@ class CognitionSchemaRequest(BaseModel):
     collection_name: constr(to_lower=True, strip_whitespace=True)
 
 
+class CollectionDataRequest(BaseModel):
+    data: dict
+    is_secure: list = []
+    collection_name: constr(to_lower=True, strip_whitespace=True)
+
+    @root_validator
+    def check(cls, values):
+        from kairon.shared.utils import Utility
+
+        data = values.get("data")
+        is_secure = values.get("is_secure")
+        collection_name = values.get("collection_name")
+        if Utility.check_empty_string(collection_name):
+            raise ValueError("collection_name should not be empty!")
+
+        if not isinstance(is_secure, list):
+            raise ValueError("is_secure should be list of keys!")
+
+        if is_secure:
+            if not data or not isinstance(data, dict):
+                raise ValueError("data cannot be empty and should be of type dict!")
+            data_keys = set(data.keys())
+            is_secure_set = set(is_secure)
+
+            if not is_secure_set.issubset(data_keys):
+                raise ValueError("is_secure contains keys that are not present in data")
+        return values
+
+
 class CognitiveDataRequest(BaseModel):
     data: Any
     content_type: CognitionDataType = CognitionDataType.text.value

kairon/shared/cognition/data_objects.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 
 from mongoengine import EmbeddedDocument, StringField, BooleanField, ValidationError, ListField, EmbeddedDocumentField, \
-    DateTimeField, SequenceField, DynamicField
+    DateTimeField, SequenceField, DynamicField, DictField
 
 from kairon.shared.data.audit.data_objects import Auditlog
 from kairon.shared.data.signals import push_notification, auditlogger
@@ -83,3 +83,40 @@ def validate(self, clean=True):
     def clean(self):
         if self.collection:
             self.collection = self.collection.strip().lower()
+
+
+class CollectionData(Auditlog):
+    collection_name = StringField(required=True)
+    is_secure = ListField(StringField(), default=[])
+    data = DictField()
+    user = StringField(required=True)
+    bot = StringField(required=True)
+    timestamp = DateTimeField(default=datetime.utcnow)
+    status = BooleanField(default=True)
+
+    meta = {"indexes": [{"fields": ["bot"]}]}
+
+    def validate(self, clean=True):
+        from kairon import Utility
+
+        if clean:
+            self.clean()
+
+        if Utility.check_empty_string(self.collection_name):
+            raise ValidationError("collection_name should not be empty")
+
+        if not isinstance(self.is_secure, list):
+            raise ValidationError("is_secure should be list of keys")
+
+        if self.is_secure:
+            if not self.data or not isinstance(self.data, dict):
+                raise ValidationError("data cannot be empty and should be of type dict")
+            data_keys = set(self.data.keys())
+            is_secure_set = set(self.is_secure)
+
+            if not is_secure_set.issubset(data_keys):
+                raise ValidationError("is_secure contains keys that are not present in data")
+
+    def clean(self):
+        if self.collection_name:
+            self.collection_name = self.collection_name.strip().lower()

kairon/shared/cognition/processor.py

@@ -7,7 +7,7 @@
 from kairon import Utility
 from kairon.exceptions import AppException
 from kairon.shared.actions.data_objects import PromptAction, DatabaseAction
-from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema, ColumnMetadata
+from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema, ColumnMetadata, CollectionData
 from kairon.shared.data.processor import MongoProcessor
 from kairon.shared.models import CognitionDataType, CognitionMetadataType
 
@@ -112,6 +112,130 @@ def list_cognition_schema(self, bot: Text):
             final_data['collection_name'] = collection
             yield final_data
 
+    def delete_collection_data(self, collection_id: str, bot: Text, user: Text):
+        try:
+            collection = CollectionData.objects(bot=bot, id=collection_id).get()
+            collection.delete(user=user)
+        except DoesNotExist:
+            raise AppException("Collection Data does not exists!")
+
+    @staticmethod
+    def validate_collection_payload(collection_name, is_secure, data):
+        if not collection_name:
+            raise AppException("collection name is empty")
+
+        if not isinstance(is_secure, list):
+            raise AppException("is_secure should be list of keys")
+
+        if is_secure:
+            if not data or not isinstance(data, dict):
+                raise AppException("Invalid value for data")
+
+    def save_collection_data(self, payload: Dict, user: Text, bot: Text):
+        collection_name = payload.get("collection_name", None)
+        data = payload.get('data')
+        is_secure = payload.get('is_secure')
+        CognitionDataProcessor.validate_collection_payload(collection_name, is_secure, data)
+
+        data = CognitionDataProcessor.prepare_encrypted_data(data, is_secure)
+
+        collection_obj = CollectionData()
+        collection_obj.data = data
+        collection_obj.is_secure = is_secure
+        collection_obj.collection_name = collection_name
+        collection_obj.user = user
+        collection_obj.bot = bot
+        collection_id = collection_obj.save().to_mongo().to_dict()["_id"].__str__()
+        return collection_id
+
+    def update_collection_data(self, collection_id: str, payload: Dict, user: Text, bot: Text):
+        collection_name = payload.get("collection_name", None)
+        data = payload.get('data')
+        is_secure = payload.get('is_secure')
+        CognitionDataProcessor.validate_collection_payload(collection_name, is_secure, data)
+
+        data = CognitionDataProcessor.prepare_encrypted_data(data, is_secure)
+
+        try:
+            collection_obj = CollectionData.objects(bot=bot, id=collection_id, collection_name=collection_name).get()
+            collection_obj.data = data
+            collection_obj.collection_name = collection_name
+            collection_obj.is_secure = is_secure
+            collection_obj.user = user
+            collection_obj.timestamp = datetime.utcnow()
+            collection_obj.save()
+        except DoesNotExist:
+            raise AppException("Collection Data with given id and collection_name not found!")
+        return collection_id
+
+    @staticmethod
+    def prepare_encrypted_data(data, is_secure):
+        encrypted_data = {}
+        for key, value in data.items():
+            if key in is_secure:
+                encrypted_data[key] = Utility.encrypt_message(value)
+            else:
+                encrypted_data[key] = value
+        return encrypted_data
+
+    @staticmethod
+    def prepare_decrypted_data(data, is_secure):
+        decrypted_data = {}
+        for key, value in data.items():
+            if key in is_secure:
+                decrypted_data[key] = Utility.decrypt_message(value)
+            else:
+                decrypted_data[key] = value
+        return decrypted_data
+
+    def list_collection_data(self, bot: Text):
+        """
+        fetches collection data
+
+        :param bot: bot id
+        :return: yield dict
+        """
+        for value in CollectionData.objects(bot=bot):
+            final_data = {}
+            item = value.to_mongo().to_dict()
+            collection_name = item.pop('collection_name', None)
+            is_secure = item.pop('is_secure')
+            data = item.pop('data')
+            data = CognitionDataProcessor.prepare_decrypted_data(data, is_secure)
+            final_data["_id"] = item["_id"].__str__()
+            final_data['collection_name'] = collection_name
+            final_data['is_secure'] = is_secure
+            final_data['data'] = data
+            yield final_data
+
+    def get_collection_data(self, bot: Text, **kwargs):
+        """
+        fetches collection data based on the filters provided
+
+        :param bot: bot id
+        :return: yield dict
+        """
+        collection_name = kwargs.pop("collection_name")
+        collection_name = collection_name.lower()
+        key = kwargs.pop("key", None)
+        value = kwargs.pop("value", None)
+        query = {"bot": bot, "collection_name": collection_name}
+        if key is not None and value is not None:
+            query.update({f"data__{key}": value})
+
+        for value in CollectionData.objects(**query):
+            final_data = {}
+            item = value.to_mongo().to_dict()
+            collection_name = item.pop('collection_name', None)
+            is_secure = item.pop('is_secure')
+            data = item.pop('data')
+            data = CognitionDataProcessor.prepare_decrypted_data(data, is_secure)
+            final_data["_id"] = item["_id"].__str__()
+            final_data['collection_name'] = collection_name
+            final_data['is_secure'] = is_secure
+            final_data['data'] = data
+            yield final_data
+
     @staticmethod
     def validate_metadata_and_payload(bot, payload):
         data = payload.get('data')