From d378aa77b4e1c3b1bc303cc1594fbabddcc440c8 Mon Sep 17 00:00:00 2001 From: Nupur Khare <78532321+nupur-khare@users.noreply.github.com> Date: Fri, 27 Oct 2023 19:43:16 +0530 Subject: [PATCH] Api fixtures. (#1067) * Api fixtures. * Api fixtures. * Api fixtures. * Api fixtures. --------- Co-authored-by: Nupur Khare --- kairon/api/app/routers/bot/data.py | 19 +++-- kairon/shared/cognition/processor.py | 22 ++++-- tests/integration_test/services_test.py | 72 ++++++++++--------- .../data_processor/data_processor_test.py | 44 +++++++++--- 4 files changed, 102 insertions(+), 55 deletions(-) diff --git a/kairon/api/app/routers/bot/data.py b/kairon/api/app/routers/bot/data.py index 9b31a86c2..41979c101 100644 --- a/kairon/api/app/routers/bot/data.py +++ b/kairon/api/app/routers/bot/data.py @@ -117,9 +117,9 @@ async def save_cognition_data( } -@router.put("/cognition/{cognition_id}", response_model=Response) +@router.put("/cognition/{row_id}", response_model=Response) async def update_cognition_data( - cognition_id: str, + row_id: str, cognition: CognitiveDataRequest, current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS), ): @@ -130,7 +130,7 @@ async def update_cognition_data( "message": "Record updated!", "data": { "_id": cognition_processor.update_cognition_data( - cognition_id, + row_id, cognition.dict(), current_user.get_user(), current_user.get_bot(), @@ -139,15 +139,15 @@ async def update_cognition_data( } -@router.delete("/cognition/{cognition_id}", response_model=Response) +@router.delete("/cognition/{row_id}", response_model=Response) async def delete_cognition_data( - cognition_id: str, + row_id: str, current_user: User = Security(Authentication.get_current_user_and_bot, scopes=DESIGNER_ACCESS), ): """ Deletes cognition content of the bot """ - cognition_processor.delete_cognition_data(cognition_id, current_user.get_bot()) + cognition_processor.delete_cognition_data(row_id, current_user.get_bot()) return { "message": "Record deleted!" } @@ -163,4 +163,9 @@ async def list_cognition_data( Fetches cognition content of the bot """ kwargs = request.query_params._dict.copy() - return {"data": list(cognition_processor.list_cognition_data(current_user.get_bot(), start_idx, page_size, **kwargs))} + cognition_data, row_cnt = cognition_processor.get_cognition_data(current_user.get_bot(), start_idx, page_size, **kwargs) + data = { + "data": cognition_data, + "row_count": row_cnt + } + return Response(data=data) diff --git a/kairon/shared/cognition/processor.py b/kairon/shared/cognition/processor.py index e9468f936..3a7cf6596 100644 --- a/kairon/shared/cognition/processor.py +++ b/kairon/shared/cognition/processor.py @@ -142,19 +142,19 @@ def save_cognition_data(self, payload: Dict, user: Text, bot: Text): payload_id = payload_obj.save().to_mongo().to_dict()["_id"].__str__() return payload_id - def update_cognition_data(self, payload_id: str, payload: Dict, user: Text, bot: Text): + def update_cognition_data(self, row_id: str, payload: Dict, user: Text, bot: Text): from kairon import Utility data = payload['data'] content_type = payload['content_type'] if payload.get('content_type') == CognitionDataType.text.value and len(payload.get('data').split()) < 10: raise AppException("Content should contain atleast 10 words.") - Utility.is_exist(CognitionData, bot=bot, id__ne=payload_id, data=data, + Utility.is_exist(CognitionData, bot=bot, id__ne=row_id, data=data, exp_message="Payload data already exists!") if payload.get('collection') and not Utility.is_exist(CognitionSchema, bot=bot, collection_name=payload.get('collection'), raise_error=False): raise AppException('Collection does not exist!') try: - payload_obj = CognitionData.objects(bot=bot, id=payload_id).get() + payload_obj = CognitionData.objects(bot=bot, id=row_id).get() if content_type == CognitionDataType.json.value: CognitionDataProcessor.validate_metadata_and_payload(bot, payload) payload_obj.data = data @@ -166,9 +166,9 @@ def update_cognition_data(self, payload_id: str, payload: Dict, user: Text, bot: except DoesNotExist: raise AppException("Payload with given id not found!") - def delete_cognition_data(self, payload_id: str, bot: Text): + def delete_cognition_data(self, row_id: str, bot: Text): try: - payload = CognitionData.objects(bot=bot, id=payload_id).get() + payload = CognitionData.objects(bot=bot, id=row_id).get() payload.delete() except DoesNotExist: raise AppException("Payload does not exists!") @@ -189,12 +189,12 @@ def list_cognition_data(self, bot: Text, start_idx: int = 0, page_size: int = 10 cognition_data = CognitionData.objects(**kwargs) if search: cognition_data = cognition_data.search_text(search) - for value in cognition_data.skip(start_idx).limit(page_size): + for value in cognition_data.skip(start_idx).limit(page_size).order_by('-id'): final_data = {} item = value.to_mongo().to_dict() data = item.pop("data") data_type = item.pop("content_type") - final_data["_id"] = item["_id"].__str__() + final_data["row_id"] = item["_id"].__str__() final_data['data'] = data final_data['content_type'] = data_type final_data['collection'] = item.get('collection', None) @@ -202,6 +202,14 @@ def list_cognition_data(self, bot: Text, start_idx: int = 0, page_size: int = 10 final_data['bot'] = item.get('bot') yield final_data + def get_cognition_data(self, bot: Text, start_idx: int = 0, page_size: int = 10, **kwargs): + processor = MongoProcessor() + collection = kwargs.pop('collection', None) + kwargs['collection'] = collection + cognition_data = list(self.list_cognition_data(bot, start_idx, page_size, **kwargs)) + row_cnt = processor.get_row_count(CognitionData, bot, **kwargs) + return cognition_data, row_cnt + @staticmethod def validate_column_values(data: Any, schema: Dict): if schema and isinstance(data, dict): diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index cbc6d4719..d2fb0a1a1 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -1662,7 +1662,7 @@ def test_content_update_api(): response = client.put( url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.content_id_text}", json={ - "cognition_id": pytest.content_id_text, + "row_id": pytest.content_id_text, "data": "AWS Fargate is a serverless compute engine for containers that allows you to run " "Docker containers without having to manage the underlying EC2 instances. With Fargate, " "you can focus on developing and deploying your applications rather than managing the infrastructure.", @@ -1682,7 +1682,7 @@ def test_content_update_api_collection_does_not_exist(): response = client.put( url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.content_id_text}", json={ - "cognition_id": pytest.content_id_text, + "row_id": pytest.content_id_text, "data": "Docker containers without having to manage the underlying EC2 instances.", "collection": "test_content_update_api_collection_does_not_exist", "content_type": "text" @@ -1700,7 +1700,7 @@ def test_content_update_api_invalid(): response = client.put( url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.content_id_text}", json={ - "cognition_id": pytest.content_id_text, + "row_id": pytest.content_id_text, "data": "AWS Fargate is a serverless compute engine.", "collection": "details", "content_type": "text" @@ -1720,7 +1720,7 @@ def test_content_update_api_already_exist(): response = client.put( url=f"/api/bot/{pytest.bot}/data/cognition/{content_id}", json={ - "cognition_id": content_id, + "row_id": content_id, "data": "AWS Fargate is a serverless compute engine for containers that allows you to run " "Docker containers without having to manage the underlying EC2 instances. With Fargate, " "you can focus on developing and deploying your applications rather than managing the infrastructure.", @@ -1742,7 +1742,7 @@ def test_content_update_api_id_not_found(): response = client.put( url=f"/api/bot/{pytest.bot}/data/cognition/{content_id}", json={ - "cognition_id": content_id, + "row_id": content_id, "data": "Artificial intelligence (AI) involves using computers to do things that traditionally require human " "intelligence. AI can process large amounts of data in ways that humans cannot. The goal for AI is " "to be able to do things like recognize patterns, make decisions, and judge like humans.", @@ -1760,15 +1760,22 @@ def test_content_update_api_id_not_found(): @mock.patch('kairon.shared.cognition.processor.CognitionDataProcessor.list_cognition_data', autospec=True) -def test_list_cognition_data(mock_list_cognition_data): - def _list_cognition_data(*args, **kwargs): - return [{'vector_id': 1, - '_id': '65266ff16f0190ca4fd09898', +@mock.patch('kairon.shared.cognition.processor.CognitionDataProcessor.get_cognition_data', autospec=True) +def test_list_cognition_data(mock_get_cognition_data, mock_list_cognition_data): + cognition_data = [{'vector_id': 1, + 'row_id': '65266ff16f0190ca4fd09898', 'data': 'AWS Fargate is a serverless compute engine for containers that allows you to run Docker containers without having to manage the underlying EC2 instances. With Fargate, you can focus on developing and deploying your applications rather than managing the infrastructure.', 'content_type': 'text', - 'collection': 'aws', 'user': '"integration@demo.ai"', 'bot': pytest.bot,}] - + 'collection': 'aws', 'user': '"integration@demo.ai"', 'bot': pytest.bot}] + row_count = 1 + def _list_cognition_data(*args, **kwargs): + return cognition_data mock_list_cognition_data.return_value = _list_cognition_data() + + def _get_cognition_data(*args, **kwargs): + return cognition_data, row_count + mock_get_cognition_data.return_value = _get_cognition_data() + filter_query = 'without having to manage' response = client.get( url=f"/api/bot/{pytest.bot}/data/cognition?data={filter_query}", @@ -1778,8 +1785,8 @@ def _list_cognition_data(*args, **kwargs): print(actual) assert actual["success"] assert actual["error_code"] == 0 - assert actual["data"] - assert actual["data"][0]['collection'] + assert actual["data"]['data'][0]['collection'] == 'aws' + assert actual["data"]['row_count'] == 1 def test_get_content_without_data(): @@ -1791,17 +1798,16 @@ def test_get_content_without_data(): print(actual) assert actual["success"] assert actual["error_code"] == 0 - assert actual["data"] - assert actual["data"][0]['collection'] == 'details' - assert actual["data"][0]['data'] == 'AWS Fargate is a serverless compute engine for containers that allows you to run Docker containers without having to manage the underlying EC2 instances. With Fargate, you can focus on developing and deploying your applications rather than managing the infrastructure.' - assert actual["data"][1]['data'] == 'Blockchain technology is an advanced database mechanism that allows transparent information sharing within a business network.' + assert actual["data"]['data'][0]['collection'] == None + assert actual["data"]['data'][0]['data'] == 'Blockchain technology is an advanced database mechanism that allows transparent information sharing within a business network.' + assert actual["data"]['row_count'] == 1 def test_delete_content(): response_one = client.delete( url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.content_id_no_collection}", json={ - "cognition_id": pytest.content_id_no_collection, + "row_id": pytest.content_id_no_collection, }, headers={"Authorization": pytest.token_type + " " + pytest.access_token} ) @@ -1814,7 +1820,7 @@ def test_delete_content(): response = client.delete( url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.content_id_text}", json={ - "cognition_id": pytest.content_id_text, + "row_id": pytest.content_id_text, }, headers={"Authorization": pytest.token_type + " " + pytest.access_token} ) @@ -1830,7 +1836,7 @@ def test_delete_content_does_not_exist(): response = client.delete( url=f"/api/bot/{pytest.bot}/data/cognition/{content_id}", json={ - "cognition_id": content_id, + "row_id": content_id, }, headers={"Authorization": pytest.token_type + " " + pytest.access_token} ) @@ -1851,7 +1857,8 @@ def test_get_content_not_exists(): assert actual["success"] assert actual["message"] is None assert actual["error_code"] == 0 - assert actual["data"] == [] + assert actual["data"]['data'] == [] + assert actual["data"]['row_count'] == 0 def test_payload_upload_api_with_gpt_feature_disabled(): @@ -1973,7 +1980,7 @@ def test_payload_updated_api_collection_does_not_exists(): response = client.put( url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.payload_id}", json={ - "cognition_id": pytest.payload_id, + "row_id": pytest.payload_id, "data": {"details": "data science"}, "collection": "test_payload_updated_api_collection_does_not_exists", "content_type": "json" @@ -2010,7 +2017,7 @@ def test_payload_updated_api(): response = client.put( url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.payload_id}", json={ - "cognition_id": pytest.payload_id, + "row_id": pytest.payload_id, "data": {"details": "data science"}, "collection": "Details", "content_type": "json" @@ -2033,7 +2040,7 @@ def _mock_get_bot_settings(*args, **kwargs): response = client.put( url=f"/api/bot/{pytest.bot}/data/cognition/{payload_id}", json={ - "cognition_id": payload_id, + "row_id": payload_id, "data": {"details": "data science"}, "content_type": "json", }, @@ -2053,7 +2060,7 @@ def test_payload_content_update_api_id_not_found(): response = client.put( url=f"/api/bot/{pytest.bot}/data/cognition/{payload_id}", json={ - "cognition_id": payload_id, + "row_id": payload_id, "data": {"details": "data"}, "content_type": "json", }, @@ -2070,23 +2077,23 @@ def test_payload_content_update_api_id_not_found(): def test_get_payload_content(): response = client.get( - url=f"/api/bot/{pytest.bot}/data/cognition", + url=f"/api/bot/{pytest.bot}/data/cognition?collection=Details", headers={"Authorization": pytest.token_type + " " + pytest.access_token} ) actual = response.json() print(actual) assert actual["success"] assert actual["error_code"] == 0 - assert actual["data"][0]['data'] == {'details': 'data science'} - - assert actual["data"][0]['collection'] == 'Details' + assert actual["data"]['data'][0]['data'] == {'details': 'data science'} + assert actual["data"]['data'][0]['collection'] == 'Details' + assert actual["data"]['row_count'] == 1 def test_delete_payload_content(): response = client.delete( url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.payload_id}", json={ - "cognition_id": pytest.payload_id, + "row_id": pytest.payload_id, }, headers={"Authorization": pytest.token_type + " " + pytest.access_token} ) @@ -2103,7 +2110,7 @@ def test_delete_payload_content_does_not_exist(): response = client.delete( url=f"/api/bot/{pytest.bot}/data/cognition/{payload_id}", json={ - "cognition_id": payload_id, + "row_id": payload_id, }, headers={"Authorization": pytest.token_type + " " + pytest.access_token} ) @@ -2123,7 +2130,8 @@ def test_get_payload_content_not_exists(): assert actual["success"] assert actual["message"] is None assert actual["error_code"] == 0 - assert actual["data"] == [] + assert actual["data"]['data'] == [] + assert actual["data"]['row_count'] == 0 def test_get_kairon_faq_action_with_no_actions(): diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 64db9b030..976b36b17 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -14594,15 +14594,23 @@ def _list_cognition_data(*args, **kwargs): assert list(processor.list_cognition_data(bot, **kwargs)) == [] @patch("kairon.shared.cognition.processor.CognitionDataProcessor.list_cognition_data", autospec=True) - def test_list_cognition_data(self, mock_list_cognition_data): - def _list_cognition_data(*args, **kwargs): - return [{'vector_id': 1, - '_id': '65266ff16f0190ca4fd09898', + @patch("kairon.shared.cognition.processor.CognitionDataProcessor.get_cognition_data", autospec=True) + def test_list_cognition_data(self, mock_get_cognition_data, mock_list_cognition_data): + cognition_data = [{'vector_id': 1, + 'row_id': '65266ff16f0190ca4fd09898', 'data': 'Unit testing is a software testing technique in which individual units or components of a software application are tested in isolation to ensure that each unit functions as expected. ', 'content_type': 'text', 'collection': 'testing', 'user': 'testUser', 'bot': 'test'}] + row_count = 1 + def _list_cognition_data(*args, **kwargs): + return cognition_data mock_list_cognition_data.return_value = _list_cognition_data() + + def _get_cognition_data(*args, **kwargs): + return cognition_data, row_count + + mock_get_cognition_data.return_value = _get_cognition_data() processor = CognitionDataProcessor() bot = 'test' user = 'testUser' @@ -14614,20 +14622,36 @@ def _list_cognition_data(*args, **kwargs): "content_type": "text", "collection": collection} pytest.content_id_unit = processor.save_cognition_data(payload, user, bot) - kwargs = {'data': 'Unit testing'} + kwargs = {'collection': 'testing', 'data': 'Unit testing'} data = list(processor.list_cognition_data(bot, **kwargs)) + print(data) assert data[0][ 'data'] == 'Unit testing is a software testing technique in which individual units or components of a ' \ 'software application are tested in isolation to ensure that each unit functions as expected. ' - assert data[0]['_id'] + assert data[0]['row_id'] assert data[0]['collection'] == 'testing' + log, count = processor.get_cognition_data(bot, **kwargs) + assert log[0][ + 'data'] == 'Unit testing is a software testing technique in which individual units or components of a ' \ + 'software application are tested in isolation to ensure that each unit functions as expected. ' + assert log[0]['row_id'] + assert log[0]['collection'] == 'testing' + assert count == 1 kwargs = {} actual = list(processor.list_cognition_data(bot, **kwargs)) + print(actual) assert actual[0][ 'data'] == 'Unit testing is a software testing technique in which individual units or components of a ' \ 'software application are tested in isolation to ensure that each unit functions as expected. ' - assert actual[0]['_id'] + assert actual[0]['row_id'] assert actual[0]['collection'] == 'testing' + cognition_data, row_count = processor.get_cognition_data(bot, **kwargs) + assert cognition_data[0][ + 'data'] == 'Unit testing is a software testing technique in which individual units or components of a ' \ + 'software application are tested in isolation to ensure that each unit functions as expected. ' + assert cognition_data[0]['row_id'] + assert cognition_data[0]['collection'] == 'testing' + assert row_count == 1 def test_delete_content_for_action(self): processor = CognitionDataProcessor() @@ -14790,11 +14814,13 @@ def test_get_payload_content(self): processor = CognitionDataProcessor() bot = 'test' user = 'testUser' - data = list(processor.list_cognition_data(bot)) + kwargs = {'collection': 'test_save_payload_content'} + data, row_count = list(processor.get_cognition_data(bot, **kwargs)) print(data) + assert row_count == 1 assert data[0][ 'data'] == {"name": "Digite", "city": "Mumbai"} - assert data[0]['_id'] + assert data[0]['row_id'] assert data[0]['collection'] == 'test_save_payload_content' def test_delete_payload_content(self):