From 4f7076c25773fad93feec1ded6a5f8ba86f7ed78 Mon Sep 17 00:00:00 2001 From: himanshu_gupta Date: Thu, 12 Dec 2024 20:39:11 +0530 Subject: [PATCH] Fixed floating point data type check in knowledge vault --- kairon/shared/cognition/processor.py | 9 +- tests/integration_test/services_test.py | 157 ++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 3 deletions(-) diff --git a/kairon/shared/cognition/processor.py b/kairon/shared/cognition/processor.py index 93cb2a53a..9228eaf63 100644 --- a/kairon/shared/cognition/processor.py +++ b/kairon/shared/cognition/processor.py @@ -268,7 +268,10 @@ def validate_metadata_and_payload(bot, payload): collection = payload.get('collection', None) matched_metadata = CognitionDataProcessor.find_matching_metadata(bot, data, collection) for metadata_dict in matched_metadata['metadata']: - CognitionDataProcessor.validate_column_values(data, metadata_dict) + column_name = metadata_dict['column_name'] + if column_name in data: + data[column_name] = CognitionDataProcessor.validate_column_values(data, metadata_dict) + return data def save_cognition_data(self, payload: Dict, user: Text, bot: Text): from kairon import Utility @@ -289,7 +292,7 @@ def save_cognition_data(self, payload: Dict, user: Text, bot: Text): collection_name__iexact=payload.get('collection'), raise_error=False): raise AppException('Text content type does not have schema!') if payload.get('content_type') == CognitionDataType.json.value: - CognitionDataProcessor.validate_metadata_and_payload(bot, payload) + payload['data'] = CognitionDataProcessor.validate_metadata_and_payload(bot, payload) payload_obj = CognitionData() payload_obj.data = payload.get('data') @@ -316,7 +319,7 @@ def update_cognition_data(self, row_id: str, payload: Dict, user: Text, bot: Tex try: payload_obj = CognitionData.objects(bot=bot, id=row_id).get() if content_type == CognitionDataType.json.value: - CognitionDataProcessor.validate_metadata_and_payload(bot, payload) + payload['data'] = CognitionDataProcessor.validate_metadata_and_payload(bot, payload) payload_obj.data = data payload_obj.content_type = content_type payload_obj.collection = payload.get('collection', None) diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 61e217988..0bbd8146c 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -1379,6 +1379,163 @@ def test_logout(): assert audit_log_data['user'] == 'integration@demo.ai' assert audit_log_data['data']['username'] == 'integration@demo.ai' +def test_payload_upload_api_with_float_field(monkeypatch): + bot_settings = BotSettings.objects(bot=pytest.bot).get() + bot_settings.cognition_collections_limit = 20 + bot_settings.llm_settings['enable_faq'] = True + bot_settings.save() + metadata = { + "metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True}, + {"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}], + "collection_name": "with_float_field", + } + response = client.post( + url=f"/api/bot/{pytest.bot}/data/cognition/schema", + json=metadata, + headers={"Authorization": pytest.token_type + " " + pytest.access_token} + ) + payload = { + "data": {"item": "Box","price":54.02}, + "content_type": "json", + "collection": "with_float_field" + } + response = client.post( + url=f"/api/bot/{pytest.bot}/data/cognition", + json=payload, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + pytest.payload_id = actual["data"]["_id"] + assert actual["message"] == "Record saved!" + assert actual["data"]["_id"] + assert actual["error_code"] == 0 + + cognition_data = CognitionData.objects(bot=pytest.bot, collection="with_float_field").first() + assert cognition_data is not None + data_dict = cognition_data.to_mongo().to_dict() + assert data_dict['data']['item'] == 'Box' + assert data_dict['data']['price'] == 54.02 + CognitionData.objects(bot=pytest.bot, collection="with_float_field").delete() + bot_settings = BotSettings.objects(bot=pytest.bot).get() + bot_settings.llm_settings['enable_faq'] = False + bot_settings.save() + CognitionSchema.objects(bot=pytest.bot, collection_name="with_float_field").delete() + +def test_payload_upload_api_with_float_field_value_integer(monkeypatch): + bot_settings = BotSettings.objects(bot=pytest.bot).get() + bot_settings.content_importer_limit_per_day = 10 + bot_settings.cognition_collections_limit = 20 + bot_settings.llm_settings['enable_faq'] = True + bot_settings.save() + metadata = { + "metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True}, + {"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}], + "collection_name": "with_float_field_value_integer", + } + response = client.post( + url=f"/api/bot/{pytest.bot}/data/cognition/schema", + json=metadata, + headers={"Authorization": pytest.token_type + " " + pytest.access_token} + ) + payload = { + "data": {"item": "Box","price":54}, + "content_type": "json", + "collection": "with_float_field_value_integer" + } + response = client.post( + url=f"/api/bot/{pytest.bot}/data/cognition", + json=payload, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + pytest.payload_id = actual["data"]["_id"] + assert actual["message"] == "Record saved!" + assert actual["data"]["_id"] + assert actual["error_code"] == 0 + + cognition_data = CognitionData.objects(bot=pytest.bot, collection="with_float_field_value_integer").first() + assert cognition_data is not None + data_dict = cognition_data.to_mongo().to_dict() + print(data_dict) + assert data_dict['data']['item'] == 'Box' + assert isinstance(data_dict['data']['price'], float) + assert data_dict['data']['price'] == 54.0 + CognitionData.objects(bot=pytest.bot, collection="with_float_field_value_integer").delete() + bot_settings = BotSettings.objects(bot=pytest.bot).get() + bot_settings.llm_settings['enable_faq'] = False + bot_settings.save() + CognitionSchema.objects(bot=pytest.bot, collection_name="with_float_field_value_integer").delete() + +def test_update_payload_upload_api_with_float_field_value_integer(monkeypatch): + bot_settings = BotSettings.objects(bot=pytest.bot).get() + bot_settings.content_importer_limit_per_day = 10 + bot_settings.cognition_collections_limit = 20 + bot_settings.llm_settings['enable_faq'] = True + bot_settings.save() + metadata = { + "metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True}, + {"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}], + "collection_name": "update_with_float_field_value_integer", + } + response = client.post( + url=f"/api/bot/{pytest.bot}/data/cognition/schema", + json=metadata, + headers={"Authorization": pytest.token_type + " " + pytest.access_token} + ) + + payload = { + "data": {"item": "Box","price":54.08}, + "content_type": "json", + "collection": "update_with_float_field_value_integer" + } + response = client.post( + url=f"/api/bot/{pytest.bot}/data/cognition", + json=payload, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + pytest.payload_id = actual["data"]["_id"] + assert actual["message"] == "Record saved!" + assert actual["data"]["_id"] + assert actual["error_code"] == 0 + + cognition_data = CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").first() + assert cognition_data is not None + data_dict = cognition_data.to_mongo().to_dict() + print(data_dict) + assert data_dict['data']['item'] == 'Box' + assert isinstance(data_dict['data']['price'], float) + assert data_dict['data']['price'] == 54.08 + + + + update_payload = { + "data": {"item": "Box", "price": 27}, + "content_type": "json", + "collection": "update_with_float_field_value_integer" + } + response = client.put( + url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.payload_id}", + json=update_payload, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + assert actual["message"] == "Record updated!" + assert actual["error_code"] == 0 + + cognition_data = CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").first() + assert cognition_data is not None + data_dict = cognition_data.to_mongo().to_dict() + print(data_dict) + assert data_dict['data']['item'] == 'Box' + assert isinstance(data_dict['data']['price'], float) + assert data_dict['data']['price'] == 27.0 + CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").delete() + bot_settings = BotSettings.objects(bot=pytest.bot).get() + bot_settings.llm_settings['enable_faq'] = False + bot_settings.save() + CognitionSchema.objects(bot=pytest.bot, collection_name="update_with_float_field_value_integer").delete() + @responses.activate def test_default_values():