Fixed floating point data type check in knowledge vault (#1648)

kairon/shared/cognition/processor.py

@@ -268,7 +268,10 @@ def validate_metadata_and_payload(bot, payload):
         collection = payload.get('collection', None)
         matched_metadata = CognitionDataProcessor.find_matching_metadata(bot, data, collection)
         for metadata_dict in matched_metadata['metadata']:
-            CognitionDataProcessor.validate_column_values(data, metadata_dict)
+            column_name = metadata_dict['column_name']
+            if column_name in data:
+                data[column_name] = CognitionDataProcessor.validate_column_values(data, metadata_dict)
+        return data
 
     def save_cognition_data(self, payload: Dict, user: Text, bot: Text):
         from kairon import Utility
@@ -289,7 +292,7 @@ def save_cognition_data(self, payload: Dict, user: Text, bot: Text):
                                          collection_name__iexact=payload.get('collection'), raise_error=False):
                 raise AppException('Text content type does not have schema!')
         if payload.get('content_type') == CognitionDataType.json.value:
-            CognitionDataProcessor.validate_metadata_and_payload(bot, payload)
+            payload['data'] = CognitionDataProcessor.validate_metadata_and_payload(bot, payload)
 
         payload_obj = CognitionData()
         payload_obj.data = payload.get('data')
@@ -316,7 +319,7 @@ def update_cognition_data(self, row_id: str, payload: Dict, user: Text, bot: Tex
         try:
             payload_obj = CognitionData.objects(bot=bot, id=row_id).get()
             if content_type == CognitionDataType.json.value:
-                CognitionDataProcessor.validate_metadata_and_payload(bot, payload)
+                payload['data'] = CognitionDataProcessor.validate_metadata_and_payload(bot, payload)
             payload_obj.data = data
             payload_obj.content_type = content_type
             payload_obj.collection = payload.get('collection', None)

tests/integration_test/services_test.py

@@ -1379,6 +1379,163 @@ def test_logout():
     assert audit_log_data['user'] == '[email protected]'
     assert audit_log_data['data']['username'] == '[email protected]'
 
+def test_payload_upload_api_with_float_field(monkeypatch):
+    bot_settings = BotSettings.objects(bot=pytest.bot).get()
+    bot_settings.cognition_collections_limit = 20
+    bot_settings.llm_settings['enable_faq'] = True
+    bot_settings.save()
+    metadata = {
+        "metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True},
+                     {"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}],
+        "collection_name": "with_float_field",
+    }
+    response = client.post(
+        url=f"/api/bot/{pytest.bot}/data/cognition/schema",
+        json=metadata,
+        headers={"Authorization": pytest.token_type + " " + pytest.access_token}
+    )
+    payload = {
+        "data": {"item": "Box","price":54.02},
+        "content_type": "json",
+        "collection": "with_float_field"
+    }
+    response = client.post(
+        url=f"/api/bot/{pytest.bot}/data/cognition",
+        json=payload,
+        headers={"Authorization": pytest.token_type + " " + pytest.access_token},
+    )
+    actual = response.json()
+    pytest.payload_id = actual["data"]["_id"]
+    assert actual["message"] == "Record saved!"
+    assert actual["data"]["_id"]
+    assert actual["error_code"] == 0
+
+    cognition_data = CognitionData.objects(bot=pytest.bot, collection="with_float_field").first()
+    assert cognition_data is not None
+    data_dict = cognition_data.to_mongo().to_dict()
+    assert data_dict['data']['item'] == 'Box'
+    assert data_dict['data']['price'] == 54.02
+    CognitionData.objects(bot=pytest.bot, collection="with_float_field").delete()
+    bot_settings = BotSettings.objects(bot=pytest.bot).get()
+    bot_settings.llm_settings['enable_faq'] = False
+    bot_settings.save()
+    CognitionSchema.objects(bot=pytest.bot, collection_name="with_float_field").delete()
+
+def test_payload_upload_api_with_float_field_value_integer(monkeypatch):
+    bot_settings = BotSettings.objects(bot=pytest.bot).get()
+    bot_settings.content_importer_limit_per_day = 10
+    bot_settings.cognition_collections_limit = 20
+    bot_settings.llm_settings['enable_faq'] = True
+    bot_settings.save()
+    metadata = {
+        "metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True},
+                     {"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}],
+        "collection_name": "with_float_field_value_integer",
+    }
+    response = client.post(
+        url=f"/api/bot/{pytest.bot}/data/cognition/schema",
+        json=metadata,
+        headers={"Authorization": pytest.token_type + " " + pytest.access_token}
+    )
+    payload = {
+        "data": {"item": "Box","price":54},
+        "content_type": "json",
+        "collection": "with_float_field_value_integer"
+    }
+    response = client.post(
+        url=f"/api/bot/{pytest.bot}/data/cognition",
+        json=payload,
+        headers={"Authorization": pytest.token_type + " " + pytest.access_token},
+    )
+    actual = response.json()
+    pytest.payload_id = actual["data"]["_id"]
+    assert actual["message"] == "Record saved!"
+    assert actual["data"]["_id"]
+    assert actual["error_code"] == 0
+
+    cognition_data = CognitionData.objects(bot=pytest.bot, collection="with_float_field_value_integer").first()
+    assert cognition_data is not None
+    data_dict = cognition_data.to_mongo().to_dict()
+    print(data_dict)
+    assert data_dict['data']['item'] == 'Box'
+    assert isinstance(data_dict['data']['price'], float)
+    assert data_dict['data']['price'] == 54.0
+    CognitionData.objects(bot=pytest.bot, collection="with_float_field_value_integer").delete()
+    bot_settings = BotSettings.objects(bot=pytest.bot).get()
+    bot_settings.llm_settings['enable_faq'] = False
+    bot_settings.save()
+    CognitionSchema.objects(bot=pytest.bot, collection_name="with_float_field_value_integer").delete()
+
+def test_update_payload_upload_api_with_float_field_value_integer(monkeypatch):
+    bot_settings = BotSettings.objects(bot=pytest.bot).get()
+    bot_settings.content_importer_limit_per_day = 10
+    bot_settings.cognition_collections_limit = 20
+    bot_settings.llm_settings['enable_faq'] = True
+    bot_settings.save()
+    metadata = {
+        "metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True},
+                     {"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}],
+        "collection_name": "update_with_float_field_value_integer",
+    }
+    response = client.post(
+        url=f"/api/bot/{pytest.bot}/data/cognition/schema",
+        json=metadata,
+        headers={"Authorization": pytest.token_type + " " + pytest.access_token}
+    )
+
+    payload = {
+        "data": {"item": "Box","price":54.08},
+        "content_type": "json",
+        "collection": "update_with_float_field_value_integer"
+    }
+    response = client.post(
+        url=f"/api/bot/{pytest.bot}/data/cognition",
+        json=payload,
+        headers={"Authorization": pytest.token_type + " " + pytest.access_token},
+    )
+    actual = response.json()
+    pytest.payload_id = actual["data"]["_id"]
+    assert actual["message"] == "Record saved!"
+    assert actual["data"]["_id"]
+    assert actual["error_code"] == 0
+
+    cognition_data = CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").first()
+    assert cognition_data is not None
+    data_dict = cognition_data.to_mongo().to_dict()
+    print(data_dict)
+    assert data_dict['data']['item'] == 'Box'
+    assert isinstance(data_dict['data']['price'], float)
+    assert data_dict['data']['price'] == 54.08
+
+
+
+    update_payload = {
+        "data": {"item": "Box", "price": 27},
+        "content_type": "json",
+        "collection": "update_with_float_field_value_integer"
+    }
+    response = client.put(
+        url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.payload_id}",
+        json=update_payload,
+        headers={"Authorization": pytest.token_type + " " + pytest.access_token},
+    )
+    actual = response.json()
+    assert actual["message"] == "Record updated!"
+    assert actual["error_code"] == 0
+
+    cognition_data = CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").first()
+    assert cognition_data is not None
+    data_dict = cognition_data.to_mongo().to_dict()
+    print(data_dict)
+    assert data_dict['data']['item'] == 'Box'
+    assert isinstance(data_dict['data']['price'], float)
+    assert data_dict['data']['price'] == 27.0
+    CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").delete()
+    bot_settings = BotSettings.objects(bot=pytest.bot).get()
+    bot_settings.llm_settings['enable_faq'] = False
+    bot_settings.save()
+    CognitionSchema.objects(bot=pytest.bot, collection_name="update_with_float_field_value_integer").delete()
+
 
 @responses.activate
 def test_default_values():