Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed floating point data type check in knowledge vault #1648

Merged
merged 1 commit into from
Dec 12, 2024
+163 −3
Merged

Fixed floating point data type check in knowledge vault #1648

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions kairon/shared/cognition/processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,10 @@ def validate_metadata_and_payload(bot, payload):
collection = payload.get('collection', None)
matched_metadata = CognitionDataProcessor.find_matching_metadata(bot, data, collection)
for metadata_dict in matched_metadata['metadata']:
CognitionDataProcessor.validate_column_values(data, metadata_dict)
column_name = metadata_dict['column_name']
if column_name in data:
data[column_name] = CognitionDataProcessor.validate_column_values(data, metadata_dict)
return data

def save_cognition_data(self, payload: Dict, user: Text, bot: Text):
from kairon import Utility
Expand All @@ -289,7 +292,7 @@ def save_cognition_data(self, payload: Dict, user: Text, bot: Text):
collection_name__iexact=payload.get('collection'), raise_error=False):
raise AppException('Text content type does not have schema!')
if payload.get('content_type') == CognitionDataType.json.value:
CognitionDataProcessor.validate_metadata_and_payload(bot, payload)
payload['data'] = CognitionDataProcessor.validate_metadata_and_payload(bot, payload)

payload_obj = CognitionData()
payload_obj.data = payload.get('data')
Expand All @@ -316,7 +319,7 @@ def update_cognition_data(self, row_id: str, payload: Dict, user: Text, bot: Tex
try:
payload_obj = CognitionData.objects(bot=bot, id=row_id).get()
if content_type == CognitionDataType.json.value:
CognitionDataProcessor.validate_metadata_and_payload(bot, payload)
payload['data'] = CognitionDataProcessor.validate_metadata_and_payload(bot, payload)
payload_obj.data = data
payload_obj.content_type = content_type
payload_obj.collection = payload.get('collection', None)
Expand Down
157 changes: 157 additions & 0 deletions tests/integration_test/services_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1379,6 +1379,163 @@ def test_logout():
assert audit_log_data['user'] == '[email protected]'
assert audit_log_data['data']['username'] == '[email protected]'

def test_payload_upload_api_with_float_field(monkeypatch):
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.cognition_collections_limit = 20
bot_settings.llm_settings['enable_faq'] = True
bot_settings.save()
metadata = {
"metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True},
{"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}],
"collection_name": "with_float_field",
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition/schema",
json=metadata,
headers={"Authorization": pytest.token_type + " " + pytest.access_token}
)
payload = {
"data": {"item": "Box","price":54.02},
"content_type": "json",
"collection": "with_float_field"
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition",
json=payload,
headers={"Authorization": pytest.token_type + " " + pytest.access_token},
)
actual = response.json()
pytest.payload_id = actual["data"]["_id"]
assert actual["message"] == "Record saved!"
assert actual["data"]["_id"]
assert actual["error_code"] == 0

cognition_data = CognitionData.objects(bot=pytest.bot, collection="with_float_field").first()
assert cognition_data is not None
data_dict = cognition_data.to_mongo().to_dict()
assert data_dict['data']['item'] == 'Box'
assert data_dict['data']['price'] == 54.02
CognitionData.objects(bot=pytest.bot, collection="with_float_field").delete()
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.llm_settings['enable_faq'] = False
bot_settings.save()
CognitionSchema.objects(bot=pytest.bot, collection_name="with_float_field").delete()

def test_payload_upload_api_with_float_field_value_integer(monkeypatch):
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.content_importer_limit_per_day = 10
bot_settings.cognition_collections_limit = 20
bot_settings.llm_settings['enable_faq'] = True
bot_settings.save()
metadata = {
"metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True},
{"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}],
"collection_name": "with_float_field_value_integer",
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition/schema",
json=metadata,
headers={"Authorization": pytest.token_type + " " + pytest.access_token}
)
payload = {
"data": {"item": "Box","price":54},
"content_type": "json",
"collection": "with_float_field_value_integer"
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition",
json=payload,
headers={"Authorization": pytest.token_type + " " + pytest.access_token},
)
actual = response.json()
pytest.payload_id = actual["data"]["_id"]
assert actual["message"] == "Record saved!"
assert actual["data"]["_id"]
assert actual["error_code"] == 0

cognition_data = CognitionData.objects(bot=pytest.bot, collection="with_float_field_value_integer").first()
assert cognition_data is not None
data_dict = cognition_data.to_mongo().to_dict()
print(data_dict)
assert data_dict['data']['item'] == 'Box'
assert isinstance(data_dict['data']['price'], float)
assert data_dict['data']['price'] == 54.0
CognitionData.objects(bot=pytest.bot, collection="with_float_field_value_integer").delete()
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.llm_settings['enable_faq'] = False
bot_settings.save()
CognitionSchema.objects(bot=pytest.bot, collection_name="with_float_field_value_integer").delete()

def test_update_payload_upload_api_with_float_field_value_integer(monkeypatch):
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.content_importer_limit_per_day = 10
bot_settings.cognition_collections_limit = 20
bot_settings.llm_settings['enable_faq'] = True
bot_settings.save()
metadata = {
"metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True},
{"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}],
"collection_name": "update_with_float_field_value_integer",
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition/schema",
json=metadata,
headers={"Authorization": pytest.token_type + " " + pytest.access_token}
)

payload = {
"data": {"item": "Box","price":54.08},
"content_type": "json",
"collection": "update_with_float_field_value_integer"
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition",
json=payload,
headers={"Authorization": pytest.token_type + " " + pytest.access_token},
)
actual = response.json()
pytest.payload_id = actual["data"]["_id"]
assert actual["message"] == "Record saved!"
assert actual["data"]["_id"]
assert actual["error_code"] == 0

cognition_data = CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").first()
assert cognition_data is not None
data_dict = cognition_data.to_mongo().to_dict()
print(data_dict)
assert data_dict['data']['item'] == 'Box'
assert isinstance(data_dict['data']['price'], float)
assert data_dict['data']['price'] == 54.08



update_payload = {
"data": {"item": "Box", "price": 27},
"content_type": "json",
"collection": "update_with_float_field_value_integer"
}
response = client.put(
url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.payload_id}",
json=update_payload,
headers={"Authorization": pytest.token_type + " " + pytest.access_token},
)
actual = response.json()
assert actual["message"] == "Record updated!"
assert actual["error_code"] == 0

cognition_data = CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").first()
assert cognition_data is not None
data_dict = cognition_data.to_mongo().to_dict()
print(data_dict)
assert data_dict['data']['item'] == 'Box'
assert isinstance(data_dict['data']['price'], float)
assert data_dict['data']['price'] == 27.0
CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").delete()
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.llm_settings['enable_faq'] = False
bot_settings.save()
CognitionSchema.objects(bot=pytest.bot, collection_name="update_with_float_field_value_integer").delete()


@responses.activate
def test_default_values():
Expand Down
Loading