Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Knowledge_vault_floating_point_fix #1643

Closed
wants to merge 3 commits into from
Closed

Knowledge_vault_floating_point_fix #1643

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
909aa25
Fixed floating point data type check in knowledge vault
himanshugt16 Dec 12, 2024
dd20324
Updated test cases
himanshugt16 Dec 12, 2024
3e21ba3
Updated test cases
himanshugt16 Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions kairon/shared/cognition/processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,10 @@ def validate_metadata_and_payload(bot, payload):
collection = payload.get('collection', None)
matched_metadata = CognitionDataProcessor.find_matching_metadata(bot, data, collection)
for metadata_dict in matched_metadata['metadata']:
CognitionDataProcessor.validate_column_values(data, metadata_dict)
column_name = metadata_dict['column_name']
if column_name in data:
data[column_name] = CognitionDataProcessor.validate_column_values(data, metadata_dict)
return data

def save_cognition_data(self, payload: Dict, user: Text, bot: Text):
from kairon import Utility
Expand All @@ -289,7 +292,7 @@ def save_cognition_data(self, payload: Dict, user: Text, bot: Text):
collection_name__iexact=payload.get('collection'), raise_error=False):
raise AppException('Text content type does not have schema!')
if payload.get('content_type') == CognitionDataType.json.value:
CognitionDataProcessor.validate_metadata_and_payload(bot, payload)
payload['data'] = CognitionDataProcessor.validate_metadata_and_payload(bot, payload)

payload_obj = CognitionData()
payload_obj.data = payload.get('data')
Expand All @@ -316,7 +319,7 @@ def update_cognition_data(self, row_id: str, payload: Dict, user: Text, bot: Tex
try:
payload_obj = CognitionData.objects(bot=bot, id=row_id).get()
if content_type == CognitionDataType.json.value:
CognitionDataProcessor.validate_metadata_and_payload(bot, payload)
payload['data'] = CognitionDataProcessor.validate_metadata_and_payload(bot, payload)
payload_obj.data = data
payload_obj.content_type = content_type
payload_obj.collection = payload.get('collection', None)
Expand Down
160 changes: 159 additions & 1 deletion tests/integration_test/services_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1379,6 +1379,164 @@ def test_logout():
assert audit_log_data['user'] == '[email protected]'
assert audit_log_data['data']['username'] == '[email protected]'

def test_payload_upload_api_with_float_field(monkeypatch):
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.cognition_collections_limit = 20
bot_settings.llm_settings['enable_faq'] = True
bot_settings.save()
metadata = {
"metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True},
{"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}],
"collection_name": "with_float_field",
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition/schema",
json=metadata,
headers={"Authorization": pytest.token_type + " " + pytest.access_token}
)
payload = {
"data": {"item": "Box","price":54.02},
"content_type": "json",
"collection": "with_float_field"
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition",
json=payload,
headers={"Authorization": pytest.token_type + " " + pytest.access_token},
)
actual = response.json()
pytest.payload_id = actual["data"]["_id"]
assert actual["message"] == "Record saved!"
assert actual["data"]["_id"]
assert actual["error_code"] == 0

cognition_data = CognitionData.objects(bot=pytest.bot, collection="with_float_field").first()
assert cognition_data is not None
data_dict = cognition_data.to_mongo().to_dict()
assert data_dict['data']['item'] == 'Box'
assert data_dict['data']['price'] == 54.02
CognitionData.objects(bot=pytest.bot, collection="with_float_field").delete()
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.llm_settings['enable_faq'] = False
bot_settings.save()
CognitionSchema.objects(bot=pytest.bot, collection_name="with_float_field").delete()

def test_payload_upload_api_with_float_field_value_integer(monkeypatch):
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.content_importer_limit_per_day = 10
bot_settings.cognition_collections_limit = 20
bot_settings.llm_settings['enable_faq'] = True
bot_settings.save()
metadata = {
"metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True},
{"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}],
"collection_name": "with_float_field_value_integer",
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition/schema",
json=metadata,
headers={"Authorization": pytest.token_type + " " + pytest.access_token}
)
payload = {
"data": {"item": "Box","price":54},
"content_type": "json",
"collection": "with_float_field_value_integer"
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition",
json=payload,
headers={"Authorization": pytest.token_type + " " + pytest.access_token},
)
actual = response.json()
pytest.payload_id = actual["data"]["_id"]
assert actual["message"] == "Record saved!"
assert actual["data"]["_id"]
assert actual["error_code"] == 0

cognition_data = CognitionData.objects(bot=pytest.bot, collection="with_float_field_value_integer").first()
assert cognition_data is not None
data_dict = cognition_data.to_mongo().to_dict()
print(data_dict)
assert data_dict['data']['item'] == 'Box'
assert isinstance(data_dict['data']['price'], float)
assert data_dict['data']['price'] == 54.0
CognitionData.objects(bot=pytest.bot, collection="with_float_field_value_integer").delete()
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.llm_settings['enable_faq'] = False
bot_settings.save()
CognitionSchema.objects(bot=pytest.bot, collection_name="with_float_field_value_integer").delete()

def test_update_payload_upload_api_with_float_field_value_integer(monkeypatch):
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.content_importer_limit_per_day = 10
bot_settings.cognition_collections_limit = 20
bot_settings.llm_settings['enable_faq'] = True
bot_settings.save()
metadata = {
"metadata": [{"column_name": "item", "data_type": "str", "enable_search": True, "create_embeddings": True},
{"column_name": "price", "data_type": "float", "enable_search": True, "create_embeddings": True}],
"collection_name": "update_with_float_field_value_integer",
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition/schema",
json=metadata,
headers={"Authorization": pytest.token_type + " " + pytest.access_token}
)

payload = {
"data": {"item": "Box","price":54.08},
"content_type": "json",
"collection": "update_with_float_field_value_integer"
}
response = client.post(
url=f"/api/bot/{pytest.bot}/data/cognition",
json=payload,
headers={"Authorization": pytest.token_type + " " + pytest.access_token},
)
actual = response.json()
pytest.payload_id = actual["data"]["_id"]
assert actual["message"] == "Record saved!"
assert actual["data"]["_id"]
assert actual["error_code"] == 0

cognition_data = CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").first()
assert cognition_data is not None
data_dict = cognition_data.to_mongo().to_dict()
print(data_dict)
assert data_dict['data']['item'] == 'Box'
assert isinstance(data_dict['data']['price'], float)
assert data_dict['data']['price'] == 54.08



update_payload = {
"data": {"item": "Box", "price": 27},
"content_type": "json",
"collection": "update_with_float_field_value_integer"
}
response = client.put(
url=f"/api/bot/{pytest.bot}/data/cognition/{pytest.payload_id}",
json=update_payload,
headers={"Authorization": pytest.token_type + " " + pytest.access_token},
)
actual = response.json()
assert actual["message"] == "Record updated!"
assert actual["error_code"] == 0

cognition_data = CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").first()
assert cognition_data is not None
data_dict = cognition_data.to_mongo().to_dict()
print(data_dict)
assert data_dict['data']['item'] == 'Box'
assert isinstance(data_dict['data']['price'], float)
assert data_dict['data']['price'] == 27.0
CognitionData.objects(bot=pytest.bot, collection="update_with_float_field_value_integer").delete()
bot_settings = BotSettings.objects(bot=pytest.bot).get()
bot_settings.llm_settings['enable_faq'] = False
bot_settings.save()
CognitionSchema.objects(bot=pytest.bot, collection_name="update_with_float_field_value_integer").delete()



@responses.activate
def test_default_values():
Expand Down Expand Up @@ -29682,4 +29840,4 @@ def test_leave_non_existent_bot_1():
actual = response.json()
assert actual["message"] == "Access to bot is denied"
assert actual["error_code"] == 422
assert not actual["success"]
assert not actual["success"]
Loading