Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test case #39

Merged
merged 4 commits into from
Feb 14, 2022
Merged

Test case #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fa74e31
Test case added for runAllDimension
PraveenCuebook Feb 14, 2022
3bd0e8a
Test case added
PraveenCuebook Feb 14, 2022
5961c79
Test case fixes
PraveenCuebook Feb 14, 2022
b0069bf
Merge branch 'main' into TestCases
PraveenCuebook Feb 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions api/cueSearch/elasticSearch/elastic_search_indexing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ def deleteOldIndex(indexName: str, aliasIndex: str):
esClient.indices.delete(index=key, ignore=[400, 404])

@staticmethod
def deleteAllIndexed():
def deleteAllIndex():
logging.info("Deleting all indexes")
esClient = ESIndexingUtils._getESClient()

Expand Down Expand Up @@ -156,8 +156,6 @@ def runAllIndexDimension():
The child thread assumes an index existing with a predefined unaltered indexDefinition
"""
logging.info("Indexing starts on global dimension action")
cardIndexer = threading.Thread(target=ESIndexingUtils.indexGlobalDimensionName)
cardIndexer.start()
cardIndexer1 = threading.Thread(
target=ESIndexingUtils.indexGlobalDimensionsDataForSearchSuggestion
)
Expand Down
166 changes: 156 additions & 10 deletions api/cueSearch/tests/test_elasticSearch_indexing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@ def test_elastic_search_indexing(client, mocker):
"""
Method to create test dataset
"""
ESIndexingUtils.deleteAllIndexed()

ESIndexingUtils.deleteAllIndex()
connection = mixer.blend("dataset.connection")
testDataset = mixer.blend(
"dataset.dataset",
Expand Down Expand Up @@ -114,6 +113,67 @@ def test_elastic_search_indexing(client, mocker):

assert result == expectedResult


@pytest.mark.django_db(transaction=True)
def testIndexGlobalDimensinData(client, mocker):
"""Method to test index global dimension data"""
ESIndexingUtils.deleteAllIndex()
connection = mixer.blend("dataset.connection")
testDataset = mixer.blend(
"dataset.dataset",
name="orders",
id=1,
dimensions='["Brand", "Color", "State"]',
metrics='["Orders", "OrderAmount", "OrderQuantity"]',
granularity="day",
timestampColumn="TestDate",
sql="Select * from testTable",
)
mockResponse = mocker.patch(
"cueSearch.elasticSearch.elastic_search_indexing.ESIndexingUtils.runAllIndexDimension",
new=mock.MagicMock(autospec=True, return_value=True),
)
mockResponse.start()
path = reverse("createDataset")
data = {
"name": "demo_dataset",
"sql": "SELECT * from TEST_TABLE",
"connectionId": connection.id,
"metrics": ["Amount", "Quantity"],
"dimensions": ["Category", "Region"],
"timestamp": "CreatedAt",
"granularity": "day",
"isNonRollup": False,
}
response = client.post(path, data=data, content_type="application/json")

# create dimension for testing
dataset = Dataset.objects.get(id=1)
mockResponse.start()
path = reverse("globalDimensionCreate")
gd_data = {
"name": "test",
"dimensionalValues": [
{
"datasetId": dataset.id,
"dataset": "Returns",
"dimension": "WarehouseCode",
}
],
}
response = client.post(path, gd_data, content_type="application/json")
assert response.data["success"] == True
assert response.status_code == 200

globalDimsId = GlobalDimensionServices.getGlobalDimensions()
globalDimensionId = globalDimsId.data[0]["values"][0]["id"]

# Publishing global dimension by id
path = reverse("pubGlobalDimension")
payload = {"id": globalDimensionId, "published": True}
response = client.post(path, payload)
mockResponse.stop()

##################### Global dimension data index ######################
# Creating a index value
res = {"success": True, "data": ["TestData", "TestDataOne"]}
Expand All @@ -124,6 +184,7 @@ def test_elastic_search_indexing(client, mocker):
)
ESIndexingUtils.indexGlobalDimensionsData()
mockResponse.stop()
query = "TestData"
result = ESQueryingUtils.findGlobalDimensionResults(query=query)
count = 0
while not result:
Expand All @@ -148,6 +209,68 @@ def test_elastic_search_indexing(client, mocker):
]
assert result == expectedResults


@pytest.mark.django_db(transaction=True)
def testNonGlobalDimensionDataIndex(client, mocker):
"""Method to test non global dimension index"""
ESIndexingUtils.deleteAllIndex()

connection = mixer.blend("dataset.connection")
testDataset = mixer.blend(
"dataset.dataset",
name="orders",
id=1,
dimensions='["Brand", "Color", "State"]',
metrics='["Orders", "OrderAmount", "OrderQuantity"]',
granularity="day",
timestampColumn="TestDate",
sql="Select * from testTable",
)
mockResponse = mocker.patch(
"cueSearch.elasticSearch.elastic_search_indexing.ESIndexingUtils.runAllIndexDimension",
new=mock.MagicMock(autospec=True, return_value=True),
)
mockResponse.start()
path = reverse("createDataset")
data = {
"name": "demo_dataset",
"sql": "SELECT * from TEST_TABLE",
"connectionId": connection.id,
"metrics": ["Amount", "Quantity"],
"dimensions": ["Category", "Region"],
"timestamp": "CreatedAt",
"granularity": "day",
"isNonRollup": False,
}
response = client.post(path, data=data, content_type="application/json")

# create dimension for testing
dataset = Dataset.objects.get(id=1)
mockResponse.start()
path = reverse("globalDimensionCreate")
gd_data = {
"name": "test",
"dimensionalValues": [
{
"datasetId": dataset.id,
"dataset": "Returns",
"dimension": "WarehouseCode",
}
],
}
response = client.post(path, gd_data, content_type="application/json")
assert response.data["success"] == True
assert response.status_code == 200

globalDimsId = GlobalDimensionServices.getGlobalDimensions()
globalDimensionId = globalDimsId.data[0]["values"][0]["id"]

# Publishing global dimension by id
path = reverse("pubGlobalDimension")
payload = {"id": globalDimensionId, "published": True}
response = client.post(path, payload)
mockResponse.stop()

################################ Global dimension data index #################

listToIndex = [
Expand Down Expand Up @@ -184,31 +307,31 @@ def test_elastic_search_indexing(client, mocker):
"value": "TestData",
"user_entity_identifier": "Test data_Brand",
"id": "Brand_TestData_1",
"datasetId": 1,
"datasetId": testDataset.id,
"globalDimensionId": "Brand_TestData_1",
"type": "DATASETDIMENSION",
},
{
"value": "TestData",
"user_entity_identifier": "Test data_WarehouseCode",
"id": "WarehouseCode_TestData_1",
"datasetId": 1,
"datasetId": testDataset.id,
"globalDimensionId": "WarehouseCode_TestData_1",
"type": "DATASETDIMENSION",
},
{
"value": "TestDataOne",
"user_entity_identifier": "Test data_Brand",
"id": "Brand_TestDataOne_1",
"datasetId": 1,
"datasetId": testDataset.id,
"globalDimensionId": "Brand_TestDataOne_1",
"type": "DATASETDIMENSION",
},
{
"value": "TestDataOne",
"user_entity_identifier": "Test data_WarehouseCode",
"id": "WarehouseCode_TestDataOne_1",
"datasetId": 1,
"datasetId": testDataset.id,
"globalDimensionId": "WarehouseCode_TestDataOne_1",
"type": "DATASETDIMENSION",
},
Expand All @@ -223,7 +346,7 @@ def test_elastic_search_indexing(client, mocker):
"user_entity_identifier": "Test data_Brand",
"id": "Brand_TestData_1",
"dataset": "Test data",
"datasetId": 1,
"datasetId": testDataset.id,
"type": "DATASETDIMENSION",
},
{
Expand All @@ -233,7 +356,7 @@ def test_elastic_search_indexing(client, mocker):
"user_entity_identifier": "Test data_WarehouseCode",
"id": "WarehouseCode_TestData_1",
"dataset": "Test data",
"datasetId": 1,
"datasetId": testDataset.id,
"type": "DATASETDIMENSION",
},
]
Expand All @@ -248,6 +371,29 @@ def test_elastic_search_indexing(client, mocker):
return result

assert result == expectedResults

################################ Delete all indexes ##############
ESIndexingUtils.deleteAllIndexed()


def testRunAllIndexing(client, mocker):

mockResponse = mocker.patch(
"cueSearch.elasticSearch.elastic_search_indexing.ESIndexingUtils.indexGlobalDimensionsDataForSearchSuggestion",
new=mock.MagicMock(autospec=True, return_value=True),
)
mockResponse.start()
mockResponse1 = mocker.patch(
"cueSearch.elasticSearch.elastic_search_indexing.ESIndexingUtils.indexGlobalDimensionsData",
new=mock.MagicMock(autospec=True, return_value=True),
)
mockResponse1.start()
mockResponse2 = mocker.patch(
"cueSearch.elasticSearch.elastic_search_indexing.ESIndexingUtils.indexNonGlobalDimensionsDataForSearchSuggestion",
new=mock.MagicMock(autospec=True, return_value=True),
)
mockResponse2.start()

ESIndexingUtils.runAllIndexDimension()

mockResponse.stop()
mockResponse1.stop()
mockResponse2.stop()