From fa74e31be989623cfe7281699c67619988d90316 Mon Sep 17 00:00:00 2001 From: praveencuebok Date: Mon, 14 Feb 2022 18:27:34 +0530 Subject: [PATCH 1/3] Test case added for runAllDimension --- .../elasticSearch/elastic_search_indexing.py | 2 -- .../tests/test_elasticSearch_indexing.py | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/api/cueSearch/elasticSearch/elastic_search_indexing.py b/api/cueSearch/elasticSearch/elastic_search_indexing.py index 1448aefe..4b1172e2 100644 --- a/api/cueSearch/elasticSearch/elastic_search_indexing.py +++ b/api/cueSearch/elasticSearch/elastic_search_indexing.py @@ -156,8 +156,6 @@ def runAllIndexDimension(): The child thread assumes an index existing with a predefined unaltered indexDefinition """ logging.info("Indexing starts on global dimension action") - cardIndexer = threading.Thread(target=ESIndexingUtils.indexGlobalDimensionName) - cardIndexer.start() cardIndexer1 = threading.Thread( target=ESIndexingUtils.indexGlobalDimensionsDataForSearchSuggestion ) diff --git a/api/cueSearch/tests/test_elasticSearch_indexing.py b/api/cueSearch/tests/test_elasticSearch_indexing.py index d4fdfd35..59581c81 100644 --- a/api/cueSearch/tests/test_elasticSearch_indexing.py +++ b/api/cueSearch/tests/test_elasticSearch_indexing.py @@ -251,3 +251,28 @@ def test_elastic_search_indexing(client, mocker): ################################ Delete all indexes ############## ESIndexingUtils.deleteAllIndexed() + + +def testRunAllIndexing(client, mocker): + + mockResponse = mocker.patch( + "cueSearch.elasticSearch.elastic_search_indexing.ESIndexingUtils.indexGlobalDimensionsDataForSearchSuggestion", + new=mock.MagicMock(autospec=True, return_value=True), + ) + mockResponse.start() + mockResponse1 = mocker.patch( + "cueSearch.elasticSearch.elastic_search_indexing.ESIndexingUtils.indexGlobalDimensionsData", + new=mock.MagicMock(autospec=True, return_value=True), + ) + mockResponse1.start() + mockResponse2 = mocker.patch( + "cueSearch.elasticSearch.elastic_search_indexing.ESIndexingUtils.indexNonGlobalDimensionsDataForSearchSuggestion", + new=mock.MagicMock(autospec=True, return_value=True), + ) + mockResponse2.start() + + ESIndexingUtils.runAllIndexDimension() + + mockResponse.stop() + mockResponse1.stop() + mockResponse2.stop() From 3bd0e8a558f724d9c0f7ab2afef82607c7dc2325 Mon Sep 17 00:00:00 2001 From: praveencuebok Date: Mon, 14 Feb 2022 19:12:37 +0530 Subject: [PATCH 2/3] Test case added --- .../tests/test_elasticSearch_indexing.py | 137 +++++++++++++++++- 1 file changed, 129 insertions(+), 8 deletions(-) diff --git a/api/cueSearch/tests/test_elasticSearch_indexing.py b/api/cueSearch/tests/test_elasticSearch_indexing.py index 59581c81..ccdd214c 100644 --- a/api/cueSearch/tests/test_elasticSearch_indexing.py +++ b/api/cueSearch/tests/test_elasticSearch_indexing.py @@ -14,7 +14,7 @@ def test_elastic_search_indexing(client, mocker): """ Method to create test dataset """ - ESIndexingUtils.deleteAllIndexed() + # ESIndexingUtils.deleteAllIndexed() connection = mixer.blend("dataset.connection") testDataset = mixer.blend( @@ -114,6 +114,67 @@ def test_elastic_search_indexing(client, mocker): assert result == expectedResult + +@pytest.mark.django_db(transaction=True) +def testIndexGlobalDimensinData(client, mocker): + """Method to test index global dimension data""" + + connection = mixer.blend("dataset.connection") + testDataset = mixer.blend( + "dataset.dataset", + name="orders", + id=1, + dimensions='["Brand", "Color", "State"]', + metrics='["Orders", "OrderAmount", "OrderQuantity"]', + granularity="day", + timestampColumn="TestDate", + sql="Select * from testTable", + ) + mockResponse = mocker.patch( + "cueSearch.elasticSearch.elastic_search_indexing.ESIndexingUtils.runAllIndexDimension", + new=mock.MagicMock(autospec=True, return_value=True), + ) + mockResponse.start() + path = reverse("createDataset") + data = { + "name": "demo_dataset", + "sql": "SELECT * from TEST_TABLE", + "connectionId": connection.id, + "metrics": ["Amount", "Quantity"], + "dimensions": ["Category", "Region"], + "timestamp": "CreatedAt", + "granularity": "day", + "isNonRollup": False, + } + response = client.post(path, data=data, content_type="application/json") + + # create dimension for testing + dataset = Dataset.objects.get(id=1) + mockResponse.start() + path = reverse("globalDimensionCreate") + gd_data = { + "name": "test", + "dimensionalValues": [ + { + "datasetId": dataset.id, + "dataset": "Returns", + "dimension": "WarehouseCode", + } + ], + } + response = client.post(path, gd_data, content_type="application/json") + assert response.data["success"] == True + assert response.status_code == 200 + + globalDimsId = GlobalDimensionServices.getGlobalDimensions() + globalDimensionId = globalDimsId.data[0]["values"][0]["id"] + + # Publishing global dimension by id + path = reverse("pubGlobalDimension") + payload = {"id": globalDimensionId, "published": True} + response = client.post(path, payload) + mockResponse.stop() + ##################### Global dimension data index ###################### # Creating a index value res = {"success": True, "data": ["TestData", "TestDataOne"]} @@ -124,6 +185,7 @@ def test_elastic_search_indexing(client, mocker): ) ESIndexingUtils.indexGlobalDimensionsData() mockResponse.stop() + query = "TestData" result = ESQueryingUtils.findGlobalDimensionResults(query=query) count = 0 while not result: @@ -148,6 +210,66 @@ def test_elastic_search_indexing(client, mocker): ] assert result == expectedResults + +@pytest.mark.django_db(transaction=True) +def testNonGlobalDimensionDataIndex(client, mocker): + """Method test""" + connection = mixer.blend("dataset.connection") + testDataset = mixer.blend( + "dataset.dataset", + name="orders", + id=1, + dimensions='["Brand", "Color", "State"]', + metrics='["Orders", "OrderAmount", "OrderQuantity"]', + granularity="day", + timestampColumn="TestDate", + sql="Select * from testTable", + ) + mockResponse = mocker.patch( + "cueSearch.elasticSearch.elastic_search_indexing.ESIndexingUtils.runAllIndexDimension", + new=mock.MagicMock(autospec=True, return_value=True), + ) + mockResponse.start() + path = reverse("createDataset") + data = { + "name": "demo_dataset", + "sql": "SELECT * from TEST_TABLE", + "connectionId": connection.id, + "metrics": ["Amount", "Quantity"], + "dimensions": ["Category", "Region"], + "timestamp": "CreatedAt", + "granularity": "day", + "isNonRollup": False, + } + response = client.post(path, data=data, content_type="application/json") + + # create dimension for testing + dataset = Dataset.objects.get(id=1) + mockResponse.start() + path = reverse("globalDimensionCreate") + gd_data = { + "name": "test", + "dimensionalValues": [ + { + "datasetId": dataset.id, + "dataset": "Returns", + "dimension": "WarehouseCode", + } + ], + } + response = client.post(path, gd_data, content_type="application/json") + assert response.data["success"] == True + assert response.status_code == 200 + + globalDimsId = GlobalDimensionServices.getGlobalDimensions() + globalDimensionId = globalDimsId.data[0]["values"][0]["id"] + + # Publishing global dimension by id + path = reverse("pubGlobalDimension") + payload = {"id": globalDimensionId, "published": True} + response = client.post(path, payload) + mockResponse.stop() + ################################ Global dimension data index ################# listToIndex = [ @@ -184,7 +306,7 @@ def test_elastic_search_indexing(client, mocker): "value": "TestData", "user_entity_identifier": "Test data_Brand", "id": "Brand_TestData_1", - "datasetId": 1, + "datasetId": testDataset.id, "globalDimensionId": "Brand_TestData_1", "type": "DATASETDIMENSION", }, @@ -192,7 +314,7 @@ def test_elastic_search_indexing(client, mocker): "value": "TestData", "user_entity_identifier": "Test data_WarehouseCode", "id": "WarehouseCode_TestData_1", - "datasetId": 1, + "datasetId": testDataset.id, "globalDimensionId": "WarehouseCode_TestData_1", "type": "DATASETDIMENSION", }, @@ -200,7 +322,7 @@ def test_elastic_search_indexing(client, mocker): "value": "TestDataOne", "user_entity_identifier": "Test data_Brand", "id": "Brand_TestDataOne_1", - "datasetId": 1, + "datasetId": testDataset.id, "globalDimensionId": "Brand_TestDataOne_1", "type": "DATASETDIMENSION", }, @@ -208,7 +330,7 @@ def test_elastic_search_indexing(client, mocker): "value": "TestDataOne", "user_entity_identifier": "Test data_WarehouseCode", "id": "WarehouseCode_TestDataOne_1", - "datasetId": 1, + "datasetId": testDataset.id, "globalDimensionId": "WarehouseCode_TestDataOne_1", "type": "DATASETDIMENSION", }, @@ -223,7 +345,7 @@ def test_elastic_search_indexing(client, mocker): "user_entity_identifier": "Test data_Brand", "id": "Brand_TestData_1", "dataset": "Test data", - "datasetId": 1, + "datasetId": testDataset.id, "type": "DATASETDIMENSION", }, { @@ -233,7 +355,7 @@ def test_elastic_search_indexing(client, mocker): "user_entity_identifier": "Test data_WarehouseCode", "id": "WarehouseCode_TestData_1", "dataset": "Test data", - "datasetId": 1, + "datasetId": testDataset.id, "type": "DATASETDIMENSION", }, ] @@ -250,7 +372,6 @@ def test_elastic_search_indexing(client, mocker): assert result == expectedResults ################################ Delete all indexes ############## - ESIndexingUtils.deleteAllIndexed() def testRunAllIndexing(client, mocker): From 5961c79b6a39fc9fad7abc82b7b3cf545a6bde8b Mon Sep 17 00:00:00 2001 From: praveencuebok Date: Mon, 14 Feb 2022 19:19:46 +0530 Subject: [PATCH 3/3] Test case fixes --- api/cueSearch/elasticSearch/elastic_search_indexing.py | 2 +- api/cueSearch/tests/test_elasticSearch_indexing.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/cueSearch/elasticSearch/elastic_search_indexing.py b/api/cueSearch/elasticSearch/elastic_search_indexing.py index 4b1172e2..70c7b33e 100644 --- a/api/cueSearch/elasticSearch/elastic_search_indexing.py +++ b/api/cueSearch/elasticSearch/elastic_search_indexing.py @@ -115,7 +115,7 @@ def deleteOldIndex(indexName: str, aliasIndex: str): esClient.indices.delete(index=key, ignore=[400, 404]) @staticmethod - def deleteAllIndexed(): + def deleteAllIndex(): logging.info("Deleting all indexes") esClient = ESIndexingUtils._getESClient() diff --git a/api/cueSearch/tests/test_elasticSearch_indexing.py b/api/cueSearch/tests/test_elasticSearch_indexing.py index ccdd214c..16872153 100644 --- a/api/cueSearch/tests/test_elasticSearch_indexing.py +++ b/api/cueSearch/tests/test_elasticSearch_indexing.py @@ -14,8 +14,7 @@ def test_elastic_search_indexing(client, mocker): """ Method to create test dataset """ - # ESIndexingUtils.deleteAllIndexed() - + ESIndexingUtils.deleteAllIndex() connection = mixer.blend("dataset.connection") testDataset = mixer.blend( "dataset.dataset", @@ -118,7 +117,7 @@ def test_elastic_search_indexing(client, mocker): @pytest.mark.django_db(transaction=True) def testIndexGlobalDimensinData(client, mocker): """Method to test index global dimension data""" - + ESIndexingUtils.deleteAllIndex() connection = mixer.blend("dataset.connection") testDataset = mixer.blend( "dataset.dataset", @@ -213,7 +212,9 @@ def testIndexGlobalDimensinData(client, mocker): @pytest.mark.django_db(transaction=True) def testNonGlobalDimensionDataIndex(client, mocker): - """Method test""" + """Method to test non global dimension index""" + ESIndexingUtils.deleteAllIndex() + connection = mixer.blend("dataset.connection") testDataset = mixer.blend( "dataset.dataset", @@ -370,7 +371,6 @@ def testNonGlobalDimensionDataIndex(client, mocker): return result assert result == expectedResults - ################################ Delete all indexes ##############