From dae6f90e3f5c704c794750f3fc7073d14a734c72 Mon Sep 17 00:00:00 2001 From: Rafalz13 Date: Thu, 12 Oct 2023 13:08:28 +0200 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20test=20and=20removed?= =?UTF-8?q?=20sensitive=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/integration/flows/test_prefect_logs.py | 52 +++----------------- 1 file changed, 6 insertions(+), 46 deletions(-) diff --git a/tests/integration/flows/test_prefect_logs.py b/tests/integration/flows/test_prefect_logs.py index 60e3f799b..82cf3e0cd 100644 --- a/tests/integration/flows/test_prefect_logs.py +++ b/tests/integration/flows/test_prefect_logs.py @@ -8,18 +8,13 @@ def expectation_suite(): expectation_suite = { "data": { "project": [ - { - "id": "6f413380-e228-4d64-8e1b-41c6cd434a2a", - "name": "Installer Engagement", - "flows": [], - }, { "id": "223a8acf-4cf0-4cf7-ae1f-b66f78e28813", - "name": "oso_reporting", + "name": "Admin", "flows": [ { "id": "b13dcc6d-b621-4acd-88be-2cf28715a7c5", - "name": "1-raw dakvenster_order_prod_info extract", + "name": "1-raw table_catalog extract", "version": 3, "flow_runs": [ { @@ -34,23 +29,18 @@ def expectation_suite(): }, { "id": "14b1a89e-f902-48a1-b6df-43cacdb91e1a", - "name": "1-raw dakvenster_order_prod_info extract", + "name": "1-raw table_catalog extract", "version": 2, "flow_runs": [], }, { "id": "a1eace09-38b4-46bf-bacf-a5d29bdbb633", - "name": "1-raw dakvenster_order_prod_info extract", + "name": "1-raw table_catalog extract", "version": 1, "flow_runs": [], }, ], }, - { - "id": "844372db-2d22-495d-a343-b8f8cbcf8963", - "name": "sap", - "flows": [], - }, { "id": "512d0f29-2ceb-4177-b7d8-c5908da666ef", "name": "integrations", @@ -61,16 +51,6 @@ def expectation_suite(): "name": "dev_cdl", "flows": [], }, - { - "id": "e2a926e2-ec86-4900-a24e-330a44b6cb19", - "name": "cic_test", - "flows": [], - }, - { - "id": "667d5026-2f01-452a-b6fe-5437ca833066", - "name": "cic_dev", - "flows": [], - }, { "id": "eac9b6d4-725a-4354-bf8f-25e7828ea2d8", "name": "Admin", @@ -86,31 +66,11 @@ def expectation_suite(): "name": "cic", "flows": [], }, - { - "id": "dd2ccc32-2163-4f55-a746-1dbc6b28aaa4", - "name": "Hyperlocal", - "flows": [], - }, - { - "id": "7131c357-bad7-43cf-aabc-87f9cf045384", - "name": "Installer Segmentation", - "flows": [], - }, - { - "id": "94a8b8bf-14fa-4b64-ab78-af1d332dedd4", - "name": "Marketing KPI", - "flows": [], - }, { "id": "ebe0e5aa-4add-4440-8c1a-6f9c74eb29fe", "name": "dev", "flows": [], }, - { - "id": "b5d924b0-4116-479f-a8f5-e28f9a9051ca", - "name": "velux", - "flows": [], - }, ] } } @@ -127,7 +87,7 @@ def test_prefect_logs(expectation_suite): id name flows ( - where : {name: {_eq: "1-raw google_analytics_oso_sps_gb extract"}} + where : {name: {_eq: "1-raw table_catalog extract"}} ) { id name @@ -156,7 +116,7 @@ def test_prefect_logs(expectation_suite): scheduled_start_time="2022-09-05", filter_type="_gte", local_file_path=f"prefect_extract_logs.parquet", - adls_path=f"raw/supermetrics/mp/prefect_extract_logs.parquet", + adls_path=f"raw/tests/prefect_extract_logs.parquet", ) results = flow.run() From bc19a169a551c24372477060ae56ee42bc6c1b3a Mon Sep 17 00:00:00 2001 From: Rafalz13 Date: Thu, 12 Oct 2023 13:15:47 +0200 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20`Mindful`=20tests=20?= =?UTF-8?q?-=20replaced=20auth=20with=20header?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/integration/test_mindful.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_mindful.py b/tests/integration/test_mindful.py index c7088913c..1718a9db9 100644 --- a/tests/integration/test_mindful.py +++ b/tests/integration/test_mindful.py @@ -10,9 +10,10 @@ os.system("clear") credentials_mindful = local_config["MINDFUL"] -header = { - "Authorization": f"Bearer {credentials_mindful.get('VAULT')}", -} +auth = ( + credentials_mindful["CUSTOMER_UUID"], + credentials_mindful["AUTH_TOKEN"], +) class MockClass: @@ -42,14 +43,14 @@ def json(): @pytest.mark.init def test_instance_mindful(): - mf = Mindful(header=header) + mf = Mindful(auth=auth) assert isinstance(mf, Mindful) @mock.patch("viadot.sources.mindful.handle_api_response", return_value=MockClass) @pytest.mark.connect def test_mindful_api_response(mock_connection): - mf = Mindful(header=header) + mf = Mindful(auth=auth) mf.get_interactions_list() mf.get_responses_list() mock_connection.call_count == 2 @@ -58,7 +59,7 @@ def test_mindful_api_response(mock_connection): @mock.patch("viadot.sources.mindful.handle_api_response", return_value=MockClass) @pytest.mark.connect def test_mindful_api_response2(mock_api_response): - mf = Mindful(header=header) + mf = Mindful(auth=auth) response = mf.get_interactions_list() @@ -69,7 +70,7 @@ def test_mindful_api_response2(mock_api_response): @mock.patch("viadot.sources.mindful.handle_api_response", return_value=MockClass) @pytest.mark.connect def test_mindful_api_response3(mock_api_response): - mf = Mindful(header=header) + mf = Mindful(auth=auth) response = mf.get_responses_list() @@ -80,7 +81,7 @@ def test_mindful_api_response3(mock_api_response): @mock.patch("viadot.sources.mindful.handle_api_response", return_value=MockClass) @pytest.mark.connect def test_mindful_api_response4(mock_api_response): - mf = Mindful(header=header) + mf = Mindful(auth=auth) response = mf.get_survey_list() @@ -91,7 +92,7 @@ def test_mindful_api_response4(mock_api_response): @mock.patch("viadot.sources.Mindful._mindful_api_response", return_value=MockClass) @pytest.mark.save def test_mindful_interactions(mock_connection): - mf = Mindful(header=header) + mf = Mindful(auth=auth) response = mf.get_interactions_list() mf.response_to_file(response) assert mf.endpoint == "interactions" and isinstance(mf.endpoint, str) @@ -103,7 +104,7 @@ def test_mindful_interactions(mock_connection): @mock.patch("viadot.sources.Mindful._mindful_api_response", return_value=MockClass) @pytest.mark.save def test_mindful_responses(mock_connection): - mf = Mindful(header=header) + mf = Mindful(auth=auth) response = mf.get_responses_list() mf.response_to_file(response) @@ -115,7 +116,7 @@ def test_mindful_responses(mock_connection): @mock.patch("viadot.sources.Mindful._mindful_api_response", return_value=MockClass) @pytest.mark.save def test_mindful_surveys(mock_connection): - mf = Mindful(header=header) + mf = Mindful(auth=auth) response = mf.get_survey_list() mf.response_to_file(response) From b0b048da0c71aac27d7f1f271a63612541248fa9 Mon Sep 17 00:00:00 2001 From: Rafalz13 Date: Thu, 12 Oct 2023 13:18:41 +0200 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=90=9B=20Added=20`=5Fviadot=5Fsource`?= =?UTF-8?q?=20to=20column=20list=20for=20BigQuery=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/integration/tasks/test_bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/tasks/test_bigquery.py b/tests/integration/tasks/test_bigquery.py index 3cf184d85..304c31f34 100644 --- a/tests/integration/tasks/test_bigquery.py +++ b/tests/integration/tasks/test_bigquery.py @@ -17,7 +17,7 @@ def test_bigquery_to_df_success(): credentials_key=CREDENTIALS_KEY, ) df = bigquery_to_df_task.run() - expected_column = ["my_value"] + expected_column = ["my_value", "_viadot_source"] assert isinstance(df, pd.DataFrame) assert expected_column == list(df.columns)