Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Correct tests before release #764

Merged
merged 3 commits into from
Oct 12, 2023
Merged

Correct tests before release #764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dae6f90
🐛 Fixed test and removed sensitive data
Rafalz13 Oct 12, 2023
bc19a16
🐛 Fixed `Mindful` tests - replaced auth with header
Rafalz13 Oct 12, 2023
b0b048d
🐛 Added `_viadot_source` to column list for BigQuery test
Rafalz13 Oct 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 6 additions & 46 deletions tests/integration/flows/test_prefect_logs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,13 @@ def expectation_suite():
expectation_suite = {
"data": {
"project": [
{
"id": "6f413380-e228-4d64-8e1b-41c6cd434a2a",
"name": "Installer Engagement",
"flows": [],
},
{
"id": "223a8acf-4cf0-4cf7-ae1f-b66f78e28813",
"name": "oso_reporting",
"name": "Admin",
"flows": [
{
"id": "b13dcc6d-b621-4acd-88be-2cf28715a7c5",
"name": "1-raw dakvenster_order_prod_info extract",
"name": "1-raw table_catalog extract",
"version": 3,
"flow_runs": [
{
Expand All @@ -34,23 +29,18 @@ def expectation_suite():
},
{
"id": "14b1a89e-f902-48a1-b6df-43cacdb91e1a",
"name": "1-raw dakvenster_order_prod_info extract",
"name": "1-raw table_catalog extract",
"version": 2,
"flow_runs": [],
},
{
"id": "a1eace09-38b4-46bf-bacf-a5d29bdbb633",
"name": "1-raw dakvenster_order_prod_info extract",
"name": "1-raw table_catalog extract",
"version": 1,
"flow_runs": [],
},
],
},
{
"id": "844372db-2d22-495d-a343-b8f8cbcf8963",
"name": "sap",
"flows": [],
},
{
"id": "512d0f29-2ceb-4177-b7d8-c5908da666ef",
"name": "integrations",
Expand All @@ -61,16 +51,6 @@ def expectation_suite():
"name": "dev_cdl",
"flows": [],
},
{
"id": "e2a926e2-ec86-4900-a24e-330a44b6cb19",
"name": "cic_test",
"flows": [],
},
{
"id": "667d5026-2f01-452a-b6fe-5437ca833066",
"name": "cic_dev",
"flows": [],
},
{
"id": "eac9b6d4-725a-4354-bf8f-25e7828ea2d8",
"name": "Admin",
Expand All @@ -86,31 +66,11 @@ def expectation_suite():
"name": "cic",
"flows": [],
},
{
"id": "dd2ccc32-2163-4f55-a746-1dbc6b28aaa4",
"name": "Hyperlocal",
"flows": [],
},
{
"id": "7131c357-bad7-43cf-aabc-87f9cf045384",
"name": "Installer Segmentation",
"flows": [],
},
{
"id": "94a8b8bf-14fa-4b64-ab78-af1d332dedd4",
"name": "Marketing KPI",
"flows": [],
},
{
"id": "ebe0e5aa-4add-4440-8c1a-6f9c74eb29fe",
"name": "dev",
"flows": [],
},
{
"id": "b5d924b0-4116-479f-a8f5-e28f9a9051ca",
"name": "velux",
"flows": [],
},
]
}
}
Expand All @@ -127,7 +87,7 @@ def test_prefect_logs(expectation_suite):
id
name
flows (
where : {name: {_eq: "1-raw google_analytics_oso_sps_gb extract"}}
where : {name: {_eq: "1-raw table_catalog extract"}}
) {
id
name
Expand Down Expand Up @@ -156,7 +116,7 @@ def test_prefect_logs(expectation_suite):
scheduled_start_time="2022-09-05",
filter_type="_gte",
local_file_path=f"prefect_extract_logs.parquet",
adls_path=f"raw/supermetrics/mp/prefect_extract_logs.parquet",
adls_path=f"raw/tests/prefect_extract_logs.parquet",
)

results = flow.run()
Expand Down
2 changes: 1 addition & 1 deletion tests/integration/tasks/test_bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def test_bigquery_to_df_success():
credentials_key=CREDENTIALS_KEY,
)
df = bigquery_to_df_task.run()
expected_column = ["my_value"]
expected_column = ["my_value", "_viadot_source"]

assert isinstance(df, pd.DataFrame)
assert expected_column == list(df.columns)
Expand Down
23 changes: 12 additions & 11 deletions tests/integration/test_mindful.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
os.system("clear")

credentials_mindful = local_config["MINDFUL"]
header = {
"Authorization": f"Bearer {credentials_mindful.get('VAULT')}",
}
auth = (
credentials_mindful["CUSTOMER_UUID"],
credentials_mindful["AUTH_TOKEN"],
)


class MockClass:
Expand Down Expand Up @@ -42,14 +43,14 @@ def json():

@pytest.mark.init
def test_instance_mindful():
mf = Mindful(header=header)
mf = Mindful(auth=auth)
assert isinstance(mf, Mindful)


@mock.patch("viadot.sources.mindful.handle_api_response", return_value=MockClass)
@pytest.mark.connect
def test_mindful_api_response(mock_connection):
mf = Mindful(header=header)
mf = Mindful(auth=auth)
mf.get_interactions_list()
mf.get_responses_list()
mock_connection.call_count == 2
Expand All @@ -58,7 +59,7 @@ def test_mindful_api_response(mock_connection):
@mock.patch("viadot.sources.mindful.handle_api_response", return_value=MockClass)
@pytest.mark.connect
def test_mindful_api_response2(mock_api_response):
mf = Mindful(header=header)
mf = Mindful(auth=auth)

response = mf.get_interactions_list()

Expand All @@ -69,7 +70,7 @@ def test_mindful_api_response2(mock_api_response):
@mock.patch("viadot.sources.mindful.handle_api_response", return_value=MockClass)
@pytest.mark.connect
def test_mindful_api_response3(mock_api_response):
mf = Mindful(header=header)
mf = Mindful(auth=auth)

response = mf.get_responses_list()

Expand All @@ -80,7 +81,7 @@ def test_mindful_api_response3(mock_api_response):
@mock.patch("viadot.sources.mindful.handle_api_response", return_value=MockClass)
@pytest.mark.connect
def test_mindful_api_response4(mock_api_response):
mf = Mindful(header=header)
mf = Mindful(auth=auth)

response = mf.get_survey_list()

Expand All @@ -91,7 +92,7 @@ def test_mindful_api_response4(mock_api_response):
@mock.patch("viadot.sources.Mindful._mindful_api_response", return_value=MockClass)
@pytest.mark.save
def test_mindful_interactions(mock_connection):
mf = Mindful(header=header)
mf = Mindful(auth=auth)
response = mf.get_interactions_list()
mf.response_to_file(response)
assert mf.endpoint == "interactions" and isinstance(mf.endpoint, str)
Expand All @@ -103,7 +104,7 @@ def test_mindful_interactions(mock_connection):
@mock.patch("viadot.sources.Mindful._mindful_api_response", return_value=MockClass)
@pytest.mark.save
def test_mindful_responses(mock_connection):
mf = Mindful(header=header)
mf = Mindful(auth=auth)
response = mf.get_responses_list()
mf.response_to_file(response)

Expand All @@ -115,7 +116,7 @@ def test_mindful_responses(mock_connection):
@mock.patch("viadot.sources.Mindful._mindful_api_response", return_value=MockClass)
@pytest.mark.save
def test_mindful_surveys(mock_connection):
mf = Mindful(header=header)
mf = Mindful(auth=auth)
response = mf.get_survey_list()
mf.response_to_file(response)

Expand Down
Loading