diff --git a/CHANGELOG.md b/CHANGELOG.md index b2419e6d6a..b0c2381ae2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Fixed - Fixed pagination bugs on some tables [#5819](https://github.com/ethyca/fides/pull/5819) +- Fixed load_samples to wrap variables in quotes to prevent YAML parsing errors [#5857](https://github.com/ethyca/fides/pull/5857) ## [2.56.2](https://github.com/ethyca/fides/compare/2.56.1...2.56.2) diff --git a/src/fides/data/sample_project/sample_connections/sample_connections.yml b/src/fides/data/sample_project/sample_connections/sample_connections.yml index 43cdf37753..a66e1ec16c 100644 --- a/src/fides/data/sample_project/sample_connections/sample_connections.yml +++ b/src/fides/data/sample_project/sample_connections/sample_connections.yml @@ -6,12 +6,12 @@ connection: dataset: postgres_example_test_dataset system_key: cookie_house_postgresql_database secrets: - host: $FIDES_DEPLOY__CONNECTORS__POSTGRES__HOST - port: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT - dbname: $FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME - username: $FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME - password: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD - ssh_required: $FIDES_DEPLOY__CONNECTORS__POSTGRES__SSH_REQUIRED + host: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__HOST" + port: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT" + dbname: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME" + username: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME" + password: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD" + ssh_required: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__SSH_REQUIRED" - key: cookie_house_loyalty_database name: Postgres Connector (Loyalty) connection_type: postgres @@ -19,11 +19,11 @@ connection: dataset: postgres_example_test_extended_dataset system_key: cookie_house_loyalty_database secrets: - host: $FIDES_DEPLOY__CONNECTORS__POSTGRES_LOYALTY__HOST - port: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT - dbname: $FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME - username: $FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME - password: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD + host: "$FIDES_DEPLOY__CONNECTORS__POSTGRES_LOYALTY__HOST" + port: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT" + dbname: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME" + username: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME" + password: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD" disabled: True - key: cookie_house_customer_database_mongodb name: MongoDB Connector @@ -32,43 +32,43 @@ connection: dataset: mongo_test system_key: cookie_house_customer_database secrets: - host: $FIDES_DEPLOY__CONNECTORS__MONGO_HOST - port: $FIDES_DEPLOY__CONNECTORS__MONGO_PORT - defaultauthdb: $FIDES_DEPLOY__CONNECTORS__MONGO_DEFAULTAUTHDB - username: $FIDES_DEPLOY__CONNECTORS__MONGO_USERNAME - password: $FIDES_DEPLOY__CONNECTORS__MONGO_PASSWORD + host: "$FIDES_DEPLOY__CONNECTORS__MONGO_HOST" + port: "$FIDES_DEPLOY__CONNECTORS__MONGO_PORT" + defaultauthdb: "$FIDES_DEPLOY__CONNECTORS__MONGO_DEFAULTAUTHDB" + username: "$FIDES_DEPLOY__CONNECTORS__MONGO_USERNAME" + password: "$FIDES_DEPLOY__CONNECTORS__MONGO_PASSWORD" - key: mailchimp_connector name: Mailchimp Connector connection_type: saas saas_connector_type: mailchimp access: write secrets: - domain: $FIDES_DEPLOY__CONNECTORS__MAILCHIMP__DOMAIN - username: $FIDES_DEPLOY__CONNECTORS__MAILCHIMP__USERNAME - api_key: $FIDES_DEPLOY__CONNECTORS__MAILCHIMP__API_KEY + domain: "$FIDES_DEPLOY__CONNECTORS__MAILCHIMP__DOMAIN" + username: "$FIDES_DEPLOY__CONNECTORS__MAILCHIMP__USERNAME" + api_key: "$FIDES_DEPLOY__CONNECTORS__MAILCHIMP__API_KEY" - key: stripe_connector name: Stripe Connector connection_type: saas saas_connector_type: stripe access: write secrets: - domain: $FIDES_DEPLOY__CONNECTORS__STRIPE__DOMAIN - api_key: $FIDES_DEPLOY__CONNECTORS__STRIPE__API_KEY + domain: "$FIDES_DEPLOY__CONNECTORS__STRIPE__DOMAIN" + api_key: "$FIDES_DEPLOY__CONNECTORS__STRIPE__API_KEY" - key: hubspot_connector name: Hubspot Connector connection_type: saas saas_connector_type: hubspot access: write secrets: - domain: $FIDES_DEPLOY__CONNECTORS__HUBSPOT__DOMAIN - private_app_token: $FIDES_DEPLOY__CONNECTORS__HUBSPOT__PRIVATE_APP_TOKEN + domain: "$FIDES_DEPLOY__CONNECTORS__HUBSPOT__DOMAIN" + private_app_token: "$FIDES_DEPLOY__CONNECTORS__HUBSPOT__PRIVATE_APP_TOKEN" - key: mailchimp_transactional_connector name: Mailchimp Transactional Connector connection_type: saas saas_connector_type: mailchimp_transactional access: write secrets: - api_key: $FIDES_DEPLOY__CONNECTORS__MAILCHIMP_TRANSACTIONAL_API_KEY + api_key: "$FIDES_DEPLOY__CONNECTORS__MAILCHIMP_TRANSACTIONAL_API_KEY" - key: cookie_house_custom_request_fields_database name: Postgres Connector (Custom Request Fields) connection_type: postgres @@ -76,8 +76,8 @@ connection: dataset: postgres_example_custom_request_field_dataset system_key: cookie_house_custom_request_fields_database secrets: - host: $FIDES_DEPLOY__CONNECTORS__POSTGRES__HOST - port: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT - dbname: $FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME - username: $FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME - password: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD + host: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__HOST" + port: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT" + dbname: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME" + username: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME" + password: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD" diff --git a/tests/ctl/api/test_seed.py b/tests/ctl/api/test_seed.py index 9bd78ba18a..3a7d90d8ce 100644 --- a/tests/ctl/api/test_seed.py +++ b/tests/ctl/api/test_seed.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest +import yaml from fideslang.default_taxonomy import DEFAULT_TAXONOMY from fideslang.models import DataCategory, Organization from sqlalchemy.ext.asyncio import AsyncSession @@ -459,7 +460,7 @@ class TestLoadSamples: "FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT": "9090", "FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME": "test-var-db", "FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME": "test-var-user", - "FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD": "test-var-password", + "FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD": "&anchor!-test-password", "FIDES_DEPLOY__CONNECTORS__POSTGRES__SSH_REQUIRED": "false", "FIDES_DEPLOY__CONNECTORS__STRIPE__DOMAIN": "test-stripe-domain", "FIDES_DEPLOY__CONNECTORS__STRIPE__API_KEY": "test-stripe-api-key", @@ -467,7 +468,7 @@ class TestLoadSamples: "FIDES_DEPLOY__CONNECTORS__MONGO_PORT": "9090", "FIDES_DEPLOY__CONNECTORS__MONGO_DEFAULTAUTHDB": "test-var-db", "FIDES_DEPLOY__CONNECTORS__MONGO_USERNAME": "test-var-user", - "FIDES_DEPLOY__CONNECTORS__MONGO_PASSWORD": "test-var-password", + "FIDES_DEPLOY__CONNECTORS__MONGO_PASSWORD": "&anchor!-test-password", } @patch.dict(os.environ, SAMPLE_ENV_VARS, clear=True) @@ -613,7 +614,8 @@ async def test_load_sample_connections(self): 0 ].model_dump(mode="json") assert postgres["secrets"]["host"] == "test-var-expansion" - assert postgres["secrets"]["port"] == 9090 + assert postgres["secrets"]["port"] == "9090" + assert postgres["secrets"]["password"] == "&anchor!-test-password" @patch.dict( os.environ, @@ -657,3 +659,47 @@ async def test_load_sample_yaml_file(self): assert sample_connection["secrets"]["dbname"] == "var-2" assert sample_connection["secrets"]["username"] == "user-var-2" assert sample_connection["secrets"]["password"] == "var-1-var-2" + + @patch.dict( + os.environ, + { + "TEST_PASSWORD": "&anchor!'quote'!@#$%^&*", + }, + clear=True, + ) + async def test_load_sample_yaml_with_special_chars(self): + """Test that YAML parsing requires proper quoting for environment variables with special characters""" + # Test safe usage with quotes + safe_yaml = dedent( + """\ + connection: + - key: test_connection + name: Test Connection + connection_type: postgres + access: write + secrets: + password: "$TEST_PASSWORD" + """ + ) + sample_file = io.StringIO(safe_yaml) + sample_dict = samples.load_sample_yaml_file(sample_file) + assert ( + sample_dict["connection"][0]["secrets"]["password"] + == "&anchor!'quote'!@#$%^&*" + ) + + # Test unsafe usage without quotes - should raise YAML parsing error + unsafe_yaml = dedent( + """\ + connection: + - key: test_connection + name: Test Connection + connection_type: postgres + access: write + secrets: + password: $TEST_PASSWORD + """ + ) + sample_file = io.StringIO(unsafe_yaml) + with pytest.raises(yaml.scanner.ScannerError): + samples.load_sample_yaml_file(sample_file)