Skip to content

Commit 238bad5

Browse files
NevilleSNevilleS
authored
Fixed load_samples to wrap variables in quotes to prevent YAML parsing errors (#5858)
1 parent 7f21d2e commit 238bad5

File tree

3 files changed

+79
-32
lines changed

3 files changed

+79
-32
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o
3737

3838
### Fixed
3939
- Fixed pagination bugs on some tables [#5819](https://github.com/ethyca/fides/pull/5819)
40+
- Fixed load_samples to wrap variables in quotes to prevent YAML parsing errors [#5857](https://github.com/ethyca/fides/pull/5857)
4041

4142
## [2.56.2](https://github.com/ethyca/fides/compare/2.56.1...2.56.2)
4243

‎src/fides/data/sample_project/sample_connections/sample_connections.yml

+29-29
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,24 @@ connection:
66
dataset: postgres_example_test_dataset
77
system_key: cookie_house_postgresql_database
88
secrets:
9-
host: $FIDES_DEPLOY__CONNECTORS__POSTGRES__HOST
10-
port: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT
11-
dbname: $FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME
12-
username: $FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME
13-
password: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD
14-
ssh_required: $FIDES_DEPLOY__CONNECTORS__POSTGRES__SSH_REQUIRED
9+
host: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__HOST"
10+
port: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT"
11+
dbname: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME"
12+
username: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME"
13+
password: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD"
14+
ssh_required: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__SSH_REQUIRED"
1515
- key: cookie_house_loyalty_database
1616
name: Postgres Connector (Loyalty)
1717
connection_type: postgres
1818
access: write
1919
dataset: postgres_example_test_extended_dataset
2020
system_key: cookie_house_loyalty_database
2121
secrets:
22-
host: $FIDES_DEPLOY__CONNECTORS__POSTGRES_LOYALTY__HOST
23-
port: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT
24-
dbname: $FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME
25-
username: $FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME
26-
password: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD
22+
host: "$FIDES_DEPLOY__CONNECTORS__POSTGRES_LOYALTY__HOST"
23+
port: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT"
24+
dbname: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME"
25+
username: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME"
26+
password: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD"
2727
disabled: True
2828
- key: cookie_house_customer_database_mongodb
2929
name: MongoDB Connector
@@ -32,52 +32,52 @@ connection:
3232
dataset: mongo_test
3333
system_key: cookie_house_customer_database
3434
secrets:
35-
host: $FIDES_DEPLOY__CONNECTORS__MONGO_HOST
36-
port: $FIDES_DEPLOY__CONNECTORS__MONGO_PORT
37-
defaultauthdb: $FIDES_DEPLOY__CONNECTORS__MONGO_DEFAULTAUTHDB
38-
username: $FIDES_DEPLOY__CONNECTORS__MONGO_USERNAME
39-
password: $FIDES_DEPLOY__CONNECTORS__MONGO_PASSWORD
35+
host: "$FIDES_DEPLOY__CONNECTORS__MONGO_HOST"
36+
port: "$FIDES_DEPLOY__CONNECTORS__MONGO_PORT"
37+
defaultauthdb: "$FIDES_DEPLOY__CONNECTORS__MONGO_DEFAULTAUTHDB"
38+
username: "$FIDES_DEPLOY__CONNECTORS__MONGO_USERNAME"
39+
password: "$FIDES_DEPLOY__CONNECTORS__MONGO_PASSWORD"
4040
- key: mailchimp_connector
4141
name: Mailchimp Connector
4242
connection_type: saas
4343
saas_connector_type: mailchimp
4444
access: write
4545
secrets:
46-
domain: $FIDES_DEPLOY__CONNECTORS__MAILCHIMP__DOMAIN
47-
username: $FIDES_DEPLOY__CONNECTORS__MAILCHIMP__USERNAME
48-
api_key: $FIDES_DEPLOY__CONNECTORS__MAILCHIMP__API_KEY
46+
domain: "$FIDES_DEPLOY__CONNECTORS__MAILCHIMP__DOMAIN"
47+
username: "$FIDES_DEPLOY__CONNECTORS__MAILCHIMP__USERNAME"
48+
api_key: "$FIDES_DEPLOY__CONNECTORS__MAILCHIMP__API_KEY"
4949
- key: stripe_connector
5050
name: Stripe Connector
5151
connection_type: saas
5252
saas_connector_type: stripe
5353
access: write
5454
secrets:
55-
domain: $FIDES_DEPLOY__CONNECTORS__STRIPE__DOMAIN
56-
api_key: $FIDES_DEPLOY__CONNECTORS__STRIPE__API_KEY
55+
domain: "$FIDES_DEPLOY__CONNECTORS__STRIPE__DOMAIN"
56+
api_key: "$FIDES_DEPLOY__CONNECTORS__STRIPE__API_KEY"
5757
- key: hubspot_connector
5858
name: Hubspot Connector
5959
connection_type: saas
6060
saas_connector_type: hubspot
6161
access: write
6262
secrets:
63-
domain: $FIDES_DEPLOY__CONNECTORS__HUBSPOT__DOMAIN
64-
private_app_token: $FIDES_DEPLOY__CONNECTORS__HUBSPOT__PRIVATE_APP_TOKEN
63+
domain: "$FIDES_DEPLOY__CONNECTORS__HUBSPOT__DOMAIN"
64+
private_app_token: "$FIDES_DEPLOY__CONNECTORS__HUBSPOT__PRIVATE_APP_TOKEN"
6565
- key: mailchimp_transactional_connector
6666
name: Mailchimp Transactional Connector
6767
connection_type: saas
6868
saas_connector_type: mailchimp_transactional
6969
access: write
7070
secrets:
71-
api_key: $FIDES_DEPLOY__CONNECTORS__MAILCHIMP_TRANSACTIONAL_API_KEY
71+
api_key: "$FIDES_DEPLOY__CONNECTORS__MAILCHIMP_TRANSACTIONAL_API_KEY"
7272
- key: cookie_house_custom_request_fields_database
7373
name: Postgres Connector (Custom Request Fields)
7474
connection_type: postgres
7575
access: write
7676
dataset: postgres_example_custom_request_field_dataset
7777
system_key: cookie_house_custom_request_fields_database
7878
secrets:
79-
host: $FIDES_DEPLOY__CONNECTORS__POSTGRES__HOST
80-
port: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT
81-
dbname: $FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME
82-
username: $FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME
83-
password: $FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD
79+
host: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__HOST"
80+
port: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT"
81+
dbname: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME"
82+
username: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME"
83+
password: "$FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD"

‎tests/ctl/api/test_seed.py

+49-3
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from unittest.mock import patch
66

77
import pytest
8+
import yaml
89
from fideslang.default_taxonomy import DEFAULT_TAXONOMY
910
from fideslang.models import DataCategory, Organization
1011
from sqlalchemy.ext.asyncio import AsyncSession
@@ -459,15 +460,15 @@ class TestLoadSamples:
459460
"FIDES_DEPLOY__CONNECTORS__POSTGRES__PORT": "9090",
460461
"FIDES_DEPLOY__CONNECTORS__POSTGRES__DBNAME": "test-var-db",
461462
"FIDES_DEPLOY__CONNECTORS__POSTGRES__USERNAME": "test-var-user",
462-
"FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD": "test-var-password",
463+
"FIDES_DEPLOY__CONNECTORS__POSTGRES__PASSWORD": "&anchor!-test-password",
463464
"FIDES_DEPLOY__CONNECTORS__POSTGRES__SSH_REQUIRED": "false",
464465
"FIDES_DEPLOY__CONNECTORS__STRIPE__DOMAIN": "test-stripe-domain",
465466
"FIDES_DEPLOY__CONNECTORS__STRIPE__API_KEY": "test-stripe-api-key",
466467
"FIDES_DEPLOY__CONNECTORS__MONGO_HOST": "test-var-expansion",
467468
"FIDES_DEPLOY__CONNECTORS__MONGO_PORT": "9090",
468469
"FIDES_DEPLOY__CONNECTORS__MONGO_DEFAULTAUTHDB": "test-var-db",
469470
"FIDES_DEPLOY__CONNECTORS__MONGO_USERNAME": "test-var-user",
470-
"FIDES_DEPLOY__CONNECTORS__MONGO_PASSWORD": "test-var-password",
471+
"FIDES_DEPLOY__CONNECTORS__MONGO_PASSWORD": "&anchor!-test-password",
471472
}
472473

473474
@patch.dict(os.environ, SAMPLE_ENV_VARS, clear=True)
@@ -613,7 +614,8 @@ async def test_load_sample_connections(self):
613614
0
614615
].model_dump(mode="json")
615616
assert postgres["secrets"]["host"] == "test-var-expansion"
616-
assert postgres["secrets"]["port"] == 9090
617+
assert postgres["secrets"]["port"] == "9090"
618+
assert postgres["secrets"]["password"] == "&anchor!-test-password"
617619

618620
@patch.dict(
619621
os.environ,
@@ -657,3 +659,47 @@ async def test_load_sample_yaml_file(self):
657659
assert sample_connection["secrets"]["dbname"] == "var-2"
658660
assert sample_connection["secrets"]["username"] == "user-var-2"
659661
assert sample_connection["secrets"]["password"] == "var-1-var-2"
662+
663+
@patch.dict(
664+
os.environ,
665+
{
666+
"TEST_PASSWORD": "&anchor!'quote'!@#$%^&*",
667+
},
668+
clear=True,
669+
)
670+
async def test_load_sample_yaml_with_special_chars(self):
671+
"""Test that YAML parsing requires proper quoting for environment variables with special characters"""
672+
# Test safe usage with quotes
673+
safe_yaml = dedent(
674+
"""\
675+
connection:
676+
- key: test_connection
677+
name: Test Connection
678+
connection_type: postgres
679+
access: write
680+
secrets:
681+
password: "$TEST_PASSWORD"
682+
"""
683+
)
684+
sample_file = io.StringIO(safe_yaml)
685+
sample_dict = samples.load_sample_yaml_file(sample_file)
686+
assert (
687+
sample_dict["connection"][0]["secrets"]["password"]
688+
== "&anchor!'quote'!@#$%^&*"
689+
)
690+
691+
# Test unsafe usage without quotes - should raise YAML parsing error
692+
unsafe_yaml = dedent(
693+
"""\
694+
connection:
695+
- key: test_connection
696+
name: Test Connection
697+
connection_type: postgres
698+
access: write
699+
secrets:
700+
password: $TEST_PASSWORD
701+
"""
702+
)
703+
sample_file = io.StringIO(unsafe_yaml)
704+
with pytest.raises(yaml.scanner.ScannerError):
705+
samples.load_sample_yaml_file(sample_file)

0 commit comments

Comments
 (0)