diff --git a/myds.yml b/myds.yml new file mode 100644 index 000000000..43bfe4004 --- /dev/null +++ b/myds.yml @@ -0,0 +1,10 @@ +type: postgres +name: postgres_ds +connection: + host: localhost + user: ${POSTGRES_USERNAME} + password: ${POSTGRES_PASSWORD} + database: your_postgres_db, +format_regexes: + # Example named regex format + single_digit_test_format: ^[0-9]$ \ No newline at end of file diff --git a/soda-core/src/soda_core/cli/soda.py b/soda-core/src/soda_core/cli/soda.py index 447273125..aaf35e2bc 100644 --- a/soda-core/src/soda_core/cli/soda.py +++ b/soda-core/src/soda_core/cli/soda.py @@ -1,9 +1,36 @@ from __future__ import annotations import argparse +import logging +import sys +from os.path import dirname, exists +from pathlib import Path from textwrap import dedent -from soda_core.contracts.contract_verification import ContractVerification, ContractVerificationBuilder +from soda_core.contracts.contract_verification import ContractVerification, ContractVerificationBuilder, \ + ContractVerificationResult + + +def configure_logging(): + sys.stderr = sys.stdout + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("botocore").setLevel(logging.WARNING) + logging.getLogger("pyathena").setLevel(logging.WARNING) + logging.getLogger("faker").setLevel(logging.ERROR) + logging.getLogger("snowflake").setLevel(logging.WARNING) + logging.getLogger("matplotlib").setLevel(logging.WARNING) + logging.getLogger("pyspark").setLevel(logging.ERROR) + logging.getLogger("pyhive").setLevel(logging.ERROR) + logging.getLogger("py4j").setLevel(logging.INFO) + logging.getLogger("segment").setLevel(logging.WARNING) + logging.basicConfig( + level=logging.DEBUG, + force=True, # Override any previously set handlers. + # https://docs.python.org/3/library/logging.html#logrecord-attributes + # %(name)s + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], + ) def verify_contract( @@ -43,12 +70,14 @@ def verify_contract( contract_verification_builder.with_soda_cloud_yaml_file(soda_cloud_file_path) if skip_publish: print(f"\u274C Not publishing the contract on Soda Cloud") - contract_verification_builder.skip_publish() + contract_verification_builder.with_soda_cloud_skip_publish() else: print(f"\u2705 Publishing contract to Soda Cloud \U0001F325") else: print(f"\u274C Not sending results to Soda Cloud") + contract_verification_result: ContractVerificationResult = contract_verification_builder.execute() + def publish_contract(contract_file_paths: list[str] | None): print( @@ -56,15 +85,63 @@ def publish_contract(contract_file_paths: list[str] | None): ) -def test_data_source(data_source_name: str): - print(f"Testing data source {data_source_name}") +def create_data_source(data_source_file_path: str, data_source_type: str): + print(f"Creating {data_source_type} data source YAML file '{data_source_file_path}'") + if exists(data_source_file_path): + print(f"\U0001F92F I'm suppose to create data source file '{data_source_file_path}' but it already exists") + return + if data_source_type!= "postgres": + print(f"\U0001F92F Only type postgres is supported atm") + return + dir: str = dirname(data_source_file_path) + Path(dir).mkdir(parents=True, exist_ok=True) + with open(data_source_file_path, "w") as text_file: + text_file.write(dedent( + """ + type: postgres + name: postgres_ds + connection: + host: localhost + user: ${POSTGRES_USERNAME} + password: ${POSTGRES_PASSWORD} + database: your_postgres_db, + format_regexes: + # Example named regex format + single_digit_test_format: ^[0-9]$ + """ + ).strip()) + print(f"\u2705 Created data source file '{data_source_file_path}'") + + +def test_data_source(data_source_file_path: str): + print(f"Testing data source configuration file {data_source_file_path}") + +def create_soda_cloud(soda_cloud_file_path: str): + print(f"Creating Soda Cloud YAML file '{soda_cloud_file_path}'") + if exists(soda_cloud_file_path): + print(f"\U0001F92F I'm suppose to create soda cloud file '{soda_cloud_file_path}' but it already exists") + dir: str = dirname(soda_cloud_file_path) + Path(dir).mkdir(parents=True, exist_ok=True) + with open(soda_cloud_file_path, "w") as text_file: + text_file.write(dedent( + """ + soda_cloud: + host: cloud.soda.io + api_key_id: ${SODA_CLOUD_API_KEY_ID} + api_key_secret: ${SODA_CLOUD_API_KEY_SECRET} + """ + ).strip()) + print(f"\u2705 Created Soda Cloud configuration file '{soda_cloud_file_path}'") -def test_soda_cloud(soda_cloud_path: str): - print(f"Testing soda cloud {soda_cloud_path}") + +def test_soda_cloud(soda_cloud_file_path: str): + print(f"Testing soda cloud file {soda_cloud_file_path}") def main(): + configure_logging() + print(dedent(""" __| _ \| \ \\ \__ \ ( | | _ \\ @@ -116,12 +193,44 @@ def main(): help="One or more contract file paths." ) + create_data_source_parser = sub_parsers.add_parser( + name="create-data-source", + help="Create a data source YAML configuration file" + ) + create_data_source_parser.add_argument( + "-f", "--file", + type=str, + help="The path to the file to be created. (directories will be created if needed)" + ) + create_data_source_parser.add_argument( + "-t", "--type", + type=str, + default="postgres", + help="Type of the data source. Eg postgres" + ) + test_parser = sub_parsers.add_parser('test-data-source', help='Test a data source connection') test_parser.add_argument( "-ds", "--data-source", type=str, help="The name of a configured data source to test." ) + test_parser = sub_parsers.add_parser('test-data-source', help='Test a data source connection') + test_parser.add_argument( + "-ds", "--data-source", + type=str, + help="The name of a configured data source to test." + ) + + create_soda_cloud_parser = sub_parsers.add_parser( + name="create-soda-cloud", + help="Create a Soda Cloud YAML configuration file" + ) + create_soda_cloud_parser.add_argument( + "-f", "--file", + type=str, + help="The path to the file to be created. (directories will be created if needed)" + ) test_parser = sub_parsers.add_parser('test-soda-cloud', help='Test the Soda Cloud connection') test_parser.add_argument( @@ -137,8 +246,12 @@ def main(): verify_contract(args.contract, args.soda_cloud, args.skip_publish, args.use_agent) elif args.command == "publish": publish_contract(args.contract) + elif args.command == "create-data-source": + create_data_source(args.file, args.type) elif args.command == "test-data-source": test_data_source(args.data_source) + elif args.command == "create-soda-cloud": + create_soda_cloud(args.file) elif args.command == "test-soda-cloud": test_soda_cloud(args.soda_cloud) else: diff --git a/soda-core/src/soda_core/common/soda_cloud.py b/soda-core/src/soda_core/common/soda_cloud.py index e3f3e7a27..d703a3353 100644 --- a/soda-core/src/soda_core/common/soda_cloud.py +++ b/soda-core/src/soda_core/common/soda_cloud.py @@ -98,6 +98,7 @@ def __init__( self.logs = logs self.soda_cloud_trace_ids = {} self._organization_configuration = None + self.skip_publish: bool = False def send_contract_result(self, contract_result: ContractResult): contract_yaml_source_str = contract_result.contract_info.source.source_content_str diff --git a/soda-core/src/soda_core/contracts/contract_verification.py b/soda-core/src/soda_core/contracts/contract_verification.py index 4e0252568..fd4513660 100644 --- a/soda-core/src/soda_core/contracts/contract_verification.py +++ b/soda-core/src/soda_core/contracts/contract_verification.py @@ -23,6 +23,7 @@ def __init__(self): self.soda_cloud: Optional['SodaCloud'] = None self.soda_cloud_yaml_source: Optional[YamlSource] = None self.variables: dict[str, str] = {} + self.soda_cloud_skip_publish: bool = False self.logs: Logs = Logs() def with_contract_yaml_file(self, contract_yaml_file_path: str) -> ContractVerificationBuilder: @@ -85,6 +86,13 @@ def with_variables(self, variables: dict[str, str]) -> ContractVerificationBuild self.variables.update(variables) return self + def with_soda_cloud_skip_publish(self) -> ContractVerificationBuilder: + """ + Skips contract publication on Soda Cloud. + """ + self.soda_cloud_skip_publish = True + return self + def build(self) -> ContractVerification: return ContractVerification(contract_verification_builder=self) @@ -108,6 +116,7 @@ def __init__(self, contract_verification_builder: ContractVerificationBuilder): soda_cloud=contract_verification_builder.soda_cloud, soda_cloud_yaml_source=contract_verification_builder.soda_cloud_yaml_source, variables=contract_verification_builder.variables, + skip_publish=contract_verification_builder.soda_cloud_skip_publish, logs=contract_verification_builder.logs, ) diff --git a/soda-core/src/soda_core/contracts/impl/contract_verification_impl.py b/soda-core/src/soda_core/contracts/impl/contract_verification_impl.py index 84ed58434..0eeab6b14 100644 --- a/soda-core/src/soda_core/contracts/impl/contract_verification_impl.py +++ b/soda-core/src/soda_core/contracts/impl/contract_verification_impl.py @@ -42,6 +42,7 @@ def __init__( soda_cloud: 'SodaCloud' | None, soda_cloud_yaml_source: Optional[YamlSource], variables: dict[str, str], + skip_publish: bool, logs: Logs = Logs(), ): self.logs: Logs = logs @@ -70,6 +71,8 @@ def __init__( logs=logs ) self.soda_cloud = SodaCloud.from_file(soda_cloud_yaml_file_content) + if self.soda_cloud: + self.soda_cloud.skip_publish = skip_publish for contract_yaml_source in contract_yaml_sources: contract_yaml: ContractYaml = ContractYaml.parse(contract_yaml_source=contract_yaml_source, variables=variables, logs=logs)