From fbde221f6b8a95eeaf4c9b5648ca09febe1995cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 12 Sep 2023 15:09:58 -0600 Subject: [PATCH 1/3] test: Test with singer-sdk @ main --- poetry.lock | 30 +++++++++++++++++------------- pyproject.toml | 4 ++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6bfcaa9..37be0e6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1583,36 +1583,34 @@ files = [ [[package]] name = "singer-sdk" -version = "0.42.1" +version = "0.43.0b1.post1.dev0+d3dcc8e9" description = "A framework for building Singer taps" optional = false -python-versions = ">=3.8" -files = [ - {file = "singer_sdk-0.42.1-py3-none-any.whl", hash = "sha256:cc588927ff87b7aea0087c57c37c85bbe95f9b7c52ebb265abb50a184cec6ad8"}, - {file = "singer_sdk-0.42.1.tar.gz", hash = "sha256:92019c3f2e476f5c17eaa381e35d3c0ff428814587f8424149a4b28904f75ca0"}, -] +python-versions = ">=3.9" +files = [] +develop = false [package.dependencies] backoff = {version = ">=2.0.0", markers = "python_version < \"4\""} backports-datetime-fromisoformat = {version = ">=2.0.1", markers = "python_version < \"3.11\""} -click = ">=8.0,<9.0" -faker = {version = ">=22.5", optional = true, markers = "extra == \"faker\""} +click = "~=8.0" +faker = {version = ">=22.5", optional = true} fs = ">=2.4.16" fsspec = ">=2024.9.0" importlib-metadata = {version = "<9.0.0", markers = "python_version < \"3.12\""} -importlib-resources = {version = ">=5.12.0,<6.2.0 || >6.2.0,<6.3.0 || >6.3.0,<6.3.1 || >6.3.1", markers = "python_version < \"3.10\""} +importlib-resources = {version = ">=5.12.0,!=6.2.0,!=6.3.0,!=6.3.1", markers = "python_version < \"3.10\""} inflection = ">=0.5.1" joblib = ">=1.3.0" jsonpath-ng = ">=1.5.3" jsonschema = ">=4.16.0" packaging = ">=23.1" -pytest = {version = ">=7.2.1", optional = true, markers = "extra == \"docs\" or extra == \"testing\""} +pytest = {version = ">=7.2.1", optional = true} python-dotenv = ">=0.20" PyYAML = ">=6.0" referencing = ">=0.30.0" requests = ">=2.25.1" setuptools = "<=70.3.0" -simpleeval = {version = ">=0.9.13,<1.0.1 || >1.0.1", markers = "python_version >= \"3.9\""} +simpleeval = ">=0.9.13,!=1.0.1" simplejson = ">=3.17.6" sqlalchemy = ">=1.4,<3.0" typing-extensions = ">=4.5.0" @@ -1621,11 +1619,17 @@ typing-extensions = ">=4.5.0" docs = ["furo (>=2024.5.6)", "myst-parser (>=3)", "pytest (>=7.2.1)", "sphinx (>=7)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinx-notfound-page (>=1.0.0)", "sphinx-reredirects (>=0.1.5)"] faker = ["faker (>=22.5)"] jwt = ["PyJWT (>=2.4,<3.0)", "cryptography (>=3.4.6)"] -parquet = ["numpy (>=1.22)", "numpy (>=1.22,<1.25)", "numpy (>=1.22,<2.1)", "pyarrow (>=13)", "pyarrow (>=13,<18)"] +parquet = ["numpy (>=1.22)", "numpy (>=1.22,<2.1)", "pyarrow (>=13)"] s3 = ["fs-s3fs (>=1.1.1)", "s3fs (>=2024.9.0)"] ssh = ["paramiko (>=3.3.0)"] testing = ["pytest (>=7.2.1)"] +[package.source] +type = "git" +url = "https://github.com/meltano/sdk.git" +reference = "HEAD" +resolved_reference = "d3dcc8e9a73cf795c438cd6d8294ff02dc704511" + [[package]] name = "six" version = "1.16.0" @@ -1972,4 +1976,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9" -content-hash = "96b08db846bc177f796945342a58a19b8aab39a6bc69ea2fe59497c4b6a5dc73" +content-hash = "c786e23bd1f0d53c579f362d073af686d3452cee6c257c9a9fedbd64230fa3f5" diff --git a/pyproject.toml b/pyproject.toml index 53e53e4..6ae99e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ sqlalchemy = "2.0.36" sshtunnel = "0.4.0" [tool.poetry.dependencies.singer-sdk] -version = "~=0.42.1" +git = "https://github.com/meltano/sdk.git" extras = ["faker"] [tool.poetry.group.dev.dependencies] @@ -57,7 +57,7 @@ types-jsonschema = ">=4.19.0.3" types-psycopg2 = ">=2.9.21.20240118" [tool.poetry.dev-dependencies.singer-sdk] -version = "*" +git = "https://github.com/meltano/sdk.git" extras = ["testing"] [tool.mypy] From 00fd1c1aa87c17d364b5984677948d66ad18bdd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Thu, 28 Nov 2024 16:43:25 -0600 Subject: [PATCH 2/3] refactor: Use `sql_to_jsonschema_converter` --- tap_postgres/client.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tap_postgres/client.py b/tap_postgres/client.py index 7023716..a861e2b 100644 --- a/tap_postgres/client.py +++ b/tap_postgres/client.py @@ -35,12 +35,20 @@ class PostgresSQLToJSONSchema(SQLToJSONSchema): """Custom SQL to JSON Schema conversion for Postgres.""" - def __init__(self, dates_as_string: bool, json_as_object: bool, *args, **kwargs): + def __init__(self, *, dates_as_string: bool, json_as_object: bool, **kwargs): """Initialize the SQL to JSON Schema converter.""" - super().__init__(*args, **kwargs) + super().__init__(**kwargs) self.dates_as_string = dates_as_string self.json_as_object = json_as_object + @classmethod + def from_config(cls, config: dict) -> PostgresSQLToJSONSchema: + """Instantiate the SQL to JSON Schema converter from a config dictionary.""" + return cls( + dates_as_string=config["dates_as_string"], + json_as_object=config["json_as_object"], + ) + @functools.singledispatchmethod def to_jsonschema(self, column_type: t.Any) -> dict: """Customize the JSON Schema for Postgres types.""" @@ -132,6 +140,8 @@ def patched_conform(elem: t.Any, property_schema: dict) -> t.Any: class PostgresConnector(SQLConnector): """Connects to the Postgres SQL source.""" + sql_to_jsonschema_converter = PostgresSQLToJSONSchema + def __init__( self, config: dict | None = None, @@ -160,14 +170,6 @@ def __init__( super().__init__(config=config, sqlalchemy_url=sqlalchemy_url) - @functools.cached_property - def sql_to_jsonschema(self): - """Return a mapping of SQL types to JSON Schema types.""" - return PostgresSQLToJSONSchema( - dates_as_string=self.config["dates_as_string"], - json_as_object=self.config["json_as_object"], - ) - def get_schema_names(self, engine: Engine, inspected: Inspector) -> list[str]: """Return a list of schema names in DB, or overrides with user-provided values. From 257984ca00043882e826cc92551c0588e919187e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 3 Dec 2024 10:08:12 -0600 Subject: [PATCH 3/3] fix: Exclude `information_schema` and `pg_catalog` from discovery --- tap_postgres/tap.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tap_postgres/tap.py b/tap_postgres/tap.py index 0e1e48f..4cdab8a 100644 --- a/tap_postgres/tap.py +++ b/tap_postgres/tap.py @@ -42,6 +42,11 @@ class TapPostgres(SQLTap): name = "tap-postgres" default_stream_class = PostgresStream + exclude_schemas = ( + "information_schema", + "pg_catalog", + ) + def __init__( self, *args, @@ -547,7 +552,11 @@ def catalog_dict(self) -> dict: return self.input_catalog.to_dict() result: dict[str, list[dict]] = {"streams": []} - result["streams"].extend(self.connector.discover_catalog_entries()) + result["streams"].extend( + self.connector.discover_catalog_entries( + exclude_schemas=self.exclude_schemas, + ) + ) self._catalog_dict: dict = result return self._catalog_dict