Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: SQLTap connector instance shared with streams #1861

Merged
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
bef91bf
add default_connector_class to SQLTap
BuzzCutNorman Jul 17, 2023
eb6634e
add tap_connector to SQLTap
BuzzCutNorman Jul 17, 2023
f0c1c4d
updated catalog_dict to use tap_connector
BuzzCutNorman Jul 17, 2023
923e820
update discover_streams to use tap_connector
BuzzCutNorman Jul 17, 2023
f36cc56
update cookiecutter tap.py
BuzzCutNorman Jul 17, 2023
bdd3801
Merge branch 'main' of https://github.com/BuzzCutNorman/sdk into 1733…
BuzzCutNorman Jul 19, 2023
4858a09
drop default_connector_class
BuzzCutNorman Jul 20, 2023
c011d47
revert cookiecutter changes
BuzzCutNorman Jul 20, 2023
eb350fe
removed _tap_connector type hint
BuzzCutNorman Jul 20, 2023
15fd7ad
add _tap_connector type hint
BuzzCutNorman Jul 20, 2023
e61c8bd
Merge branch 'main' into 1733-tap-level-connector-instance-shared-wit…
BuzzCutNorman Jul 20, 2023
5ed7e53
Merge branch 'main' into 1733-tap-level-connector-instance-shared-wit…
edgarrmondragon Jul 21, 2023
0e72b2b
Merge branch 'main' into 1733-tap-level-connector-instance-shared-wit…
BuzzCutNorman Jul 28, 2023
f926397
Merge branch 'main' into 1733-tap-level-connector-instance-shared-wit…
BuzzCutNorman Aug 4, 2023
40d8dbb
Merge branch 'main' into 1733-tap-level-connector-instance-shared-wit…
BuzzCutNorman Aug 7, 2023
c940836
Merge branch 'main' into 1733-tap-level-connector-instance-shared-wit…
BuzzCutNorman Aug 8, 2023
582eb29
Merge branch 'main' into 1733-tap-level-connector-instance-shared-wit…
BuzzCutNorman Aug 15, 2023
8fe9eb8
Merge branch 'main' into 1733-tap-level-connector-instance-shared-wit…
edgarrmondragon Aug 16, 2023
ace7142
Merge branch 'main' into 1733-tap-level-connector-instance-shared-wit…
edgarrmondragon Aug 17, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions singer_sdk/tap_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
if t.TYPE_CHECKING:
from pathlib import PurePath

from singer_sdk.connectors import SQLConnector
from singer_sdk.mapper import PluginMapper
from singer_sdk.streams import SQLStream, Stream

Expand Down Expand Up @@ -612,6 +613,8 @@ class SQLTap(Tap):
# Stream class used to initialize new SQL streams from their catalog declarations.
default_stream_class: type[SQLStream]

_tap_connector: SQLConnector | None = None

def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
"""Initialize the SQL tap.

Expand All @@ -624,6 +627,19 @@ def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
self._catalog_dict: dict | None = None
super().__init__(*args, **kwargs)

@property
def tap_connector(self) -> SQLConnector:
"""The connector object.

Returns:
The connector object.
"""
if self._tap_connector is None:
self._tap_connector = self.default_stream_class.connector_class(
dict(self.config),
)
return self._tap_connector

@property
def catalog_dict(self) -> dict:
"""Get catalog dictionary.
Expand All @@ -637,7 +653,7 @@ def catalog_dict(self) -> dict:
if self.input_catalog:
return self.input_catalog.to_dict()

connector = self.default_stream_class.connector_class(dict(self.config))
connector = self.tap_connector

result: dict[str, list[dict]] = {"streams": []}
result["streams"].extend(connector.discover_catalog_entries())
Expand All @@ -653,6 +669,12 @@ def discover_streams(self) -> list[Stream]:
"""
result: list[Stream] = []
for catalog_entry in self.catalog_dict["streams"]:
result.append(self.default_stream_class(self, catalog_entry))
result.append(
self.default_stream_class(
tap=self,
catalog_entry=catalog_entry,
connector=self.tap_connector,
),
)

return result