Merge branch 'main' into standardize_load_methods

.github/ISSUE_TEMPLATE/bug.yml

@@ -15,7 +15,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.30.0"
+      placeholder: "0.31.0"
     validations:
       required: true
   - type: checkboxes

CHANGELOG.md

@@ -5,6 +5,36 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v0.31.0 (2023-08-07)
+
+### ✨ New
+
+- [#1892](https://github.com/meltano/sdk/issues/1892) Add a mapper cookiecutter template
+- [#1864](https://github.com/meltano/sdk/issues/1864) SQLTarget connector instance shared with sinks -- _**Thanks @BuzzCutNorman!**_
+- [#1878](https://github.com/meltano/sdk/issues/1878) Add `_sdc_sync_started_at` metadata column to indicate the start of the target process
+- [#1484](https://github.com/meltano/sdk/issues/1484) Bump latest supported sqlalchemy from `1.*` to `2.*`
+
+### 🐛 Fixes
+
+- [#1898](https://github.com/meltano/sdk/issues/1898) Correctly serialize `decimal.Decimal` in JSON fields of SQL targets
+- [#1881](https://github.com/meltano/sdk/issues/1881) Expose `add_record_metadata` as a builtin target setting
+- [#1880](https://github.com/meltano/sdk/issues/1880) Append batch config if target supports the batch capability
+- [#1865](https://github.com/meltano/sdk/issues/1865) Handle missing record properties in SQL sinks
+- [#1838](https://github.com/meltano/sdk/issues/1838) Add deprecation warning when importing legacy testing helpers
+- [#1842](https://github.com/meltano/sdk/issues/1842) Ensure all expected tap parameters are passed to `SQLTap` initializer
+- [#1853](https://github.com/meltano/sdk/issues/1853) Check against the unconformed key properties when validating record keys
+- [#1843](https://github.com/meltano/sdk/issues/1843) Target template should not reference `tap_id`
+- [#1708](https://github.com/meltano/sdk/issues/1708) Finalize and write last state message with dedupe
+- [#1835](https://github.com/meltano/sdk/issues/1835) Avoid setting up mapper in discovery mode
+
+### ⚙️ Under the Hood
+
+- [#1877](https://github.com/meltano/sdk/issues/1877) Use `importlib.resources` instead of `__file__` to retrieve sample Singer output files
+
+### 📚 Documentation Improvements
+
+- [#1852](https://github.com/meltano/sdk/issues/1852) Fix stale `pip_url` example that uses shell script workaround for editable installation
+
 ## v0.30.0 (2023-07-10)
 
 ### ✨ New

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml

@@ -22,7 +22,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "<3.12,>=3.7.1"
-singer-sdk = { version="^0.30.0" }
+singer-sdk = { version="^0.31.0" }
 fs-s3fs = { version = "^1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]

cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml

@@ -21,7 +21,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "<3.12,>=3.7.1"
-singer-sdk = { version="^0.30.0" }
+singer-sdk = { version="^0.31.0" }
 fs-s3fs = { version = "^1.1.1", optional = true }
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
 requests = "^2.31.0"
@@ -32,7 +32,7 @@ cached-property = "^1" # Remove after Python 3.7 support is dropped
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.0"
-singer-sdk = { version="^0.30.0", extras = ["testing"] }
+singer-sdk = { version="^0.31.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml

@@ -21,15 +21,15 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "<3.12,>=3.7.1"
-singer-sdk = { version="^0.30.0" }
+singer-sdk = { version="^0.31.0" }
 fs-s3fs = { version = "^1.1.1", optional = true }
 {%- if cookiecutter.serialization_method != "SQL" %}
 requests = "^2.31.0"
 {%- endif %}
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.4.0"
-singer-sdk = { version="^0.30.0", extras = ["testing"] }
+singer-sdk = { version="^0.31.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]

docs/conf.py

@@ -26,7 +26,7 @@
 author = "Meltano Core Team and Contributors"
 
 # The full version, including alpha/beta/rc tags
-release = "0.30.0"
+release = "0.31.0"
 
 
 # -- General configuration ---------------------------------------------------

pyproject.toml

@@ -1,9 +1,9 @@
 [tool.poetry]
 name = "singer-sdk"
-version = "0.30.0"
+version = "0.31.0"
 description = "A framework for building Singer taps"
-authors = ["Meltano Team and Contributors"]
-maintainers = ["Meltano Team and Contributors"]
+authors = ["Meltano Team and Contributors <[email protected]>"]
+maintainers = ["Meltano Team and Contributors <[email protected]>"]
 readme = "README.md"
 homepage = "https://sdk.meltano.com/en/latest/"
 repository = "https://github.com/meltano/sdk"
@@ -141,13 +141,14 @@ norecursedirs = "cookiecutter"
 
 [tool.commitizen]
 name = "cz_version_bump"
-version = "0.30.0"
+version = "0.31.0"
 tag_format = "v$major.$minor.$patch$prerelease"
 version_files = [
-    "docs/conf.py",
+    "docs/conf.py:^release =",
     "pyproject.toml:^version =",
     "cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml:singer-sdk",
     "cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml:singer-sdk",
+    "cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml:singer-sdk",
     ".github/ISSUE_TEMPLATE/bug.yml:^      placeholder:",
 ]
 

singer_sdk/connectors/sql.py

@@ -2,13 +2,16 @@
 
 from __future__ import annotations
 
+import decimal
+import json
 import logging
 import typing as t
 import warnings
 from contextlib import contextmanager
 from datetime import datetime
 from functools import lru_cache
 
+import simplejson
 import sqlalchemy
 from sqlalchemy.engine import Engine
 from sqlalchemy.sql import text
@@ -319,7 +322,12 @@ def create_engine(self) -> Engine:
         Returns:
             A new SQLAlchemy Engine.
         """
-        return sqlalchemy.create_engine(self.sqlalchemy_url, echo=False)
+        return sqlalchemy.create_engine(
+            self.sqlalchemy_url,
+            echo=False,
+            json_serializer=self.serialize_json,
+            json_deserializer=self.deserialize_json,
+        )
 
     def quote(self, name: str) -> str:
         """Quote a name if it needs quoting, using '.' as a name-part delimiter.
@@ -1168,3 +1176,35 @@ def _adapt_column_type(
         )
         with self._connect() as conn:
             conn.execute(alter_column_ddl)
+
+    def serialize_json(self, obj: object) -> str:
+        """Serialize an object to a JSON string.
+
+        Target connectors may override this method to provide custom serialization logic
+        for JSON types.
+
+        Args:
+            obj: The object to serialize.
+
+        Returns:
+            The JSON string.
+
+        .. versionadded:: 0.31.0
+        """
+        return simplejson.dumps(obj, use_decimal=True)
+
+    def deserialize_json(self, json_str: str) -> object:
+        """Deserialize a JSON string to an object.
+
+        Tap connectors may override this method to provide custom deserialization
+        logic for JSON types.
+
+        Args:
+            json_str: The JSON string to deserialize.
+
+        Returns:
+            The deserialized object.
+
+        .. versionadded:: 0.31.0
+        """
+        return json.loads(json_str, parse_float=decimal.Decimal)

tests/core/test_connector_sql.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from decimal import Decimal
 from unittest import mock
 
 import pytest
@@ -258,3 +259,27 @@ def test_merge_generic_sql_types(
     ):
         merged_type = connector.merge_sql_types(types)
         assert isinstance(merged_type, expected_type)
+
+    def test_engine_json_serialization(self, connector: SQLConnector):
+        engine = connector._engine
+        meta = sqlalchemy.MetaData()
+        table = sqlalchemy.Table(
+            "test_table",
+            meta,
+            sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True),
+            sqlalchemy.Column("attrs", sqlalchemy.JSON),
+        )
+        meta.create_all(engine)
+        with engine.connect() as conn:
+            conn.execute(
+                table.insert(),
+                [
+                    {"attrs": {"x": Decimal("1.0")}},
+                    {"attrs": {"x": Decimal("2.0"), "y": [1, 2, 3]}},
+                ],
+            )
+            result = conn.execute(table.select())
+            assert result.fetchall() == [
+                (1, {"x": Decimal("1.0")}),
+                (2, {"x": Decimal("2.0"), "y": [1, 2, 3]}),
+            ]