Merge branch 'main' into http-connector

.github/workflows/release.yml

@@ -19,6 +19,7 @@ jobs:
   publish:
     name: Publish to PyPI
     runs-on: ubuntu-latest
+    needs: [build]
     environment:
       name: publishing
       url: https://pypi.org/p/singer-sdk

.gitignore

@@ -1,3 +1,7 @@
+# Local Poetry configuration file
+
+poetry.toml
+
 # CI
 
 _changelog_fragment.md

.pre-commit-config.yaml

@@ -45,7 +45,7 @@ repos:
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.11
+  rev: v0.1.13
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

cookiecutter/mapper-template/cookiecutter.json

@@ -7,13 +7,15 @@
   "variant": "None (Skip)",
   "include_ci_files": ["GitHub", "None (Skip)"],
   "license": ["Apache-2.0"],
+  "ide": ["VSCode", "None"],
   "__prompts__": {
     "name": "The name of the mapper, in CamelCase",
     "admin_name": "Provide your [bold yellow]full name[/]",
     "admin_email": "Provide your [bold yellow]email[/]",
     "mapper_id": "The ID of the tap, in kebab-case",
     "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
     "include_ci_files": "Whether to include CI files for a common CI services",
-    "license": "The license for the project"
+    "license": "The license for the project",
+    "ide": "Add configuration files for your preferred IDE"
   }
 }

cookiecutter/mapper-template/hooks/post_gen_project.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+from pathlib import Path
+import shutil
+
+
+BASE_PATH = Path("{{cookiecutter.library_name}}")
+
+
+if __name__ == "__main__":
+    if "{{ cookiecutter.license }}" != "Apache-2.0":
+        Path("LICENSE").unlink()
+
+    if "{{ cookiecutter.include_ci_files }}" != "GitHub":
+        shutil.rmtree(Path(".github"))
+
+    if "{{ cookiecutter.ide }}" != "VSCode":
+        shutil.rmtree(".vscode")

cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/__main__.py

@@ -0,0 +1,7 @@
+"""{{ cookiecutter.name }} entry point."""
+
+from __future__ import annotations
+
+from {{ cookiecutter.library_name }}.mapper import {{ cookiecutter.name }}Mapper
+
+{{ cookiecutter.name }}Mapper.cli()

cookiecutter/tap-template/cookiecutter.json

@@ -16,6 +16,7 @@
   ],
   "include_ci_files": ["GitHub", "None"],
   "license": ["Apache-2.0", "None"],
+  "ide": ["VSCode", "None"],
   "__prompts__": {
     "source_name": "The name of the source, in CamelCase",
     "admin_name": "Provide your [bold yellow]full name[/]",
@@ -25,6 +26,7 @@
     "stream_type": "The type of stream the source provides",
     "auth_method": "The [bold red]authentication[/] method used by the source, for REST and GraphQL sources",
     "include_ci_files": "Whether to include CI files for a common CI services",
-    "license": "The license for the project"
+    "license": "The license for the project",
+    "ide": "Add configuration files for your preferred IDE"
   }
 }

cookiecutter/tap-template/hooks/post_gen_project.py

@@ -26,3 +26,6 @@
 
     if "{{ cookiecutter.include_ci_files }}" != "GitHub":
         shutil.rmtree(".github")
+
+    if "{{ cookiecutter.ide }}" != "VSCode":
+        shutil.rmtree(".vscode")

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__main__.py

@@ -0,0 +1,7 @@
+"""{{ cookiecutter.source_name }} entry point."""
+
+from __future__ import annotations
+
+from {{ cookiecutter.library_name }}.tap import Tap{{ cookiecutter.source_name }}
+
+Tap{{ cookiecutter.source_name }}.cli()

cookiecutter/target-template/cookiecutter.json

@@ -8,6 +8,7 @@
   "serialization_method": ["Per record", "Per batch", "SQL"],
   "include_ci_files": ["GitHub", "None (Skip)"],
   "license": ["Apache-2.0"],
+  "ide": ["VSCode", "None"],
   "__prompts__": {
     "name": "The name of the mapper, in CamelCase",
     "admin_name": "Provide your [bold yellow]full name[/]",
@@ -16,6 +17,7 @@
     "library_name": "The name of the library, in snake_case. This is how the library will be imported in Python.",
     "serialization_method": "The serialization method to use for loading data",
     "include_ci_files": "Whether to include CI files for a common CI services",
-    "license": "The license for the project"
+    "license": "The license for the project",
+    "ide": "Add configuration files for your preferred IDE"
   }
 }

cookiecutter/target-template/hooks/post_gen_project.py

@@ -12,3 +12,6 @@
 
     if "{{ cookiecutter.include_ci_files }}" != "GitHub":
         shutil.rmtree(Path(".github"))
+
+    if "{{ cookiecutter.ide }}" != "VSCode":
+        shutil.rmtree(".vscode")

cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/__main__.py

@@ -0,0 +1,7 @@
+"""{{ cookiecutter.destination_name }} entry point."""
+
+from __future__ import annotations
+
+from {{ cookiecutter.library_name }}.target import Target{{ cookiecutter.destination_name }}
+
+Target{{ cookiecutter.destination_name }}.cli()

docs/parent_streams.md

@@ -8,18 +8,21 @@ from a parent record each time the child stream is invoked.
 
 1. Set `parent_stream_type` in the child-stream's class to the class of the parent.
 2. Implement one of the below methods to pass context from the parent to the child:
-   1. If using `get_records(context)` you can simply return a tuple instead of a `record`
+   1. If using [`get_records`](singer_sdk.Stream.get_child_context) you can simply return a tuple instead of a `record`
       dictionary. A tuple return value will be interpreted by the SDK as
       `(record: dict, child_context: dict)`.
-   1. Override `get_child_context(record, context: Dict) -> dict` to return a new
+   2. Override [`get_child_context`](singer_sdk.Stream.get_child_context) to return a new
       child context object based on records and any existing context from the parent stream.
+   3. If you need to sync more than one child stream per parent record, you can override
+      [`generate_child_contexts`](singer_sdk.Stream.generate_child_contexts) to yield as many
+      contexts as you need.
 3. If the parent stream's replication key won't get updated when child items are changed,
    indicate this by adding `ignore_parent_replication_key = True` in the child stream
    class declaration.
 4. If the number of _parent_ items is very large (thousands or tens of thousands), you can
-   optionally set `state_partitioning_keys` on the child stream to specify a subset of context keys to use
+   optionally set [`state_partitioning_keys`](singer_sdk.Stream.state_partitioning_keys) on the child stream to specify a subset of context keys to use
    in state bookmarks. (When not set, the number of bookmarks will be equal to the number
-   of parent items.) If you do not wish to store any state bookmarks for the child stream, set `state_partitioning_keys` to `[]`.
+   of parent items.) If you do not wish to store any state bookmarks for the child stream, set[`state_partitioning_keys`](singer_sdk.Stream.state_partitioning_keys) to `[]`.
 
 ## Example parent-child implementation
 

e2e-tests/cookiecutters/mapper-base.json

@@ -8,6 +8,7 @@
       "variant": "None (Skip)",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../mapper-template/",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/tap-graphql-jwt.json

@@ -10,6 +10,7 @@
       "auth_method": "JWT",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/tap-other-custom.json

@@ -10,6 +10,7 @@
       "auth_method": "Custom or N/A",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/tap-rest-api_key-github.json

@@ -10,6 +10,7 @@
       "auth_method": "API Key",
       "include_ci_files": "GitHub",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/tap-rest-basic_auth.json

@@ -10,6 +10,7 @@
       "auth_method": "Basic Auth",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/tap-rest-bearer_token.json

@@ -10,6 +10,7 @@
       "auth_method": "Bearer Token",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/tap-rest-custom.json

@@ -10,6 +10,7 @@
       "auth_method": "Custom or N/A",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/tap-rest-jwt.json

@@ -10,6 +10,7 @@
       "auth_method": "JWT",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/tap-rest-oauth2.json

@@ -10,6 +10,7 @@
       "auth_method": "OAuth2",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/tap-sql-custom.json

@@ -10,6 +10,7 @@
       "auth_method": "Custom or N/A",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "../tap-template/",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/target-per_record.json

@@ -9,6 +9,7 @@
       "serialization_method": "Per record",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "./sdk/cookiecutter/target-template",
       "_output_dir": "."
     }

e2e-tests/cookiecutters/target-sql.json

@@ -9,6 +9,7 @@
       "serialization_method": "SQL",
       "include_ci_files": "None (Skip)",
       "license": "Apache-2.0",
+      "ide": "VSCode",
       "_template": "./sdk/cookiecutter/target-template",
       "_output_dir": "."
     }

noxfile.py

@@ -42,24 +42,19 @@
     "coverage[toml]",
     "duckdb",
     "duckdb-engine",
+    "pyarrow",
     "pytest",
     "pytest-benchmark",
     "pytest-durations",
     "pytest-httpserver",
     "pytest-snapshot",
-    "pyarrow",
+    "pytz",
     "requests-mock",
+    "rfc3339-validator",
     "time-machine",
 ]
 
 
-def _clean_py312_deps(session: Session, dependencies: list[str]) -> None:
-    """Clean dependencies for Python 3.12."""
-    if session.python == "3.12":
-        dependencies.remove("duckdb")
-        dependencies.remove("duckdb-engine")
-
-
 @session(python=main_python_version)
 def mypy(session: Session) -> None:
     """Check types with mypy."""
@@ -85,7 +80,6 @@ def mypy(session: Session) -> None:
 @session(python=python_versions)
 def tests(session: Session) -> None:
     """Execute pytest tests and compute coverage."""
-    _clean_py312_deps(session, test_dependencies)
     session.install(".[s3,parquet]")
     session.install(*test_dependencies)
 
@@ -119,7 +113,6 @@ def tests(session: Session) -> None:
 @session(python=main_python_version)
 def benches(session: Session) -> None:
     """Run benchmarks."""
-    _clean_py312_deps(session, test_dependencies)
     session.install(".[s3]")
     session.install(*test_dependencies)
     sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION")
@@ -142,7 +135,6 @@ def update_snapshots(session: Session) -> None:
     """Update pytest snapshots."""
     args = session.posargs or ["-m", "snapshot"]
 
-    _clean_py312_deps(session, test_dependencies)
     session.install(".")
     session.install(*test_dependencies)
     session.run("pytest", "--snapshot-update", *args)