feat: Support other content-types in REST streams (#2762)

* WIP: Support other content-types for HTTP streams

* Add test

* Fix types

* Specifically test legacy pagination

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/graphql-client.py

@@ -45,14 +45,11 @@ def http_headers(self) -> dict:
         Returns:
             A dictionary of HTTP headers.
         """
-        headers = {}
-        if "user_agent" in self.config:
-            headers["User-Agent"] = self.config.get("user_agent")
 {%- if cookiecutter.auth_method not in ("OAuth2", "JWT") %}
         # If not using an authenticator, you may also provide inline auth headers:
         # headers["Private-Token"] = self.config.get("auth_token")
 {%- endif %}
-        return headers
+        return {}
 
     def parse_response(self, response: requests.Response) -> t.Iterable[dict]:
         """Parse the response and return an iterator of result records.

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/rest-client.py

@@ -132,14 +132,11 @@ def http_headers(self) -> dict:
         Returns:
             A dictionary of HTTP headers.
         """
-        headers = {}
-        if "user_agent" in self.config:
-            headers["User-Agent"] = self.config.get("user_agent")
 {%- if cookiecutter.auth_method not in ("OAuth2", "JWT") %}
         # If not using an authenticator, you may also provide inline auth headers:
         # headers["Private-Token"] = self.config.get("auth_token")  # noqa: ERA001
 {%- endif %}
-        return headers
+        return {}
 
     def get_new_paginator(self) -> BaseAPIPaginator:
         """Create a new pagination helper instance.

docs/_templates/stream_class.rst

@@ -0,0 +1,10 @@
+{{ fullname }}
+{{ "=" * fullname|length }}
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ name }}
+    :members:
+    :show-inheritance:
+    :inherited-members: Stream
+    :special-members: __init__

docs/classes/singer_sdk.GraphQLStream.rst

@@ -5,4 +5,6 @@
 
 .. autoclass:: GraphQLStream
     :members:
-    :special-members: __init__, __call__
+    :show-inheritance:
+    :inherited-members: Stream
+    :special-members: __init__

docs/classes/singer_sdk.RESTStream.rst

@@ -5,4 +5,6 @@
 
 .. autoclass:: RESTStream
     :members:
-    :special-members: __init__, __call__
+    :show-inheritance:
+    :inherited-members: Stream
+    :special-members: __init__

docs/classes/singer_sdk.SQLStream.rst

@@ -5,4 +5,6 @@
 
 .. autoclass:: SQLStream
     :members:
-    :special-members: __init__, __call__
+    :show-inheritance:
+    :inherited-members: Stream
+    :special-members: __init__

docs/classes/singer_sdk.Stream.rst

@@ -5,4 +5,6 @@
 
 .. autoclass:: Stream
     :members:
-    :special-members: __init__, __call__
+    :show-inheritance:
+    :inherited-members: Stream
+    :special-members: __init__

docs/reference.rst

@@ -21,7 +21,7 @@ Stream Classes
 
 .. autosummary::
     :toctree: classes
-    :template: class.rst
+    :template: stream_class.rst
 
     Stream
     RESTStream

pyproject.toml

@@ -170,7 +170,7 @@ filterwarnings = [
     # https://github.com/meltano/sdk/issues/1354
     "ignore:The function singer_sdk.testing.get_standard_tap_tests is deprecated:DeprecationWarning",
     # https://github.com/meltano/sdk/issues/2744
-    "ignore::singer_sdk.helpers._compat.SingerSDKDeprecationWarning",
+    "default::singer_sdk.helpers._compat.SingerSDKDeprecationWarning",
     # TODO: Address this SQLite warning in Python 3.13+
     "ignore::ResourceWarning",
 ]

singer_sdk/authenticators.py

@@ -22,7 +22,7 @@
 if t.TYPE_CHECKING:
     import logging
 
-    from singer_sdk.streams.rest import RESTStream
+    from singer_sdk.streams.rest import _HTTPStream
 
 
 def _add_parameters(initial_url: str, extra_parameters: dict) -> str:
@@ -91,7 +91,7 @@ class APIAuthenticatorBase:
         auth_params: URL query parameters for authentication.
     """
 
-    def __init__(self, stream: RESTStream) -> None:
+    def __init__(self, stream: _HTTPStream) -> None:
         """Init authenticator.
 
         Args:
@@ -156,7 +156,7 @@ class SimpleAuthenticator(APIAuthenticatorBase):
 
     def __init__(
         self,
-        stream: RESTStream,
+        stream: _HTTPStream,
         auth_headers: dict | None = None,
     ) -> None:
         """Create a new authenticator.
@@ -186,7 +186,7 @@ class APIKeyAuthenticator(APIAuthenticatorBase):
 
     def __init__(
         self,
-        stream: RESTStream,
+        stream: _HTTPStream,
         key: str,
         value: str,
         location: str = "header",
@@ -221,7 +221,7 @@ def __init__(
     @classmethod
     def create_for_stream(
         cls: type[APIKeyAuthenticator],
-        stream: RESTStream,
+        stream: _HTTPStream,
         key: str,
         value: str,
         location: str,
@@ -249,7 +249,7 @@ class BearerTokenAuthenticator(APIAuthenticatorBase):
     'Bearer '. The token will be merged with HTTP headers on the stream.
     """
 
-    def __init__(self, stream: RESTStream, token: str) -> None:
+    def __init__(self, stream: _HTTPStream, token: str) -> None:
         """Create a new authenticator.
 
         Args:
@@ -266,7 +266,7 @@ def __init__(self, stream: RESTStream, token: str) -> None:
     @classmethod
     def create_for_stream(
         cls: type[BearerTokenAuthenticator],
-        stream: RESTStream,
+        stream: _HTTPStream,
         token: str,
     ) -> BearerTokenAuthenticator:
         """Create an Authenticator object specific to the Stream class.
@@ -299,7 +299,7 @@ class BasicAuthenticator(APIAuthenticatorBase):
 
     def __init__(
         self,
-        stream: RESTStream,
+        stream: _HTTPStream,
         username: str,
         password: str,
     ) -> None:
@@ -323,7 +323,7 @@ def __init__(
     @classmethod
     def create_for_stream(
         cls: type[BasicAuthenticator],
-        stream: RESTStream,
+        stream: _HTTPStream,
         username: str,
         password: str,
     ) -> BasicAuthenticator:
@@ -346,7 +346,7 @@ class OAuthAuthenticator(APIAuthenticatorBase):
 
     def __init__(
         self,
-        stream: RESTStream,
+        stream: _HTTPStream,
         auth_endpoint: str | None = None,
         oauth_scopes: str | None = None,
         default_expiration: int | None = None,