From bdf7f83372622d8f731692e83e6203738ad730cf Mon Sep 17 00:00:00 2001 From: "TAHRI Ahmed R." Date: Fri, 25 Oct 2024 10:03:16 +0200 Subject: [PATCH] Fix in session defaults for verify, and stream (#167) 3.10.2 (2024-10-25) ------------------ **Fixed** - Ensure `stream`, and `verify` both defaults to your ``Session`` parameters. (#165) --- .github/workflows/run-tests.yml | 7 ++++ HISTORY.md | 6 +++ docs/dev/migrate.rst | 8 +++- docs/user/advanced.rst | 23 ++++++++++++ docs/user/quickstart.rst | 26 ------------- src/niquests/__version__.py | 4 +- src/niquests/_async.py | 66 ++++++++++++++++----------------- src/niquests/sessions.py | 44 ++++++++++++++-------- tests/test_testserver.py | 26 +++++++++++++ 9 files changed, 133 insertions(+), 77 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 8637203783..f1280b4cae 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -30,6 +30,13 @@ jobs: os: ubuntu-22.04 - python-version: pypy-3.8 os: macOS-13 + exclude: + # pypy 3.9 and 3.10 suffers from a wierd bug, probably due to gc + # this bug prevent us from running the suite on Windows. + - python-version: pypy-3.9 + os: windows-latest + - python-version: pypy-3.10 + os: windows-latest steps: - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 diff --git a/HISTORY.md b/HISTORY.md index e1d5e75d17..c82ecb8ce8 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,12 @@ Release History =============== +3.10.2 (2024-10-25) +------------------ + +**Fixed** +- Ensure `stream`, and `verify` both defaults to your ``Session`` parameters. + 3.10.1 (2024-10-22) ------------------ diff --git a/docs/dev/migrate.rst b/docs/dev/migrate.rst index 54c570bfb3..511e821c72 100644 --- a/docs/dev/migrate.rst +++ b/docs/dev/migrate.rst @@ -45,7 +45,7 @@ The library itself (sources) should be really easy to migrate (cf. developer mig but the tests may be harder to adapt. The main reason behind this difficulty is often related to a strong tie with third-party -mocking library such as ``response``. +mocking library such as ``responses``. To overcome this, we will introduce you to a clever bypass. If you are using pytest, do the following in your ``conftest.py``, see https://docs.pytest.org/en/6.2.x/fixture.html#conftest-py-sharing-fixtures-across-multiple-files @@ -65,3 +65,9 @@ for more information. (The goal would simply to execute the following piece of c modules["requests.packages.urllib3"] = urllib3 .. warning:: This code sample is only to be executed in a development environment, it permit to fool the third-party dependencies that have a strong tie on Requests. + +.. warning:: Some pytest plugins may load/import Requests at startup. + Disable the plugin auto-loading first by either passing ``PYTEST_DISABLE_PLUGIN_AUTOLOAD=1`` (in environment) + or ``pytest -p "no:pytest-betamax"`` in CLI parameters. Replace ``pytest-betamax`` by the name of the target plugin. + To find out the name of the plugin auto-loaded, execute ``pytest --trace-config`` as the name aren't usually what + you would expect them to be. diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 89f4f0b6f9..96b85fcc5f 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -379,6 +379,29 @@ Note that connections are only released back to the pool for reuse once all body data has been read; be sure to either set ``stream`` to ``False`` or read the ``content`` property of the ``Response`` object. +.. note:: Available since Niquests v3.10 and before this only HTTP/1.1 were kept alive properly. + +Niquests can automatically make sure that your HTTP connection is kept alive +no matter the used protocol using a discrete scheduled task for each host. + +.. code-block:: python + + import niquests + + sess = niquests.Session(keepalive_delay=300, keepalive_idle_window=60) # already the defaults!, you don't need to specify anything + +In that example, we indicate that we wish to keep a connection alive for 5 minutes and +eventually send ping every 60s after the connection was idle. (Those values are the default ones!) + +The pings are only sent when using HTTP/2 or HTTP/3 over QUIC. Any connection activity is considered as used, therefor +making the ping only 60s after zero activity. If the connection receive unsolicited data, it is also considered used. + +.. note:: Setting either keepalive_delay or keepalive_idle_window to None disable this feature. + +.. warning:: We do not recommend setting anything lower than 30s for keepalive_idle_window. Anything lower than 1s is considered to be 1s. High frequency ping will lower the performance of your connection pool. And probably end up by getting kicked out by the server. + +Once the ``keepalive_delay`` passed, we do not close the connection, we simply cease to ensure it is alive. This is purely for backward compatibility with our predecessor, as some host may retain the connection for hours. + .. _streaming-uploads: Streaming Uploads diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 66cd37842a..8b0690d5b1 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -1249,32 +1249,6 @@ See:: .. note:: The given example are really basic ones. You may adjust at will the settings and algorithm to match your requisites. -Keep-Alive ----------- - -.. note:: Available since Niquests v3.10 and before this only HTTP/1.1 were kept alive properly. - -Niquests can automatically make sure that your HTTP connection is kept alive -no matter the used protocol using a discrete scheduled task for each host. - -.. code-block:: python - - import niquests - - sess = niquests.Session(keepalive_delay=300, keepalive_idle_window=60) # already the defaults!, you don't need to specify anything - -In that example, we indicate that we wish to keep a connection alive for 5 minutes and -eventually send ping every 60s after the connection was idle. (Those values are the default ones!) - -The pings are only sent when using HTTP/2 or HTTP/3 over QUIC. Any connection activity is considered as used, therefor -making the ping only 60s after zero activity. If the connection receive unsolicited data, it is also considered used. - -.. note:: Setting either keepalive_delay or keepalive_idle_window to None disable this feature. - -.. warning:: We do not recommend setting anything lower than 30s for keepalive_idle_window. Anything lower than 1s is considered to be 1s. High frequency ping will lower the performance of your connection pool. And probably end up by getting kicked out by the server. - -Once the ``keepalive_delay`` passed, we do not close the connection, we simply cease to ensure it is alive. This is purely for backward compatibility with our predecessor, as some host may retain the connection for hours. - ----------------------- Ready for more? Check out the :ref:`advanced ` section. diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py index 99d104e593..7d87f81569 100644 --- a/src/niquests/__version__.py +++ b/src/niquests/__version__.py @@ -9,9 +9,9 @@ __url__: str = "https://niquests.readthedocs.io" __version__: str -__version__ = "3.10.1" +__version__ = "3.10.2" -__build__: int = 0x031001 +__build__: int = 0x031002 __author__: str = "Kenneth Reitz" __author_email__: str = "me@kennethreitz.org" __license__: str = "Apache-2.0" diff --git a/src/niquests/_async.py b/src/niquests/_async.py index 4af57cf567..61f47747db 100644 --- a/src/niquests/_async.py +++ b/src/niquests/_async.py @@ -862,7 +862,7 @@ async def get( proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., verify: TLSVerifyType = ..., - stream: Literal[False] = ..., + stream: Literal[False] | Literal[None] = ..., cert: TLSClientCertType | None = ..., **kwargs: typing.Any, ) -> Response: ... @@ -898,8 +898,8 @@ async def get( # type: ignore[override] allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, **kwargs: typing.Any, ) -> Response | AsyncResponse: @@ -933,8 +933,8 @@ async def options( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., - stream: Literal[False] = ..., + verify: TLSVerifyType | None = ..., + stream: Literal[False] | Literal[None] = ..., cert: TLSClientCertType | None = ..., **kwargs: typing.Any, ) -> Response: ... @@ -952,7 +952,7 @@ async def options( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., + verify: TLSVerifyType | None = ..., stream: Literal[True], cert: TLSClientCertType | None = ..., **kwargs: typing.Any, @@ -970,8 +970,8 @@ async def options( # type: ignore[override] allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, **kwargs: typing.Any, ) -> Response | AsyncResponse: @@ -1005,8 +1005,8 @@ async def head( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., - stream: Literal[False] = ..., + verify: TLSVerifyType | None = ..., + stream: Literal[False] | Literal[None] = ..., cert: TLSClientCertType | None = ..., **kwargs: typing.Any, ) -> Response: ... @@ -1024,7 +1024,7 @@ async def head( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., + verify: TLSVerifyType | None = ..., stream: Literal[True], cert: TLSClientCertType | None = ..., **kwargs: typing.Any, @@ -1042,8 +1042,8 @@ async def head( # type: ignore[override] allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, **kwargs: typing.Any, ) -> Response | AsyncResponse: @@ -1080,8 +1080,8 @@ async def post( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., - stream: Literal[False] = ..., + verify: TLSVerifyType | None = ..., + stream: Literal[False] | Literal[None] = ..., cert: TLSClientCertType | None = ..., ) -> Response: ... @@ -1101,7 +1101,7 @@ async def post( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., + verify: TLSVerifyType | None = ..., stream: Literal[True], cert: TLSClientCertType | None = ..., ) -> AsyncResponse: ... @@ -1121,8 +1121,8 @@ async def post( # type: ignore[override] allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, ) -> Response | AsyncResponse: return await self.request( # type: ignore[call-overload,misc] @@ -1160,8 +1160,8 @@ async def put( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., - stream: Literal[False] = ..., + verify: TLSVerifyType | None = ..., + stream: Literal[False] | Literal[None] = ..., cert: TLSClientCertType | None = ..., ) -> Response: ... @@ -1181,7 +1181,7 @@ async def put( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., + verify: TLSVerifyType | None = ..., stream: Literal[True], cert: TLSClientCertType | None = ..., ) -> AsyncResponse: ... @@ -1201,8 +1201,8 @@ async def put( # type: ignore[override] allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, ) -> Response | AsyncResponse: return await self.request( # type: ignore[call-overload,misc] @@ -1240,8 +1240,8 @@ async def patch( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., - stream: Literal[False] = ..., + verify: TLSVerifyType | None = ..., + stream: Literal[False] | Literal[None] = ..., cert: TLSClientCertType | None = ..., ) -> Response: ... @@ -1261,7 +1261,7 @@ async def patch( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., + verify: TLSVerifyType | None = ..., stream: Literal[True], cert: TLSClientCertType | None = ..., ) -> AsyncResponse: ... @@ -1281,8 +1281,8 @@ async def patch( # type: ignore[override] allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, ) -> Response | AsyncResponse: return await self.request( # type: ignore[call-overload,misc] @@ -1317,8 +1317,8 @@ async def delete( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., - stream: Literal[False] = ..., + verify: TLSVerifyType | None = ..., + stream: Literal[False] | Literal[None] = ..., cert: TLSClientCertType | None = ..., **kwargs: typing.Any, ) -> Response: ... @@ -1336,7 +1336,7 @@ async def delete( allow_redirects: bool = ..., proxies: ProxyType | None = ..., hooks: HookType[PreparedRequest | Response] | None = ..., - verify: TLSVerifyType = ..., + verify: TLSVerifyType | None = ..., stream: Literal[True], cert: TLSClientCertType | None = ..., **kwargs: typing.Any, @@ -1354,8 +1354,8 @@ async def delete( # type: ignore[override] allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, **kwargs: typing.Any, ) -> Response | AsyncResponse: diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py index 0186e0df18..56d25cb66d 100644 --- a/src/niquests/sessions.py +++ b/src/niquests/sessions.py @@ -259,12 +259,21 @@ def __init__( :param quic_cache_layer: Provide an external cache mechanism to store HTTP/3 host capabilities. :param retries: Configure a number of times a request must be automatically retried before giving up. :param multiplexed: Enable or disable concurrent request when the remote host support HTTP/2 onward. + :param disable_http1: Toggle to disable negotiating HTTP/1 with remote peers. Set it to True so that + you may be able to force HTTP/2 over cleartext (h2c). :param disable_http2: Toggle to disable negotiating HTTP/2 with remote peers. :param disable_http3: Toggle to disable negotiating HTTP/3 with remote peers. :param disable_ipv6: Toggle to disable using IPv6 even if the remote host supports IPv6. :param disable_ipv4: Toggle to disable using IPv4 even if the remote host supports IPv4. - :param pool_connections: Number of concurrent hosts to be handled by this Session at a maximum. + :param pool_connections: Number of concurrent hosts to be kept alive by this Session at a maximum. :param pool_maxsize: Maximum number of concurrent connections per (single) host at a time. + :param happy_eyeballs: Use IETF Happy Eyeballs algorithm when trying to connect to a remote host by issuing + concurrent connection using available IPs. Tries IPv6/IPv4 at the same time or multiple IPv6 / IPv4. + The domain name must yield multiple A or AAAA records for this to be used. + :param keepalive_delay: Delay expressed in seconds, in which we should keep a connection alive by sending PING + frame. This only applies to HTTP/2 onward. + :param keepalive_idle_window: Delay expressed in seconds, in which we should send a PING frame after the connection + being completely idle. This only applies to HTTP/2 onward. """ if [disable_ipv4, disable_ipv6].count(True) == 2: raise RuntimeError("Cannot disable both IPv4 and IPv6") @@ -399,6 +408,7 @@ def __init__( source_address=source_address, disable_http1=disable_http1, disable_http2=disable_http2, + disable_http3=disable_http3, disable_ipv4=disable_ipv4, disable_ipv6=disable_ipv6, pool_connections=pool_connections, @@ -576,8 +586,8 @@ def get( allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, **kwargs: typing.Any, ) -> Response: @@ -644,8 +654,8 @@ def options( allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, **kwargs: typing.Any, ) -> Response: @@ -712,8 +722,8 @@ def head( allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, **kwargs: typing.Any, ) -> Response: @@ -783,8 +793,8 @@ def post( allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, ) -> Response: r"""Sends a POST request. Returns :class:`Response` object. @@ -861,8 +871,8 @@ def put( allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, ) -> Response: r"""Sends a PUT request. Returns :class:`Response` object. @@ -939,8 +949,8 @@ def patch( allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, ) -> Response: r"""Sends a PATCH request. Returns :class:`Response` object. @@ -1014,8 +1024,8 @@ def delete( allow_redirects: bool = True, proxies: ProxyType | None = None, hooks: HookType[PreparedRequest | Response] | None = None, - verify: TLSVerifyType = True, - stream: bool = False, + verify: TLSVerifyType | None = None, + stream: bool | None = None, cert: TLSClientCertType | None = None, **kwargs: typing.Any, ) -> Response: @@ -1447,6 +1457,7 @@ def __setstate__(self, state): HTTPAdapter( quic_cache_layer=self.quic_cache_layer, max_retries=self.retries, + disable_http1=self._disable_http1, disable_http2=self._disable_http2, disable_http3=self._disable_http3, source_address=self.source_address, @@ -1464,6 +1475,9 @@ def __setstate__(self, state): "http://", HTTPAdapter( max_retries=self.retries, + disable_http1=self._disable_http1, + disable_http2=self._disable_http2, + disable_http3=self._disable_http3, source_address=self.source_address, disable_ipv4=self._disable_ipv4, disable_ipv6=self._disable_ipv6, diff --git a/tests/test_testserver.py b/tests/test_testserver.py index d2266bf027..b81d825999 100644 --- a/tests/test_testserver.py +++ b/tests/test_testserver.py @@ -54,6 +54,32 @@ def test_text_response(self): assert r.text == "roflol" assert r.headers["Content-Length"] == "6" + def test_early_response_caught(self): + server = Server.text_response_server( + "HTTP/1.1 100 CONTINUE\r\n\r\nHTTP/1.1 200 OK\r\n" + "Content-Length: 6\r\n" + "\r\nroflol" + ) + + with server as (host, port): + early_r = None + + def catch_early_response(*args): + nonlocal early_r + early_r = args[0] + + r = niquests.get( + f"http://{host}:{port}", + hooks={"early_response": [catch_early_response]}, + ) + + assert early_r is not None + assert early_r.status_code == 100 + + assert r.status_code == 200 + assert r.text == "roflol" + assert r.headers["Content-Length"] == "6" + def test_invalid_location_response(self): server = Server.text_response_server( "HTTP/1.1 302 PERMANENT-REDIRECTION\r\n"