From 0ab877e61e45627a7ad2e74217c6b2e0a5eefb5a Mon Sep 17 00:00:00 2001 From: TAHRI Ahmed R Date: Sun, 7 Jul 2024 19:56:19 +0100 Subject: [PATCH] :bookmark: Release 2.8.902 (#129) - Added support for async iterable yielding either bytes or str when passing a body into your requests. - Added dummy module (e.g. http2 and emscriptem) like upstream without serving any of them. Those modules won't be served and are empty as we diverged since. - Added a better error message for http3 handshake failure to help out users figuring out what is happening. - Added official support for Python 3.13 --- .github/workflows/ci.yml | 3 +- CHANGES.rst | 8 ++++ dev-requirements.txt | 4 +- docs/async.rst | 5 +++ noxfile.py | 4 +- pyproject.toml | 2 +- src/urllib3/_async/connection.py | 40 +++++++++++++------ src/urllib3/_async/connectionpool.py | 20 ++++++---- src/urllib3/_request_methods.py | 14 +++---- src/urllib3/_typing.py | 5 +++ src/urllib3/_version.py | 2 +- src/urllib3/backend/_async/hface.py | 16 ++++++-- src/urllib3/backend/hface.py | 16 ++++++-- src/urllib3/connection.py | 4 ++ src/urllib3/contrib/emscripten/__init__.py | 24 +++++++++++ src/urllib3/http2.py | 22 ++++++++++ src/urllib3/util/request.py | 27 +++++++++++-- .../asynchronous/test_connectionpool.py | 36 +++++++++++++++++ 18 files changed, 209 insertions(+), 43 deletions(-) create mode 100644 src/urllib3/contrib/emscripten/__init__.py create mode 100644 src/urllib3/http2.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c2b48baf8..e7bb24d226 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] os: - macos-12 - windows-latest @@ -242,6 +242,7 @@ jobs: uses: "actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1" with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: "Install dependencies" run: python -m pip install --upgrade pip setuptools nox diff --git a/CHANGES.rst b/CHANGES.rst index a8d2902cca..1b35da9861 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,11 @@ +2.8.902 (2024-07-07) +==================== + +- Added support for async iterable yielding either bytes or str when passing a body into your requests. +- Added dummy module (e.g. http2 and emscriptem) like upstream without serving any of them. Those modules won't be served and are empty as we diverged since. +- Added a better error message for http3 handshake failure to help out users figuring out what is happening. +- Added official support for Python 3.13 + 2.8.901 (2024-06-27) ==================== diff --git a/dev-requirements.txt b/dev-requirements.txt index 90bd6766e3..3a5278e69a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ -coverage>=7.2.7,<=7.4 +coverage>=7.2.7,<=7.4.1 tornado>=6.2,<=6.4 python-socks==2.4.4 pytest==7.4.4 @@ -11,3 +11,5 @@ cryptography==42.0.5;implementation_name!="pypy" or implementation_version>="7.3 backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==21.9.0 pytest-asyncio>=0.21.1,<=0.23.5.post1 +# unblock cffi for Python 3.13 +cffi==1.17.0rc1; python_version > "3.12" \ No newline at end of file diff --git a/docs/async.rst b/docs/async.rst index 462c26cbbb..a73a2571e0 100644 --- a/docs/async.rst +++ b/docs/async.rst @@ -147,3 +147,8 @@ The following properties and methods are awaitable: In addition to that, ``AsyncHTTPResponse`` ships with an async iterator. +Sending async iterable +---------------------- + +In our asynchronous APIs, you can send async iterable using the ``body=...`` keyword argument. +It is most useful when trying to send files that are IO bound, thus blocking your event loop needlessly. diff --git a/noxfile.py b/noxfile.py index f5903a70af..1bbd6cbf94 100644 --- a/noxfile.py +++ b/noxfile.py @@ -8,7 +8,7 @@ def tests_impl( session: nox.Session, - extras: str = "socks,brotli,zstd", + extras: str = "socks,brotli", byte_string_comparisons: bool = False, ) -> None: # Install deps and the package itself. @@ -44,7 +44,7 @@ def tests_impl( ) -@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy"]) +@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "pypy"]) def test(session: nox.Session) -> None: tests_impl(session) diff --git a/pyproject.toml b/pyproject.toml index c254946ad8..b6cb6ae293 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ authors = [ {name = "Andrey Petrov", email = "andrey.petrov@shazow.net"} ] maintainers = [ - {name = "Ahmed R. TAHRI", email="ahmed.tahri@cloudnursery.dev"}, + {name = "Ahmed R. TAHRI", email="tahri.ahmed@proton.me"}, ] classifiers = [ "Environment :: Web Environment", diff --git a/src/urllib3/_async/connection.py b/src/urllib3/_async/connection.py index 4038dadfec..9f640d9cb0 100644 --- a/src/urllib3/_async/connection.py +++ b/src/urllib3/_async/connection.py @@ -18,6 +18,7 @@ _TYPE_SOCKET_OPTIONS, _TYPE_TIMEOUT_INTERNAL, ProxyConfig, + _TYPE_ASYNC_BODY, ) from ..util._async.traffic_police import AsyncTrafficPolice @@ -324,7 +325,7 @@ async def request( self, method: str, url: str, - body: _TYPE_BODY | None = None, + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = None, headers: typing.Mapping[str, str] | None = None, *, chunked: bool = False, @@ -459,17 +460,32 @@ async def request( try: # If we're given a body we start sending that in chunks. if chunks is not None: - for chunk in chunks: - # Sending empty chunks isn't allowed for TE: chunked - # as it indicates the end of the body. - if not chunk: - continue - if isinstance(chunk, str): - chunk = chunk.encode("utf-8") - await self.send(chunk) - total_sent += len(chunk) - if on_upload_body is not None: - await on_upload_body(total_sent, content_length, False, False) + if hasattr(chunks, "__aiter__"): + async for chunk in chunks: + if not chunk: + continue + if isinstance(chunk, str): + chunk = chunk.encode("utf-8") + await self.send(chunk) + total_sent += len(chunk) + if on_upload_body is not None: + await on_upload_body( + total_sent, content_length, False, False + ) + else: + for chunk in chunks: + # Sending empty chunks isn't allowed for TE: chunked + # as it indicates the end of the body. + if not chunk: + continue + if isinstance(chunk, str): + chunk = chunk.encode("utf-8") + await self.send(chunk) + total_sent += len(chunk) + if on_upload_body is not None: + await on_upload_body( + total_sent, content_length, False, False + ) try: rp = await self.send(b"", eot=True) except TypeError: diff --git a/src/urllib3/_async/connectionpool.py b/src/urllib3/_async/connectionpool.py index 608977baa7..51c8b35740 100644 --- a/src/urllib3/_async/connectionpool.py +++ b/src/urllib3/_async/connectionpool.py @@ -16,7 +16,13 @@ from .._collections import HTTPHeaderDict from .._request_methods import AsyncRequestMethods -from .._typing import _TYPE_BODY, _TYPE_BODY_POSITION, _TYPE_TIMEOUT, ProxyConfig +from .._typing import ( + _TYPE_ASYNC_BODY, + _TYPE_BODY, + _TYPE_BODY_POSITION, + _TYPE_TIMEOUT, + ProxyConfig, +) from ..backend import ConnectionInfo, ResponsePromise from ..connection import _wrap_proxy_error from ..connectionpool import _normalize_host @@ -851,7 +857,7 @@ async def _make_request( conn: AsyncHTTPConnection, method: str, url: str, - body: _TYPE_BODY | None = ..., + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = ..., headers: typing.Mapping[str, str] | None = ..., retries: Retry | None = ..., timeout: _TYPE_TIMEOUT = ..., @@ -876,7 +882,7 @@ async def _make_request( conn: AsyncHTTPConnection, method: str, url: str, - body: _TYPE_BODY | None = ..., + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = ..., headers: typing.Mapping[str, str] | None = ..., retries: Retry | None = ..., timeout: _TYPE_TIMEOUT = ..., @@ -900,7 +906,7 @@ async def _make_request( conn: AsyncHTTPConnection, method: str, url: str, - body: _TYPE_BODY | None = None, + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, @@ -1151,7 +1157,7 @@ async def urlopen( self, method: str, url: str, - body: _TYPE_BODY | None = ..., + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = ..., headers: typing.Mapping[str, str] | None = ..., retries: Retry | bool | int | None = ..., redirect: bool = ..., @@ -1179,7 +1185,7 @@ async def urlopen( self, method: str, url: str, - body: _TYPE_BODY | None = ..., + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = ..., headers: typing.Mapping[str, str] | None = ..., retries: Retry | bool | int | None = ..., redirect: bool = ..., @@ -1206,7 +1212,7 @@ async def urlopen( self, method: str, url: str, - body: _TYPE_BODY | None = None, + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, diff --git a/src/urllib3/_request_methods.py b/src/urllib3/_request_methods.py index e60edc7527..38bb5b8699 100644 --- a/src/urllib3/_request_methods.py +++ b/src/urllib3/_request_methods.py @@ -6,7 +6,7 @@ from ._async.response import AsyncHTTPResponse from ._collections import HTTPHeaderDict -from ._typing import _TYPE_BODY, _TYPE_ENCODE_URL_FIELDS, _TYPE_FIELDS +from ._typing import _TYPE_ASYNC_BODY, _TYPE_BODY, _TYPE_ENCODE_URL_FIELDS, _TYPE_FIELDS from .filepost import encode_multipart_formdata from .response import HTTPResponse @@ -377,7 +377,7 @@ async def urlopen( self, method: str, url: str, - body: _TYPE_BODY | None = None, + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = None, headers: typing.Mapping[str, str] | None = None, encode_multipart: bool = True, multipart_boundary: str | None = None, @@ -392,7 +392,7 @@ async def urlopen( self, method: str, url: str, - body: _TYPE_BODY | None = None, + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = None, headers: typing.Mapping[str, str] | None = None, encode_multipart: bool = True, multipart_boundary: str | None = None, @@ -406,7 +406,7 @@ async def urlopen( self, method: str, url: str, - body: _TYPE_BODY | None = None, + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = None, headers: typing.Mapping[str, str] | None = None, encode_multipart: bool = True, multipart_boundary: str | None = None, @@ -422,7 +422,7 @@ async def request( self, method: str, url: str, - body: _TYPE_BODY | None = ..., + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = ..., fields: _TYPE_FIELDS | None = ..., headers: typing.Mapping[str, str] | None = ..., json: typing.Any | None = ..., @@ -437,7 +437,7 @@ async def request( self, method: str, url: str, - body: _TYPE_BODY | None = ..., + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = ..., fields: _TYPE_FIELDS | None = ..., headers: typing.Mapping[str, str] | None = ..., json: typing.Any | None = ..., @@ -451,7 +451,7 @@ async def request( self, method: str, url: str, - body: _TYPE_BODY | None = None, + body: _TYPE_BODY | _TYPE_ASYNC_BODY | None = None, fields: _TYPE_FIELDS | None = None, headers: typing.Mapping[str, str] | None = None, json: typing.Any | None = None, diff --git a/src/urllib3/_typing.py b/src/urllib3/_typing.py index 91b04be461..ff9bb848ee 100644 --- a/src/urllib3/_typing.py +++ b/src/urllib3/_typing.py @@ -30,6 +30,11 @@ class _TYPE_PEER_CERT_RET_DICT(TypedDict, total=False): AsyncLowLevelResponse, ] +_TYPE_ASYNC_BODY: typing.TypeAlias = typing.Union[ + typing.AsyncIterable[bytes], + typing.AsyncIterable[str], +] + _TYPE_FIELD_VALUE: typing.TypeAlias = typing.Union[str, bytes] _TYPE_FIELD_VALUE_TUPLE: typing.TypeAlias = typing.Union[ _TYPE_FIELD_VALUE, diff --git a/src/urllib3/_version.py b/src/urllib3/_version.py index c8fb8156d7..e223f4a66d 100644 --- a/src/urllib3/_version.py +++ b/src/urllib3/_version.py @@ -1,4 +1,4 @@ # This file is protected via CODEOWNERS from __future__ import annotations -__version__ = "2.8.901" +__version__ = "2.8.902" diff --git a/src/urllib3/backend/_async/hface.py b/src/urllib3/backend/_async/hface.py index 4803f62fb3..1b191007dd 100644 --- a/src/urllib3/backend/_async/hface.py +++ b/src/urllib3/backend/_async/hface.py @@ -452,10 +452,18 @@ async def _post_conn(self) -> None: # type: ignore[override] return # it may be required to send some initial data, aka. magic header (PRI * HTTP/2..) - await self.__exchange_until( - HandshakeCompleted, - receive_first=False, - ) + try: + await self.__exchange_until( + HandshakeCompleted, + receive_first=False, + ) + except ProtocolError as e: + if isinstance(self._protocol, HTTPOverQUICProtocol): + raise ProtocolError( + "It is likely that the server yielded its support for HTTP/3 through the Alt-Svc header while unable to do so. " + "To remediate that issue, either disable http3 or reach out to the server admin." + ) from e + raise if isinstance(self._protocol, HTTPOverQUICProtocol): self.conn_info.certificate_der = self._protocol.getpeercert( diff --git a/src/urllib3/backend/hface.py b/src/urllib3/backend/hface.py index d0ab9546ac..80cbca0089 100644 --- a/src/urllib3/backend/hface.py +++ b/src/urllib3/backend/hface.py @@ -513,10 +513,18 @@ def _post_conn(self) -> None: return # it may be required to send some initial data, aka. magic header (PRI * HTTP/2..) - self.__exchange_until( - HandshakeCompleted, - receive_first=False, - ) + try: + self.__exchange_until( + HandshakeCompleted, + receive_first=False, + ) + except ProtocolError as e: + if isinstance(self._protocol, HTTPOverQUICProtocol): + raise ProtocolError( + "It is likely that the server yielded its support for HTTP/3 through the Alt-Svc header while unable to do so. " + "To remediate that issue, either disable http3 or reach out to the server admin." + ) from e + raise #: Populating ConnectionInfo using QUIC TLS interfaces if isinstance(self._protocol, HTTPOverQUICProtocol): diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index ef6494c76b..e607fff8c1 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -467,6 +467,10 @@ def request( try: # If we're given a body we start sending that in chunks. if chunks is not None: + if hasattr(chunks, "__aiter__"): + raise RuntimeError( + "Unable to send an async iterable through a synchronous connection" + ) for chunk in chunks: # Sending empty chunks isn't allowed for TE: chunked # as it indicates the end of the body. diff --git a/src/urllib3/contrib/emscripten/__init__.py b/src/urllib3/contrib/emscripten/__init__.py new file mode 100644 index 0000000000..3a535177b8 --- /dev/null +++ b/src/urllib3/contrib/emscripten/__init__.py @@ -0,0 +1,24 @@ +# Dummy file to match upstream modules +# without actually serving them. +# urllib3-future diverged from urllib3. +# only the top-level (public API) are guaranteed to be compatible. +# in-fact urllib3-future propose a better way to migrate/transition toward +# newer protocols. + +from __future__ import annotations + +import warnings + + +def inject_into_urllib3() -> None: + warnings.warn( + "urllib3-future do not have a emscripten module as it is irrelevant to urllib3 nature. " + "wasm support will be brought in Niquests (replacement for Requests). " + "One does not simply ship an addon that essentially kills 90% of its other features and alter the 10 " + "remaining percents.", + UserWarning, + ) + + +def extract_from_urllib3() -> None: + pass diff --git a/src/urllib3/http2.py b/src/urllib3/http2.py new file mode 100644 index 0000000000..7dcf25640c --- /dev/null +++ b/src/urllib3/http2.py @@ -0,0 +1,22 @@ +# Dummy file to match upstream modules +# without actually serving them. +# urllib3-future diverged from urllib3. +# only the top-level (public API) are guaranteed to be compatible. +# in-fact urllib3-future propose a better way to migrate/transition toward +# newer protocols. + +from __future__ import annotations + +import warnings + + +def inject_into_urllib3() -> None: + warnings.warn( + "urllib3-future do not propose the http2 module as it is useless to us. " + "enjoy all three protocols. urllib3-future just works out of the box with all protocols.", + UserWarning, + ) + + +def extract_from_urllib3() -> None: + pass diff --git a/src/urllib3/util/request.py b/src/urllib3/util/request.py index 61aa6c36b1..2611228ecd 100644 --- a/src/urllib3/util/request.py +++ b/src/urllib3/util/request.py @@ -194,7 +194,7 @@ def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) - class ChunksAndContentLength(typing.NamedTuple): - chunks: typing.Iterable[bytes] | None + chunks: typing.Iterable[bytes] | typing.AsyncIterable[bytes] | None content_length: int | None is_string: bool @@ -214,7 +214,7 @@ def body_to_chunks( for framing instead. """ - chunks: typing.Iterable[bytes] | None + chunks: typing.Iterable[bytes] | typing.AsyncIterable[bytes] | None content_length: int | None # No body, we need to make a recommendation on 'Content-Length' @@ -244,6 +244,25 @@ def chunk_readable() -> typing.Iterable[bytes]: datablock = datablock.encode("utf-8") yield datablock + async def chunk_areadable() -> typing.AsyncIterable[bytes]: + nonlocal body, blocksize + assert body is not None and hasattr(body, "__aiter__") + encode: bool | None = None + + buf = b"" + + async for block in body: + if encode is None: + encode = isinstance(block, str) + if len(buf) >= blocksize: + yield buf + buf = b"" + continue + buf += block.encode("utf-8") if encode else block + + if buf: + yield buf + # Bytes or strings become bytes if isinstance(body, (str, bytes)): converted = to_bytes(body) @@ -259,7 +278,9 @@ def chunk_readable() -> typing.Iterable[bytes]: elif hasattr(body, "read"): chunks = chunk_readable() content_length = None - + elif hasattr(body, "__aiter__"): + chunks = chunk_areadable() + content_length = None # Otherwise we need to start checking via duck-typing. else: try: diff --git a/test/with_dummyserver/asynchronous/test_connectionpool.py b/test/with_dummyserver/asynchronous/test_connectionpool.py index 768f91874a..2b648eec77 100644 --- a/test/with_dummyserver/asynchronous/test_connectionpool.py +++ b/test/with_dummyserver/asynchronous/test_connectionpool.py @@ -288,6 +288,42 @@ async def test_request_method_body(self) -> None: with pytest.raises(TypeError): await pool.request("POST", "/echo", body=body, fields=fields) + async def test_sending_async_iterable_orig_bytes(self) -> None: + async with AsyncHTTPConnectionPool(self.host, self.port) as pool: + + async def abody() -> typing.AsyncIterable[bytes]: + await asyncio.sleep(0.01) + yield b"foo" + await asyncio.sleep(0.01) + yield b"bar" + + r = await pool.request("POST", "/echo", body=abody()) + assert await r.data == b"foobar" + + async def test_sending_async_iterable_orig_str(self) -> None: + async with AsyncHTTPConnectionPool(self.host, self.port) as pool: + + async def abody() -> typing.AsyncIterable[str]: + await asyncio.sleep(0.01) + yield "foo" + await asyncio.sleep(0.01) + yield "bar" + + r = await pool.request("POST", "/echo", body=abody()) + assert await r.data == b"foobar" + + async def test_sending_async_iterable_orig_str_non_ascii(self) -> None: + async with AsyncHTTPConnectionPool(self.host, self.port) as pool: + + async def abody() -> typing.AsyncIterable[str]: + await asyncio.sleep(0.01) + yield "hélloà" + await asyncio.sleep(0.01) + yield "bar" + + r = await pool.request("POST", "/echo", body=abody()) + assert await r.data == "hélloàbar".encode() + async def test_unicode_upload(self) -> None: fieldname = "myfile" filename = "\xe2\x99\xa5.txt"