From fa15acecd112c3edb3d634002ff523d17836f798 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 24 Oct 2024 09:39:35 -1000 Subject: [PATCH 1/9] Add some simple benchmarks This will get the ball rolling so we know when we have solved #9520 --- .github/workflows/ci-cd.yml | 41 +++++++++++++++++++++++++ requirements/test.in | 1 + requirements/test.txt | 6 ++++ tests/test_client_request_benchmarks.py | 22 +++++++++++++ tests/test_cookiejar_benchmarks.py | 27 ++++++++++++++++ 5 files changed, 97 insertions(+) create mode 100644 tests/test_client_request_benchmarks.py create mode 100644 tests/test_cookiejar_benchmarks.py diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 454a1bc146e..7bc9283f10d 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -254,6 +254,47 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + benchmark: + name: Benchmark + needs: gen_llhttp + + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout project + uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Python 3.12 + id: python-install + uses: actions/setup-python@v5 + with: + python-version: 3.12 + cache: pip + cache-dependency-path: requirements/*.txt + - name: Update pip, wheel, setuptools, build, twine + run: | + python -m pip install -U pip wheel setuptools build twine + - name: Install dependencies + run: | + python -m pip install -r requirements/test.in -c requirements/test.txt + - name: Restore llhttp generated files + uses: actions/download-artifact@v3 + with: + name: llhttp + path: vendor/llhttp/build/ + - name: Cythonize + run: | + make cythonize + - name: Install self + run: python -m pip install -e . + - name: Run benchmarks + uses: CodSpeedHQ/action@v3 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: python -Im pytest --no-cov -vvvvv --codspeed + + check: # This job does nothing and is only used for the branch protection if: always() diff --git a/requirements/test.in b/requirements/test.in index 18423f3743c..a98d9edf28d 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -7,6 +7,7 @@ proxy.py >= 2.4.4rc4 pytest pytest-cov pytest-mock +pytest_codspeed python-on-whales setuptools-git trustme; platform_machine != "i686" # no 32-bit wheels diff --git a/requirements/test.txt b/requirements/test.txt index b7d3c7db665..1c994962aba 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -22,6 +22,7 @@ cffi==1.17.1 # via # cryptography # pycares + # pytest-codspeed charset-normalizer==3.4.0 # via requests click==8.1.7 @@ -36,6 +37,8 @@ cryptography==43.0.3 # via trustme exceptiongroup==1.2.2 # via pytest +filelock==3.16.1 + # via pytest-codspeed freezegun==1.5.1 # via -r requirements/test.in frozenlist==1.5.0 @@ -88,8 +91,11 @@ pygments==2.18.0 pytest==8.1.1 # via # -r requirements/test.in + # pytest-codspeed # pytest-cov # pytest-mock +pytest-codspeed==2.2.1 + # via -r requirements/test.in pytest-cov==5.0.0 # via -r requirements/test.in pytest-mock==3.14.0 diff --git a/tests/test_client_request_benchmarks.py b/tests/test_client_request_benchmarks.py new file mode 100644 index 00000000000..7e4aff6a536 --- /dev/null +++ b/tests/test_client_request_benchmarks.py @@ -0,0 +1,22 @@ +"""codspeed benchmarks for client requests.""" + +import asyncio +from http.cookies import Morsel + +from pytest_codspeed import BenchmarkFixture # type: ignore[import-untyped] +from yarl import URL + +from aiohttp.client_reqrep import ClientRequest + + +def test_update_cookies( + loop: asyncio.AbstractEventLoop, benchmark: BenchmarkFixture +) -> None: + req = ClientRequest("get", URL("http://python.org"), loop=loop) + morsel: "Morsel[str]" = Morsel() + morsel.set(key="string", val="Another string", coded_val="really") + morsel_cookie = {"str": morsel} + + @benchmark + def _run() -> None: + req.update_cookies(cookies=morsel_cookie) diff --git a/tests/test_cookiejar_benchmarks.py b/tests/test_cookiejar_benchmarks.py new file mode 100644 index 00000000000..2f25e550dfc --- /dev/null +++ b/tests/test_cookiejar_benchmarks.py @@ -0,0 +1,27 @@ +"""codspeed benchmarks for cookies.""" + +from http.cookies import BaseCookie + +from pytest_codspeed import BenchmarkFixture # type: ignore[import-untyped] +from yarl import URL + +from aiohttp.cookiejar import CookieJar + + +async def test_process_cookies(benchmark: BenchmarkFixture) -> None: + """Benchmark for creating a temp CookieJar and filtering by URL. + + This benchmark matches what the client request does when cookies + are passed to the request. + """ + all_cookies: BaseCookie[str] = BaseCookie() + url = URL("http://example.com") + cookies = {"cookie1": "value1", "cookie2": "value2"} + + @benchmark + def _run() -> None: + tmp_cookie_jar = CookieJar() + tmp_cookie_jar.update_cookies(cookies) + req_cookies = tmp_cookie_jar.filter_cookies(url) + if req_cookies: + all_cookies.load(req_cookies) From 3fb08a13d30b76b6ea5ca255e0807439ddcff516 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 24 Oct 2024 09:43:28 -1000 Subject: [PATCH 2/9] fix lint --- requirements/lint.in | 1 + requirements/lint.txt | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/requirements/lint.in b/requirements/lint.in index 29c5810916a..999f41b7a40 100644 --- a/requirements/lint.in +++ b/requirements/lint.in @@ -5,6 +5,7 @@ pre-commit proxy.py pytest pytest-mock +pytest_codspeed python-on-whales slotscheck trustme diff --git a/requirements/lint.txt b/requirements/lint.txt index 2912c944a84..614f10e7bbf 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -16,6 +16,7 @@ cffi==1.17.1 # via # cryptography # pycares + # pytest-codspeed cfgv==3.4.0 # via pre-commit charset-normalizer==3.4.0 @@ -31,7 +32,9 @@ distlib==0.3.9 exceptiongroup==1.2.2 # via pytest filelock==3.16.1 - # via virtualenv + # via + # pytest-codspeed + # virtualenv freezegun==1.5.1 # via -r requirements/lint.in identify==2.6.1 @@ -75,7 +78,10 @@ pygments==2.18.0 pytest==8.1.1 # via # -r requirements/lint.in + # pytest-codspeed # pytest-mock +pytest-codspeed==2.2.1 + # via -r requirements/lint.in pytest-mock==3.14.0 # via -r requirements/lint.in python-dateutil==2.9.0.post0 From 265dea80fb88178b6b7253d63286d725fdc9b0ad Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 24 Oct 2024 09:47:02 -1000 Subject: [PATCH 3/9] mypy --- .mypy.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.mypy.ini b/.mypy.ini index 70261107033..2f022559800 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -34,3 +34,8 @@ ignore_missing_imports = True [mypy-gunicorn.*] ignore_missing_imports = True + +[mypy-test_*_benchmarks] +disable_error_code = + no-any-unimported, + misc From 6a5e3061176f215938a980b7ae44fa91eaa1e201 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 24 Oct 2024 09:49:24 -1000 Subject: [PATCH 4/9] rework file names --- .mypy.ini | 2 +- ..._request_benchmarks.py => test_benchmarks_client_request.py} | 0 ...st_cookiejar_benchmarks.py => tests_benchmarks_cookiejar.py} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename tests/{test_client_request_benchmarks.py => test_benchmarks_client_request.py} (100%) rename tests/{test_cookiejar_benchmarks.py => tests_benchmarks_cookiejar.py} (100%) diff --git a/.mypy.ini b/.mypy.ini index 2f022559800..ee844d478a1 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -35,7 +35,7 @@ ignore_missing_imports = True [mypy-gunicorn.*] ignore_missing_imports = True -[mypy-test_*_benchmarks] +[mypy-benchmarks_*] disable_error_code = no-any-unimported, misc diff --git a/tests/test_client_request_benchmarks.py b/tests/test_benchmarks_client_request.py similarity index 100% rename from tests/test_client_request_benchmarks.py rename to tests/test_benchmarks_client_request.py diff --git a/tests/test_cookiejar_benchmarks.py b/tests/tests_benchmarks_cookiejar.py similarity index 100% rename from tests/test_cookiejar_benchmarks.py rename to tests/tests_benchmarks_cookiejar.py From 6e3ab1c334c96f311c1a14c5be95f59977bebf59 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 24 Oct 2024 09:51:08 -1000 Subject: [PATCH 5/9] improve naming since namespace for benchmark is per project --- tests/test_benchmarks_client_request.py | 2 +- ...sts_benchmarks_cookiejar.py => test_benchmarks_cookiejar.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/{tests_benchmarks_cookiejar.py => test_benchmarks_cookiejar.py} (90%) diff --git a/tests/test_benchmarks_client_request.py b/tests/test_benchmarks_client_request.py index 7e4aff6a536..63c77dfcdc8 100644 --- a/tests/test_benchmarks_client_request.py +++ b/tests/test_benchmarks_client_request.py @@ -9,7 +9,7 @@ from aiohttp.client_reqrep import ClientRequest -def test_update_cookies( +def test_client_request_update_cookies( loop: asyncio.AbstractEventLoop, benchmark: BenchmarkFixture ) -> None: req = ClientRequest("get", URL("http://python.org"), loop=loop) diff --git a/tests/tests_benchmarks_cookiejar.py b/tests/test_benchmarks_cookiejar.py similarity index 90% rename from tests/tests_benchmarks_cookiejar.py rename to tests/test_benchmarks_cookiejar.py index 2f25e550dfc..2df6397aed1 100644 --- a/tests/tests_benchmarks_cookiejar.py +++ b/tests/test_benchmarks_cookiejar.py @@ -8,7 +8,7 @@ from aiohttp.cookiejar import CookieJar -async def test_process_cookies(benchmark: BenchmarkFixture) -> None: +async def test_load_cookies_into_temp_cookiejar(benchmark: BenchmarkFixture) -> None: """Benchmark for creating a temp CookieJar and filtering by URL. This benchmark matches what the client request does when cookies From d94f8f14c64101abc4fa5c71550b6c591b135ef0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 24 Oct 2024 09:53:03 -1000 Subject: [PATCH 6/9] write out mypy ignores --- .mypy.ini | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.mypy.ini b/.mypy.ini index ee844d478a1..6a33da66b2b 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -35,7 +35,12 @@ ignore_missing_imports = True [mypy-gunicorn.*] ignore_missing_imports = True -[mypy-benchmarks_*] +[mypy-benchmarks_client_request] +disable_error_code = + no-any-unimported, + misc + +[mypy-benchmarks_cookiejar] disable_error_code = no-any-unimported, misc From 722a4f1f855ef40c4e5a48e51deb3f71d3b4986c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 24 Oct 2024 09:57:38 -1000 Subject: [PATCH 7/9] write out mypy ignores --- .mypy.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.mypy.ini b/.mypy.ini index 6a33da66b2b..782a51fd15a 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -35,12 +35,12 @@ ignore_missing_imports = True [mypy-gunicorn.*] ignore_missing_imports = True -[mypy-benchmarks_client_request] +[mypy-test_benchmarks_client_request] disable_error_code = no-any-unimported, misc -[mypy-benchmarks_cookiejar] +[mypy-test_benchmarks_cookiejar] disable_error_code = no-any-unimported, misc From 6340033b143004677ff111af66dd00c60c6a682c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 24 Oct 2024 10:22:16 -1000 Subject: [PATCH 8/9] make unconditional --- tests/test_benchmarks_cookiejar.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_benchmarks_cookiejar.py b/tests/test_benchmarks_cookiejar.py index 2df6397aed1..508b49f68cb 100644 --- a/tests/test_benchmarks_cookiejar.py +++ b/tests/test_benchmarks_cookiejar.py @@ -23,5 +23,4 @@ def _run() -> None: tmp_cookie_jar = CookieJar() tmp_cookie_jar.update_cookies(cookies) req_cookies = tmp_cookie_jar.filter_cookies(url) - if req_cookies: - all_cookies.load(req_cookies) + all_cookies.load(req_cookies) From 19e6bd302fa43f617456f124c3a067e57b3a7d15 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 25 Oct 2024 11:35:23 -1000 Subject: [PATCH 9/9] add a comment about missing py.typed for codspeed --- .mypy.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.mypy.ini b/.mypy.ini index 782a51fd15a..7eb7908efe3 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -35,6 +35,10 @@ ignore_missing_imports = True [mypy-gunicorn.*] ignore_missing_imports = True +# Benchmark configuration is because pytest_codspeed is missing +# a py.typed file. Can be removed once the following PR is merged +# and released: +# https://github.com/CodSpeedHQ/pytest-codspeed/pull/53 [mypy-test_benchmarks_client_request] disable_error_code = no-any-unimported,