From 3c4b1510fd240df1f78788665785fdc6ea8a1fe4 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Fri, 20 Oct 2023 18:40:30 +0800 Subject: [PATCH 01/14] dev(narugo): add support for headers and binary body in recorder --- .gitignore | 2 + responses/__init__.py | 10 +- responses/_recorder.py | 20 +++- responses/tests/test_recorder.py | 171 +++++++++++++++++++++++++++++-- 4 files changed, 190 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 3534373a..28dd7e51 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ venv /.pytest_cache /.tox /.artifacts +/test_* +/*.yaml diff --git a/responses/__init__.py b/responses/__init__.py index 78a3a436..ee384523 100644 --- a/responses/__init__.py +++ b/responses/__init__.py @@ -1,3 +1,4 @@ +import base64 import inspect import json as json_module import logging @@ -821,11 +822,18 @@ def _add_from_file(self, file_path: "Union[str, bytes, os.PathLike[Any]]") -> No for rsp in data["responses"]: rsp = rsp["response"] + headers = dict(rsp.get("headers") or {}) + if "Content-Type" in headers: + headers.pop("Content-Type") + body = rsp["body"] + if rsp.get("body_encoded"): + body = base64.urlsafe_b64decode(body) self.add( method=rsp["method"], url=rsp["url"], - body=rsp["body"], + body=body, status=rsp["status"], + headers=headers, content_type=rsp["content_type"], auto_calculate_content_length=rsp["auto_calculate_content_length"], ) diff --git a/responses/_recorder.py b/responses/_recorder.py index 93603ff7..92759098 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -1,3 +1,4 @@ +import base64 from functools import wraps from typing import TYPE_CHECKING @@ -22,6 +23,7 @@ import yaml +from responses import _UNSET from responses import RequestsMock from responses import Response from responses import _real_send @@ -45,12 +47,19 @@ def _dump( for rsp in registered: try: content_length = rsp.auto_calculate_content_length # type: ignore[attr-defined] + body = rsp.body # type: ignore[attr-defined] + if isinstance(body, bytes): + body = base64.urlsafe_b64encode(body).decode() + body_encoded = True + else: + body_encoded = False data["responses"].append( { "response": { "method": rsp.method, "url": rsp.url, - "body": rsp.body, # type: ignore[attr-defined] + "body": body, # type: ignore[attr-defined] + "body_encoded": body_encoded, "status": rsp.status, # type: ignore[attr-defined] "headers": rsp.headers, "content_type": rsp.content_type, @@ -116,11 +125,18 @@ def _on_request( request.params = self._parse_request_params(request.path_url) # type: ignore[attr-defined] request.req_kwargs = kwargs # type: ignore[attr-defined] requests_response = _real_send(adapter, request, **kwargs) + requests_headers = dict(requests_response.headers) + if "Content-Type" in requests_headers: + requests_content_type = requests_headers.pop("Content-Type") + else: + requests_content_type = _UNSET responses_response = Response( method=str(request.method), url=str(requests_response.request.url), status=requests_response.status_code, - body=requests_response.text, + headers=requests_headers, + body=requests_response.content, + content_type=requests_content_type, ) self._registry.add(responses_response) return requests_response diff --git a/responses/tests/test_recorder.py b/responses/tests/test_recorder.py index d892845c..18581242 100644 --- a/responses/tests/test_recorder.py +++ b/responses/tests/test_recorder.py @@ -1,3 +1,4 @@ +import collections.abc from pathlib import Path import pytest @@ -15,15 +16,130 @@ # python 3.11 import tomllib as _toml # type: ignore[no-redef] +_NOT_CARE = object() + + +class _CompareDict: + def __init__(self, obj): + self.obj = obj + + def __eq__(self, other): + if self.obj is _NOT_CARE: + return True + elif isinstance(self.obj, collections.abc.Mapping): + if not isinstance(other, collections.abc.Mapping): + return False + if sorted(self.obj.keys()) != sorted(other.keys()): + return False + for key in self.obj.keys(): + if _CompareDict(self.obj[key]) != other[key]: + return False + return True + elif isinstance(self.obj, list): + if not isinstance(other, list): + return False + if len(self.obj) != len(other): + return False + for i, (obj_item, other_item) in enumerate(zip(self.obj, other)): + if _CompareDict(obj_item) != other_item: + return False + return True + else: + return self.obj == other + + +def get_data_for_cmp(host, port): + data = { + "responses": [ + { + "response": { + "method": "GET", + "url": f"http://{host}:{port}/404", + "body": "NDA0IE5vdCBGb3VuZA==", + "body_encoded": True, + "headers": { + "Connection": "keep-alive", + "Content-Length": "13", + "Date": _NOT_CARE, + "Keep-Alive": _NOT_CARE, + "Proxy-Connection": "keep-alive", + "Server": _NOT_CARE, + }, + "status": 404, + "content_type": "text/plain", + "auto_calculate_content_length": False, + } + }, + { + "response": { + "method": "GET", + "url": f"http://{host}:{port}/status/wrong", + "body": "SW52YWxpZCBzdGF0dXMgY29kZQ==", + "body_encoded": True, + "headers": { + "Connection": "keep-alive", + "Content-Length": "19", + "Date": _NOT_CARE, + "Keep-Alive": _NOT_CARE, + "Proxy-Connection": "keep-alive", + "Server": _NOT_CARE, + }, + "status": 400, + "content_type": "text/plain", + "auto_calculate_content_length": False, + } + }, + { + "response": { + "method": "GET", + "url": f"http://{host}:{port}/500", + "body": "NTAwIEludGVybmFsIFNlcnZlciBFcnJvcg==", + "body_encoded": True, + "headers": { + "Connection": "keep-alive", + "Content-Length": "25", + "Date": _NOT_CARE, + "Keep-Alive": _NOT_CARE, + "Proxy-Connection": "keep-alive", + "Server": _NOT_CARE, + }, + "status": 500, + "content_type": "text/plain", + "auto_calculate_content_length": False, + } + }, + { + "response": { + "method": "PUT", + "url": f"http://{host}:{port}/202", + "body": "T0s=", + "body_encoded": True, + "headers": { + "Connection": "keep-alive", + "Content-Length": "2", + "Date": _NOT_CARE, + "Keep-Alive": _NOT_CARE, + "Proxy-Connection": "keep-alive", + "Server": _NOT_CARE, + }, + "status": 202, + "content_type": "text/plain", + "auto_calculate_content_length": False, + } + }, + ] + } + return _CompareDict(data) + -def get_data(host, port): +def get_data_for_dump(host, port): data = { "responses": [ { "response": { "method": "GET", "url": f"http://{host}:{port}/404", - "body": "404 Not Found", + "body": "404 Not Found", # test the backward support for 0.23.3 "status": 404, "content_type": "text/plain", "auto_calculate_content_length": False, @@ -33,7 +149,16 @@ def get_data(host, port): "response": { "method": "GET", "url": f"http://{host}:{port}/status/wrong", - "body": "Invalid status code", + "body": "SW52YWxpZCBzdGF0dXMgY29kZQ==", + "body_encoded": True, + "headers": { + "Connection": "keep-alive", + "Content-Length": "19", + "Date": "Fri, 20 Oct 2023 10:12:13 " "GMT", + "Keep-Alive": "timeout=4", + "Proxy-Connection": "keep-alive", + "Server": "Werkzeug/3.0.0 " "Python/3.8.10", + }, "status": 400, "content_type": "text/plain", "auto_calculate_content_length": False, @@ -43,7 +168,16 @@ def get_data(host, port): "response": { "method": "GET", "url": f"http://{host}:{port}/500", - "body": "500 Internal Server Error", + "body": "NTAwIEludGVybmFsIFNlcnZlciBFcnJvcg==", + "body_encoded": True, + "headers": { + "Connection": "keep-alive", + "Content-Length": "25", + "Date": "Fri, 20 Oct 2023 10:12:13 " "GMT", + "Keep-Alive": "timeout=4", + "Proxy-Connection": "keep-alive", + "Server": "Werkzeug/3.0.0 " "Python/3.8.10", + }, "status": 500, "content_type": "text/plain", "auto_calculate_content_length": False, @@ -53,7 +187,16 @@ def get_data(host, port): "response": { "method": "PUT", "url": f"http://{host}:{port}/202", - "body": "OK", + "body": "T0s=", + "body_encoded": True, + "headers": { + "Connection": "keep-alive", + "Content-Length": "2", + "Date": "Fri, 20 Oct 2023 10:12:13 " "GMT", + "Keep-Alive": "timeout=4", + "Proxy-Connection": "keep-alive", + "Server": "Werkzeug/3.0.0 " "Python/3.8.10", + }, "status": 202, "content_type": "text/plain", "auto_calculate_content_length": False, @@ -90,7 +233,10 @@ def run(): with open(self.out_file) as file: data = yaml.safe_load(file) - assert data == get_data(httpserver.host, httpserver.port) + import pprint + + pprint.pprint(data) + assert data == get_data_for_cmp(httpserver.host, httpserver.port) def test_recorder_toml(self, httpserver): custom_recorder = _recorder.Recorder() @@ -118,7 +264,7 @@ def run(): with open(self.out_file, "rb") as file: data = _toml.load(file) - assert data == get_data(httpserver.host, httpserver.port) + assert data == get_data_for_cmp(httpserver.host, httpserver.port) def prepare_server(self, httpserver): httpserver.expect_request("/500").respond_with_data( @@ -154,10 +300,10 @@ def teardown_method(self): def test_add_from_file(self, parser): if parser == yaml: with open(self.out_file, "w") as file: - parser.dump(get_data("example.com", "8080"), file) + parser.dump(get_data_for_dump("example.com", "8080"), file) else: with open(self.out_file, "wb") as file: - parser.dump(get_data("example.com", "8080"), file) + parser.dump(get_data_for_dump("example.com", "8080"), file) @responses.activate def run(): @@ -188,10 +334,15 @@ def _parse_response_file(file_path): assert responses.registered()[4].method == "PUT" assert responses.registered()[5].method == "POST" + assert responses.registered()[1].status == 404 assert responses.registered()[2].status == 400 assert responses.registered()[3].status == 500 - assert responses.registered()[3].body == "500 Internal Server Error" + assert ( + responses.registered()[1].body == "404 Not Found" + ) # test the backward support for 0.23.3 + assert responses.registered()[2].body == b"Invalid status code" + assert responses.registered()[3].body == b"500 Internal Server Error" assert responses.registered()[3].content_type == "text/plain" From 2444a0c10e98bce92aeedd243a9c079f2809a867 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Fri, 20 Oct 2023 19:34:24 +0800 Subject: [PATCH 02/14] dev(narugo): fix bug when Content-Encoding: gzip --- responses/_recorder.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/responses/_recorder.py b/responses/_recorder.py index 92759098..cc3eeb90 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -130,12 +130,15 @@ def _on_request( requests_content_type = requests_headers.pop("Content-Type") else: requests_content_type = _UNSET + # do not use requests_response.content as body here + # because Content-Encoding: gzip (or some other format) may be in the headers + # the raw binary data must be used to make sure it can be decompressed properly responses_response = Response( method=str(request.method), url=str(requests_response.request.url), status=requests_response.status_code, headers=requests_headers, - body=requests_response.content, + body=requests_response.raw.read(), content_type=requests_content_type, ) self._registry.add(responses_response) From d369a3136089a6c24d810e1e1fd6bc6a720160d1 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Fri, 20 Oct 2023 20:14:28 +0800 Subject: [PATCH 03/14] dev(narugo): try fix gzip problem by removing Content-Encoding --- responses/_recorder.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/responses/_recorder.py b/responses/_recorder.py index cc3eeb90..ba10bc99 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -130,15 +130,16 @@ def _on_request( requests_content_type = requests_headers.pop("Content-Type") else: requests_content_type = _UNSET - # do not use requests_response.content as body here - # because Content-Encoding: gzip (or some other format) may be in the headers - # the raw binary data must be used to make sure it can be decompressed properly + # Content-Encoding should be removed to + # avoid 'Content-Encoding: gzip' causing the error in requests + if "Content-Encoding" in requests_headers: + requests_headers.pop("Content-Encoding") responses_response = Response( method=str(request.method), url=str(requests_response.request.url), status=requests_response.status_code, headers=requests_headers, - body=requests_response.raw.read(), + body=requests_response.content, content_type=requests_content_type, ) self._registry.add(responses_response) From c8c0b24e4acf952d1dd0d4a2b72f67b64e20e718 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Sat, 21 Oct 2023 18:28:52 +0800 Subject: [PATCH 04/14] dev(narugo): move binary data to files --- .gitignore | 2 ++ example_bins/202.bin | 1 + example_bins/400.bin | 1 + example_bins/500.bin | 1 + responses/__init__.py | 12 +++++---- responses/_recorder.py | 42 +++++++++++++++++++++++--------- responses/tests/test_recorder.py | 40 ++++++++++++++++++------------ 7 files changed, 66 insertions(+), 33 deletions(-) create mode 100644 example_bins/202.bin create mode 100644 example_bins/400.bin create mode 100644 example_bins/500.bin diff --git a/.gitignore b/.gitignore index 28dd7e51..426296cc 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ venv /.artifacts /test_* /*.yaml +*.bin +!/example_bins/*.bin diff --git a/example_bins/202.bin b/example_bins/202.bin new file mode 100644 index 00000000..d86bac9d --- /dev/null +++ b/example_bins/202.bin @@ -0,0 +1 @@ +OK diff --git a/example_bins/400.bin b/example_bins/400.bin new file mode 100644 index 00000000..86fc79f4 --- /dev/null +++ b/example_bins/400.bin @@ -0,0 +1 @@ +Invalid status code diff --git a/example_bins/500.bin b/example_bins/500.bin new file mode 100644 index 00000000..e534a491 --- /dev/null +++ b/example_bins/500.bin @@ -0,0 +1 @@ +500 Internal Server Error diff --git a/responses/__init__.py b/responses/__init__.py index ee384523..4daa6dd3 100644 --- a/responses/__init__.py +++ b/responses/__init__.py @@ -1,7 +1,7 @@ -import base64 import inspect import json as json_module import logging +import os from collections import namedtuple from functools import partialmethod from functools import wraps @@ -68,7 +68,6 @@ if TYPE_CHECKING: # pragma: no cover # import only for linter run - import os from typing import Protocol from unittest.mock import _patch as _mock_patcher @@ -819,15 +818,18 @@ def _parse_response_file( def _add_from_file(self, file_path: "Union[str, bytes, os.PathLike[Any]]") -> None: data = self._parse_response_file(file_path) + parent_directory = os.path.dirname(os.path.abspath(file_path)) for rsp in data["responses"]: rsp = rsp["response"] headers = dict(rsp.get("headers") or {}) if "Content-Type" in headers: headers.pop("Content-Type") - body = rsp["body"] - if rsp.get("body_encoded"): - body = base64.urlsafe_b64decode(body) + if "body_file" in rsp: + with open(os.path.join(parent_directory, rsp["body_file"]), "rb") as f: + body = f.read() + else: + body = rsp["body"] self.add( method=rsp["method"], url=rsp["url"], diff --git a/responses/_recorder.py b/responses/_recorder.py index ba10bc99..1c6be4ad 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -1,10 +1,9 @@ -import base64 +import os +import uuid from functools import wraps from typing import TYPE_CHECKING if TYPE_CHECKING: # pragma: no cover - import os - from typing import Any from typing import BinaryIO from typing import Callable @@ -40,26 +39,44 @@ def _remove_nones(d: "Any") -> "Any": def _dump( registered: "List[BaseResponse]", - destination: "Union[BinaryIO, TextIOWrapper]", + config_file: "Union[str, os.PathLike]", dumper: "Callable[[Union[Dict[Any, Any], List[Any]], Union[BinaryIO, TextIOWrapper]], Any]", -) -> None: + dumper_mode: "str" = "w", +): data: Dict[str, Any] = {"responses": []} + + # e.g. config_file = 'my/dir/responses.yaml' + # parent_directory = 'my/dir' + # binary_directory = 'my/dir/responses' + fname, fext = os.path.splitext(os.path.basename(config_file)) + parent_directory = os.path.dirname(os.path.abspath(config_file)) + binary_directory = os.path.join( + parent_directory, fname if fext else f"{fname}_bins" + ) + for rsp in registered: try: content_length = rsp.auto_calculate_content_length # type: ignore[attr-defined] body = rsp.body # type: ignore[attr-defined] if isinstance(body, bytes): - body = base64.urlsafe_b64encode(body).decode() - body_encoded = True + os.makedirs(binary_directory, exist_ok=True) + bin_file = os.path.join(binary_directory, f"{uuid.uuid4()}.bin") + with open(bin_file, "wb") as bf: + bf.write(body) + + # make sure the stored binary file path is relative to config file + # or the config file and binary directory will be hard to move + body_file = os.path.relpath(bin_file, parent_directory) + body = None else: - body_encoded = False + body_file = None data["responses"].append( { "response": { "method": rsp.method, "url": rsp.url, "body": body, # type: ignore[attr-defined] - "body_encoded": body_encoded, + "body_file": body_file, "status": rsp.status, # type: ignore[attr-defined] "headers": rsp.headers, "content_type": rsp.content_type, @@ -72,7 +89,9 @@ def _dump( "Cannot dump response object." "Probably you use custom Response object that is missing required attributes" ) from exc - dumper(_remove_nones(data), destination) + + with open(config_file, dumper_mode) as cfile: + dumper(_remove_nones(data), cfile) class Recorder(RequestsMock): @@ -111,8 +130,7 @@ def dump_to_file( file_path: "Union[str, bytes, os.PathLike[Any]]", registered: "List[BaseResponse]", ) -> None: - with open(file_path, "w") as file: - _dump(registered, file, yaml.dump) + _dump(registered, file_path, yaml.dump) def _on_request( self, diff --git a/responses/tests/test_recorder.py b/responses/tests/test_recorder.py index 18581242..413d3a1a 100644 --- a/responses/tests/test_recorder.py +++ b/responses/tests/test_recorder.py @@ -1,4 +1,5 @@ import collections.abc +import shutil from pathlib import Path import pytest @@ -55,8 +56,7 @@ def get_data_for_cmp(host, port): "response": { "method": "GET", "url": f"http://{host}:{port}/404", - "body": "NDA0IE5vdCBGb3VuZA==", - "body_encoded": True, + "body_file": _NOT_CARE, "headers": { "Connection": "keep-alive", "Content-Length": "13", @@ -74,8 +74,7 @@ def get_data_for_cmp(host, port): "response": { "method": "GET", "url": f"http://{host}:{port}/status/wrong", - "body": "SW52YWxpZCBzdGF0dXMgY29kZQ==", - "body_encoded": True, + "body_file": _NOT_CARE, "headers": { "Connection": "keep-alive", "Content-Length": "19", @@ -93,8 +92,7 @@ def get_data_for_cmp(host, port): "response": { "method": "GET", "url": f"http://{host}:{port}/500", - "body": "NTAwIEludGVybmFsIFNlcnZlciBFcnJvcg==", - "body_encoded": True, + "body_file": _NOT_CARE, "headers": { "Connection": "keep-alive", "Content-Length": "25", @@ -112,8 +110,7 @@ def get_data_for_cmp(host, port): "response": { "method": "PUT", "url": f"http://{host}:{port}/202", - "body": "T0s=", - "body_encoded": True, + "body_file": _NOT_CARE, "headers": { "Connection": "keep-alive", "Content-Length": "2", @@ -149,8 +146,7 @@ def get_data_for_dump(host, port): "response": { "method": "GET", "url": f"http://{host}:{port}/status/wrong", - "body": "SW52YWxpZCBzdGF0dXMgY29kZQ==", - "body_encoded": True, + "body_file": "example_bins/400.bin", "headers": { "Connection": "keep-alive", "Content-Length": "19", @@ -168,8 +164,7 @@ def get_data_for_dump(host, port): "response": { "method": "GET", "url": f"http://{host}:{port}/500", - "body": "NTAwIEludGVybmFsIFNlcnZlciBFcnJvcg==", - "body_encoded": True, + "body_file": "example_bins/500.bin", "headers": { "Connection": "keep-alive", "Content-Length": "25", @@ -187,8 +182,7 @@ def get_data_for_dump(host, port): "response": { "method": "PUT", "url": f"http://{host}:{port}/202", - "body": "T0s=", - "body_encoded": True, + "body_file": "example_bins/202.bin", "headers": { "Connection": "keep-alive", "Content-Length": "2", @@ -213,6 +207,18 @@ def setup_method(self): if self.out_file.exists(): self.out_file.unlink() # pragma: no cover + self.out_bins_dir = Path("response_record_bins") + if self.out_bins_dir.exists(): + shutil.rmtree(self.out_bins_dir) # pragma: no cover + + assert not self.out_file.exists() + + def teardown_method(self): + if self.out_file.exists(): + self.out_file.unlink() + if self.out_bins_dir.exists(): + shutil.rmtree(self.out_bins_dir) + assert not self.out_file.exists() def test_recorder(self, httpserver): @@ -242,8 +248,7 @@ def test_recorder_toml(self, httpserver): custom_recorder = _recorder.Recorder() def dump_to_file(file_path, registered): - with open(file_path, "wb") as file: - _dump(registered, file, tomli_w.dump) + _dump(registered, file_path, tomli_w.dump, "wb") custom_recorder.dump_to_file = dump_to_file @@ -289,10 +294,13 @@ def prepare_server(self, httpserver): class TestReplay: def setup_method(self): self.out_file = Path("response_record") + self.out_bins_dir = Path("response_record_bins") def teardown_method(self): if self.out_file.exists(): self.out_file.unlink() + if self.out_bins_dir.exists(): + shutil.rmtree(self.out_bins_dir) assert not self.out_file.exists() From 86e65a80eea1f9a110314cabcd80c12ffcac7cd8 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Sat, 21 Oct 2023 18:48:02 +0800 Subject: [PATCH 05/14] dev(narugo): fix problem in example_bins --- .pre-commit-config.yaml | 1 + example_bins/202.bin | 2 +- example_bins/400.bin | 2 +- example_bins/500.bin | 2 +- responses/tests/test_recorder.py | 3 --- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 899de43a..0dd56c49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +exclude: ^example_bins repos: - repo: https://github.com/psf/black rev: 23.3.0 diff --git a/example_bins/202.bin b/example_bins/202.bin index d86bac9d..a0aba931 100644 --- a/example_bins/202.bin +++ b/example_bins/202.bin @@ -1 +1 @@ -OK +OK \ No newline at end of file diff --git a/example_bins/400.bin b/example_bins/400.bin index 86fc79f4..bc4a7469 100644 --- a/example_bins/400.bin +++ b/example_bins/400.bin @@ -1 +1 @@ -Invalid status code +Invalid status code \ No newline at end of file diff --git a/example_bins/500.bin b/example_bins/500.bin index e534a491..87a218a5 100644 --- a/example_bins/500.bin +++ b/example_bins/500.bin @@ -1 +1 @@ -500 Internal Server Error +500 Internal Server Error \ No newline at end of file diff --git a/responses/tests/test_recorder.py b/responses/tests/test_recorder.py index 413d3a1a..9f4562c3 100644 --- a/responses/tests/test_recorder.py +++ b/responses/tests/test_recorder.py @@ -239,9 +239,6 @@ def run(): with open(self.out_file) as file: data = yaml.safe_load(file) - import pprint - - pprint.pprint(data) assert data == get_data_for_cmp(httpserver.host, httpserver.port) def test_recorder_toml(self, httpserver): From 7d6b6e5e72db4f7d876a56c901cf1e1e35d295e0 Mon Sep 17 00:00:00 2001 From: narugo1992 <117186571+narugo1992@users.noreply.github.com> Date: Mon, 23 Oct 2023 23:13:33 +0800 Subject: [PATCH 06/14] Update responses/_recorder.py Co-authored-by: Mark Story --- responses/_recorder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/responses/_recorder.py b/responses/_recorder.py index 1c6be4ad..30db1336 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -51,7 +51,7 @@ def _dump( fname, fext = os.path.splitext(os.path.basename(config_file)) parent_directory = os.path.dirname(os.path.abspath(config_file)) binary_directory = os.path.join( - parent_directory, fname if fext else f"{fname}_bins" + parent_directory, fname if fext else f"{fname}_bodies" ) for rsp in registered: From 32bcf32c82aa6c78fea364d13d1c3ce31095024c Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Mon, 23 Oct 2023 23:13:06 +0800 Subject: [PATCH 07/14] dev(narugo): fix unittest when not using socks proxy --- responses/tests/test_recorder.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/responses/tests/test_recorder.py b/responses/tests/test_recorder.py index 9f4562c3..2121e3f3 100644 --- a/responses/tests/test_recorder.py +++ b/responses/tests/test_recorder.py @@ -58,11 +58,9 @@ def get_data_for_cmp(host, port): "url": f"http://{host}:{port}/404", "body_file": _NOT_CARE, "headers": { - "Connection": "keep-alive", + "Connection": _NOT_CARE, "Content-Length": "13", "Date": _NOT_CARE, - "Keep-Alive": _NOT_CARE, - "Proxy-Connection": "keep-alive", "Server": _NOT_CARE, }, "status": 404, @@ -76,11 +74,9 @@ def get_data_for_cmp(host, port): "url": f"http://{host}:{port}/status/wrong", "body_file": _NOT_CARE, "headers": { - "Connection": "keep-alive", + "Connection": _NOT_CARE, "Content-Length": "19", "Date": _NOT_CARE, - "Keep-Alive": _NOT_CARE, - "Proxy-Connection": "keep-alive", "Server": _NOT_CARE, }, "status": 400, @@ -94,11 +90,9 @@ def get_data_for_cmp(host, port): "url": f"http://{host}:{port}/500", "body_file": _NOT_CARE, "headers": { - "Connection": "keep-alive", + "Connection": _NOT_CARE, "Content-Length": "25", "Date": _NOT_CARE, - "Keep-Alive": _NOT_CARE, - "Proxy-Connection": "keep-alive", "Server": _NOT_CARE, }, "status": 500, @@ -112,11 +106,9 @@ def get_data_for_cmp(host, port): "url": f"http://{host}:{port}/202", "body_file": _NOT_CARE, "headers": { - "Connection": "keep-alive", + "Connection": _NOT_CARE, "Content-Length": "2", "Date": _NOT_CARE, - "Keep-Alive": _NOT_CARE, - "Proxy-Connection": "keep-alive", "Server": _NOT_CARE, }, "status": 202, From f9deb24317fffbb978794330cba73ba110394148 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Mon, 23 Oct 2023 23:34:39 +0800 Subject: [PATCH 08/14] dev(narugo): fix some lint problems --- responses/_recorder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/responses/_recorder.py b/responses/_recorder.py index 30db1336..fabd8091 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -39,10 +39,10 @@ def _remove_nones(d: "Any") -> "Any": def _dump( registered: "List[BaseResponse]", - config_file: "Union[str, os.PathLike]", + config_file: "Union[str, bytes, os.PathLike[Any]]", dumper: "Callable[[Union[Dict[Any, Any], List[Any]], Union[BinaryIO, TextIOWrapper]], Any]", dumper_mode: "str" = "w", -): +) -> None: data: Dict[str, Any] = {"responses": []} # e.g. config_file = 'my/dir/responses.yaml' @@ -75,7 +75,7 @@ def _dump( "response": { "method": rsp.method, "url": rsp.url, - "body": body, # type: ignore[attr-defined] + "body": body, "body_file": body_file, "status": rsp.status, # type: ignore[attr-defined] "headers": rsp.headers, From 56ac7983c750a9892363db2daf6da7e7e18caac9 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Tue, 24 Oct 2023 00:27:37 +0800 Subject: [PATCH 09/14] dev(narugo): fix remaining lint problems --- responses/_recorder.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/responses/_recorder.py b/responses/_recorder.py index fabd8091..e5403fda 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -5,12 +5,12 @@ if TYPE_CHECKING: # pragma: no cover from typing import Any - from typing import BinaryIO from typing import Callable from typing import Dict from typing import List from typing import Type from typing import Union + from typing import IO from responses import FirstMatchRegistry from responses import HTTPAdapter from responses import PreparedRequest @@ -18,7 +18,6 @@ from responses import _F from responses import BaseResponse - from io import TextIOWrapper import yaml @@ -39,8 +38,8 @@ def _remove_nones(d: "Any") -> "Any": def _dump( registered: "List[BaseResponse]", - config_file: "Union[str, bytes, os.PathLike[Any]]", - dumper: "Callable[[Union[Dict[Any, Any], List[Any]], Union[BinaryIO, TextIOWrapper]], Any]", + config_file: "Union[str, os.PathLike[str]]", + dumper: "Callable[[Union[Dict[Any, Any], List[Any]], Union[IO[Any]]], Any]", dumper_mode: "str" = "w", ) -> None: data: Dict[str, Any] = {"responses": []} @@ -107,7 +106,7 @@ def reset(self) -> None: self._registry = OrderedRegistry() def record( - self, *, file_path: "Union[str, bytes, os.PathLike[Any]]" = "response.yaml" + self, *, file_path: "Union[str, os.PathLike[str]]" = "response.yaml" ) -> "Union[Callable[[_F], _F], _F]": def deco_record(function: "_F") -> "Callable[..., Any]": @wraps(function) @@ -127,7 +126,7 @@ def wrapper(*args: "Any", **kwargs: "Any") -> "Any": # type: ignore[misc] def dump_to_file( self, *, - file_path: "Union[str, bytes, os.PathLike[Any]]", + file_path: "Union[str, os.PathLike[str]]", registered: "List[BaseResponse]", ) -> None: _dump(registered, file_path, yaml.dump) @@ -147,7 +146,7 @@ def _on_request( if "Content-Type" in requests_headers: requests_content_type = requests_headers.pop("Content-Type") else: - requests_content_type = _UNSET + requests_content_type = _UNSET # type: ignore[assignment] # Content-Encoding should be removed to # avoid 'Content-Encoding: gzip' causing the error in requests if "Content-Encoding" in requests_headers: From 7461aa6fead5d34c95770b65bcd92a7da58326cd Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Tue, 24 Oct 2023 16:04:57 +0800 Subject: [PATCH 10/14] dev(narugo): use pathlib --- responses/_recorder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/responses/_recorder.py b/responses/_recorder.py index e5403fda..4314bc24 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -1,4 +1,5 @@ import os +import pathlib import uuid from functools import wraps from typing import TYPE_CHECKING @@ -47,11 +48,10 @@ def _dump( # e.g. config_file = 'my/dir/responses.yaml' # parent_directory = 'my/dir' # binary_directory = 'my/dir/responses' - fname, fext = os.path.splitext(os.path.basename(config_file)) - parent_directory = os.path.dirname(os.path.abspath(config_file)) - binary_directory = os.path.join( - parent_directory, fname if fext else f"{fname}_bodies" - ) + config_file = pathlib.Path(config_file) + fname, fext = os.path.splitext(config_file.name) + parent_directory = config_file.absolute().parent + binary_directory = parent_directory / (fname if fext else f"{fname}_bodies") for rsp in registered: try: From 523857ea31175f0d19174473336f8d8716f843a6 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Wed, 25 Oct 2023 20:48:24 +0800 Subject: [PATCH 11/14] dev(narugo): pass request method to function _form_response --- responses/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/responses/__init__.py b/responses/__init__.py index 94e8e603..9f05ef8c 100644 --- a/responses/__init__.py +++ b/responses/__init__.py @@ -504,6 +504,7 @@ def matches(self, request: "PreparedRequest") -> Tuple[bool, str]: def _form_response( + method: Optional[str], body: Union[BufferedReader, BytesIO], headers: Optional[Mapping[str, str]], status: int, @@ -528,6 +529,7 @@ def _form_response( headers=headers, original_response=orig_response, preload_content=False, + request_method=method, ) @@ -593,7 +595,7 @@ def get_response(self, request: "PreparedRequest") -> HTTPResponse: content_length = len(body.getvalue()) headers["Content-Length"] = str(content_length) - return _form_response(body, headers, status) + return _form_response(request.method, body, headers, status) def __repr__(self) -> str: return ( @@ -655,7 +657,7 @@ def get_response(self, request: "PreparedRequest") -> HTTPResponse: body = _handle_body(body) headers.extend(r_headers) - return _form_response(body, headers, status) + return _form_response(request.method, body, headers, status) class PassthroughResponse(BaseResponse): From 58768f72901793a0bfe78eafc7d2e4e5db232ae9 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Thu, 26 Oct 2023 00:01:51 +0800 Subject: [PATCH 12/14] dev(narugo): use the decompressed length when Content-Encoding is gzip or something --- responses/_recorder.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/responses/_recorder.py b/responses/_recorder.py index 4314bc24..eaa09517 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -151,6 +151,12 @@ def _on_request( # avoid 'Content-Encoding: gzip' causing the error in requests if "Content-Encoding" in requests_headers: requests_headers.pop("Content-Encoding") + + # When something like 'Content-Encoding: gzip' is used + # the 'Content-Length' may be the length of compressed data, + # so we need to replace it with decompressed length + if "Content-Length" in requests_headers: + requests_headers["Content-Length"] = str(len(requests_response.content)) responses_response = Response( method=str(request.method), url=str(requests_response.request.url), From f6303e1afed1d7803d1927e41051d07f2f1de9b2 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Fri, 27 Oct 2023 16:12:23 +0800 Subject: [PATCH 13/14] dev(narugo): fix issue with lower-cased headers from some websites --- responses/_recorder.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/responses/_recorder.py b/responses/_recorder.py index eaa09517..e402b993 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -1,3 +1,4 @@ +import copy import os import pathlib import uuid @@ -19,7 +20,6 @@ from responses import _F from responses import BaseResponse - import yaml from responses import _UNSET @@ -142,7 +142,12 @@ def _on_request( request.params = self._parse_request_params(request.path_url) # type: ignore[attr-defined] request.req_kwargs = kwargs # type: ignore[attr-defined] requests_response = _real_send(adapter, request, **kwargs) - requests_headers = dict(requests_response.headers) + # the object is a requests.structures.CaseInsensitiveDict object, + # if you re-construct the headers with a primitive dict object, + # some lower case headers like 'content-type' will not be able to be processed properly + # the deepcopy is for making sure the original headers object + # not changed by the following operations + requests_headers = copy.deepcopy(requests_response.headers) if "Content-Type" in requests_headers: requests_content_type = requests_headers.pop("Content-Type") else: @@ -161,7 +166,7 @@ def _on_request( method=str(request.method), url=str(requests_response.request.url), status=requests_response.status_code, - headers=requests_headers, + headers=dict(requests_headers), body=requests_response.content, content_type=requests_content_type, ) From 2ecf38c12f28241ce385af106acfd8d9da0565a6 Mon Sep 17 00:00:00 2001 From: narugo1992 Date: Thu, 16 Nov 2023 16:24:58 +0800 Subject: [PATCH 14/14] dev(narugo): fix type hint issues --- responses/_recorder.py | 6 +++--- responses/tests/test_recorder.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/responses/_recorder.py b/responses/_recorder.py index e402b993..04e2f20e 100644 --- a/responses/_recorder.py +++ b/responses/_recorder.py @@ -40,7 +40,7 @@ def _remove_nones(d: "Any") -> "Any": def _dump( registered: "List[BaseResponse]", config_file: "Union[str, os.PathLike[str]]", - dumper: "Callable[[Union[Dict[Any, Any], List[Any]], Union[IO[Any]]], Any]", + dumper: "Callable[[Union[Dict[Any, Any], List[Any]], Union[IO[Any]]], None]", dumper_mode: "str" = "w", ) -> None: data: Dict[str, Any] = {"responses": []} @@ -56,7 +56,7 @@ def _dump( for rsp in registered: try: content_length = rsp.auto_calculate_content_length # type: ignore[attr-defined] - body = rsp.body # type: ignore[attr-defined] + body = rsp.body if isinstance(body, bytes): os.makedirs(binary_directory, exist_ok=True) bin_file = os.path.join(binary_directory, f"{uuid.uuid4()}.bin") @@ -76,7 +76,7 @@ def _dump( "url": rsp.url, "body": body, "body_file": body_file, - "status": rsp.status, # type: ignore[attr-defined] + "status": rsp.status, "headers": rsp.headers, "content_type": rsp.content_type, "auto_calculate_content_length": content_length, diff --git a/responses/tests/test_recorder.py b/responses/tests/test_recorder.py index ed170d1b..57c66877 100644 --- a/responses/tests/test_recorder.py +++ b/responses/tests/test_recorder.py @@ -237,7 +237,7 @@ def test_recorder_toml(self, httpserver): custom_recorder = _recorder.Recorder() def dump_to_file(file_path, registered): - _dump(registered, file_path, tomli_w.dump, "wb") + _dump(registered, file_path, tomli_w.dump, "wb") # type: ignore[arg-type] custom_recorder.dump_to_file = dump_to_file # type: ignore[method-assign] @@ -299,7 +299,7 @@ def test_add_from_file(self, parser): # type: ignore[misc] with open(self.out_file, "w") as file: parser.dump(get_data_for_dump("example.com", "8080"), file) else: - with open(self.out_file, "wb") as file: + with open(self.out_file, "wb") as file: # type: ignore[assignment] parser.dump(get_data_for_dump("example.com", "8080"), file) @responses.activate