From 92aa75f7383baf20a4f5810709710f2706602358 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Sun, 24 Feb 2019 14:47:20 +0100 Subject: [PATCH 01/13] fix interaction between Task & Task2; add python.Placeholder Any OnTask plugin that changes Task2.request has no effect because that field is not taken into account in Task2.statements. Fixing this by introducing python.Placeholder, a syntax node representing a non-syntax node (like a Request instance, in our case). Special care has to be taken when the Task being converted into a Task2 already has a Task.locust_request, which is kind of a copy of Task.request with a different interface. Fix #31. Signed-off-by: Thibaut Le Page --- poetry.lock | 11 +- pyproject.toml | 1 + transformer/plugins/sanitize_headers.py | 17 +-- transformer/python.py | 44 +++++++ transformer/scenario.py | 1 - transformer/task.py | 150 +++++++++++++++--------- 6 files changed, 157 insertions(+), 67 deletions(-) diff --git a/poetry.lock b/poetry.lock index b3d0ebe..57a0ae7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -97,6 +97,14 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, <4" version = "4.5.2" +[[package]] +category = "main" +description = "A backport of the dataclasses module for Python 3.6" +name = "dataclasses" +optional = false +python-versions = "*" +version = "0.6" + [[package]] category = "main" description = "Pythonic argument parser, that will make you smile" @@ -488,7 +496,7 @@ python-versions = "*" version = "0.14.1" [metadata] -content-hash = "07875f7d656cdf2a43285874140662ab7ac017bfc395f0619fad99f246fcf71a" +content-hash = "b571ac733fdcaec72508313548443753bb6d648385ef50aaa26af52a8e1a1594" python-versions = "^3.6" [metadata.hashes] @@ -503,6 +511,7 @@ chevron = ["95b0a055ef0ada5eb061d60be64a7f70670b53372ccd221d1b88adf1c41a9094", " click = ["2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", "5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"] colorama = ["05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", "f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"] coverage = ["06123b58a1410873e22134ca2d88bd36680479fe354955b3579fb8ff150e4d27", "09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", "0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", "0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", "0d34245f824cc3140150ab7848d08b7e2ba67ada959d77619c986f2062e1f0e8", "10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", "1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", "1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", "258b21c5cafb0c3768861a6df3ab0cfb4d8b495eee5ec660e16f928bf7385390", "2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", "3ad59c84c502cd134b0088ca9038d100e8fb5081bbd5ccca4863f3804d81f61d", "447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", "46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", "4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", "510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", "5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", "5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", "5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", "6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", "6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", "71afc1f5cd72ab97330126b566bbf4e8661aab7449f08895d21a5d08c6b051ff", "7349c27128334f787ae63ab49d90bf6d47c7288c63a0a5dfaa319d4b4541dd2c", "77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", "828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", "859714036274a75e6e57c7bab0c47a4602d2a8cfaaa33bbdb68c8359b2ed4f5c", "85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", "869ef4a19f6e4c6987e18b315721b8b971f7048e6eaea29c066854242b4e98d9", "8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", "977e2d9a646773cc7428cdd9a34b069d6ee254fadfb4d09b3f430e95472f3cf3", "99bd767c49c775b79fdcd2eabff405f1063d9d959039c0bdd720527a7738748a", "a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", "aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", "ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", "b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", "bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", "c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", "d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", "d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", "da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", "ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", "ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9"] +dataclasses = ["454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f", "6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84"] docopt = ["49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"] ecological = ["536cca0c6d6f0a85dcf3c825ddc31189a5c78428b6deed0eac00637d450d4f89", "ed0f6e5a6ae5bd0db32591c0ed0cb62816ae5e0f0a0677ce1192ebf00b7d83eb"] entrypoints = ["589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", "c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"] diff --git a/pyproject.toml b/pyproject.toml index b33de7d..4c1a675 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ pendulum = "^2.0" chevron = "^0.13" docopt = "^0.6.2" ecological = "^1.6" +dataclasses = "^0.6.0" [tool.poetry.dev-dependencies] locustio = "^0.9.0" diff --git a/transformer/plugins/sanitize_headers.py b/transformer/plugins/sanitize_headers.py index 588d895..06df633 100644 --- a/transformer/plugins/sanitize_headers.py +++ b/transformer/plugins/sanitize_headers.py @@ -1,5 +1,5 @@ -from transformer.helpers import zip_kv_pairs from transformer.plugins import plugin, Contract +from transformer.request import Header from transformer.task import Task2 @@ -10,16 +10,11 @@ def plugin(task: Task2) -> Task2: Converts header names to lowercase to simplify further overriding. Removes the cookie header as it is handled by Locust's HttpSession. """ - headers = task.request.headers - - if not isinstance(headers, dict): - headers = zip_kv_pairs(headers) - - sanitized_headers = { - k.lower(): v - for (k, v) in headers.items() - if not k.startswith(":") and k.lower() != "cookie" - } + sanitized_headers = [ + Header(name=h.name.lower(), value=h.value) + for h in task.request.headers + if not h.name.startswith(":") and h.name.lower() != "cookie" + ] task.request = task.request._replace(headers=sanitized_headers) diff --git a/transformer/python.py b/transformer/python.py index 5a15f6c..5d4a0bc 100644 --- a/transformer/python.py +++ b/transformer/python.py @@ -11,8 +11,12 @@ Tuple, cast, Iterable, + Callable, + TypeVar, ) +from dataclasses import dataclass + IMMUTABLE_EMPTY_DICT = MappingProxyType({}) @@ -686,3 +690,43 @@ def __repr__(self) -> str: self.alias, self.comments, ) + + +_T = TypeVar("_T") + + +@dataclass +class Placeholder(Expression): + """ + The promise of an Expression representing an object currently not in + Expression format. + + The Placeholder allows to mix non-Expression objects in the syntax tree, + along with a function capable of transforming these objects into actual + Expression objects at any time. + This is useful when there is a simpler representation than Expression. + + For instance, any Request object can be converted into an equivalent + Expression, but Request has a simpler API than Expression for all + request-oriented operations like accessing the URL, etc. + Embedding a Request in a Placeholder allows to pretend that the Request is + already in Expression format (with all associated benefits) but still use + the Request API. + + `target` is a callable returning the non-Expression object. That callable + allows to specify as target some mutable field of an object, rather than a + fixed reference to an object. See for example task.Task2, which contains a + Placeholder to its own "request" field; if the value of that field is + changed, the Placeholder will refer to the new value instead of keeping a + reference to the old value. + + `name` is purely descriptive: it can make inspection of data structures + containing Placeholder objects more comfortable. + """ + + name: str + target: Callable[[], _T] + converter: Callable[[_T], Expression] + + def __str__(self) -> str: + return str(self.converter(self.target())) diff --git a/transformer/scenario.py b/transformer/scenario.py index 08da472..abd7cd3 100644 --- a/transformer/scenario.py +++ b/transformer/scenario.py @@ -14,7 +14,6 @@ NamedTuple, ) - import transformer.plugins as plug from transformer.naming import to_identifier from transformer.plugins.contracts import Plugin diff --git a/transformer/task.py b/transformer/task.py index 5e88067..253bd27 100644 --- a/transformer/task.py +++ b/transformer/task.py @@ -5,7 +5,18 @@ import json from types import MappingProxyType -from typing import Iterable, NamedTuple, Iterator, Sequence, Optional, Mapping, List +from typing import ( + Iterable, + NamedTuple, + Iterator, + Sequence, + Optional, + Mapping, + List, + Dict, +) + +from dataclasses import dataclass import transformer.python as py from transformer.blacklist import on_blacklist @@ -38,45 +49,21 @@ def from_request(cls, r: Request) -> "LocustRequest": query=r.query, ) - NOOP_HTTP_METHODS = {HttpMethod.GET, HttpMethod.OPTIONS, HttpMethod.DELETE} - - def as_locust_action(self) -> str: - args = { - "url": self.url, - "name": self.url, - "headers": self.headers, - "timeout": TIMEOUT, - "allow_redirects": False, - } - if self.method is HttpMethod.POST: - post_data = _parse_post_data(self.post_data) - args[post_data["key"]] = post_data["data"] - elif self.method is HttpMethod.PUT: - post_data = _parse_post_data(self.post_data) - args["params"] = zip_kv_pairs(self.query) - args[post_data["key"]] = post_data["data"] - elif self.method not in self.NOOP_HTTP_METHODS: - raise ValueError(f"unsupported HTTP method: {self.method!r}") - - method = self.method.name.lower() - named_args = ", ".join(f"{k}={v}" for k, v in args.items()) - return f"response = self.client.{method}({named_args})" - +@dataclass class Task2: - def __init__( - self, - name: str, - request: Request, - statements: Sequence[py.Statement] = (), - # TODO: Replace me with a plugin framework that accesses the full tree. - # See https://github.com/zalando-incubator/Transformer/issues/11. - global_code_blocks: Mapping[str, Sequence[str]] = IMMUTABLE_EMPTY_DICT, - ) -> None: - self.name = name - self.request = request - self.statements = list(statements) - self.global_code_blocks = {k: list(v) for k, v in global_code_blocks.items()} + name: str + request: Request + statements: Sequence[py.Statement] = () + # TODO: Replace me with a plugin framework that accesses the full tree. + # See https://github.com/zalando-incubator/Transformer/issues/11. + global_code_blocks: Mapping[str, Sequence[str]] = IMMUTABLE_EMPTY_DICT + + def __post_init__(self,) -> None: + self.statements = list(self.statements) + self.global_code_blocks = { + k: list(v) for k, v in self.global_code_blocks.items() + } @classmethod def from_requests(cls, requests: Iterable[Request]) -> Iterator["Task2"]: @@ -88,7 +75,8 @@ def from_requests(cls, requests: Iterable[Request]) -> Iterator["Task2"]: corresponding request. """ # TODO: Update me when merging Task with Task2: "statements" needs to - # contain the equivalent of LocustRequest. + # contain a Placeholder to Task2.request. + # See what is done in from_task (but without the LocustRequest part). # See https://github.com/zalando-incubator/Transformer/issues/11. for req in sorted(requests, key=lambda r: r.timestamp): if not on_blacklist(req.url.netloc): @@ -99,21 +87,75 @@ def from_task(cls, task: "Task") -> "Task2": # TODO: Remove me as soon as the old Task is no longer used and Task2 is # renamed to Task. # See https://github.com/zalando-incubator/Transformer/issues/11. - locust_request = task.locust_request - if locust_request is None: - locust_request = LocustRequest.from_request(task.request) - return cls( - name=task.name, - request=task.request, - statements=[ - py.OpaqueBlock(block) - for block in [ - *task.locust_preprocessing, - locust_request.as_locust_action(), - *task.locust_postprocessing, - ] - ], - ) + t = cls(name=task.name, request=task.request) + if task.locust_request: + placeholder = py.Placeholder( + name="this task's request field", + target=lambda: task.locust_request, + converter=lreq_to_expr, + ) + else: + placeholder = py.Placeholder( + name="this task's request field", + target=lambda: t.request, + converter=req_to_expr, + ) + t.statements = [ + *[py.OpaqueBlock(x) for x in task.locust_preprocessing], + py.Assignment("response", placeholder), + *[py.OpaqueBlock(x) for x in task.locust_postprocessing], + ] + return t + + +NOOP_HTTP_METHODS = {HttpMethod.GET, HttpMethod.OPTIONS, HttpMethod.DELETE} + + +def req_to_expr(r: Request) -> py.FunctionCall: + url = py.Literal(str(r.url.geturl())) + args: Dict[str, py.Expression] = { + "url": url, + "name": url, + "headers": py.Literal(zip_kv_pairs(r.headers)), + "timeout": py.Literal(TIMEOUT), + "allow_redirects": py.Symbol("False"), + } + if r.method is HttpMethod.POST: + post_data = _parse_post_data(r.post_data) + args[post_data["key"]] = post_data["data"] + elif r.method is HttpMethod.PUT: + post_data = _parse_post_data(r.post_data) + args[post_data["key"]] = post_data["data"] + args["params"] = zip_kv_pairs(r.query) + elif r.method not in NOOP_HTTP_METHODS: + raise ValueError(f"unsupported HTTP method: {r.method!r}") + + method = r.method.name.lower() + return py.FunctionCall(name=f"self.client.{method}", named_args=args) + + +def lreq_to_expr(lr: LocustRequest) -> py.FunctionCall: + # TODO: Remove me once LocustRequest no longer exists. + # See https://github.com/zalando-incubator/Transformer/issues/11. + args = { + "url": lr.url, + "name": lr.url, + "headers": lr.headers, + "timeout": TIMEOUT, + "allow_redirects": False, + } + if lr.method is HttpMethod.POST: + post_data = _parse_post_data(lr.post_data) + args[post_data["key"]] = post_data["data"] + elif lr.method is HttpMethod.PUT: + post_data = _parse_post_data(lr.post_data) + args["params"] = zip_kv_pairs(lr.query) + args[post_data["key"]] = post_data["data"] + elif lr.method not in NOOP_HTTP_METHODS: + raise ValueError(f"unsupported HTTP method: {lr.method!r}") + + method = lr.method.name.lower() + return py.FunctionCall(name=f"self.client.{method}", named_args=args) class Task(NamedTuple): From d0bef76f04b4fcba5ad5934a0e6a3401d8f14c90 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Sun, 24 Feb 2019 16:06:34 +0100 Subject: [PATCH 02/13] test_sanitize_headers: fix test It was relying on a type error all along, because Request.headers is supposed to be a List[Header], not a dict. Signed-off-by: Thibaut Le Page --- transformer/plugins/test_sanitize_headers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformer/plugins/test_sanitize_headers.py b/transformer/plugins/test_sanitize_headers.py index f76e475..bd565db 100644 --- a/transformer/plugins/test_sanitize_headers.py +++ b/transformer/plugins/test_sanitize_headers.py @@ -38,7 +38,8 @@ def test_it_removes_headers_beginning_with_a_colon(): def test_it_downcases_header_names(): task = task_with_header("Some Name", "some value") sanitized_headers = plugin(task).request.headers - assert "some name" in sanitized_headers + header_names = {h.name for h in sanitized_headers} + assert "some name" in header_names def test_it_removes_cookies(): From 14c16ff918d5fbc8d1b205f3528b7215f4cf56d9 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Sun, 24 Feb 2019 16:42:52 +0100 Subject: [PATCH 03/13] fix LocustRequest handling in Task2.from_task; add tests Signed-off-by: Thibaut Le Page --- transformer/task.py | 36 +-- transformer/test_task.py | 547 ++++++++++++++++++++++----------------- 2 files changed, 332 insertions(+), 251 deletions(-) diff --git a/transformer/task.py b/transformer/task.py index 253bd27..d124b72 100644 --- a/transformer/task.py +++ b/transformer/task.py @@ -4,6 +4,7 @@ """ import json +from collections import OrderedDict from types import MappingProxyType from typing import ( Iterable, @@ -113,13 +114,13 @@ def from_task(cls, task: "Task") -> "Task2": def req_to_expr(r: Request) -> py.FunctionCall: url = py.Literal(str(r.url.geturl())) - args: Dict[str, py.Expression] = { - "url": url, - "name": url, - "headers": py.Literal(zip_kv_pairs(r.headers)), - "timeout": py.Literal(TIMEOUT), - "allow_redirects": py.Symbol("False"), - } + args: Dict[str, py.Expression] = OrderedDict( + url=url, + name=url, + headers=py.Literal(zip_kv_pairs(r.headers)), + timeout=py.Literal(TIMEOUT), + allow_redirects=py.Symbol("False"), + ) if r.method is HttpMethod.POST: post_data = _parse_post_data(r.post_data) args[post_data["key"]] = post_data["data"] @@ -137,19 +138,24 @@ def req_to_expr(r: Request) -> py.FunctionCall: def lreq_to_expr(lr: LocustRequest) -> py.FunctionCall: # TODO: Remove me once LocustRequest no longer exists. # See https://github.com/zalando-incubator/Transformer/issues/11. - args = { - "url": lr.url, - "name": lr.url, - "headers": lr.headers, - "timeout": TIMEOUT, - "allow_redirects": False, - } + if lr.url.startswith("f"): + url = py.FString(lr.url[2:-1]) + else: + url = py.Literal(lr.url[1:-1]) + + args: Dict[str, py.Expression] = OrderedDict( + url=url, + name=url, + headers=py.Literal(lr.headers), + timeout=py.Literal(TIMEOUT), + allow_redirects=py.Symbol("False"), + ) if lr.method is HttpMethod.POST: post_data = _parse_post_data(lr.post_data) args[post_data["key"]] = post_data["data"] elif lr.method is HttpMethod.PUT: - post_data = _parse_post_data(lr.post_data) args["params"] = zip_kv_pairs(lr.query) + post_data = _parse_post_data(lr.post_data) args[post_data["key"]] = post_data["data"] elif lr.method not in NOOP_HTTP_METHODS: raise ValueError(f"unsupported HTTP method: {lr.method!r}") diff --git a/transformer/test_task.py b/transformer/test_task.py index c20554a..b2e2265 100644 --- a/transformer/test_task.py +++ b/transformer/test_task.py @@ -2,6 +2,7 @@ import io import json +from datetime import datetime from unittest.mock import MagicMock from unittest.mock import patch from urllib.parse import urlparse @@ -9,6 +10,7 @@ import pytest from transformer.request import Header +from transformer import python as py from transformer.task import ( Task, Request, @@ -16,242 +18,315 @@ QueryPair, TIMEOUT, LocustRequest, + Task2, ) -class TestFromRequests: - def test_it_returns_a_task(self): - request = MagicMock() - request.timestamp = 1 - second_request = MagicMock() - second_request.timestamp = 2 - assert all( - isinstance(t, Task) for t in Task.from_requests([request, second_request]) - ) - - @patch("builtins.open") - def test_it_doesnt_create_a_task_if_the_url_is_on_the_blacklist(self, mock_open): - mock_open.return_value = io.StringIO("amazon") - request = MagicMock() - request.url = MagicMock() - request.url.netloc = "www.amazon.com" - task = Task.from_requests([request]) - assert len(list(task)) == 0 - - @patch("builtins.open") - def test_it_creates_a_task_if_the_path_not_host_is_on_the_blacklist( - self, mock_open - ): - mock_open.return_value = io.StringIO("search\namazon") - request = MagicMock() - request.url = urlparse("https://www.google.com/search?&q=amazon") - task = Task.from_requests([request]) - assert len(list(task)) == 1 - - -class TestAsLocustAction: - def test_it_returns_an_error_given_an_unsupported_http_method(self): - a_request_with_an_unsupported_http_method = MagicMock() - task = Task("some_task", a_request_with_an_unsupported_http_method) - with pytest.raises(ValueError): - task.as_locust_action() - - def test_it_returns_a_string(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - assert isinstance(task.as_locust_action(), str) - - def test_it_returns_action_from_locust_request(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - locust_request = LocustRequest( - method=HttpMethod.GET, url=repr("http://locust-task"), headers={} - ) - task = Task("some_task", request=a_request, locust_request=locust_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.get(url='http://locust-task'") - - def test_it_returns_task_using_get_given_a_get_http_method(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.get(") - - def test_it_returns_a_task_using_post_given_a_post_http_method(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.POST - a_request.post_data = {} - task = Task("some_task", a_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.post(") - - def test_it_returns_a_task_using_put_given_a_put_http_method(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.PUT - a_request.post_data = {"text": "{'some key': 'some value'}"} - a_request.query = [QueryPair(name="some name", value="some value")] - task = Task("some_task", a_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.put(") - assert "params={'some name': 'some value'}" in action - assert "data=b\"{'some key': 'some value'}\"" in action - - def test_it_returns_a_task_using_options_given_an_options_http_method(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.OPTIONS - a_request.headers = [Header(name="Access-Control-Request-Method", value="POST")] - task = Task("some_task", a_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.options(") - assert "headers={'Access-Control-Request-Method': 'POST'" in action - - def test_it_returns_a_task_using_delete_given_a_delete_http_method(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.DELETE - a_request.url = urlparse("http://www.some.web.site/?some_name=some_value") - task = Task("some_task", a_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.delete(") - assert "?some_name=some_value" in action - - def test_it_provides_timeout_to_requests(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - action = task.as_locust_action() - assert f"timeout={TIMEOUT}" in action - - def test_it_injects_headers(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - a_request.headers = [Header(name="some_header", value="some_value")] - task = Task("some_task", a_request) - action = task.as_locust_action() - assert "some_value" in action - - def test_it_encodes_data_in_task_for_text_mime(self): - decoded_value = '{"formatted": "54,95 €"}' - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.POST - a_request.post_data = {"text": decoded_value} - task = Task("some_task", a_request) - action = task.as_locust_action() - assert str(decoded_value.encode()) in action - - def test_it_encodes_data_in_task_for_json_mime(self): - decoded_value = '{"formatted": "54,95 €"}' - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.POST - a_request.post_data = {"text": decoded_value, "mimeType": "application/json"} - task = Task("some_task", a_request) - action = task.as_locust_action() - assert str(json.loads(decoded_value)) in action - - def test_it_converts_post_params_to_post_text(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.POST - a_request.post_data = { - "mimeType": "application/json", - "params": [ - {"name": "username", "value": "some user"}, - {"name": "password", "value": "some password"}, - ], - } - task = Task("some task", a_request) - action = task.as_locust_action() - assert "'username': 'some user'" in action - assert "'password': 'some password'" in action - - def test_it_creates_a_locust_request_when_there_is_none(self): - task = Task(name="some name", request=MagicMock()) - - modified_task = Task.inject_headers(task, {}) - - assert modified_task.locust_request - - def test_it_returns_a_task_with_the_injected_headers(self): - locust_request = LocustRequest( - method=MagicMock(), url=MagicMock(), headers={"x-forwarded-for": ""} - ) - task = Task( - name="some name", request=MagicMock(), locust_request=locust_request - ) - expected_headers = {"x-forwarded-for": "1.2.3.4"} - modified_task = Task.inject_headers(task, headers=expected_headers) - - assert isinstance(modified_task, Task) - - headers = modified_task.locust_request.headers - assert len(headers) == 1 - assert headers == expected_headers - - -class TestIndentation: - def test_pre_processing_returns_an_indented_string_given_an_indentation(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - new_pre_processings = (*task.locust_preprocessing, "def some_function():") - task = task._replace(locust_preprocessing=new_pre_processings) - action = task.as_locust_action(indentation=2) - assert action.startswith(" def some_function():") - - def test_post_processing_returns_an_indented_string_given_an_indentation(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - new_post_processings = (*task.locust_postprocessing, "def some_function():") - task = task._replace(locust_postprocessing=new_post_processings) - action = task.as_locust_action(indentation=2) - assert " def some_function():" in action - - def test_it_applies_indentation_to_all_pre_processings(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - new_pre_processings = ( - *task.locust_preprocessing, - "def some_function():", - "def some_other_function():", - ) - task = task._replace(locust_preprocessing=new_pre_processings) - action = task.as_locust_action(indentation=2) - assert action.startswith( - " def some_function():\n\n def some_other_function():" - ) - - def test_it_respects_sub_indentation_levels(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - new_pre_processings = ( - *task.locust_preprocessing, - "\n def function():\n if True:\n print(True)", - ) - task = task._replace(locust_preprocessing=new_pre_processings) - action = task.as_locust_action(indentation=1) - assert action.startswith(" \n def function():\n if True:\n print(True)") - - -class TestReplaceURL: - def test_it_creates_a_locust_request_when_there_is_none(self): - task = Task(name="some name", request=MagicMock()) - - modified_task = Task.replace_url(task, "") - - assert modified_task.locust_request - - def test_it_returns_a_task_with_the_replaced_url(self): - locust_request = LocustRequest( - method=MagicMock(), url=MagicMock(), headers=MagicMock() - ) - task = Task( - name="some name", request=MagicMock(), locust_request=locust_request - ) - expected_url = 'f"http://a.b.c/{some.value}/"' - - modified_task = Task.replace_url(task, expected_url) - - assert modified_task.locust_request.url == expected_url +class TestTask: + class TestFromRequests: + def test_it_returns_a_task(self): + request = MagicMock() + request.timestamp = 1 + second_request = MagicMock() + second_request.timestamp = 2 + assert all( + isinstance(t, Task) + for t in Task.from_requests([request, second_request]) + ) + + @patch("builtins.open") + def test_it_doesnt_create_a_task_if_the_url_is_on_the_blacklist( + self, mock_open + ): + mock_open.return_value = io.StringIO("amazon") + request = MagicMock() + request.url = MagicMock() + request.url.netloc = "www.amazon.com" + task = Task.from_requests([request]) + assert len(list(task)) == 0 + + @patch("builtins.open") + def test_it_creates_a_task_if_the_path_not_host_is_on_the_blacklist( + self, mock_open + ): + mock_open.return_value = io.StringIO("search\namazon") + request = MagicMock() + request.url = urlparse("https://www.google.com/search?&q=amazon") + task = Task.from_requests([request]) + assert len(list(task)) == 1 + + class TestAsLocustAction: + def test_it_returns_an_error_given_an_unsupported_http_method(self): + a_request_with_an_unsupported_http_method = MagicMock() + task = Task("some_task", a_request_with_an_unsupported_http_method) + with pytest.raises(ValueError): + task.as_locust_action() + + def test_it_returns_a_string(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.GET + task = Task("some_task", a_request) + assert isinstance(task.as_locust_action(), str) + + def test_it_returns_action_from_locust_request(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.GET + locust_request = LocustRequest( + method=HttpMethod.GET, url=repr("http://locust-task"), headers={} + ) + task = Task("some_task", request=a_request, locust_request=locust_request) + action = task.as_locust_action() + assert action.startswith( + "response = self.client.get(url='http://locust-task'" + ) + + def test_it_returns_task_using_get_given_a_get_http_method(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.GET + task = Task("some_task", a_request) + action = task.as_locust_action() + assert action.startswith("response = self.client.get(") + + def test_it_returns_a_task_using_post_given_a_post_http_method(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.POST + a_request.post_data = {} + task = Task("some_task", a_request) + action = task.as_locust_action() + assert action.startswith("response = self.client.post(") + + def test_it_returns_a_task_using_put_given_a_put_http_method(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.PUT + a_request.post_data = {"text": "{'some key': 'some value'}"} + a_request.query = [QueryPair(name="some name", value="some value")] + task = Task("some_task", a_request) + action = task.as_locust_action() + assert action.startswith("response = self.client.put(") + assert "params={'some name': 'some value'}" in action + assert "data=b\"{'some key': 'some value'}\"" in action + + def test_it_returns_a_task_using_options_given_an_options_http_method(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.OPTIONS + a_request.headers = [ + Header(name="Access-Control-Request-Method", value="POST") + ] + task = Task("some_task", a_request) + action = task.as_locust_action() + assert action.startswith("response = self.client.options(") + assert "headers={'Access-Control-Request-Method': 'POST'" in action + + def test_it_returns_a_task_using_delete_given_a_delete_http_method(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.DELETE + a_request.url = urlparse("http://www.some.web.site/?some_name=some_value") + task = Task("some_task", a_request) + action = task.as_locust_action() + assert action.startswith("response = self.client.delete(") + assert "?some_name=some_value" in action + + def test_it_provides_timeout_to_requests(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.GET + task = Task("some_task", a_request) + action = task.as_locust_action() + assert f"timeout={TIMEOUT}" in action + + def test_it_injects_headers(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.GET + a_request.headers = [Header(name="some_header", value="some_value")] + task = Task("some_task", a_request) + action = task.as_locust_action() + assert "some_value" in action + + def test_it_encodes_data_in_task_for_text_mime(self): + decoded_value = '{"formatted": "54,95 €"}' + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.POST + a_request.post_data = {"text": decoded_value} + task = Task("some_task", a_request) + action = task.as_locust_action() + assert str(decoded_value.encode()) in action + + def test_it_encodes_data_in_task_for_json_mime(self): + decoded_value = '{"formatted": "54,95 €"}' + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.POST + a_request.post_data = { + "text": decoded_value, + "mimeType": "application/json", + } + task = Task("some_task", a_request) + action = task.as_locust_action() + assert str(json.loads(decoded_value)) in action + + def test_it_converts_post_params_to_post_text(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.POST + a_request.post_data = { + "mimeType": "application/json", + "params": [ + {"name": "username", "value": "some user"}, + {"name": "password", "value": "some password"}, + ], + } + task = Task("some task", a_request) + action = task.as_locust_action() + assert "'username': 'some user'" in action + assert "'password': 'some password'" in action + + def test_it_creates_a_locust_request_when_there_is_none(self): + task = Task(name="some name", request=MagicMock()) + + modified_task = Task.inject_headers(task, {}) + + assert modified_task.locust_request + + def test_it_returns_a_task_with_the_injected_headers(self): + locust_request = LocustRequest( + method=MagicMock(), url=MagicMock(), headers={"x-forwarded-for": ""} + ) + task = Task( + name="some name", request=MagicMock(), locust_request=locust_request + ) + expected_headers = {"x-forwarded-for": "1.2.3.4"} + modified_task = Task.inject_headers(task, headers=expected_headers) + + assert isinstance(modified_task, Task) + + headers = modified_task.locust_request.headers + assert len(headers) == 1 + assert headers == expected_headers + + class TestIndentation: + def test_pre_processing_returns_an_indented_string_given_an_indentation(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.GET + task = Task("some_task", a_request) + new_pre_processings = (*task.locust_preprocessing, "def some_function():") + task = task._replace(locust_preprocessing=new_pre_processings) + action = task.as_locust_action(indentation=2) + assert action.startswith(" def some_function():") + + def test_post_processing_returns_an_indented_string_given_an_indentation(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.GET + task = Task("some_task", a_request) + new_post_processings = (*task.locust_postprocessing, "def some_function():") + task = task._replace(locust_postprocessing=new_post_processings) + action = task.as_locust_action(indentation=2) + assert " def some_function():" in action + + def test_it_applies_indentation_to_all_pre_processings(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.GET + task = Task("some_task", a_request) + new_pre_processings = ( + *task.locust_preprocessing, + "def some_function():", + "def some_other_function():", + ) + task = task._replace(locust_preprocessing=new_pre_processings) + action = task.as_locust_action(indentation=2) + assert action.startswith( + " def some_function():\n\n def some_other_function():" + ) + + def test_it_respects_sub_indentation_levels(self): + a_request = MagicMock(spec_set=Request) + a_request.method = HttpMethod.GET + task = Task("some_task", a_request) + new_pre_processings = ( + *task.locust_preprocessing, + "\n def function():\n if True:\n print(True)", + ) + task = task._replace(locust_preprocessing=new_pre_processings) + action = task.as_locust_action(indentation=1) + assert action.startswith(" \n def function():\n if True:\n print(True)") + + class TestReplaceURL: + def test_it_creates_a_locust_request_when_there_is_none(self): + task = Task(name="some name", request=MagicMock()) + + modified_task = Task.replace_url(task, "") + + assert modified_task.locust_request + + def test_it_returns_a_task_with_the_replaced_url(self): + locust_request = LocustRequest( + method=MagicMock(), url=MagicMock(), headers=MagicMock() + ) + task = Task( + name="some name", request=MagicMock(), locust_request=locust_request + ) + expected_url = 'f"http://a.b.c/{some.value}/"' + + modified_task = Task.replace_url(task, expected_url) + + assert modified_task.locust_request.url == expected_url + + +class TestTask2: + class TestFromTask: + def test_without_locust_request(self): + url = "https://abc.de" + req = Request( + timestamp=datetime.now(), + method=HttpMethod.GET, + url=urlparse(url), + headers=[Header("a", "b")], + post_data={}, + query=[], + ) + task = Task(name="T", request=req) + task2 = Task2.from_task(task) + + assert task2.name == "T" + assert task2.request == req + assert len(task2.statements) == 1 + + assign = task2.statements[0] + assert isinstance(assign, py.Assignment) + assert assign.lhs == "response" + + assert isinstance(assign.rhs, py.Placeholder) + assert assign.rhs.target() == task2.request + + assert str(assign.rhs) == ( + f"self.client.get(url={url!r}, name={url!r}," + f" headers={{'a': 'b'}}, timeout={TIMEOUT}, allow_redirects=False)" + ) + + def test_with_locust_request(self): + url = "https://abc.de" + req = Request( + timestamp=datetime.now(), + method=HttpMethod.GET, + url=urlparse(url), + headers=[Header("a", "b")], + post_data={}, + query=[], + ) + lr = LocustRequest.from_request(req) + lr = lr._replace(url="f" + lr.url.replace("de", "{tld}")) + task = Task(name="T", request=req, locust_request=lr) + task2 = Task2.from_task(task) + + assert task2.name == "T" + assert task2.request == req + assert len(task2.statements) == 1 + + assign = task2.statements[0] + assert isinstance(assign, py.Assignment) + assert assign.lhs == "response" + + assert isinstance(assign.rhs, py.Placeholder) + assert assign.rhs.target() == lr + + expected_url = """ f'https://abc.{tld}' """.strip() + assert str(assign.rhs) == ( + f"self.client.get(url={expected_url}, name={expected_url}," + f" headers={{'a': 'b'}}, timeout={TIMEOUT}, allow_redirects=False)" + ) From 04df83bb9eb161e611d28d1c7627396a9e06e3b3 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Sun, 24 Feb 2019 16:46:04 +0100 Subject: [PATCH 04/13] remove Task.as_locust_action: unused; remove associated tests Signed-off-by: Thibaut Le Page --- transformer/task.py | 32 +------ transformer/test_task.py | 191 --------------------------------------- 2 files changed, 1 insertion(+), 222 deletions(-) diff --git a/transformer/task.py b/transformer/task.py index d124b72..9d15bc1 100644 --- a/transformer/task.py +++ b/transformer/task.py @@ -6,16 +6,7 @@ import json from collections import OrderedDict from types import MappingProxyType -from typing import ( - Iterable, - NamedTuple, - Iterator, - Sequence, - Optional, - Mapping, - List, - Dict, -) +from typing import Iterable, NamedTuple, Iterator, Sequence, Optional, Mapping, Dict from dataclasses import dataclass @@ -189,27 +180,6 @@ def from_requests(cls, requests: Iterable[Request]) -> Iterator["Task"]: else: yield cls(name=req.task_name(), request=req) - def as_locust_action(self, indentation=ACTION_INDENTATION_LEVEL) -> str: - """ - Converts a Task into a Locust Action. - """ - action: List[str] = [] - - for preprocessing in self.locust_preprocessing: - action.append(_indent(preprocessing, indentation)) - - if self.locust_request is None: - locust_request = LocustRequest.from_request(self.request) - else: - locust_request = self.locust_request - - action.append(locust_request.as_locust_action()) - - for postprocessing in self.locust_postprocessing: - action.append(_indent(postprocessing, indentation)) - - return "\n".join(action) - def inject_headers(self, headers: dict): if self.locust_request is None: original_locust_request = LocustRequest.from_request(self.request) diff --git a/transformer/test_task.py b/transformer/test_task.py index b2e2265..edcfeaf 100644 --- a/transformer/test_task.py +++ b/transformer/test_task.py @@ -55,197 +55,6 @@ def test_it_creates_a_task_if_the_path_not_host_is_on_the_blacklist( task = Task.from_requests([request]) assert len(list(task)) == 1 - class TestAsLocustAction: - def test_it_returns_an_error_given_an_unsupported_http_method(self): - a_request_with_an_unsupported_http_method = MagicMock() - task = Task("some_task", a_request_with_an_unsupported_http_method) - with pytest.raises(ValueError): - task.as_locust_action() - - def test_it_returns_a_string(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - assert isinstance(task.as_locust_action(), str) - - def test_it_returns_action_from_locust_request(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - locust_request = LocustRequest( - method=HttpMethod.GET, url=repr("http://locust-task"), headers={} - ) - task = Task("some_task", request=a_request, locust_request=locust_request) - action = task.as_locust_action() - assert action.startswith( - "response = self.client.get(url='http://locust-task'" - ) - - def test_it_returns_task_using_get_given_a_get_http_method(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.get(") - - def test_it_returns_a_task_using_post_given_a_post_http_method(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.POST - a_request.post_data = {} - task = Task("some_task", a_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.post(") - - def test_it_returns_a_task_using_put_given_a_put_http_method(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.PUT - a_request.post_data = {"text": "{'some key': 'some value'}"} - a_request.query = [QueryPair(name="some name", value="some value")] - task = Task("some_task", a_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.put(") - assert "params={'some name': 'some value'}" in action - assert "data=b\"{'some key': 'some value'}\"" in action - - def test_it_returns_a_task_using_options_given_an_options_http_method(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.OPTIONS - a_request.headers = [ - Header(name="Access-Control-Request-Method", value="POST") - ] - task = Task("some_task", a_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.options(") - assert "headers={'Access-Control-Request-Method': 'POST'" in action - - def test_it_returns_a_task_using_delete_given_a_delete_http_method(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.DELETE - a_request.url = urlparse("http://www.some.web.site/?some_name=some_value") - task = Task("some_task", a_request) - action = task.as_locust_action() - assert action.startswith("response = self.client.delete(") - assert "?some_name=some_value" in action - - def test_it_provides_timeout_to_requests(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - action = task.as_locust_action() - assert f"timeout={TIMEOUT}" in action - - def test_it_injects_headers(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - a_request.headers = [Header(name="some_header", value="some_value")] - task = Task("some_task", a_request) - action = task.as_locust_action() - assert "some_value" in action - - def test_it_encodes_data_in_task_for_text_mime(self): - decoded_value = '{"formatted": "54,95 €"}' - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.POST - a_request.post_data = {"text": decoded_value} - task = Task("some_task", a_request) - action = task.as_locust_action() - assert str(decoded_value.encode()) in action - - def test_it_encodes_data_in_task_for_json_mime(self): - decoded_value = '{"formatted": "54,95 €"}' - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.POST - a_request.post_data = { - "text": decoded_value, - "mimeType": "application/json", - } - task = Task("some_task", a_request) - action = task.as_locust_action() - assert str(json.loads(decoded_value)) in action - - def test_it_converts_post_params_to_post_text(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.POST - a_request.post_data = { - "mimeType": "application/json", - "params": [ - {"name": "username", "value": "some user"}, - {"name": "password", "value": "some password"}, - ], - } - task = Task("some task", a_request) - action = task.as_locust_action() - assert "'username': 'some user'" in action - assert "'password': 'some password'" in action - - def test_it_creates_a_locust_request_when_there_is_none(self): - task = Task(name="some name", request=MagicMock()) - - modified_task = Task.inject_headers(task, {}) - - assert modified_task.locust_request - - def test_it_returns_a_task_with_the_injected_headers(self): - locust_request = LocustRequest( - method=MagicMock(), url=MagicMock(), headers={"x-forwarded-for": ""} - ) - task = Task( - name="some name", request=MagicMock(), locust_request=locust_request - ) - expected_headers = {"x-forwarded-for": "1.2.3.4"} - modified_task = Task.inject_headers(task, headers=expected_headers) - - assert isinstance(modified_task, Task) - - headers = modified_task.locust_request.headers - assert len(headers) == 1 - assert headers == expected_headers - - class TestIndentation: - def test_pre_processing_returns_an_indented_string_given_an_indentation(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - new_pre_processings = (*task.locust_preprocessing, "def some_function():") - task = task._replace(locust_preprocessing=new_pre_processings) - action = task.as_locust_action(indentation=2) - assert action.startswith(" def some_function():") - - def test_post_processing_returns_an_indented_string_given_an_indentation(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - new_post_processings = (*task.locust_postprocessing, "def some_function():") - task = task._replace(locust_postprocessing=new_post_processings) - action = task.as_locust_action(indentation=2) - assert " def some_function():" in action - - def test_it_applies_indentation_to_all_pre_processings(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - new_pre_processings = ( - *task.locust_preprocessing, - "def some_function():", - "def some_other_function():", - ) - task = task._replace(locust_preprocessing=new_pre_processings) - action = task.as_locust_action(indentation=2) - assert action.startswith( - " def some_function():\n\n def some_other_function():" - ) - - def test_it_respects_sub_indentation_levels(self): - a_request = MagicMock(spec_set=Request) - a_request.method = HttpMethod.GET - task = Task("some_task", a_request) - new_pre_processings = ( - *task.locust_preprocessing, - "\n def function():\n if True:\n print(True)", - ) - task = task._replace(locust_preprocessing=new_pre_processings) - action = task.as_locust_action(indentation=1) - assert action.startswith(" \n def function():\n if True:\n print(True)") - class TestReplaceURL: def test_it_creates_a_locust_request_when_there_is_none(self): task = Task(name="some name", request=MagicMock()) From 250e16384e65ed15c27324e9d40bb4d6856d7e4d Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Sun, 24 Feb 2019 16:53:07 +0100 Subject: [PATCH 05/13] add test for python.Placeholder Signed-off-by: Thibaut Le Page --- transformer/test_python.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/transformer/test_python.py b/transformer/test_python.py index 8719b8d..733295d 100644 --- a/transformer/test_python.py +++ b/transformer/test_python.py @@ -926,3 +926,13 @@ def test_repr(self): repr(stmt) == f"Import(targets=['a', 'b'], source=None, alias=None, comments=['hi'])" ) + + +class TestPlaceholder: + def test_wraps_int_into_literal(self): + def f(x: int) -> py.Literal: + return py.Literal(x * 2) + + p = py.Placeholder(name="hello", target=lambda: 7, converter=f) + assert p.converter(p.target()) == py.Literal(14) + assert str(p) == "14" From d91b30641e6e71e090cb4dab38d523fb056057b2 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Sun, 24 Feb 2019 17:10:47 +0100 Subject: [PATCH 06/13] update CHANGELOG.md Signed-off-by: Thibaut Le Page --- CHANGELOG.md | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f275850..94dd669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,29 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [1.0.2] - 2019-02-21 +### Added + + - `transformer.python.Placeholder`: An Expression that wraps a non-Expression + (e.g. a Request instance), similarly to how Standalone is a Statement that + wraps an Expression. A Placeholder has a `target` (the wrapped object), a + `converter` (function capable of transforming the target into an Expression), + and a `name` for inspection purposes. + +### Fixed + + - A bug in the conversion between Task and Task2 makes Transformer ignore all + changes made by plugins to `Task2.request` in the generated locustfile. + Thank you [@xinke2411][] for reporting this! (#33) + +### Removed + + - `transformer.task.Task.as_locust_action`: As part of the merge between Task + and Task2 (#11). `as_locust_action` generates locustfile code as a string, + which is made obsolete by the `transformer.python` syntax tree framework. (#33) + +## [1.0.2][] - 2019-02-21 ### Added @@ -38,14 +56,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `transformer.transform.transform` (#14) - `transformer.locust.locustfile` (#14) -## [1.0.1] - 2019-02-12 +## [1.0.1][] - 2019-02-12 ### Fixed - Fix `transformer` command-line crash due to a missing version identifier. (#17) - Publish development releases to PyPI for every merge to the `master` branch. (#17) -## [1.0.0] - 2019-02-11 +## [1.0.0][] - 2019-02-11 ### Added @@ -72,3 +90,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.0.2]: https://github.com/zalando-incubator/transformer/compare/v1.0.1...v1.0.2 [1.0.1]: https://github.com/zalando-incubator/transformer/compare/v1.0.0...v1.0.1 [1.0.0]: https://github.com/zalando-incubator/transformer/compare/f842c4163e037dc345eaf1992187f58126b7d909...v1.0.0 + +[@xinke2411]: https://github.com/xinke2411 From 8d4491742a356c749b6d0ca4437d566561944be6 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Mon, 25 Feb 2019 14:14:12 +0100 Subject: [PATCH 07/13] remove unused _indent since Task.as_locust_action no longer exists Signed-off-by: Thibaut Le Page --- transformer/task.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/transformer/task.py b/transformer/task.py index 9d15bc1..b82393f 100644 --- a/transformer/task.py +++ b/transformer/task.py @@ -203,33 +203,6 @@ def replace_url(self, url: str): return self._replace(locust_request=new_locust_request) -def _indent(input_string: str, requested_indentation: int) -> str: - output_string = "" - indentation = requested_indentation - initial_leading_spaces = 0 - for i, line in enumerate(input_string.splitlines()): - - leading_spaces = len(line) - len(line.lstrip()) - if leading_spaces > 0: - - # We need to check the indentation of the second line in order to - # account for the case where the existing indentation is greater than - # the requested; it is used for reapplying sub-level-indentation e.g. - # to if statements. - if i == 1: - initial_leading_spaces = leading_spaces - else: - indentation = requested_indentation + ( - leading_spaces - initial_leading_spaces - ) - - line = line.lstrip() - - output_string += line.rjust(len(line) + indentation, " ") + "\n" - - return output_string - - def _parse_post_data(post_data: dict) -> dict: data = post_data.get("text") mime: str = post_data.get("mimeType") From 9421589bd751180b1ce4681f503a172d67d5ee44 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Tue, 26 Feb 2019 00:03:26 +0100 Subject: [PATCH 08/13] rework task.py for simpler testing Signed-off-by: Thibaut Le Page --- transformer/plugins/sanitize_headers.py | 4 +- transformer/request.py | 26 +- transformer/task.py | 184 +++++++++--- transformer/test_task.py | 368 +++++++++++++++++++++--- 4 files changed, 497 insertions(+), 85 deletions(-) diff --git a/transformer/plugins/sanitize_headers.py b/transformer/plugins/sanitize_headers.py index 06df633..57e3f07 100644 --- a/transformer/plugins/sanitize_headers.py +++ b/transformer/plugins/sanitize_headers.py @@ -10,12 +10,10 @@ def plugin(task: Task2) -> Task2: Converts header names to lowercase to simplify further overriding. Removes the cookie header as it is handled by Locust's HttpSession. """ - sanitized_headers = [ + task.request.headers = [ Header(name=h.name.lower(), value=h.value) for h in task.request.headers if not h.name.startswith(":") and h.name.lower() != "cookie" ] - task.request = task.request._replace(headers=sanitized_headers) - return task diff --git a/transformer/request.py b/transformer/request.py index 21ee96c..c5b585f 100644 --- a/transformer/request.py +++ b/transformer/request.py @@ -5,10 +5,11 @@ import enum from datetime import datetime -from typing import Iterator, NamedTuple, List +from typing import Iterator, NamedTuple, List, Optional from urllib.parse import urlparse, SplitResult import pendulum +from dataclasses import dataclass from transformer.naming import to_identifier @@ -34,7 +35,8 @@ class Header(NamedTuple): value: str -class QueryPair(NamedTuple): +@dataclass +class QueryPair: """ Query String as recorded in HAR file. """ @@ -43,17 +45,27 @@ class QueryPair(NamedTuple): value: str -class Request(NamedTuple): +@dataclass +class Request: """ An HTTP request as recorded in a HAR file. + + Note that *post_data*, if present, will be a dict of the same format as read + in the HAR file. + Although not consistently followed by HAR generators, his format is + documented here: http://www.softwareishard.com/blog/har-12-spec/#postData. """ timestamp: datetime method: HttpMethod url: SplitResult - headers: List[Header] - post_data: dict - query: List[QueryPair] + headers: List[Header] = () + post_data: Optional[dict] = None + query: List[QueryPair] = () + + def __post_init__(self): + self.headers = list(self.headers) + self.query = list(self.query) @classmethod def from_har_entry(cls, entry: dict) -> "Request": @@ -107,6 +119,6 @@ def __hash__(self) -> int: self.timestamp, self.method, self.url, - tuple(self.post_data) if self.post_data else None, + tuple(self.post_data.items()) if self.post_data else None, ) ) diff --git a/transformer/task.py b/transformer/task.py index b82393f..a6ca08c 100644 --- a/transformer/task.py +++ b/transformer/task.py @@ -2,12 +2,24 @@ """ A representation of a Locust Task. """ - import json from collections import OrderedDict +from json import JSONDecodeError from types import MappingProxyType -from typing import Iterable, NamedTuple, Iterator, Sequence, Optional, Mapping, Dict - +from typing import ( + Iterable, + NamedTuple, + Iterator, + Sequence, + Optional, + Mapping, + Dict, + List, + Tuple, + cast, +) + +import dataclasses from dataclasses import dataclass import transformer.python as py @@ -110,15 +122,19 @@ def req_to_expr(r: Request) -> py.FunctionCall: name=url, headers=py.Literal(zip_kv_pairs(r.headers)), timeout=py.Literal(TIMEOUT), - allow_redirects=py.Symbol("False"), + allow_redirects=py.Literal(False), ) if r.method is HttpMethod.POST: - post_data = _parse_post_data(r.post_data) - args[post_data["key"]] = post_data["data"] + rpd = RequestsPostData.from_har_post_data(r.post_data) + args.update(rpd.as_kwargs()) elif r.method is HttpMethod.PUT: - post_data = _parse_post_data(r.post_data) - args[post_data["key"]] = post_data["data"] - args["params"] = zip_kv_pairs(r.query) + rpd = RequestsPostData.from_har_post_data(r.post_data) + args.update(rpd.as_kwargs()) + + args.setdefault("params", py.Literal({})) + cast(py.Literal, args["params"]).value.extend( + _params_from_name_value_dicts([dataclasses.asdict(q) for q in r.query]) + ) elif r.method not in NOOP_HTTP_METHODS: raise ValueError(f"unsupported HTTP method: {r.method!r}") @@ -139,15 +155,19 @@ def lreq_to_expr(lr: LocustRequest) -> py.FunctionCall: name=url, headers=py.Literal(lr.headers), timeout=py.Literal(TIMEOUT), - allow_redirects=py.Symbol("False"), + allow_redirects=py.Literal(False), ) if lr.method is HttpMethod.POST: - post_data = _parse_post_data(lr.post_data) - args[post_data["key"]] = post_data["data"] + rpd = RequestsPostData.from_har_post_data(lr.post_data) + args.update(rpd.as_kwargs()) elif lr.method is HttpMethod.PUT: - args["params"] = zip_kv_pairs(lr.query) - post_data = _parse_post_data(lr.post_data) - args[post_data["key"]] = post_data["data"] + rpd = RequestsPostData.from_har_post_data(lr.post_data) + args.update(rpd.as_kwargs()) + + args.setdefault("params", py.Literal({})) + cast(py.Literal, args["params"]).value.extend( + _params_from_name_value_dicts([dataclasses.asdict(q) for q in lr.query]) + ) elif lr.method not in NOOP_HTTP_METHODS: raise ValueError(f"unsupported HTTP method: {lr.method!r}") @@ -203,26 +223,114 @@ def replace_url(self, url: str): return self._replace(locust_request=new_locust_request) -def _parse_post_data(post_data: dict) -> dict: - data = post_data.get("text") - mime: str = post_data.get("mimeType") - if mime == "application/json": - key = "json" - # Workaround for bug in chrome-har: - # https://github.com/sitespeedio/chrome-har/issues/23 - # TODO: Remove once bug fixed. - if data is None: - params = post_data.get("params") - if params is None: - data = "" - else: - data = {} - for param in params: - data[param.get("name")] = param.get("value") - else: - data = json.loads(data) - else: - key = "data" - if data: - data = data.encode() - return {"key": key, "data": data} +@dataclass +class RequestsPostData: + """ + Data to be sent via HTTP POST, along with which API of the requests library + to use. + """ + + data: Optional[py.Literal] = None + params: Optional[py.Literal] = None + json: Optional[py.Literal] = None + + def as_kwargs(self) -> Dict[str, py.Expression]: + return {k: v for k, v in dataclasses.asdict(self).items() if v is not None} + + @classmethod + def from_har_post_data(cls, post_data: dict) -> "RequestsPostData": + """ + Converts a HAR postData object into a RequestsPostData instance. + + :param post_data: a HAR "postData" object, + see http://www.softwareishard.com/blog/har-12-spec/#postData. + :raise ValueError: if *post_data* is invalid. + """ + try: + return _from_har_post_data(post_data) + except ValueError as err: + raise ValueError(f"invalid HAR postData object: {post_data!r}") from err + + +def _from_har_post_data(post_data: dict) -> RequestsPostData: + mime_k = "mimeType" + try: + mime: str = post_data[mime_k] + except KeyError: + raise ValueError(f"missing {mime_k!r} field") from None + + rpd = RequestsPostData() + + # The "text" and "params" fields are supposed to be mutually + # exclusive (according to the HAR spec) but nobody respects that. + # Often, both text and params are provided for x-www-form-urlencoded. + text_k, params_k = "text", "params" + if text_k not in post_data and params_k not in post_data: + raise ValueError(f"should contain {text_k!r} or {params_k!r}") + + _extract_text(mime, post_data, text_k, rpd) + + try: + params = _params_from_post_data(params_k, post_data) + if params is not None: + rpd.params = py.Literal(params) + except (KeyError, UnicodeEncodeError, TypeError) as err: + raise ValueError("unreadable params field") from err + + return rpd + + +def _extract_text( + mime: str, post_data: dict, text_k: str, rpd: RequestsPostData +) -> None: + text = post_data.get(text_k) + if mime == JSON_MIME_TYPE: + if text is None: + raise ValueError(f"missing {text_k!r} field for {JSON_MIME_TYPE} content") + try: + rpd.json = py.Literal(json.loads(text)) + except JSONDecodeError as err: + raise ValueError(f"unreadable JSON from field {text_k!r}") from err + elif text is not None: # Probably application/x-www-form-urlencoded. + try: + rpd.data = py.Literal(text.encode()) + except UnicodeEncodeError as err: + raise ValueError(f"cannot encode the {text_k!r} field in UTF-8") from err + + +def _params_from_post_data( + key: str, post_data: dict +) -> Optional[List[Tuple[bytes, bytes]]]: + """ + Extracts the *key* list from *post_data* and calls + _params_from_name_value_dicts with that list. + + :raise TypeError: if the object at *key* is built using unexpected data types. + """ + params = post_data.get(key) + if params is None: + return + if not isinstance(params, list): + raise TypeError(f"the {key!r} field should be a list") + return _params_from_name_value_dicts(params) + + +def _params_from_name_value_dicts( + dicts: Iterable[Mapping[str, str]] +) -> List[Tuple[bytes, bytes]]: + """ + Converts a HAR "params" element [0] into a list of tuples that can be used + as value for requests' "params" keyword-argument. + + [0]: http://www.softwareishard.com/blog/har-12-spec/#params + [1]: http://docs.python-requests.org/en/master/user/quickstart/ + #more-complicated-post-requests + + :raise KeyError: if one of the elements doesn't contain a "name" or "value" field. + :raise UnicodeEncodeError: if an element's "name" or "value" string cannot + be encoded in UTF-8. + """ + return [(d["name"].encode(), d["value"].encode()) for d in dicts] + + +JSON_MIME_TYPE = "application/json" diff --git a/transformer/test_task.py b/transformer/test_task.py index edcfeaf..ddbfda8 100644 --- a/transformer/test_task.py +++ b/transformer/test_task.py @@ -1,24 +1,27 @@ # pylint: skip-file - +import enum import io -import json -from datetime import datetime -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock from unittest.mock import patch from urllib.parse import urlparse import pytest +from hypothesis import given +from hypothesis.strategies import composite, sampled_from, booleans -from transformer.request import Header from transformer import python as py +from transformer.request import Header, QueryPair from transformer.task import ( Task, Request, HttpMethod, - QueryPair, TIMEOUT, LocustRequest, Task2, + RequestsPostData, + JSON_MIME_TYPE, + req_to_expr, + lreq_to_expr, ) @@ -79,16 +82,8 @@ def test_it_returns_a_task_with_the_replaced_url(self): class TestTask2: class TestFromTask: - def test_without_locust_request(self): - url = "https://abc.de" - req = Request( - timestamp=datetime.now(), - method=HttpMethod.GET, - url=urlparse(url), - headers=[Header("a", "b")], - post_data={}, - query=[], - ) + def test_without_locust_request_it_proxies_the_request(self): + req = Mock(spec_set=Request) task = Task(name="T", request=req) task2 = Task2.from_task(task) @@ -102,24 +97,11 @@ def test_without_locust_request(self): assert isinstance(assign.rhs, py.Placeholder) assert assign.rhs.target() == task2.request + assert assign.rhs.converter is req_to_expr - assert str(assign.rhs) == ( - f"self.client.get(url={url!r}, name={url!r}," - f" headers={{'a': 'b'}}, timeout={TIMEOUT}, allow_redirects=False)" - ) - - def test_with_locust_request(self): - url = "https://abc.de" - req = Request( - timestamp=datetime.now(), - method=HttpMethod.GET, - url=urlparse(url), - headers=[Header("a", "b")], - post_data={}, - query=[], - ) - lr = LocustRequest.from_request(req) - lr = lr._replace(url="f" + lr.url.replace("de", "{tld}")) + def test_with_locust_request_it_proxies_it(self): + lr = Mock(spec_set=LocustRequest) + req = Mock(spec_set=Request) task = Task(name="T", request=req, locust_request=lr) task2 = Task2.from_task(task) @@ -133,9 +115,321 @@ def test_with_locust_request(self): assert isinstance(assign.rhs, py.Placeholder) assert assign.rhs.target() == lr + assert assign.rhs.converter is lreq_to_expr + + +class _KindOfDict(enum.Flag): + Text = enum.auto() + Params = enum.auto() + Both = Text | Params + + +_formats = sampled_from(("json", "www")) +_kinds_of_dicts = sampled_from(_KindOfDict) + +# From http://www.softwareishard.com/blog/har-12-spec/#postData. +@composite +def har_post_dicts(draw, format=None): + format = format or draw(_formats) + if format == "json": + d = {"mimeType": "application/json", "text": """{"a":"b", "c": "d"}"""} + if draw(booleans()): + d["params"] = [] + if draw(booleans()): + d["comment"] = "" + return d + + d = {"mimeType": "application/x-www-form-urlencoded"} + kind = draw(_kinds_of_dicts) + if kind & _KindOfDict.Text: + d["text"] = "a=b&c=d" + if draw(booleans()): + d.setdefault("params", []) + if kind & _KindOfDict.Params: + d["params"] = [{"name": "a", "value": "b"}, {"name": "c", "value": "d"}] + if draw(booleans()): + d.setdefault("text", "") + return d + + +class TestRequestPostData: + def test_as_kwargs_only_shows_defined(self): + v, w = MagicMock(), MagicMock() + assert RequestsPostData(data=v).as_kwargs() == {"data": v} + assert RequestsPostData(params=v, json=w).as_kwargs() == { + "params": v, + "json": w, + } + + class TestFromHarPostData: + @given(har_post_dicts(format="json")) + def test_it_selects_json_approach_for_json_format(self, d: dict): + rpd = RequestsPostData.from_har_post_data(d) + assert rpd.json == py.Literal({"a": "b", "c": "d"}) + assert rpd.data is None + + @given(har_post_dicts(format="www")) + def test_it_selects_data_approach_for_urlencoded_format(self, d: dict): + rpd = RequestsPostData.from_har_post_data(d) + assert rpd.json is None + assert rpd.data == py.Literal(b"a=b&c=d") or rpd.params == py.Literal( + [(b"a", b"b"), (b"c", b"d")] + ) + + @given(har_post_dicts()) + def test_it_doesnt_raise_error_on_valid_input(self, d: dict): + RequestsPostData.from_har_post_data(d) + + def test_it_raises_on_post_data_without_text_or_params(self): + with pytest.raises(ValueError): + RequestsPostData.from_har_post_data({"mimeType": "nil"}) - expected_url = """ f'https://abc.{tld}' """.strip() - assert str(assign.rhs) == ( - f"self.client.get(url={expected_url}, name={expected_url}," - f" headers={{'a': 'b'}}, timeout={TIMEOUT}, allow_redirects=False)" + def test_it_raises_on_invalid_json(self): + with pytest.raises(ValueError): + RequestsPostData.from_har_post_data( + {"mimeType": JSON_MIME_TYPE, "text": "not json"} + ) + + @pytest.mark.parametrize( + "mime,kwarg,val", + ( + (JSON_MIME_TYPE, "json", {}), + ("application/x-www-form-urlencoded", "data", b"{}"), + ), + ) + def test_it_accepts_both_params_and_text(self, mime: str, kwarg, val): + expected_fields = { + "params": py.Literal([(b"n", b"v")]), + kwarg: py.Literal(val), + } + assert RequestsPostData.from_har_post_data( + { + "mimeType": mime, + "text": "{}", + "params": [{"name": "n", "value": "v"}], + } + ) == RequestsPostData(**expected_fields) + + +class TestReqToExpr: + def test_it_supports_get_requests(self): + url = "http://abc.de" + r = Request( + timestamp=MagicMock(), + method=HttpMethod.GET, + url=urlparse(url), + headers=[Header("a", "b")], + query=[QueryPair("x", "y")], # query is currently ignored for GET + ) + assert req_to_expr(r) == py.FunctionCall( + name="self.client.get", + named_args={ + "url": py.Literal(url), + "name": py.Literal(url), + "headers": py.Literal({"a": "b"}), + "timeout": py.Literal(TIMEOUT), + "allow_redirects": py.Literal(False), + }, + ) + + def test_it_supports_urlencoded_post_requests(self): + url = "http://abc.de" + r = Request( + timestamp=MagicMock(), + method=HttpMethod.POST, + url=urlparse(url), + headers=[Header("a", "b")], + post_data={ + "mimeType": "application/x-www-form-urlencoded", + "params": [{"name": "x", "value": "y"}], + "text": "z=7", + }, + ) + assert req_to_expr(r) == py.FunctionCall( + name="self.client.post", + named_args={ + "url": py.Literal(url), + "name": py.Literal(url), + "headers": py.Literal({"a": "b"}), + "timeout": py.Literal(TIMEOUT), + "allow_redirects": py.Literal(False), + "data": py.Literal(b"z=7"), + "params": py.Literal([(b"x", b"y")]), + }, + ) + + def test_it_supports_json_post_requests(self): + url = "http://abc.de" + r = Request( + timestamp=MagicMock(), + method=HttpMethod.POST, + url=urlparse(url), + headers=[Header("a", "b")], + post_data={ + "mimeType": "application/json", + "params": [{"name": "x", "value": "y"}], + "text": """{"z": 7}""", + }, + ) + assert req_to_expr(r) == py.FunctionCall( + name="self.client.post", + named_args={ + "url": py.Literal(url), + "name": py.Literal(url), + "headers": py.Literal({"a": "b"}), + "timeout": py.Literal(TIMEOUT), + "allow_redirects": py.Literal(False), + "json": py.Literal({"z": 7}), + "params": py.Literal([(b"x", b"y")]), + }, + ) + + def test_it_supports_put_requests(self): + url = "http://abc.de" + r = Request( + timestamp=MagicMock(), + method=HttpMethod.PUT, + url=urlparse(url), + headers=[Header("a", "b")], + query=[QueryPair("c", "d")], + post_data={ + "mimeType": "application/json", + "params": [{"name": "x", "value": "y"}], + "text": """{"z": 7}""", + }, + ) + assert req_to_expr(r) == py.FunctionCall( + name="self.client.put", + named_args={ + "url": py.Literal(url), + "name": py.Literal(url), + "headers": py.Literal({"a": "b"}), + "timeout": py.Literal(TIMEOUT), + "allow_redirects": py.Literal(False), + "json": py.Literal({"z": 7}), + "params": py.Literal([(b"x", b"y"), (b"c", b"d")]), + }, + ) + + +class TestLreqToExpr: + def test_it_supports_get_requests(self): + url = "http://abc.de" + r = LocustRequest.from_request( + Request( + timestamp=MagicMock(), + method=HttpMethod.GET, + url=urlparse(url), + headers=[Header("a", "b")], + query=[QueryPair("x", "y")], # query is currently ignored for GET + ) + ) + assert lreq_to_expr(r) == py.FunctionCall( + name="self.client.get", + named_args={ + "url": py.Literal(url), + "name": py.Literal(url), + "headers": py.Literal({"a": "b"}), + "timeout": py.Literal(TIMEOUT), + "allow_redirects": py.Literal(False), + }, + ) + + def test_it_supports_fstring_urls(self): + url = "http://abc.{tld}" + r = LocustRequest(method=HttpMethod.GET, url=f"f'{url}'", headers={"a": "b"}) + assert lreq_to_expr(r) == py.FunctionCall( + name="self.client.get", + named_args={ + "url": py.FString(url), + "name": py.FString(url), + "headers": py.Literal({"a": "b"}), + "timeout": py.Literal(TIMEOUT), + "allow_redirects": py.Literal(False), + }, + ) + + def test_it_supports_urlencoded_post_requests(self): + url = "http://abc.de" + r = LocustRequest.from_request( + Request( + timestamp=MagicMock(), + method=HttpMethod.POST, + url=urlparse(url), + headers=[Header("a", "b")], + post_data={ + "mimeType": "application/x-www-form-urlencoded", + "params": [{"name": "x", "value": "y"}], + "text": "z=7", + }, + ) + ) + assert lreq_to_expr(r) == py.FunctionCall( + name="self.client.post", + named_args={ + "url": py.Literal(url), + "name": py.Literal(url), + "headers": py.Literal({"a": "b"}), + "timeout": py.Literal(TIMEOUT), + "allow_redirects": py.Literal(False), + "data": py.Literal(b"z=7"), + "params": py.Literal([(b"x", b"y")]), + }, + ) + + def test_it_supports_json_post_requests(self): + url = "http://abc.de" + r = LocustRequest.from_request( + Request( + timestamp=MagicMock(), + method=HttpMethod.POST, + url=urlparse(url), + headers=[Header("a", "b")], + post_data={ + "mimeType": "application/json", + "params": [{"name": "x", "value": "y"}], + "text": """{"z": 7}""", + }, + ) + ) + assert lreq_to_expr(r) == py.FunctionCall( + name="self.client.post", + named_args={ + "url": py.Literal(url), + "name": py.Literal(url), + "headers": py.Literal({"a": "b"}), + "timeout": py.Literal(TIMEOUT), + "allow_redirects": py.Literal(False), + "json": py.Literal({"z": 7}), + "params": py.Literal([(b"x", b"y")]), + }, + ) + + def test_it_supports_put_requests(self): + url = "http://abc.de" + r = LocustRequest.from_request( + Request( + timestamp=MagicMock(), + method=HttpMethod.PUT, + url=urlparse(url), + headers=[Header("a", "b")], + query=[QueryPair("c", "d")], + post_data={ + "mimeType": "application/json", + "params": [{"name": "x", "value": "y"}], + "text": """{"z": 7}""", + }, ) + ) + assert lreq_to_expr(r) == py.FunctionCall( + name="self.client.put", + named_args={ + "url": py.Literal(url), + "name": py.Literal(url), + "headers": py.Literal({"a": "b"}), + "timeout": py.Literal(TIMEOUT), + "allow_redirects": py.Literal(False), + "json": py.Literal({"z": 7}), + "params": py.Literal([(b"x", b"y"), (b"c", b"d")]), + }, + ) From 245da47a6e4848b76a86878162a4381d53998e66 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Tue, 26 Feb 2019 00:34:19 +0100 Subject: [PATCH 09/13] rename python.Placeholder -> python.ExpressionView Signed-off-by: Thibaut Le Page --- CHANGELOG.md | 4 ++-- transformer/python.py | 12 ++++++------ transformer/task.py | 6 +++--- transformer/test_python.py | 2 +- transformer/test_task.py | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94dd669..1fd78dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - - `transformer.python.Placeholder`: An Expression that wraps a non-Expression + - `transformer.python.ExpressionView`: An Expression that wraps a non-Expression (e.g. a Request instance), similarly to how Standalone is a Statement that - wraps an Expression. A Placeholder has a `target` (the wrapped object), a + wraps an Expression. A ExpressionView has a `target` (the wrapped object), a `converter` (function capable of transforming the target into an Expression), and a `name` for inspection purposes. diff --git a/transformer/python.py b/transformer/python.py index 5d4a0bc..72801d9 100644 --- a/transformer/python.py +++ b/transformer/python.py @@ -696,12 +696,12 @@ def __repr__(self) -> str: @dataclass -class Placeholder(Expression): +class ExpressionView(Expression): """ The promise of an Expression representing an object currently not in Expression format. - The Placeholder allows to mix non-Expression objects in the syntax tree, + The ExpressionView allows to mix non-Expression objects in the syntax tree, along with a function capable of transforming these objects into actual Expression objects at any time. This is useful when there is a simpler representation than Expression. @@ -709,19 +709,19 @@ class Placeholder(Expression): For instance, any Request object can be converted into an equivalent Expression, but Request has a simpler API than Expression for all request-oriented operations like accessing the URL, etc. - Embedding a Request in a Placeholder allows to pretend that the Request is + Embedding a Request in a ExpressionView allows to pretend that the Request is already in Expression format (with all associated benefits) but still use the Request API. `target` is a callable returning the non-Expression object. That callable allows to specify as target some mutable field of an object, rather than a fixed reference to an object. See for example task.Task2, which contains a - Placeholder to its own "request" field; if the value of that field is - changed, the Placeholder will refer to the new value instead of keeping a + ExpressionView to its own "request" field; if the value of that field is + changed, the ExpressionView will refer to the new value instead of keeping a reference to the old value. `name` is purely descriptive: it can make inspection of data structures - containing Placeholder objects more comfortable. + containing ExpressionView objects more comfortable. """ name: str diff --git a/transformer/task.py b/transformer/task.py index a6ca08c..3cd2f1c 100644 --- a/transformer/task.py +++ b/transformer/task.py @@ -79,7 +79,7 @@ def from_requests(cls, requests: Iterable[Request]) -> Iterator["Task2"]: corresponding request. """ # TODO: Update me when merging Task with Task2: "statements" needs to - # contain a Placeholder to Task2.request. + # contain a ExpressionView to Task2.request. # See what is done in from_task (but without the LocustRequest part). # See https://github.com/zalando-incubator/Transformer/issues/11. for req in sorted(requests, key=lambda r: r.timestamp): @@ -93,13 +93,13 @@ def from_task(cls, task: "Task") -> "Task2": # See https://github.com/zalando-incubator/Transformer/issues/11. t = cls(name=task.name, request=task.request) if task.locust_request: - placeholder = py.Placeholder( + placeholder = py.ExpressionView( name="this task's request field", target=lambda: task.locust_request, converter=lreq_to_expr, ) else: - placeholder = py.Placeholder( + placeholder = py.ExpressionView( name="this task's request field", target=lambda: t.request, converter=req_to_expr, diff --git a/transformer/test_python.py b/transformer/test_python.py index 733295d..8ca9472 100644 --- a/transformer/test_python.py +++ b/transformer/test_python.py @@ -933,6 +933,6 @@ def test_wraps_int_into_literal(self): def f(x: int) -> py.Literal: return py.Literal(x * 2) - p = py.Placeholder(name="hello", target=lambda: 7, converter=f) + p = py.ExpressionView(name="hello", target=lambda: 7, converter=f) assert p.converter(p.target()) == py.Literal(14) assert str(p) == "14" diff --git a/transformer/test_task.py b/transformer/test_task.py index ddbfda8..8cb1e72 100644 --- a/transformer/test_task.py +++ b/transformer/test_task.py @@ -95,7 +95,7 @@ def test_without_locust_request_it_proxies_the_request(self): assert isinstance(assign, py.Assignment) assert assign.lhs == "response" - assert isinstance(assign.rhs, py.Placeholder) + assert isinstance(assign.rhs, py.ExpressionView) assert assign.rhs.target() == task2.request assert assign.rhs.converter is req_to_expr @@ -113,7 +113,7 @@ def test_with_locust_request_it_proxies_it(self): assert isinstance(assign, py.Assignment) assert assign.lhs == "response" - assert isinstance(assign.rhs, py.Placeholder) + assert isinstance(assign.rhs, py.ExpressionView) assert assign.rhs.target() == lr assert assign.rhs.converter is lreq_to_expr From d3951892ee8e7038902716546b2f05ad80a52d19 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Tue, 26 Feb 2019 10:15:25 +0100 Subject: [PATCH 10/13] fix typo in documentation Signed-off-by: Thibaut Le Page --- transformer/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformer/request.py b/transformer/request.py index c5b585f..5dcf0fb 100644 --- a/transformer/request.py +++ b/transformer/request.py @@ -52,7 +52,7 @@ class Request: Note that *post_data*, if present, will be a dict of the same format as read in the HAR file. - Although not consistently followed by HAR generators, his format is + Although not consistently followed by HAR generators, its format is documented here: http://www.softwareishard.com/blog/har-12-spec/#postData. """ From 7ea091cb5eb20a4c8dc6ea0e06c8691d496047f0 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Tue, 26 Feb 2019 10:15:43 +0100 Subject: [PATCH 11/13] rename "placeholder" vars: Placeholder renamed as ExpressionView Signed-off-by: Thibaut Le Page --- transformer/task.py | 6 +++--- transformer/test_python.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/transformer/task.py b/transformer/task.py index 3cd2f1c..4ef65ff 100644 --- a/transformer/task.py +++ b/transformer/task.py @@ -93,20 +93,20 @@ def from_task(cls, task: "Task") -> "Task2": # See https://github.com/zalando-incubator/Transformer/issues/11. t = cls(name=task.name, request=task.request) if task.locust_request: - placeholder = py.ExpressionView( + expr_view = py.ExpressionView( name="this task's request field", target=lambda: task.locust_request, converter=lreq_to_expr, ) else: - placeholder = py.ExpressionView( + expr_view = py.ExpressionView( name="this task's request field", target=lambda: t.request, converter=req_to_expr, ) t.statements = [ *[py.OpaqueBlock(x) for x in task.locust_preprocessing], - py.Assignment("response", placeholder), + py.Assignment("response", expr_view), *[py.OpaqueBlock(x) for x in task.locust_postprocessing], ] return t diff --git a/transformer/test_python.py b/transformer/test_python.py index 8ca9472..1063619 100644 --- a/transformer/test_python.py +++ b/transformer/test_python.py @@ -928,11 +928,11 @@ def test_repr(self): ) -class TestPlaceholder: +class TestExpressionView: def test_wraps_int_into_literal(self): def f(x: int) -> py.Literal: return py.Literal(x * 2) - p = py.ExpressionView(name="hello", target=lambda: 7, converter=f) - assert p.converter(p.target()) == py.Literal(14) - assert str(p) == "14" + ev = py.ExpressionView(name="hello", target=lambda: 7, converter=f) + assert ev.converter(ev.target()) == py.Literal(14) + assert str(ev) == "14" From 80a229c384ade75a268427d3bd8a2e36180be8e8 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Tue, 26 Feb 2019 10:16:50 +0100 Subject: [PATCH 12/13] task: move JSON_MIME_TYPE definition at the top Signed-off-by: Thibaut Le Page --- transformer/task.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/transformer/task.py b/transformer/task.py index 4ef65ff..9e176e4 100644 --- a/transformer/task.py +++ b/transformer/task.py @@ -30,6 +30,7 @@ IMMUTABLE_EMPTY_DICT = MappingProxyType({}) TIMEOUT = 30 ACTION_INDENTATION_LEVEL = 12 +JSON_MIME_TYPE = "application/json" class LocustRequest(NamedTuple): @@ -331,6 +332,3 @@ def _params_from_name_value_dicts( be encoded in UTF-8. """ return [(d["name"].encode(), d["value"].encode()) for d in dicts] - - -JSON_MIME_TYPE = "application/json" From 8321601cb159d6685c48c3ab0b8c7e47a19c3998 Mon Sep 17 00:00:00 2001 From: Thibaut Le Page Date: Tue, 26 Feb 2019 11:23:28 +0100 Subject: [PATCH 13/13] CONTRIBUTORS.md: add link to contributions Signed-off-by: Thibaut Le Page --- CONTRIBUTORS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 9e3e1e5..e0acb01 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -4,6 +4,7 @@ All external contributors to the project. We are grateful for all their help! ## Contributors sorted alphabetically - - **[xinke2411](https://github.com/xinke2411)** + - [#34](https://github.com/zalando-incubator/Transformer/pull/34): + _Add attribute name for class Request_.