From 4d8f6da5f1a9c2530bffcf5b5d484d56b43d037d Mon Sep 17 00:00:00 2001 From: jenisys Date: Mon, 29 May 2023 02:52:17 +0200 Subject: [PATCH 1/5] APPLY CHANGES FROM: pull #88 * ADDED: Pipe.__or__() function * USE: self.__class__ instead of Pipe class REASON: Simlify to derive from Pipe class and extend functionality * Add test for pipeline recipe based on or-pipes. SEE ALSO: * https://github.com/JulienPalard/Pipe/pull/88 --- pipe.py | 13 ++++++++++++- tests/test_pipe.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pipe.py b/pipe.py index f6d24ef..094dd94 100644 --- a/pipe.py +++ b/pipe.py @@ -39,8 +39,19 @@ def __init__(self, function): def __ror__(self, other): return self.function(other) + # SEE: https://github.com/JulienPalard/Pipe/pull/88 + def __or__(self, other): + cls = self.__class__ + if isinstance(other, Pipe): + return cls(lambda iterable, *args2, **kwargs2: ( + other.function(self.function(iterable, *args2, **kwargs2)))) + # -- CASE: other is pipeable only + return cls(lambda iterable, *args2, **kwargs2: ( + self.function(iterable, *args2, **kwargs2) | other)) + def __call__(self, *args, **kwargs): - return Pipe( + cls = self.__class__ + return cls( lambda iterable, *args2, **kwargs2: self.function( iterable, *args, *args2, **kwargs, **kwargs2 ) diff --git a/tests/test_pipe.py b/tests/test_pipe.py index 8bba1c9..e47e41c 100644 --- a/tests/test_pipe.py +++ b/tests/test_pipe.py @@ -38,3 +38,15 @@ def test_enumerate(): data = [4, "abc", {"key": "value"}] expected = [(5, 4), (6, "abc"), (7, {"key": "value"})] assert list(data | pipe.enumerate(start=5)) == expected + + +def test_concatenate_pipes(): + data = range(10) + is_even = pipe.where(lambda x: x % 2 == 0) + higher_than_4 = pipe.where(lambda x: x > 4) + expected = [6,8] + # standard behavior + assert list(data | is_even | higher_than_4) == expected + # concatenated pipes + is_even_and_higher_than_4 = is_even | higher_than_4 + assert list(data | is_even_and_higher_than_4) == expected From 01669ef2fcb4ddd135b07cf9ef134724d6ce7ba8 Mon Sep 17 00:00:00 2001 From: jenisys Date: Mon, 29 May 2023 03:00:39 +0200 Subject: [PATCH 2/5] Add tests for pipeline recipe functionality --- tests/test_pipe_basics.py | 109 ++++++++++++++++++++++++++++ tests/test_pipeline_recipe.py | 133 ++++++++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 tests/test_pipe_basics.py create mode 100644 tests/test_pipeline_recipe.py diff --git a/tests/test_pipe_basics.py b/tests/test_pipe_basics.py new file mode 100644 index 0000000..0c5c9d0 --- /dev/null +++ b/tests/test_pipe_basics.py @@ -0,0 +1,109 @@ +""" +Explore the `pipe`_ module with some tests. + +SEE ALSO: + +* https://github.com/JulienPalard/Pipe + +RELATED: + +* pyfunctional -- operator chaining instead of pipe-expressions. +* https://github.com/sspipe/sspipe + +.. _pipe: https://github.com/JulienPalard/Pipe +""" + +from pipe import Pipe as as_pipe, where, take_while +import pytest + + +# ----------------------------------------------------------------------------- +# EXPLORATION: PIPE-ADAPTER(s) +# ----------------------------------------------------------------------------- +@as_pipe +def inspect_pipe(iterable, func=None, predicate=None): + """Inspect what items pass through this pipe.""" + if func is None: + func = print + if predicate is None: + predicate = lambda _x: True + + for item in iterable | take_while(predicate): + func(item) + yield item + +@as_pipe +def to_list(iterable): + """Pipe adapter that converts the output of this pipe-segment into a list. + + .. code-block:: + + outputs = range(5) | ... | to_list + assert isinstance(outputs, list) + """ + return list(iterable) + + + +# ----------------------------------------------------------------------------- +# TEST SUITE +# ----------------------------------------------------------------------------- +def test_pipe_basics(): + """ + IDEA:: + is_even = where(lambda x: x % 2) + result = list(range(4)) | is_even | to_list + """ + inputs = list(range(6)) + is_odd = where(lambda x: x % 2 == 1) + outputs = list(inputs | is_odd) # OTHERWISE: result is a generator + + expected = [1, 3, 5] + assert outputs == expected + assert isinstance(outputs, list) + +def test_to_list__as_pipe_adapter(): + """ + IDEA:: + is_even = where(lambda x: x % 2) + result = list(range(4)) | is_even | to_list + """ + inputs = list(range(5)) + is_even = where(lambda x: x % 2 == 0) + outputs = inputs | is_even | to_list + + expected = [0, 2, 4] + assert outputs == expected + assert isinstance(outputs, list) + +def test_pipe_sink(): + """Test how a pipe-sink can be provided. + + IDEA:: + + is_even = where(lambda x: x % 2) + result = list(range(4)) | is_even | collector + + """ + class Collector(object): + def __init__(self, data=None): + self.data = data or [] + + def append(self, item): + self.data.append(item) + + @as_pipe + def collect_to(iterable, container): + for item in iterable: + container.append(item) + return container + + inputs = list(range(5)) + is_even = where(lambda x: x % 2 == 0) + collector = Collector() + output = inputs | is_even | collect_to(collector) + + expected = [0, 2, 4] + assert output.data == expected + assert output is collector + assert isinstance(output, Collector) diff --git a/tests/test_pipeline_recipe.py b/tests/test_pipeline_recipe.py new file mode 100644 index 0000000..ddbdfbd --- /dev/null +++ b/tests/test_pipeline_recipe.py @@ -0,0 +1,133 @@ +""" +Tests related to the "pipeline recipe" idea. + +DEFINITION: Pipeline recipe + +* A prepared, often complex pipeline that should easily reusable by others + +SOLUTION: + +* Use ``or-pipes`` expression (support added to: `pipe` module) +* Use pipe-functions with yield-from expression. + ADVANTAGE: Has parametrization support. + +SEE ALSO: + +* https://github.com/JulienPalard/Pipe/pull/88 + +RELATED: + +* pyfunctional -- operator chaining instead of pipe-expressions. +* https://github.com/sspipe/sspipe + +.. _pipe: https://github.com/JulienPalard/Pipe +""" + +from functools import reduce +from pipe import Pipe as as_pipe +import pytest + + + +# ----------------------------------------------------------------------------- +# TEST SUPPORT FOR: Pipeline Recipes +# ----------------------------------------------------------------------------- +def use_pipe(iterable, the_pipe): + yield from (iterable | the_pipe) + + +# -- SIMILAR TO: functools.reduce() -- But better debuggable/explorable +# def reduce2(function, iterable): +# it = iter(iterable) +# value = next(it) +# for element in it: +# value = function(value, element) +# return value + +# -- HINT: NOT NEEDED when Pipe.__or__() is supported +def chain_pipes(*pipes): + def pipe_operator(p1, p2): + return p1 | p2 + + this_pipeline = reduce(pipe_operator, pipes) + @as_pipe + def _chain_pipes(iterable): + yield from (iterable | this_pipeline) + return _chain_pipes + + +@as_pipe +def select_numbers_modulo(iterable, modulus): + for number in iterable: + if (number % modulus) == 0: + yield number + + +@as_pipe +def multiply_by(iterable, factor): + for number in iterable: + yield number * factor + + +@as_pipe +def select_even_numbers(iterable): + return use_pipe(iterable, select_numbers_modulo(2)) + + +# ----------------------------------------------------------------------------- +# TEST SUITE: Pipeline Recipes +# ----------------------------------------------------------------------------- +class TestPipelineRecipe(object): + """A ``pipeline recipe`` is a prepared, often complex chain of pipes. + It should be easy to use this pipeline recipe without knowing the details. + """ + def test_recipe_with_one_pipe(self): + """Check pipeline-recipe that uses "yield from" can be used.""" + @as_pipe + def select_multiples_of_3(iterable): + yield from (iterable | select_numbers_modulo(3)) + + numbers = range(8) + results = list(numbers | select_multiples_of_3) + expected = [0, 3, 6] + assert results == expected + + def test_recipe_with_many_pipes(self): + """Check pipeline-recipe that uses "yield from" with many pipes.""" + @as_pipe + def select_even_numbers_and_multiply_by(iterable, factor): + yield from (iterable | select_even_numbers | multiply_by(factor)) + + numbers = range(8) + results = list(numbers | select_even_numbers_and_multiply_by(3)) + expected = [0, 6, 12, 18] + assert results == expected + + def test_recipe_with_or_pipes(self): + # REQUIRES: https://github.com/JulienPalard/Pipe/pull/88 + # HINT: or-pipes / Pipe.operator-or is provided here. + select_even_numbers_and_multiply_by_3B = select_even_numbers | multiply_by(3) + + numbers = range(8) + results = list(numbers | select_even_numbers_and_multiply_by_3B) + expected = [0, 6, 12, 18] + assert results == expected + + # -- REJECTED IDEAS: + # use_pipe() -> BETTER USE: "yield from" + # chain_pipes() -> BETTER USE: or-pipes + def test_recipe__with_chain_pipes(self): + # -- HINT: pipe-chain with "or-pipes" is simpler. + select_even_numbers_and_multiply_by_3A = chain_pipes(select_even_numbers, multiply_by(3)) + + numbers = range(8) + results = list(numbers | select_even_numbers_and_multiply_by_3A) + expected = [0, 6, 12, 18] + assert results == expected + + def test_use_pipeline_recipe_1(self): + """Check if a pipeline-recipe that uses "use_pipe()" can be used.""" + numbers = range(8) + results = list(numbers | select_even_numbers) # USES: use_pipe() internally + expected = [0, 2, 4, 6] + assert results == expected From d9f04c37e604485d93fc3e6bdce4d0e43e15ca80 Mon Sep 17 00:00:00 2001 From: jenisys Date: Mon, 29 May 2023 03:29:06 +0200 Subject: [PATCH 3/5] README: Add description for "pipeline recipes" --- README.md | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/README.md b/README.md index 919a6bb..d593ab8 100644 --- a/README.md +++ b/README.md @@ -229,9 +229,73 @@ Or now with a flag: ... Hello hello +``` + +## Pipeline Recipes + +A **pipeline recipe** provides a pipeline without its pipe-source (sequence/stream). +A pipeline may provide a complex pipeline parts. +It is easier for the user of the pipeline to just use it (without knowing the details). +In addition, the use of the **pipeline recipe**: + +* is less error-prone and +* the canned **pipeline recipe** can be tested in advance + +### SOLUTION 1: Use "yield from" expressions + +```python +# -- FILE: example1_pipeline_recipe_with_yield_from.py +from pipe import Pipe +from example_common_parts import select_even_numbers, multiply_by + +# -- PIPELINE RECIPE: Using pipe-function with yield-from +# ADVANTAGE: pipeline parametrization can be done late. +@Pipe +def select_even_numbers_and_multiply_by(iterable, factor): + yield from (iterable | select_even_numbers | multiply_by(factor)) + +numbers = range(8) +results = list(numbers | select_even_numbers_and_multiply_by(3)) +expected = [0, 6, 12, 18] +assert results == expected +``` +### SOLUTION 2: Use or-pipes expressions + +```python +# -- FILE: example2_pipeline_recipe_with_or_pipes.py +from pipe import Pipe +from example_common_parts import select_even_numbers, multiply_by + +# -- PIPELINE RECIPE: Using or-pipes expressions +# NOTE: pipeline parametrization must be done early (when pipeline is defined). +select_even_numbers_and_multiply_by_3 = select_even_numbers | multiply_by(3) + +numbers = range(8) +results = list(numbers | select_even_numbers_and_multiply_by_3) +expected = [0, 6, 12, 18] +assert results == expected ``` +```python +# -- FILE: example_common_parts.py +@Pipe +def select_numbers_modulo(iterable, modulus): + for number in iterable: + if (number % modulus) == 0: + yield number + +@Pipe +def multiply_by(iterable, factor): + for number in iterable: + yield number * factor + +@Pipe +def select_even_numbers(iterable): + yield from (iterable | select_numbers_modulo(2)) +``` + + # Deprecations of pipe 1.x In pipe 1.x a lot of functions were returning iterables and a lot From 42c09de445b808b3f2b987ae5067d23944167d45 Mon Sep 17 00:00:00 2001 From: jenisys Date: Mon, 29 May 2023 13:09:03 +0200 Subject: [PATCH 4/5] ADDED: SOLUTION 3 -- make-pipe with or-pipe expressions * CLEANUP: Remove some test parts that were rejected. --- README.md | 58 +++++++++++++++++++++--------- tests/test_pipeline_recipe.py | 66 ++++++++++------------------------- 2 files changed, 60 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index d593ab8..5fb33e8 100644 --- a/README.md +++ b/README.md @@ -229,28 +229,31 @@ Or now with a flag: ... Hello hello + ``` + ## Pipeline Recipes -A **pipeline recipe** provides a pipeline without its pipe-source (sequence/stream). -A pipeline may provide a complex pipeline parts. -It is easier for the user of the pipeline to just use it (without knowing the details). +A **pipeline recipe** (aka: partial pipe) provides a pipeline without its pipe-source (sequence/stream). +A pipeline may consist of several complex pipeline parts. +It is easier to just use a **pipeline recipe** (without need of knowing the details). In addition, the use of the **pipeline recipe**: * is less error-prone and * the canned **pipeline recipe** can be tested in advance + ### SOLUTION 1: Use "yield from" expressions ```python # -- FILE: example1_pipeline_recipe_with_yield_from.py -from pipe import Pipe -from example_common_parts import select_even_numbers, multiply_by +from pipe import Pipe as as_pipe +from example_common_pipes import select_even_numbers, multiply_by # -- PIPELINE RECIPE: Using pipe-function with yield-from -# ADVANTAGE: pipeline parametrization can be done late. -@Pipe +# ADVANTAGE: Pipeline parametrization can be done late. +@as_pipe def select_even_numbers_and_multiply_by(iterable, factor): yield from (iterable | select_even_numbers | multiply_by(factor)) @@ -260,15 +263,15 @@ expected = [0, 6, 12, 18] assert results == expected ``` -### SOLUTION 2: Use or-pipes expressions +### SOLUTION 2: Use or-pipe expressions ```python # -- FILE: example2_pipeline_recipe_with_or_pipes.py -from pipe import Pipe -from example_common_parts import select_even_numbers, multiply_by +# REQUIRES: or-pipe expressions +from example_common_pipes import select_even_numbers, multiply_by -# -- PIPELINE RECIPE: Using or-pipes expressions -# NOTE: pipeline parametrization must be done early (when pipeline is defined). +# -- PIPELINE RECIPE: Using or-pipe expressions +# NOTE: Pipeline parametrization must be done early (when pipeline is defined). select_even_numbers_and_multiply_by_3 = select_even_numbers | multiply_by(3) numbers = range(8) @@ -277,20 +280,43 @@ expected = [0, 6, 12, 18] assert results == expected ``` +### SOLUTION 3: Use make-pipe idiom with or-pipe expressions + ```python -# -- FILE: example_common_parts.py -@Pipe +# -- FILE: example3_pipeline_recipe_with_make_pipe_using_or_pipes.py +# REQUIRES: or-pipe expressions +from example_common_pipes import select_even_numbers, multiply_by + +# -- PIPELINE RECIPE: Using make_pipe4 as factory function +# ADVANTAGE: Pipeline parametrization can be done late. +def make_pipe4select_even_numbers_and_multiply_by(factor): + # -- USES: or-pipe expressions + return select_even_numbers | multiply_by(factor) + +select_even_numbers_and_multiply_by_3 = make_pipe4select_even_numbers_and_multiply_by(3) + +numbers = range(8) +results = list(numbers | select_even_numbers_and_multiply_by_3) +expected = [0, 6, 12, 18] +assert results == expected +``` + +```python +# -- FILE: example_common_pipes.py +from pipe import Pipe as as_pipe + +@as_pipe def select_numbers_modulo(iterable, modulus): for number in iterable: if (number % modulus) == 0: yield number -@Pipe +@as_pipe def multiply_by(iterable, factor): for number in iterable: yield number * factor -@Pipe +@as_pipe def select_even_numbers(iterable): yield from (iterable | select_numbers_modulo(2)) ``` diff --git a/tests/test_pipeline_recipe.py b/tests/test_pipeline_recipe.py index ddbdfbd..ea47531 100644 --- a/tests/test_pipeline_recipe.py +++ b/tests/test_pipeline_recipe.py @@ -5,11 +5,11 @@ * A prepared, often complex pipeline that should easily reusable by others -SOLUTION: +SOLUTIONS: -* Use ``or-pipes`` expression (support added to: `pipe` module) -* Use pipe-functions with yield-from expression. - ADVANTAGE: Has parametrization support. +* Use pipe-functions with ``yield-from`` expression. +* Use ``or-pipe expressions`` (support added to: `pipe` module) +* Use make-idiom with ``or-pipe expressions``. SEE ALSO: @@ -23,39 +23,14 @@ .. _pipe: https://github.com/JulienPalard/Pipe """ -from functools import reduce from pipe import Pipe as as_pipe import pytest # ----------------------------------------------------------------------------- -# TEST SUPPORT FOR: Pipeline Recipes +# TEST SUPPORT FOR: Pipeline Recipes -- Common pipes # ----------------------------------------------------------------------------- -def use_pipe(iterable, the_pipe): - yield from (iterable | the_pipe) - - -# -- SIMILAR TO: functools.reduce() -- But better debuggable/explorable -# def reduce2(function, iterable): -# it = iter(iterable) -# value = next(it) -# for element in it: -# value = function(value, element) -# return value - -# -- HINT: NOT NEEDED when Pipe.__or__() is supported -def chain_pipes(*pipes): - def pipe_operator(p1, p2): - return p1 | p2 - - this_pipeline = reduce(pipe_operator, pipes) - @as_pipe - def _chain_pipes(iterable): - yield from (iterable | this_pipeline) - return _chain_pipes - - @as_pipe def select_numbers_modulo(iterable, modulus): for number in iterable: @@ -71,7 +46,7 @@ def multiply_by(iterable, factor): @as_pipe def select_even_numbers(iterable): - return use_pipe(iterable, select_numbers_modulo(2)) + yield from (iterable | select_numbers_modulo(2)) # ----------------------------------------------------------------------------- @@ -81,7 +56,7 @@ class TestPipelineRecipe(object): """A ``pipeline recipe`` is a prepared, often complex chain of pipes. It should be easy to use this pipeline recipe without knowing the details. """ - def test_recipe_with_one_pipe(self): + def test_recipe_with_yield_from_using_one_pipe(self): """Check pipeline-recipe that uses "yield from" can be used.""" @as_pipe def select_multiples_of_3(iterable): @@ -92,7 +67,7 @@ def select_multiples_of_3(iterable): expected = [0, 3, 6] assert results == expected - def test_recipe_with_many_pipes(self): + def test_recipe_with_yield_from_using_many_pipes(self): """Check pipeline-recipe that uses "yield from" with many pipes.""" @as_pipe def select_even_numbers_and_multiply_by(iterable, factor): @@ -105,7 +80,7 @@ def select_even_numbers_and_multiply_by(iterable, factor): def test_recipe_with_or_pipes(self): # REQUIRES: https://github.com/JulienPalard/Pipe/pull/88 - # HINT: or-pipes / Pipe.operator-or is provided here. + # REQUIRES: or-pipe expressions select_even_numbers_and_multiply_by_3B = select_even_numbers | multiply_by(3) numbers = range(8) @@ -113,21 +88,16 @@ def test_recipe_with_or_pipes(self): expected = [0, 6, 12, 18] assert results == expected - # -- REJECTED IDEAS: - # use_pipe() -> BETTER USE: "yield from" - # chain_pipes() -> BETTER USE: or-pipes - def test_recipe__with_chain_pipes(self): - # -- HINT: pipe-chain with "or-pipes" is simpler. - select_even_numbers_and_multiply_by_3A = chain_pipes(select_even_numbers, multiply_by(3)) + def test_recipe_with_make_pipe_using_or_pipes(self): + # REQUIRES: https://github.com/JulienPalard/Pipe/pull/88 + # REQUIRES: or-pipe expressions + def make_pipe4select_even_numbers_and_multiply_by(factor): + # -- USES: or-pipe expressions + return select_even_numbers | multiply_by(factor) - numbers = range(8) - results = list(numbers | select_even_numbers_and_multiply_by_3A) - expected = [0, 6, 12, 18] - assert results == expected + select_even_numbers_and_multiply_by_3C = make_pipe4select_even_numbers_and_multiply_by(3) - def test_use_pipeline_recipe_1(self): - """Check if a pipeline-recipe that uses "use_pipe()" can be used.""" numbers = range(8) - results = list(numbers | select_even_numbers) # USES: use_pipe() internally - expected = [0, 2, 4, 6] + results = list(numbers | select_even_numbers_and_multiply_by_3C) + expected = [0, 6, 12, 18] assert results == expected From 171ceb00961faa54650d3c17d8f97dffb2b07687 Mon Sep 17 00:00:00 2001 From: jenisys Date: Mon, 29 May 2023 14:58:15 +0200 Subject: [PATCH 5/5] Extract "common_pipes" to "tests/common_pipes.py" * Similar to the description in the README. --- tests/__init__.py | 0 tests/common_pipes.py | 21 +++++++++++++++++++++ tests/test_pipeline_recipe.py | 24 +----------------------- 3 files changed, 22 insertions(+), 23 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/common_pipes.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/common_pipes.py b/tests/common_pipes.py new file mode 100644 index 0000000..efc9055 --- /dev/null +++ b/tests/common_pipes.py @@ -0,0 +1,21 @@ +from pipe import Pipe as as_pipe + +# ----------------------------------------------------------------------------- +# TEST SUPPORT FOR: Pipeline Recipes -- Common pipes +# ----------------------------------------------------------------------------- +@as_pipe +def select_numbers_modulo(iterable, modulus): + for number in iterable: + if (number % modulus) == 0: + yield number + + +@as_pipe +def multiply_by(iterable, factor): + for number in iterable: + yield number * factor + + +@as_pipe +def select_even_numbers(iterable): + yield from (iterable | select_numbers_modulo(2)) diff --git a/tests/test_pipeline_recipe.py b/tests/test_pipeline_recipe.py index ea47531..a09f714 100644 --- a/tests/test_pipeline_recipe.py +++ b/tests/test_pipeline_recipe.py @@ -24,29 +24,7 @@ """ from pipe import Pipe as as_pipe -import pytest - - - -# ----------------------------------------------------------------------------- -# TEST SUPPORT FOR: Pipeline Recipes -- Common pipes -# ----------------------------------------------------------------------------- -@as_pipe -def select_numbers_modulo(iterable, modulus): - for number in iterable: - if (number % modulus) == 0: - yield number - - -@as_pipe -def multiply_by(iterable, factor): - for number in iterable: - yield number * factor - - -@as_pipe -def select_even_numbers(iterable): - yield from (iterable | select_numbers_modulo(2)) +from .common_pipes import select_numbers_modulo, multiply_by, select_even_numbers # -----------------------------------------------------------------------------