Feature: Pipeline recipes

README.md

@@ -232,6 +232,96 @@ hello
 
 ```
 
+
+## Pipeline Recipes
+
+A **pipeline recipe** (aka: partial pipe) provides a pipeline without its pipe-source (sequence/stream).
+A pipeline may consist of several complex pipeline parts.
+It is easier to just use a **pipeline recipe** (without need of knowing the details).
+In addition, the use of the **pipeline recipe**:
+
+* is less error-prone and
+* the canned **pipeline recipe** can be tested in advance
+
+
+### SOLUTION 1: Use "yield from" expressions
+
+```python
+# -- FILE: example1_pipeline_recipe_with_yield_from.py
+from pipe import Pipe as as_pipe
+from example_common_pipes import select_even_numbers, multiply_by
+
+# -- PIPELINE RECIPE: Using pipe-function with yield-from
+# ADVANTAGE: Pipeline parametrization can be done late.
+@as_pipe
+def select_even_numbers_and_multiply_by(iterable, factor):
+    yield from (iterable | select_even_numbers | multiply_by(factor))
+
+numbers = range(8)
+results = list(numbers | select_even_numbers_and_multiply_by(3))
+expected = [0, 6, 12, 18]
+assert results == expected
+```
+
+### SOLUTION 2: Use or-pipe expressions
+
+```python
+# -- FILE: example2_pipeline_recipe_with_or_pipes.py
+# REQUIRES: or-pipe expressions
+from example_common_pipes import select_even_numbers, multiply_by
+
+# -- PIPELINE RECIPE: Using or-pipe expressions
+# NOTE: Pipeline parametrization must be done early (when pipeline is defined).
+select_even_numbers_and_multiply_by_3 = select_even_numbers | multiply_by(3)
+
+numbers = range(8)
+results = list(numbers | select_even_numbers_and_multiply_by_3)
+expected = [0, 6, 12, 18]
+assert results == expected
+```
+
+### SOLUTION 3: Use make-pipe idiom with or-pipe expressions
+
+```python
+# -- FILE: example3_pipeline_recipe_with_make_pipe_using_or_pipes.py
+# REQUIRES: or-pipe expressions
+from example_common_pipes import select_even_numbers, multiply_by
+
+# -- PIPELINE RECIPE: Using make_pipe4<PIPE_RECIPE> as factory function
+# ADVANTAGE: Pipeline parametrization can be done late.
+def make_pipe4select_even_numbers_and_multiply_by(factor):
+    # -- USES: or-pipe expressions
+    return select_even_numbers | multiply_by(factor)
+
+select_even_numbers_and_multiply_by_3 = make_pipe4select_even_numbers_and_multiply_by(3)
+
+numbers = range(8)
+results = list(numbers | select_even_numbers_and_multiply_by_3)
+expected = [0, 6, 12, 18]
+assert results == expected
+```
+
+```python
+# -- FILE: example_common_pipes.py
+from pipe import Pipe as as_pipe
+
+@as_pipe
+def select_numbers_modulo(iterable, modulus):
+    for number in iterable:
+        if (number % modulus) == 0:
+            yield number
+
+@as_pipe
+def multiply_by(iterable, factor):
+    for number in iterable:
+        yield number * factor
+
+@as_pipe
+def select_even_numbers(iterable):
+    yield from (iterable | select_numbers_modulo(2))
+```
+
+
 # Deprecations of pipe 1.x
 
 In pipe 1.x a lot of functions were returning iterables and a lot

pipe.py

@@ -39,8 +39,19 @@ def __init__(self, function):
     def __ror__(self, other):
         return self.function(other)
 
+    # SEE: https://github.com/JulienPalard/Pipe/pull/88
+    def __or__(self, other):
+        cls = self.__class__
+        if isinstance(other, Pipe):
+            return cls(lambda iterable, *args2, **kwargs2: (
+                other.function(self.function(iterable, *args2, **kwargs2))))
+        # -- CASE: other is pipeable only
+        return cls(lambda iterable, *args2, **kwargs2: (
+                self.function(iterable, *args2, **kwargs2) | other))
+
     def __call__(self, *args, **kwargs):
-        return Pipe(
+        cls = self.__class__
+        return cls(
             lambda iterable, *args2, **kwargs2: self.function(
                 iterable, *args, *args2, **kwargs, **kwargs2
             )

tests/common_pipes.py

@@ -0,0 +1,21 @@
+from pipe import Pipe as as_pipe
+
+# -----------------------------------------------------------------------------
+# TEST SUPPORT FOR: Pipeline Recipes -- Common pipes
+# -----------------------------------------------------------------------------
+@as_pipe
+def select_numbers_modulo(iterable, modulus):
+    for number in iterable:
+        if (number % modulus) == 0:
+            yield number
+
+
+@as_pipe
+def multiply_by(iterable, factor):
+    for number in iterable:
+        yield number * factor
+
+
+@as_pipe
+def select_even_numbers(iterable):
+    yield from (iterable | select_numbers_modulo(2))

tests/test_pipe.py

@@ -38,3 +38,15 @@ def test_enumerate():
     data = [4, "abc", {"key": "value"}]
     expected = [(5, 4), (6, "abc"), (7, {"key": "value"})]
     assert list(data | pipe.enumerate(start=5)) == expected
+
+
+def test_concatenate_pipes():
+    data = range(10)
+    is_even = pipe.where(lambda x: x % 2 == 0)
+    higher_than_4 = pipe.where(lambda x: x > 4)
+    expected = [6,8]
+    # standard behavior
+    assert list(data | is_even | higher_than_4) == expected
+    # concatenated pipes
+    is_even_and_higher_than_4 = is_even | higher_than_4
+    assert list(data | is_even_and_higher_than_4) == expected

tests/test_pipe_basics.py

@@ -0,0 +1,109 @@
+"""
+Explore the `pipe`_ module with some tests.
+
+SEE ALSO:
+
+* https://github.com/JulienPalard/Pipe
+
+RELATED:
+
+* pyfunctional -- operator chaining instead of pipe-expressions.
+* https://github.com/sspipe/sspipe
+
+.. _pipe: https://github.com/JulienPalard/Pipe
+"""
+
+from pipe import Pipe as as_pipe, where, take_while
+import pytest
+
+
+# -----------------------------------------------------------------------------
+# EXPLORATION: PIPE-ADAPTER(s)
+# -----------------------------------------------------------------------------
+@as_pipe
+def inspect_pipe(iterable, func=None, predicate=None):
+    """Inspect what items pass through this pipe."""
+    if func is None:
+        func = print
+    if predicate is None:
+        predicate = lambda _x: True
+
+    for item in iterable | take_while(predicate):
+        func(item)
+        yield item
+
+@as_pipe
+def to_list(iterable):
+    """Pipe adapter that converts the output of this pipe-segment into a list.
+
+    .. code-block::
+
+        outputs = range(5) | ... | to_list
+        assert isinstance(outputs, list)
+    """
+    return list(iterable)
+
+
+
+# -----------------------------------------------------------------------------
+# TEST SUITE
+# -----------------------------------------------------------------------------
+def test_pipe_basics():
+    """
+    IDEA::
+        is_even = where(lambda x: x % 2)
+        result = list(range(4)) | is_even | to_list
+    """
+    inputs = list(range(6))
+    is_odd = where(lambda x: x % 2 == 1)
+    outputs = list(inputs | is_odd)     # OTHERWISE: result is a generator
+
+    expected = [1, 3, 5]
+    assert outputs == expected
+    assert isinstance(outputs, list)
+
+def test_to_list__as_pipe_adapter():
+    """
+    IDEA::
+        is_even = where(lambda x: x % 2)
+        result = list(range(4)) | is_even | to_list
+    """
+    inputs = list(range(5))
+    is_even = where(lambda x: x % 2 == 0)
+    outputs = inputs | is_even | to_list
+
+    expected = [0, 2, 4]
+    assert outputs == expected
+    assert isinstance(outputs, list)
+
+def test_pipe_sink():
+    """Test how a pipe-sink can be provided.
+
+    IDEA::
+
+        is_even = where(lambda x: x % 2)
+        result = list(range(4)) | is_even | collector
+
+    """
+    class Collector(object):
+        def __init__(self, data=None):
+            self.data = data or []
+
+        def append(self, item):
+            self.data.append(item)
+
+    @as_pipe
+    def collect_to(iterable, container):
+        for item in iterable:
+            container.append(item)
+        return container
+
+    inputs = list(range(5))
+    is_even = where(lambda x: x % 2 == 0)
+    collector = Collector()
+    output = inputs | is_even | collect_to(collector)
+
+    expected = [0, 2, 4]
+    assert output.data == expected
+    assert output is collector
+    assert isinstance(output, Collector)

tests/test_pipeline_recipe.py

@@ -0,0 +1,81 @@
+"""
+Tests related to the "pipeline recipe" idea.
+
+DEFINITION: Pipeline recipe
+
+* A prepared, often complex pipeline that should easily reusable by others
+
+SOLUTIONS:
+
+* Use pipe-functions with ``yield-from`` expression.
+* Use ``or-pipe expressions`` (support added to: `pipe` module)
+* Use make-idiom with ``or-pipe expressions``.
+
+SEE ALSO:
+
+* https://github.com/JulienPalard/Pipe/pull/88
+
+RELATED:
+
+* pyfunctional -- operator chaining instead of pipe-expressions.
+* https://github.com/sspipe/sspipe
+
+.. _pipe: https://github.com/JulienPalard/Pipe
+"""
+
+from pipe import Pipe as as_pipe
+from .common_pipes import select_numbers_modulo, multiply_by, select_even_numbers
+
+
+# -----------------------------------------------------------------------------
+# TEST SUITE: Pipeline Recipes
+# -----------------------------------------------------------------------------
+class TestPipelineRecipe(object):
+    """A ``pipeline recipe`` is a prepared, often complex chain of pipes.
+    It should be easy to use this pipeline recipe without knowing the details.
+    """
+    def test_recipe_with_yield_from_using_one_pipe(self):
+        """Check pipeline-recipe that uses "yield from" can be used."""
+        @as_pipe
+        def select_multiples_of_3(iterable):
+            yield from (iterable | select_numbers_modulo(3))
+
+        numbers = range(8)
+        results = list(numbers | select_multiples_of_3)
+        expected = [0, 3, 6]
+        assert results == expected
+
+    def test_recipe_with_yield_from_using_many_pipes(self):
+        """Check pipeline-recipe that uses "yield from" with many pipes."""
+        @as_pipe
+        def select_even_numbers_and_multiply_by(iterable, factor):
+            yield from (iterable | select_even_numbers | multiply_by(factor))
+
+        numbers = range(8)
+        results = list(numbers | select_even_numbers_and_multiply_by(3))
+        expected = [0, 6, 12, 18]
+        assert results == expected
+
+    def test_recipe_with_or_pipes(self):
+        # REQUIRES: https://github.com/JulienPalard/Pipe/pull/88
+        # REQUIRES: or-pipe expressions
+        select_even_numbers_and_multiply_by_3B = select_even_numbers | multiply_by(3)
+
+        numbers = range(8)
+        results = list(numbers | select_even_numbers_and_multiply_by_3B)
+        expected = [0, 6, 12, 18]
+        assert results == expected
+
+    def test_recipe_with_make_pipe_using_or_pipes(self):
+        # REQUIRES: https://github.com/JulienPalard/Pipe/pull/88
+        # REQUIRES: or-pipe expressions
+        def make_pipe4select_even_numbers_and_multiply_by(factor):
+            # -- USES: or-pipe expressions
+            return select_even_numbers | multiply_by(factor)
+
+        select_even_numbers_and_multiply_by_3C = make_pipe4select_even_numbers_and_multiply_by(3)
+
+        numbers = range(8)
+        results = list(numbers | select_even_numbers_and_multiply_by_3C)
+        expected = [0, 6, 12, 18]
+        assert results == expected