From adac30f788931b929ce4b216f8718d619a46ef34 Mon Sep 17 00:00:00 2001 From: marcosschroh Date: Tue, 19 Dec 2023 17:41:56 +0100 Subject: [PATCH] feat: Middleware capability introduced --- README.md | 1 + docs/middleware.md | 226 ++++++++++++ examples/dlq-middleware/README.md | 46 +++ .../dlq-middleware/dlq_middleware/__init__.py | 0 examples/dlq-middleware/dlq_middleware/app.py | 43 +++ examples/dlq-middleware/poetry.lock | 348 ++++++++++++++++++ examples/dlq-middleware/pyproject.toml | 18 + examples/joker-middleware-example/README.md | 105 ++++++ .../joker_middleware_example/__init__.py | 0 .../joker_middleware_example/app.py | 36 ++ examples/joker-middleware-example/poetry.lock | 348 ++++++++++++++++++ .../joker-middleware-example/pyproject.toml | 18 + .../simple_consumer_example/app.py | 9 +- kstreams/__init__.py | 3 +- kstreams/create.py | 8 +- kstreams/engine.py | 71 ++-- kstreams/middleware/__init__.py | 12 + kstreams/middleware/middleware.py | 72 ++++ kstreams/streams.py | 138 ++++--- kstreams/types.py | 30 +- mkdocs.yml | 7 +- poetry.lock | 16 +- pyproject.toml | 1 + scripts/cluster/start | 1 + tests/middleware/conftest.py | 46 +++ tests/middleware/test_middleware.py | 318 ++++++++++++++++ 26 files changed, 1822 insertions(+), 99 deletions(-) create mode 100644 docs/middleware.md create mode 100644 examples/dlq-middleware/README.md create mode 100644 examples/dlq-middleware/dlq_middleware/__init__.py create mode 100644 examples/dlq-middleware/dlq_middleware/app.py create mode 100644 examples/dlq-middleware/poetry.lock create mode 100644 examples/dlq-middleware/pyproject.toml create mode 100644 examples/joker-middleware-example/README.md create mode 100644 examples/joker-middleware-example/joker_middleware_example/__init__.py create mode 100644 examples/joker-middleware-example/joker_middleware_example/app.py create mode 100644 examples/joker-middleware-example/poetry.lock create mode 100644 examples/joker-middleware-example/pyproject.toml create mode 100644 kstreams/middleware/__init__.py create mode 100644 kstreams/middleware/middleware.py create mode 100644 tests/middleware/conftest.py create mode 100644 tests/middleware/test_middleware.py diff --git a/README.md b/README.md index 63b6a9d8..94d77187 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ if __name__ == "__main__": - [x] Easy to integrate with any `async` framework. No tied to any library!! - [x] Yield events from streams - [x] [Opentelemetry Instrumentation](https://github.com/kpn/opentelemetry-instrumentation-kstreams) +- [x] Middlewares - [ ] Store (kafka streams pattern) - [ ] Stream Join - [ ] Windowing diff --git a/docs/middleware.md b/docs/middleware.md new file mode 100644 index 00000000..b61bbc51 --- /dev/null +++ b/docs/middleware.md @@ -0,0 +1,226 @@ +# Middleware + +Kstreams allows you to include middlewares for adding behavior to streams. + +A *middleware* is a `callable` that works with every `ConsumerRecord` (CR) *before* and *after* it is processed by a specific `stream`. `Middlewares` also have access to the `stream` and `send` function. + +- It takes each `CR` that arrives to a `kafka topic`. +- Then it can do something to the `CR` or run any needed code. +- Then it passes the `CR` to be processed by another `callable` (other middleware or stream). +- Once the `CR` is processed by the stream, the chain is "completed". +- If there is code after the `self.next_call(cr)` then it will be executed. + +Kstreams `Middleware` have the following protocol: + +::: kstreams.middleware.middleware.Middleware + +!!! note + The `__call__` method can return anything so previous calls can use the returned value. Make sure that the line `return await self.next_call(cr)` is in your method + +!!! warning + Middlewares only work with the new [Dependency Injection approach](https://kpn.github.io/kstreams/stream/#dependency-injection-and-typing) + +## Creating a middleware + +To create a middleware you have to create a class that inherits from `BaseMiddleware`. Then, the method `async def __call__` must be defined. Let's consider that we want to save the CR to `elastic` before it is processed: + +```python +import typing + +from kstreams import ConsumerRecord, middleware + +async def save_to_elastic(cr: ConsumerRecord) -> None: + ... + + +class ElasticMiddleware(middleware.BaseMiddleware): + async def __call__(self, cr: ConsumerRecord) -> typing.Any: + # save to elastic before calling the next + await save_to_elastic(cr) + + # the next call could be another middleware + return await self.next_call(cr) +``` + +Then, we have to include the middleware. It can be per `stream` or per application level (`stream_engine`): + +=== "Include middleware in stream" + ```python + from kstreams import ConsumerRecord, middleware + + from .engine import stream_engine + + + middlewares = [middleware.Middleware(ElasticMiddleware)] + + @stream_engine.stream("kstreams-topic", middlewares=middlewares) + async def processor(cr: ConsumerRecord): + ... + ``` +=== "Include middleware at application level" + ```python + from kstreams import ConsumerRecord, middleware, create + + middlewares = [middleware.Middleware(ElasticMiddleware)] + + stream_engine = kstreams.create_engine( + title="my-stream-engine", + middlewares=middlewares + ) + ``` + +!!! note + The example middlewares make sense to include it at the application level, but other ones like Dead Letter Queue (DLQ) make sense to add them at stream level + +!!! note + The `Middleware` concept also applies for `async generators` (yield from a stream) + +## Adding extra configuration to middlewares + +If you want to provide extra configuration to middleware you should override the __init__ method with the extra options as `keywargs` and then call `super().__init__(**kwargs)` + +Let's consider that we want to send an event to a spcific `topic` when a `ValueError` is raised inside a `stream` (Dead Letter Queue) + +```python +from kstreams import ConsumerRecord, types, Stream, middleware + + +class DLQMiddleware(middleware.BaseMiddleware): + def __init__(self, *, topic: str, **kwargs) -> None: + super().__init__(**kwargs) + self.topic = topic + + async def __call__(self, cr: ConsumerRecord): + try: + return await self.next_call(cr) + except ValueError: + await self.send(self.topic, key=cr.key, value=cr.value) + + +# Create the middlewares +middlewares = [ + middleware.Middleware( + DLQMiddleware, topic="kstreams-dlq-topic" + ) +] + +@stream_engine.stream("kstreams-topic", middlewares=middlewares) + async def processor(cr: ConsumerRecord): + if cr.value == b"joker": + raise ValueError("Joker received...") +``` + +## Middleware by default + +Kstreams includes one middleware by default, `ExceptionMiddleware`. This middleware adds exception handlers, for particular types of expected exception cases, for example when the `Consumer` stops (kafka disconnects), user presses `CTRL+C` or any other exception that could cause the `stream` to crash. + +::: kstreams.middleware.middleware.ExceptionMiddleware + +## Middleware chain + +It is possible to add as many middlewares as you want in order to split and reuse business logic, however the downside is extra complexity and the code might become slower. Also, there are some points to take into account: + +- The order when adding middlewares is important. +- If middlewares are added to a `stream` and `stream_engine`, then the middleware stack is build first with `stream` middlewares and then with the `stream_engines` middlewares. This means the first the `stream` middlewares are evaluated first. + + +In the example we are adding three middelwares in the following order: `DLQMiddleware`, `ElasticMiddleware`, and `S3Middleware`. The code chain execution will be: + +```mermaid +sequenceDiagram + autonumber + ExceptionMiddleware->>DLQMiddleware: + Note left of ExceptionMiddleware: Event received + alt No Processing Error + DLQMiddleware->>ElasticMiddleware: + Note right of ElasticMiddleware: Store CR on Elastic + ElasticMiddleware->>S3Middleware: + Note right of S3Middleware: Store CR on S3 + S3Middleware->>Stream: + Note right of Stream: CR processed + Stream-->>S3Middleware: + S3Middleware-->>ElasticMiddleware: + ElasticMiddleware-->>DLQMiddleware: + DLQMiddleware-->>ExceptionMiddleware: + end +``` + +```python title="Multiple middlewares example" +from kstreams import ConsumerRecord, Stream, middleware + + +class DLQMiddleware(middleware.BaseMiddleware): + async def __call__(self, cr: ConsumerRecord): + try: + return await self.next_call(cr) + except ValueError: + await dlq(cr.value) + + +class ElasticMiddleware(middleware.BaseMiddleware): + async def __call__(self, cr: ConsumerRecord): + await save_to_elastic(cr.value) + return await self.next_call(cr) + + +class S3Middleware(middleware.BaseMiddleware): + async def __call__(self, cr: ConsumerRecord): + await backup_to_s3(cr.value) + return await self.next_call(cr) + + +middlewares = [ + middleware.Middleware(DLQMiddleware), + middleware.Middleware(ElasticMiddleware), + middleware.Middleware(S3Middleware), +] + + +@stream_engine.stream("kstreams-topic", middlewares=middlewares) +async def processor(cr: ConsumerRecord): + if cr.value == event_2: + raise ValueError("Error from stream...") + await save_to_db(cr.value) +``` + +!!! note + In the example we can see that always the `cr` will be save into `elastic` and `s3` regardless an error + +## Executing Code after the CR was processed + +As mentioned in the introduction, it is possible to execute code after the `CR` is handled. To do this, we need to place code after `next_call` is called: + +```python title="Execute code after CR is handled" +from kstreams import ConsumerRecord, Stream, middleware + + +class DLQMiddleware(middleware.BaseMiddleware): + async def __call__(self, cr: ConsumerRecord): + try: + return await self.next_call(cr) + except ValueError: + await dlq(cr.value) + + +class ElasticMiddleware(middleware.BaseMiddleware): + async def __call__(self, cr: ConsumerRecord): + return await self.next_call(cr) + # This will be called after the whole chain has finished + await save_to_elastic(cr.value) + + +middlewares = [ + middleware.Middleware(DLQMiddleware), + middleware.Middleware(ElasticMiddleware), +] + + +@stream_engine.stream("kstreams-topic", middlewares=middlewares) +async def processor(cr: ConsumerRecord): + if cr.value == event_2: + raise ValueError("Error from stream...") + await save_to_db(cr.value) +``` + +!!! note + In the example we can see that only if there is not an `error` the event is saved to `elastic` diff --git a/examples/dlq-middleware/README.md b/examples/dlq-middleware/README.md new file mode 100644 index 00000000..7b274b8d --- /dev/null +++ b/examples/dlq-middleware/README.md @@ -0,0 +1,46 @@ +# Dead Letter Queue (DLQ) middleware Example + +## Requirements + +python 3.8+, poetry, docker-compose + +## Installation + +```bash +poetry install +``` + +## Explanation + +This shows how the `Middleware` concept can be applied to use a `DLQ`. In this example every time that the `ValueError` exception is raised inside the `stream`, meaning that the event was nor processed, the middleware will send an event to the topic `kstreams--dlq-topic`. + +## Usage + +1. Start the kafka cluster: From `kstreams` project root execute `./scripts/cluster/start` +2. Inside this folder execute `poetry run app` +3. From `kstreams` project root, you can use the `./scripts/cluster/events/send` to send events to the kafka cluster. A prompt will open. Enter messages to send. The command is: +```bash +./scripts/cluster/events/send "local--hello-world" +``` + +Then, on the consume side, you should see something similar to the following logs: + +```bash +❯ me@me-pc middleware-example % poetry run app + +Group Coordinator Request failed: [Error 15] CoordinatorNotAvailableError +Group Coordinator Request failed: [Error 15] CoordinatorNotAvailableError +Group Coordinator Request failed: [Error 15] CoordinatorNotAvailableError +Group Coordinator Request failed: [Error 15] CoordinatorNotAvailableError +Group Coordinator Request failed: [Error 15] CoordinatorNotAvailableError +Consumer started +Event consumed: headers: (), payload: ConsumerRecord(topic='local--hello-world', partition=0, offset=0, timestamp=1660733921761, timestamp_type=0, key=None, value=b'boo', checksum=None, serialized_key_size=-1, serialized_value_size=3, headers=()) +``` + +4. Consume from the topic `kstreams--dlq-topic`: `./scripts/cluster/events/read kstreams--dlq-topic` +5. Produce the event `joker` using the terminal opened in step `3`. Then check the terminal opened in the previous step and the event `joker` must appear + +## Note + +If you plan on using this example, pay attention to the `pyproject.toml` dependencies, where +`kstreams` is pointing to the parent folder. You will have to set the latest version. diff --git a/examples/dlq-middleware/dlq_middleware/__init__.py b/examples/dlq-middleware/dlq_middleware/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/dlq-middleware/dlq_middleware/app.py b/examples/dlq-middleware/dlq_middleware/app.py new file mode 100644 index 00000000..5ef77319 --- /dev/null +++ b/examples/dlq-middleware/dlq_middleware/app.py @@ -0,0 +1,43 @@ +import aiorun + +from kstreams import ConsumerRecord, create_engine, middleware + + +class DLQMiddleware(middleware.BaseMiddleware): + def __init__(self, *, topic: str, **kwargs) -> None: + super().__init__(**kwargs) + self.topic = topic + + async def __call__(self, cr: ConsumerRecord): + try: + return await self.next_call(cr) + except ValueError as exc: + print(f"\n Event crashed because {str(exc)} \n") + print(f"\n Producing event {cr.value} to DLQ topic {self.topic} \n") + await self.send(self.topic, key=cr.key, value=cr.value) + + +middlewares = [ + middleware.Middleware(DLQMiddleware, topic="kstreams--dlq-topic"), +] +stream_engine = create_engine(title="my-stream-engine") + + +@stream_engine.stream(topics=["local--hello-world"], middlewares=middlewares) +async def consume(cr: ConsumerRecord): + print(f"Event consumed: headers: {cr.headers}, payload: {cr}") + if cr.value == b"joker": + raise ValueError("🤡 🤡 🤡 🤡 🤡 🤡 🤡 🤡") + print("Event has been proccesses") + + +async def start(): + await stream_engine.start() + + +async def shutdown(loop): + await stream_engine.stop() + + +def main(): + aiorun.run(start(), stop_on_unhandled_errors=True, shutdown_callback=shutdown) diff --git a/examples/dlq-middleware/poetry.lock b/examples/dlq-middleware/poetry.lock new file mode 100644 index 00000000..a98633dd --- /dev/null +++ b/examples/dlq-middleware/poetry.lock @@ -0,0 +1,348 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "aiokafka" +version = "0.10.0" +description = "Kafka integration with asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiokafka-0.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ebe5be9f578e89e6db961121070f7c35662924abee00ba4ccf64557e2cdd7edf"}, + {file = "aiokafka-0.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:007f1c51f440cc07155d2491f4deea6536492324153296aa73736a74cd833d3e"}, + {file = "aiokafka-0.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22299f8d5269dcb00b1b53fdee44dbe729091d4038e1bb63d0bb2f5cdf9af47a"}, + {file = "aiokafka-0.10.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fafc95bdaed9e1810fcd80b02ac117e51c72681ffe50353e5d61e2170609e1fc"}, + {file = "aiokafka-0.10.0-cp310-cp310-win32.whl", hash = "sha256:f2f19dee69c69389f5911e6b23c361c5285366d237f782eaae118d12acc42d7f"}, + {file = "aiokafka-0.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:99127ab680f9b08b0213d00b7d1e0480c6d08601f52ad42e829350f9599db301"}, + {file = "aiokafka-0.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5efb63686562809f0f9bf0fa6d1e52f222af2d8f8441f8c412b156f15c98da43"}, + {file = "aiokafka-0.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b91109dc25f79be4d27454cc766239a5368d18b26682d4b5c6b913ca92691220"}, + {file = "aiokafka-0.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d52c25f3d0db7dd340a5d08108da302db1ba64c2190970dbdb768b79629d6add"}, + {file = "aiokafka-0.10.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1509c1b29cd1d4d920a649f257d72109bbc3d61431135505b8e0d8d488796ff2"}, + {file = "aiokafka-0.10.0-cp311-cp311-win32.whl", hash = "sha256:ffc30e4c6bfcb00356a002f623c93a51d8336ca67687ea069dd11822da07379c"}, + {file = "aiokafka-0.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e10fdee4189fe7eed36d602df822e9ff4f19535c0a514cf015f78308d206c1a"}, + {file = "aiokafka-0.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:82a75ea13d7e6e11c7ee2fb9419e9ea3541744648c69ab27b56fb6bca5b319c1"}, + {file = "aiokafka-0.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf9e241766b7f4c305807763330dacf8c220ad9e8fc7f2b22730a2db66fad61d"}, + {file = "aiokafka-0.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12d703317812262feac6577ff488f2ccddc4408da0ff608a5454062782b5a80d"}, + {file = "aiokafka-0.10.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b74aeacfb8ced9764002c63b58e4c78c94809131d89000cb936c25c298ffb1e"}, + {file = "aiokafka-0.10.0-cp312-cp312-win32.whl", hash = "sha256:de56c503b3d64e24a5b6705e55bc524a8357b0495402f859f921a71d65274cb1"}, + {file = "aiokafka-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:f4b22a31f40493cea50dddb4dfc92750dfb273635ccb094a16fde9678eb38958"}, + {file = "aiokafka-0.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7068f0beb8478cde09618dcc9a833cc18ff37bd14864fa8b60ad4e4c3dad6489"}, + {file = "aiokafka-0.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f069bda1f31e466d815b631a07bc6fad5190b29dfff5f117bcbf1948cd7a38aa"}, + {file = "aiokafka-0.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e16d8a23f0e173e5ca86c2d1c270e25a529a0eed973c77d7e8a0dfc868699aa4"}, + {file = "aiokafka-0.10.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf4a47659517000a8fe88e0fb353898b718ee214e21f62a2a949be9bf801cd9e"}, + {file = "aiokafka-0.10.0-cp38-cp38-win32.whl", hash = "sha256:781ab300214681e40667185a402abf6b31b4c4b8f1cdabbdc3549d8cf383b34d"}, + {file = "aiokafka-0.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:06060708a4bcf062be496c8641fca382c88782d3c381a34ccb5ac8677bdac695"}, + {file = "aiokafka-0.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c23ec22fbf26e2f84678f0589076bea1ff26ae6dfd3c601e6de10ad00d605261"}, + {file = "aiokafka-0.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74229a57c95e2efccec95d9b42554dc168c97a263f013e3e983202bd33ca189d"}, + {file = "aiokafka-0.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e833e4ef7fc5f3f637ba5fb4210acc7e5ea916bb7107e4b619b1b1a3e361bc62"}, + {file = "aiokafka-0.10.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9728c523f10ac4bb46719cc64f3c1d47625898872bc3901b22b9d48b6e401d1c"}, + {file = "aiokafka-0.10.0-cp39-cp39-win32.whl", hash = "sha256:05c4a7ced5d6f3dbc289767574d6a5d9b31e1c243e992dcecd34dbc40fcbbf9b"}, + {file = "aiokafka-0.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:1fe0194ea72524df37369a8cf0837263b55194ac20616e612f0ab7bfb568b76b"}, + {file = "aiokafka-0.10.0.tar.gz", hash = "sha256:7ce35563f955490b43190e3389b5f3d92d50e22b32d1a40772fd14fb1d50c5db"}, +] + +[package.dependencies] +async-timeout = "*" +packaging = "*" + +[package.extras] +all = ["cramjam", "gssapi", "lz4 (>=3.1.3)"] +gssapi = ["gssapi"] +lz4 = ["lz4 (>=3.1.3)"] +snappy = ["cramjam"] +zstd = ["cramjam"] + +[[package]] +name = "aiorun" +version = "2023.7.2" +description = "Boilerplate for asyncio applications" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiorun-2023.7.2-py3-none-any.whl", hash = "sha256:61f839265d86d3f3b37921eb4a99ef42fa2ee026241b67c54315423d7dac29c9"}, + {file = "aiorun-2023.7.2.tar.gz", hash = "sha256:95e689dc1b263aaf4556ef8fa73ccae68f38ba0f1c1017fe196ef7da244031f9"}, +] + +[package.extras] +dev = ["pytest", "pytest-cov"] + +[[package]] +name = "annotated-types" +version = "0.6.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, + {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.9\""} + +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "future" +version = "0.18.3" +description = "Clean single-source support for Python 3 and 2" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"}, +] + +[[package]] +name = "kstreams" +version = "0.15.1" +description = "Build simple kafka streams applications" +optional = false +python-versions = "^3.8" +files = [] +develop = true + +[package.dependencies] +aiokafka = "<1.0" +aiorun = "^2023.7.2" +future = "^0.18.2" +prometheus-client = "<1.0" +pydantic = "<3.0.0" +PyYAML = ">=5.4,<7.0.0" + +[package.source] +type = "directory" +url = "../.." + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "prometheus-client" +version = "0.19.0" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.8" +files = [ + {file = "prometheus_client-0.19.0-py3-none-any.whl", hash = "sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92"}, + {file = "prometheus_client-0.19.0.tar.gz", hash = "sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1"}, +] + +[package.extras] +twisted = ["twisted"] + +[[package]] +name = "pydantic" +version = "2.5.3" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-2.5.3-py3-none-any.whl", hash = "sha256:d0caf5954bee831b6bfe7e338c32b9e30c85dfe080c843680783ac2b631673b4"}, + {file = "pydantic-2.5.3.tar.gz", hash = "sha256:b3ef57c62535b0941697cce638c08900d87fcb67e29cfa99e8a68f747f393f7a"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.14.6" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.14.6" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic_core-2.14.6-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:72f9a942d739f09cd42fffe5dc759928217649f070056f03c70df14f5770acf9"}, + {file = "pydantic_core-2.14.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6a31d98c0d69776c2576dda4b77b8e0c69ad08e8b539c25c7d0ca0dc19a50d6c"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5aa90562bc079c6c290f0512b21768967f9968e4cfea84ea4ff5af5d917016e4"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:370ffecb5316ed23b667d99ce4debe53ea664b99cc37bfa2af47bc769056d534"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f85f3843bdb1fe80e8c206fe6eed7a1caeae897e496542cee499c374a85c6e08"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9862bf828112e19685b76ca499b379338fd4c5c269d897e218b2ae8fcb80139d"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:036137b5ad0cb0004c75b579445a1efccd072387a36c7f217bb8efd1afbe5245"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:92879bce89f91f4b2416eba4429c7b5ca22c45ef4a499c39f0c5c69257522c7c"}, + {file = "pydantic_core-2.14.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0c08de15d50fa190d577e8591f0329a643eeaed696d7771760295998aca6bc66"}, + {file = "pydantic_core-2.14.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:36099c69f6b14fc2c49d7996cbf4f87ec4f0e66d1c74aa05228583225a07b590"}, + {file = "pydantic_core-2.14.6-cp310-none-win32.whl", hash = "sha256:7be719e4d2ae6c314f72844ba9d69e38dff342bc360379f7c8537c48e23034b7"}, + {file = "pydantic_core-2.14.6-cp310-none-win_amd64.whl", hash = "sha256:36fa402dcdc8ea7f1b0ddcf0df4254cc6b2e08f8cd80e7010d4c4ae6e86b2a87"}, + {file = "pydantic_core-2.14.6-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:dea7fcd62915fb150cdc373212141a30037e11b761fbced340e9db3379b892d4"}, + {file = "pydantic_core-2.14.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ffff855100bc066ff2cd3aa4a60bc9534661816b110f0243e59503ec2df38421"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b027c86c66b8627eb90e57aee1f526df77dc6d8b354ec498be9a757d513b92b"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:00b1087dabcee0b0ffd104f9f53d7d3eaddfaa314cdd6726143af6bc713aa27e"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:75ec284328b60a4e91010c1acade0c30584f28a1f345bc8f72fe8b9e46ec6a96"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e1f4744eea1501404b20b0ac059ff7e3f96a97d3e3f48ce27a139e053bb370b"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2602177668f89b38b9f84b7b3435d0a72511ddef45dc14446811759b82235a1"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6c8edaea3089bf908dd27da8f5d9e395c5b4dc092dbcce9b65e7156099b4b937"}, + {file = "pydantic_core-2.14.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:478e9e7b360dfec451daafe286998d4a1eeaecf6d69c427b834ae771cad4b622"}, + {file = "pydantic_core-2.14.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b6ca36c12a5120bad343eef193cc0122928c5c7466121da7c20f41160ba00ba2"}, + {file = "pydantic_core-2.14.6-cp311-none-win32.whl", hash = "sha256:2b8719037e570639e6b665a4050add43134d80b687288ba3ade18b22bbb29dd2"}, + {file = "pydantic_core-2.14.6-cp311-none-win_amd64.whl", hash = "sha256:78ee52ecc088c61cce32b2d30a826f929e1708f7b9247dc3b921aec367dc1b23"}, + {file = "pydantic_core-2.14.6-cp311-none-win_arm64.whl", hash = "sha256:a19b794f8fe6569472ff77602437ec4430f9b2b9ec7a1105cfd2232f9ba355e6"}, + {file = "pydantic_core-2.14.6-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:667aa2eac9cd0700af1ddb38b7b1ef246d8cf94c85637cbb03d7757ca4c3fdec"}, + {file = "pydantic_core-2.14.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdee837710ef6b56ebd20245b83799fce40b265b3b406e51e8ccc5b85b9099b7"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c5bcf3414367e29f83fd66f7de64509a8fd2368b1edf4351e862910727d3e51"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:26a92ae76f75d1915806b77cf459811e772d8f71fd1e4339c99750f0e7f6324f"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a983cca5ed1dd9a35e9e42ebf9f278d344603bfcb174ff99a5815f953925140a"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cb92f9061657287eded380d7dc455bbf115430b3aa4741bdc662d02977e7d0af"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4ace1e220b078c8e48e82c081e35002038657e4b37d403ce940fa679e57113b"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ef633add81832f4b56d3b4c9408b43d530dfca29e68fb1b797dcb861a2c734cd"}, + {file = "pydantic_core-2.14.6-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7e90d6cc4aad2cc1f5e16ed56e46cebf4877c62403a311af20459c15da76fd91"}, + {file = "pydantic_core-2.14.6-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e8a5ac97ea521d7bde7621d86c30e86b798cdecd985723c4ed737a2aa9e77d0c"}, + {file = "pydantic_core-2.14.6-cp312-none-win32.whl", hash = "sha256:f27207e8ca3e5e021e2402ba942e5b4c629718e665c81b8b306f3c8b1ddbb786"}, + {file = "pydantic_core-2.14.6-cp312-none-win_amd64.whl", hash = "sha256:b3e5fe4538001bb82e2295b8d2a39356a84694c97cb73a566dc36328b9f83b40"}, + {file = "pydantic_core-2.14.6-cp312-none-win_arm64.whl", hash = "sha256:64634ccf9d671c6be242a664a33c4acf12882670b09b3f163cd00a24cffbd74e"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:24368e31be2c88bd69340fbfe741b405302993242ccb476c5c3ff48aeee1afe0"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:e33b0834f1cf779aa839975f9d8755a7c2420510c0fa1e9fa0497de77cd35d2c"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6af4b3f52cc65f8a0bc8b1cd9676f8c21ef3e9132f21fed250f6958bd7223bed"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15687d7d7f40333bd8266f3814c591c2e2cd263fa2116e314f60d82086e353a"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:095b707bb287bfd534044166ab767bec70a9bba3175dcdc3371782175c14e43c"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94fc0e6621e07d1e91c44e016cc0b189b48db053061cc22d6298a611de8071bb"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ce830e480f6774608dedfd4a90c42aac4a7af0a711f1b52f807130c2e434c06"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a306cdd2ad3a7d795d8e617a58c3a2ed0f76c8496fb7621b6cd514eb1532cae8"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:2f5fa187bde8524b1e37ba894db13aadd64faa884657473b03a019f625cee9a8"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:438027a975cc213a47c5d70672e0d29776082155cfae540c4e225716586be75e"}, + {file = "pydantic_core-2.14.6-cp37-none-win32.whl", hash = "sha256:f96ae96a060a8072ceff4cfde89d261837b4294a4f28b84a28765470d502ccc6"}, + {file = "pydantic_core-2.14.6-cp37-none-win_amd64.whl", hash = "sha256:e646c0e282e960345314f42f2cea5e0b5f56938c093541ea6dbf11aec2862391"}, + {file = "pydantic_core-2.14.6-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:db453f2da3f59a348f514cfbfeb042393b68720787bbef2b4c6068ea362c8149"}, + {file = "pydantic_core-2.14.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3860c62057acd95cc84044e758e47b18dcd8871a328ebc8ccdefd18b0d26a21b"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36026d8f99c58d7044413e1b819a67ca0e0b8ebe0f25e775e6c3d1fabb3c38fb"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ed1af8692bd8d2a29d702f1a2e6065416d76897d726e45a1775b1444f5928a7"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:314ccc4264ce7d854941231cf71b592e30d8d368a71e50197c905874feacc8a8"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:982487f8931067a32e72d40ab6b47b1628a9c5d344be7f1a4e668fb462d2da42"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dbe357bc4ddda078f79d2a36fc1dd0494a7f2fad83a0a684465b6f24b46fe80"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2f6ffc6701a0eb28648c845f4945a194dc7ab3c651f535b81793251e1185ac3d"}, + {file = "pydantic_core-2.14.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7f5025db12fc6de7bc1104d826d5aee1d172f9ba6ca936bf6474c2148ac336c1"}, + {file = "pydantic_core-2.14.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dab03ed811ed1c71d700ed08bde8431cf429bbe59e423394f0f4055f1ca0ea60"}, + {file = "pydantic_core-2.14.6-cp38-none-win32.whl", hash = "sha256:dfcbebdb3c4b6f739a91769aea5ed615023f3c88cb70df812849aef634c25fbe"}, + {file = "pydantic_core-2.14.6-cp38-none-win_amd64.whl", hash = "sha256:99b14dbea2fdb563d8b5a57c9badfcd72083f6006caf8e126b491519c7d64ca8"}, + {file = "pydantic_core-2.14.6-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:4ce8299b481bcb68e5c82002b96e411796b844d72b3e92a3fbedfe8e19813eab"}, + {file = "pydantic_core-2.14.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b9a9d92f10772d2a181b5ca339dee066ab7d1c9a34ae2421b2a52556e719756f"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd9e98b408384989ea4ab60206b8e100d8687da18b5c813c11e92fd8212a98e0"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4f86f1f318e56f5cbb282fe61eb84767aee743ebe32c7c0834690ebea50c0a6b"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86ce5fcfc3accf3a07a729779d0b86c5d0309a4764c897d86c11089be61da160"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dcf1978be02153c6a31692d4fbcc2a3f1db9da36039ead23173bc256ee3b91b"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eedf97be7bc3dbc8addcef4142f4b4164066df0c6f36397ae4aaed3eb187d8ab"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d5f916acf8afbcab6bacbb376ba7dc61f845367901ecd5e328fc4d4aef2fcab0"}, + {file = "pydantic_core-2.14.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8a14c192c1d724c3acbfb3f10a958c55a2638391319ce8078cb36c02283959b9"}, + {file = "pydantic_core-2.14.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0348b1dc6b76041516e8a854ff95b21c55f5a411c3297d2ca52f5528e49d8411"}, + {file = "pydantic_core-2.14.6-cp39-none-win32.whl", hash = "sha256:de2a0645a923ba57c5527497daf8ec5df69c6eadf869e9cd46e86349146e5975"}, + {file = "pydantic_core-2.14.6-cp39-none-win_amd64.whl", hash = "sha256:aca48506a9c20f68ee61c87f2008f81f8ee99f8d7f0104bff3c47e2d148f89d9"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d5c28525c19f5bb1e09511669bb57353d22b94cf8b65f3a8d141c389a55dec95"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:78d0768ee59baa3de0f4adac9e3748b4b1fffc52143caebddfd5ea2961595277"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b93785eadaef932e4fe9c6e12ba67beb1b3f1e5495631419c784ab87e975670"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a874f21f87c485310944b2b2734cd6d318765bcbb7515eead33af9641816506e"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b89f4477d915ea43b4ceea6756f63f0288941b6443a2b28c69004fe07fde0d0d"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:172de779e2a153d36ee690dbc49c6db568d7b33b18dc56b69a7514aecbcf380d"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:dfcebb950aa7e667ec226a442722134539e77c575f6cfaa423f24371bb8d2e94"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:55a23dcd98c858c0db44fc5c04fc7ed81c4b4d33c653a7c45ddaebf6563a2f66"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:4241204e4b36ab5ae466ecec5c4c16527a054c69f99bba20f6f75232a6a534e2"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e574de99d735b3fc8364cba9912c2bec2da78775eba95cbb225ef7dda6acea24"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1302a54f87b5cd8528e4d6d1bf2133b6aa7c6122ff8e9dc5220fbc1e07bffebd"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8e81e4b55930e5ffab4a68db1af431629cf2e4066dbdbfef65348b8ab804ea8"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c99462ffc538717b3e60151dfaf91125f637e801f5ab008f81c402f1dff0cd0f"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e4cf2d5829f6963a5483ec01578ee76d329eb5caf330ecd05b3edd697e7d768a"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:cf10b7d58ae4a1f07fccbf4a0a956d705356fea05fb4c70608bb6fa81d103cda"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:399ac0891c284fa8eb998bcfa323f2234858f5d2efca3950ae58c8f88830f145"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c6a5c79b28003543db3ba67d1df336f253a87d3112dac3a51b94f7d48e4c0e1"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:599c87d79cab2a6a2a9df4aefe0455e61e7d2aeede2f8577c1b7c0aec643ee8e"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43e166ad47ba900f2542a80d83f9fc65fe99eb63ceec4debec160ae729824052"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3a0b5db001b98e1c649dd55afa928e75aa4087e587b9524a4992316fa23c9fba"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:747265448cb57a9f37572a488a57d873fd96bf51e5bb7edb52cfb37124516da4"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:7ebe3416785f65c28f4f9441e916bfc8a54179c8dea73c23023f7086fa601c5d"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:86c963186ca5e50d5c8287b1d1c9d3f8f024cbe343d048c5bd282aec2d8641f2"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:e0641b506486f0b4cd1500a2a65740243e8670a2549bb02bc4556a83af84ae03"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71d72ca5eaaa8d38c8df16b7deb1a2da4f650c41b58bb142f3fb75d5ad4a611f"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27e524624eace5c59af499cd97dc18bb201dc6a7a2da24bfc66ef151c69a5f2a"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3dde6cac75e0b0902778978d3b1646ca9f438654395a362cb21d9ad34b24acf"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:00646784f6cd993b1e1c0e7b0fdcbccc375d539db95555477771c27555e3c556"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:23598acb8ccaa3d1d875ef3b35cb6376535095e9405d91a3d57a8c7db5d29341"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7f41533d7e3cf9520065f610b41ac1c76bc2161415955fbcead4981b22c7611e"}, + {file = "pydantic_core-2.14.6.tar.gz", hash = "sha256:1fd0c1d395372843fba13a51c28e3bb9d59bd7aebfeb17358ffaaa1e4dbbe948"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "4d73e0c41cc21bec6acf9c2266ff3f64341d6cea591bab8e13b9dd201417c4c3" diff --git a/examples/dlq-middleware/pyproject.toml b/examples/dlq-middleware/pyproject.toml new file mode 100644 index 00000000..8ee18790 --- /dev/null +++ b/examples/dlq-middleware/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "dlq-middleware" +version = "0.1.0" +description = "" +authors = ["Marcos Schroh "] +readme = "README.md" +packages = [{include = "dlq_middleware"}] + +[tool.poetry.dependencies] +python = "^3.8" +kstreams = { path = "../../.", develop = true } + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +app = "dlq_middleware.app:main" \ No newline at end of file diff --git a/examples/joker-middleware-example/README.md b/examples/joker-middleware-example/README.md new file mode 100644 index 00000000..e22f064b --- /dev/null +++ b/examples/joker-middleware-example/README.md @@ -0,0 +1,105 @@ +# Joker middleware Example + +## Requirements + +python 3.8+, poetry, docker-compose + +## Installation + +```bash +poetry install +``` + +## Explanation + +This shows how the `Middleware` concept can be used with `kstreams`. In this example we simulate a bug in the code. Every time that the event `joker` is consumed the `stream` will raise the `ValueError` exception causing it to crash. This example could also be applied to a `deserialization` exception error or any other operation that might crash the stream. + +Let's start with the `stream` code: + +```python +@stream_engine.stream(topics=["local--hello-world"]) +async def consume(cr: kstreams.ConsumerRecord): + print(f"Event consumed: headers: {cr.headers}, payload: {cr}") + if cr.value == b"joker": + raise ValueError("Stream crashed 🤡 🤡 🤡 🤡 🤡 🤡 🤡 🤡") +``` + +If there is a joker event, then we have the following error, causing a hard crash: + +```bash +Event consumed: headers: (), payload: ConsumerRecord(topic='local--hello-world', partition=0, offset=2, timestamp=1704453280808, timestamp_type=0, key=None, value=b'joker', checksum=None, serialized_key_size=-1, serialized_value_size=5, headers=()) +Stream consuming from topics ['local--hello-world'] CRASHED!!! + + Stream crashed 🤡 🤡 🤡 🤡 🤡 🤡 🤡 🤡 +Traceback (most recent call last): + File "Projects/kstreams/kstreams/middleware/middleware.py", line 60, in __call__ + await self.next_call(cr) + File "Projects/kstreams/kstreams/streams.py", line 313, in __call__ + await self.handler(cr) + File "Projects/kstreams/examples/middleware-example/middleware_example/app.py", line 22, in consume + raise ValueError("Stream crashed 🤡 🤡 🤡 🤡 🤡 🤡 🤡 🤡") +ValueError: Stream crashed 🤡 🤡 🤡 🤡 🤡 🤡 🤡 🤡 +``` + +How can we solve the problem? + +- Write buisness logic inside the `stream` to handle the `ValueError` exception +- Create a new funcion that receives the `cr` as parameter. It will also handle the `ValueError` exception, then all the buiness logic can reused and the stream will be simpler +- Create a `middleware` with the logic to handle the exception and attach it to the `stream` or the `stream_engine` + +For this case we will create a `middleware` just to show you the concept. In real use cases, this middleware could represent a `Dead Letter Queue` (DLQ) for example + +```python +class JokerBlokerMiddleware(kstreams.middleware.BaseMiddleware): + async def __call__(self, cr: kstreams.ConsumerRecord) -> None: + try: + await self.next_call(cr) + except ValueError: + print("Blocking the Joker 🤡⃤ 🤡⃤ 🤡⃤. so we can continue processing events... \n") + +middlewares = [kstreams.middleware.Middleware(JokerBlokerMiddleware)] + + +@stream_engine.stream(topics=["local--hello-world"], middlewares=middlewares) +async def consume(cr: kstreams.ConsumerRecord): + print(f"Event consumed: headers: {cr.headers}, payload: {cr}") + if cr.value == b"joker": + raise ValueError("Stream crashed 🤡 🤡 🤡 🤡 🤡 🤡 🤡 🤡") +``` + +Now that we have applied the `JokerBlokerMiddleware`, if we receive the `joker` event the `stream` will not crash!!! + +```bash +Event consumed: headers: (), payload: ConsumerRecord(topic='local--hello-world', partition=0, offset=9, timestamp=1704453577436, timestamp_type=0, key=None, value=b'joker', checksum=None, serialized_key_size=-1, serialized_value_size=5, headers=()) +Blocking the Joker 🤡⃤ 🤡⃤ 🤡⃤. so we can continue processing events... + +Event consumed: headers: (), payload: ConsumerRecord(topic='local--hello-world', partition=0, offset=10, timestamp=1704453596288, timestamp_type=0, key=None, value=b'batman', checksum=None, serialized_key_size=-1, serialized_value_size=6, headers=()) +``` + +## Usage + +1. Start the kafka cluster: From `kstreams` project root execute `./scripts/cluster/start` +2. Inside this folder execute `poetry run app` +3. From `kstreams` project root, you can use the `./scripts/cluster/events/send` to send events to the kafka cluster. A prompt will open. Enter messages to send. The command is: +```bash +./scripts/cluster/events/send "local--hello-world" +``` + +Then, on the consume side, you should see something similar to the following logs: + +```bash +❯ me@me-pc middleware-example % poetry run app + +Group Coordinator Request failed: [Error 15] CoordinatorNotAvailableError +Group Coordinator Request failed: [Error 15] CoordinatorNotAvailableError +Group Coordinator Request failed: [Error 15] CoordinatorNotAvailableError +Group Coordinator Request failed: [Error 15] CoordinatorNotAvailableError +Group Coordinator Request failed: [Error 15] CoordinatorNotAvailableError +Consumer started +Event consumed: headers: (), payload: ConsumerRecord(topic='local--hello-world', partition=0, offset=0, timestamp=1660733921761, timestamp_type=0, key=None, value=b'boo', checksum=None, serialized_key_size=-1, serialized_value_size=3, headers=()) +``` + +## Note + +If you plan on using this example, pay attention to the `pyproject.toml` dependencies, where +`kstreams` is pointing to the parent folder. You will have to set the latest version. diff --git a/examples/joker-middleware-example/joker_middleware_example/__init__.py b/examples/joker-middleware-example/joker_middleware_example/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/joker-middleware-example/joker_middleware_example/app.py b/examples/joker-middleware-example/joker_middleware_example/app.py new file mode 100644 index 00000000..830e8647 --- /dev/null +++ b/examples/joker-middleware-example/joker_middleware_example/app.py @@ -0,0 +1,36 @@ +import aiorun + +import kstreams + + +class JokerBlokerMiddleware(kstreams.middleware.BaseMiddleware): + async def __call__(self, cr: kstreams.ConsumerRecord): + try: + return await self.next_call(cr) + except ValueError: + print( + "Blocking the Joker 🤡⃤ 🤡⃤ 🤡⃤. so we can continue processing events... \n" + ) + + +middlewares = [kstreams.middleware.Middleware(JokerBlokerMiddleware)] +stream_engine = kstreams.create_engine(title="my-stream-engine") + + +@stream_engine.stream(topics=["local--hello-world"], middlewares=middlewares) +async def consume(cr: kstreams.ConsumerRecord): + print(f"Event consumed: headers: {cr.headers}, payload: {cr}") + if cr.value == b"joker": + raise ValueError("Stream crashed 🤡 🤡 🤡 🤡 🤡 🤡 🤡 🤡") + + +async def start(): + await stream_engine.start() + + +async def shutdown(loop): + await stream_engine.stop() + + +def main(): + aiorun.run(start(), stop_on_unhandled_errors=True, shutdown_callback=shutdown) diff --git a/examples/joker-middleware-example/poetry.lock b/examples/joker-middleware-example/poetry.lock new file mode 100644 index 00000000..a98633dd --- /dev/null +++ b/examples/joker-middleware-example/poetry.lock @@ -0,0 +1,348 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "aiokafka" +version = "0.10.0" +description = "Kafka integration with asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiokafka-0.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ebe5be9f578e89e6db961121070f7c35662924abee00ba4ccf64557e2cdd7edf"}, + {file = "aiokafka-0.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:007f1c51f440cc07155d2491f4deea6536492324153296aa73736a74cd833d3e"}, + {file = "aiokafka-0.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22299f8d5269dcb00b1b53fdee44dbe729091d4038e1bb63d0bb2f5cdf9af47a"}, + {file = "aiokafka-0.10.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fafc95bdaed9e1810fcd80b02ac117e51c72681ffe50353e5d61e2170609e1fc"}, + {file = "aiokafka-0.10.0-cp310-cp310-win32.whl", hash = "sha256:f2f19dee69c69389f5911e6b23c361c5285366d237f782eaae118d12acc42d7f"}, + {file = "aiokafka-0.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:99127ab680f9b08b0213d00b7d1e0480c6d08601f52ad42e829350f9599db301"}, + {file = "aiokafka-0.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5efb63686562809f0f9bf0fa6d1e52f222af2d8f8441f8c412b156f15c98da43"}, + {file = "aiokafka-0.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b91109dc25f79be4d27454cc766239a5368d18b26682d4b5c6b913ca92691220"}, + {file = "aiokafka-0.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d52c25f3d0db7dd340a5d08108da302db1ba64c2190970dbdb768b79629d6add"}, + {file = "aiokafka-0.10.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1509c1b29cd1d4d920a649f257d72109bbc3d61431135505b8e0d8d488796ff2"}, + {file = "aiokafka-0.10.0-cp311-cp311-win32.whl", hash = "sha256:ffc30e4c6bfcb00356a002f623c93a51d8336ca67687ea069dd11822da07379c"}, + {file = "aiokafka-0.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e10fdee4189fe7eed36d602df822e9ff4f19535c0a514cf015f78308d206c1a"}, + {file = "aiokafka-0.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:82a75ea13d7e6e11c7ee2fb9419e9ea3541744648c69ab27b56fb6bca5b319c1"}, + {file = "aiokafka-0.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf9e241766b7f4c305807763330dacf8c220ad9e8fc7f2b22730a2db66fad61d"}, + {file = "aiokafka-0.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12d703317812262feac6577ff488f2ccddc4408da0ff608a5454062782b5a80d"}, + {file = "aiokafka-0.10.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b74aeacfb8ced9764002c63b58e4c78c94809131d89000cb936c25c298ffb1e"}, + {file = "aiokafka-0.10.0-cp312-cp312-win32.whl", hash = "sha256:de56c503b3d64e24a5b6705e55bc524a8357b0495402f859f921a71d65274cb1"}, + {file = "aiokafka-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:f4b22a31f40493cea50dddb4dfc92750dfb273635ccb094a16fde9678eb38958"}, + {file = "aiokafka-0.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7068f0beb8478cde09618dcc9a833cc18ff37bd14864fa8b60ad4e4c3dad6489"}, + {file = "aiokafka-0.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f069bda1f31e466d815b631a07bc6fad5190b29dfff5f117bcbf1948cd7a38aa"}, + {file = "aiokafka-0.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e16d8a23f0e173e5ca86c2d1c270e25a529a0eed973c77d7e8a0dfc868699aa4"}, + {file = "aiokafka-0.10.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf4a47659517000a8fe88e0fb353898b718ee214e21f62a2a949be9bf801cd9e"}, + {file = "aiokafka-0.10.0-cp38-cp38-win32.whl", hash = "sha256:781ab300214681e40667185a402abf6b31b4c4b8f1cdabbdc3549d8cf383b34d"}, + {file = "aiokafka-0.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:06060708a4bcf062be496c8641fca382c88782d3c381a34ccb5ac8677bdac695"}, + {file = "aiokafka-0.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c23ec22fbf26e2f84678f0589076bea1ff26ae6dfd3c601e6de10ad00d605261"}, + {file = "aiokafka-0.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74229a57c95e2efccec95d9b42554dc168c97a263f013e3e983202bd33ca189d"}, + {file = "aiokafka-0.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e833e4ef7fc5f3f637ba5fb4210acc7e5ea916bb7107e4b619b1b1a3e361bc62"}, + {file = "aiokafka-0.10.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9728c523f10ac4bb46719cc64f3c1d47625898872bc3901b22b9d48b6e401d1c"}, + {file = "aiokafka-0.10.0-cp39-cp39-win32.whl", hash = "sha256:05c4a7ced5d6f3dbc289767574d6a5d9b31e1c243e992dcecd34dbc40fcbbf9b"}, + {file = "aiokafka-0.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:1fe0194ea72524df37369a8cf0837263b55194ac20616e612f0ab7bfb568b76b"}, + {file = "aiokafka-0.10.0.tar.gz", hash = "sha256:7ce35563f955490b43190e3389b5f3d92d50e22b32d1a40772fd14fb1d50c5db"}, +] + +[package.dependencies] +async-timeout = "*" +packaging = "*" + +[package.extras] +all = ["cramjam", "gssapi", "lz4 (>=3.1.3)"] +gssapi = ["gssapi"] +lz4 = ["lz4 (>=3.1.3)"] +snappy = ["cramjam"] +zstd = ["cramjam"] + +[[package]] +name = "aiorun" +version = "2023.7.2" +description = "Boilerplate for asyncio applications" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiorun-2023.7.2-py3-none-any.whl", hash = "sha256:61f839265d86d3f3b37921eb4a99ef42fa2ee026241b67c54315423d7dac29c9"}, + {file = "aiorun-2023.7.2.tar.gz", hash = "sha256:95e689dc1b263aaf4556ef8fa73ccae68f38ba0f1c1017fe196ef7da244031f9"}, +] + +[package.extras] +dev = ["pytest", "pytest-cov"] + +[[package]] +name = "annotated-types" +version = "0.6.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, + {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.9\""} + +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "future" +version = "0.18.3" +description = "Clean single-source support for Python 3 and 2" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"}, +] + +[[package]] +name = "kstreams" +version = "0.15.1" +description = "Build simple kafka streams applications" +optional = false +python-versions = "^3.8" +files = [] +develop = true + +[package.dependencies] +aiokafka = "<1.0" +aiorun = "^2023.7.2" +future = "^0.18.2" +prometheus-client = "<1.0" +pydantic = "<3.0.0" +PyYAML = ">=5.4,<7.0.0" + +[package.source] +type = "directory" +url = "../.." + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "prometheus-client" +version = "0.19.0" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.8" +files = [ + {file = "prometheus_client-0.19.0-py3-none-any.whl", hash = "sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92"}, + {file = "prometheus_client-0.19.0.tar.gz", hash = "sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1"}, +] + +[package.extras] +twisted = ["twisted"] + +[[package]] +name = "pydantic" +version = "2.5.3" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-2.5.3-py3-none-any.whl", hash = "sha256:d0caf5954bee831b6bfe7e338c32b9e30c85dfe080c843680783ac2b631673b4"}, + {file = "pydantic-2.5.3.tar.gz", hash = "sha256:b3ef57c62535b0941697cce638c08900d87fcb67e29cfa99e8a68f747f393f7a"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.14.6" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.14.6" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic_core-2.14.6-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:72f9a942d739f09cd42fffe5dc759928217649f070056f03c70df14f5770acf9"}, + {file = "pydantic_core-2.14.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6a31d98c0d69776c2576dda4b77b8e0c69ad08e8b539c25c7d0ca0dc19a50d6c"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5aa90562bc079c6c290f0512b21768967f9968e4cfea84ea4ff5af5d917016e4"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:370ffecb5316ed23b667d99ce4debe53ea664b99cc37bfa2af47bc769056d534"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f85f3843bdb1fe80e8c206fe6eed7a1caeae897e496542cee499c374a85c6e08"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9862bf828112e19685b76ca499b379338fd4c5c269d897e218b2ae8fcb80139d"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:036137b5ad0cb0004c75b579445a1efccd072387a36c7f217bb8efd1afbe5245"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:92879bce89f91f4b2416eba4429c7b5ca22c45ef4a499c39f0c5c69257522c7c"}, + {file = "pydantic_core-2.14.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0c08de15d50fa190d577e8591f0329a643eeaed696d7771760295998aca6bc66"}, + {file = "pydantic_core-2.14.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:36099c69f6b14fc2c49d7996cbf4f87ec4f0e66d1c74aa05228583225a07b590"}, + {file = "pydantic_core-2.14.6-cp310-none-win32.whl", hash = "sha256:7be719e4d2ae6c314f72844ba9d69e38dff342bc360379f7c8537c48e23034b7"}, + {file = "pydantic_core-2.14.6-cp310-none-win_amd64.whl", hash = "sha256:36fa402dcdc8ea7f1b0ddcf0df4254cc6b2e08f8cd80e7010d4c4ae6e86b2a87"}, + {file = "pydantic_core-2.14.6-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:dea7fcd62915fb150cdc373212141a30037e11b761fbced340e9db3379b892d4"}, + {file = "pydantic_core-2.14.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ffff855100bc066ff2cd3aa4a60bc9534661816b110f0243e59503ec2df38421"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b027c86c66b8627eb90e57aee1f526df77dc6d8b354ec498be9a757d513b92b"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:00b1087dabcee0b0ffd104f9f53d7d3eaddfaa314cdd6726143af6bc713aa27e"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:75ec284328b60a4e91010c1acade0c30584f28a1f345bc8f72fe8b9e46ec6a96"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e1f4744eea1501404b20b0ac059ff7e3f96a97d3e3f48ce27a139e053bb370b"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2602177668f89b38b9f84b7b3435d0a72511ddef45dc14446811759b82235a1"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6c8edaea3089bf908dd27da8f5d9e395c5b4dc092dbcce9b65e7156099b4b937"}, + {file = "pydantic_core-2.14.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:478e9e7b360dfec451daafe286998d4a1eeaecf6d69c427b834ae771cad4b622"}, + {file = "pydantic_core-2.14.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b6ca36c12a5120bad343eef193cc0122928c5c7466121da7c20f41160ba00ba2"}, + {file = "pydantic_core-2.14.6-cp311-none-win32.whl", hash = "sha256:2b8719037e570639e6b665a4050add43134d80b687288ba3ade18b22bbb29dd2"}, + {file = "pydantic_core-2.14.6-cp311-none-win_amd64.whl", hash = "sha256:78ee52ecc088c61cce32b2d30a826f929e1708f7b9247dc3b921aec367dc1b23"}, + {file = "pydantic_core-2.14.6-cp311-none-win_arm64.whl", hash = "sha256:a19b794f8fe6569472ff77602437ec4430f9b2b9ec7a1105cfd2232f9ba355e6"}, + {file = "pydantic_core-2.14.6-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:667aa2eac9cd0700af1ddb38b7b1ef246d8cf94c85637cbb03d7757ca4c3fdec"}, + {file = "pydantic_core-2.14.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdee837710ef6b56ebd20245b83799fce40b265b3b406e51e8ccc5b85b9099b7"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c5bcf3414367e29f83fd66f7de64509a8fd2368b1edf4351e862910727d3e51"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:26a92ae76f75d1915806b77cf459811e772d8f71fd1e4339c99750f0e7f6324f"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a983cca5ed1dd9a35e9e42ebf9f278d344603bfcb174ff99a5815f953925140a"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cb92f9061657287eded380d7dc455bbf115430b3aa4741bdc662d02977e7d0af"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4ace1e220b078c8e48e82c081e35002038657e4b37d403ce940fa679e57113b"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ef633add81832f4b56d3b4c9408b43d530dfca29e68fb1b797dcb861a2c734cd"}, + {file = "pydantic_core-2.14.6-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7e90d6cc4aad2cc1f5e16ed56e46cebf4877c62403a311af20459c15da76fd91"}, + {file = "pydantic_core-2.14.6-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e8a5ac97ea521d7bde7621d86c30e86b798cdecd985723c4ed737a2aa9e77d0c"}, + {file = "pydantic_core-2.14.6-cp312-none-win32.whl", hash = "sha256:f27207e8ca3e5e021e2402ba942e5b4c629718e665c81b8b306f3c8b1ddbb786"}, + {file = "pydantic_core-2.14.6-cp312-none-win_amd64.whl", hash = "sha256:b3e5fe4538001bb82e2295b8d2a39356a84694c97cb73a566dc36328b9f83b40"}, + {file = "pydantic_core-2.14.6-cp312-none-win_arm64.whl", hash = "sha256:64634ccf9d671c6be242a664a33c4acf12882670b09b3f163cd00a24cffbd74e"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:24368e31be2c88bd69340fbfe741b405302993242ccb476c5c3ff48aeee1afe0"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:e33b0834f1cf779aa839975f9d8755a7c2420510c0fa1e9fa0497de77cd35d2c"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6af4b3f52cc65f8a0bc8b1cd9676f8c21ef3e9132f21fed250f6958bd7223bed"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15687d7d7f40333bd8266f3814c591c2e2cd263fa2116e314f60d82086e353a"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:095b707bb287bfd534044166ab767bec70a9bba3175dcdc3371782175c14e43c"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94fc0e6621e07d1e91c44e016cc0b189b48db053061cc22d6298a611de8071bb"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ce830e480f6774608dedfd4a90c42aac4a7af0a711f1b52f807130c2e434c06"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a306cdd2ad3a7d795d8e617a58c3a2ed0f76c8496fb7621b6cd514eb1532cae8"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:2f5fa187bde8524b1e37ba894db13aadd64faa884657473b03a019f625cee9a8"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:438027a975cc213a47c5d70672e0d29776082155cfae540c4e225716586be75e"}, + {file = "pydantic_core-2.14.6-cp37-none-win32.whl", hash = "sha256:f96ae96a060a8072ceff4cfde89d261837b4294a4f28b84a28765470d502ccc6"}, + {file = "pydantic_core-2.14.6-cp37-none-win_amd64.whl", hash = "sha256:e646c0e282e960345314f42f2cea5e0b5f56938c093541ea6dbf11aec2862391"}, + {file = "pydantic_core-2.14.6-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:db453f2da3f59a348f514cfbfeb042393b68720787bbef2b4c6068ea362c8149"}, + {file = "pydantic_core-2.14.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3860c62057acd95cc84044e758e47b18dcd8871a328ebc8ccdefd18b0d26a21b"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36026d8f99c58d7044413e1b819a67ca0e0b8ebe0f25e775e6c3d1fabb3c38fb"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ed1af8692bd8d2a29d702f1a2e6065416d76897d726e45a1775b1444f5928a7"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:314ccc4264ce7d854941231cf71b592e30d8d368a71e50197c905874feacc8a8"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:982487f8931067a32e72d40ab6b47b1628a9c5d344be7f1a4e668fb462d2da42"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dbe357bc4ddda078f79d2a36fc1dd0494a7f2fad83a0a684465b6f24b46fe80"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2f6ffc6701a0eb28648c845f4945a194dc7ab3c651f535b81793251e1185ac3d"}, + {file = "pydantic_core-2.14.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7f5025db12fc6de7bc1104d826d5aee1d172f9ba6ca936bf6474c2148ac336c1"}, + {file = "pydantic_core-2.14.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dab03ed811ed1c71d700ed08bde8431cf429bbe59e423394f0f4055f1ca0ea60"}, + {file = "pydantic_core-2.14.6-cp38-none-win32.whl", hash = "sha256:dfcbebdb3c4b6f739a91769aea5ed615023f3c88cb70df812849aef634c25fbe"}, + {file = "pydantic_core-2.14.6-cp38-none-win_amd64.whl", hash = "sha256:99b14dbea2fdb563d8b5a57c9badfcd72083f6006caf8e126b491519c7d64ca8"}, + {file = "pydantic_core-2.14.6-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:4ce8299b481bcb68e5c82002b96e411796b844d72b3e92a3fbedfe8e19813eab"}, + {file = "pydantic_core-2.14.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b9a9d92f10772d2a181b5ca339dee066ab7d1c9a34ae2421b2a52556e719756f"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd9e98b408384989ea4ab60206b8e100d8687da18b5c813c11e92fd8212a98e0"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4f86f1f318e56f5cbb282fe61eb84767aee743ebe32c7c0834690ebea50c0a6b"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86ce5fcfc3accf3a07a729779d0b86c5d0309a4764c897d86c11089be61da160"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dcf1978be02153c6a31692d4fbcc2a3f1db9da36039ead23173bc256ee3b91b"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eedf97be7bc3dbc8addcef4142f4b4164066df0c6f36397ae4aaed3eb187d8ab"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d5f916acf8afbcab6bacbb376ba7dc61f845367901ecd5e328fc4d4aef2fcab0"}, + {file = "pydantic_core-2.14.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8a14c192c1d724c3acbfb3f10a958c55a2638391319ce8078cb36c02283959b9"}, + {file = "pydantic_core-2.14.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0348b1dc6b76041516e8a854ff95b21c55f5a411c3297d2ca52f5528e49d8411"}, + {file = "pydantic_core-2.14.6-cp39-none-win32.whl", hash = "sha256:de2a0645a923ba57c5527497daf8ec5df69c6eadf869e9cd46e86349146e5975"}, + {file = "pydantic_core-2.14.6-cp39-none-win_amd64.whl", hash = "sha256:aca48506a9c20f68ee61c87f2008f81f8ee99f8d7f0104bff3c47e2d148f89d9"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d5c28525c19f5bb1e09511669bb57353d22b94cf8b65f3a8d141c389a55dec95"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:78d0768ee59baa3de0f4adac9e3748b4b1fffc52143caebddfd5ea2961595277"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b93785eadaef932e4fe9c6e12ba67beb1b3f1e5495631419c784ab87e975670"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a874f21f87c485310944b2b2734cd6d318765bcbb7515eead33af9641816506e"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b89f4477d915ea43b4ceea6756f63f0288941b6443a2b28c69004fe07fde0d0d"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:172de779e2a153d36ee690dbc49c6db568d7b33b18dc56b69a7514aecbcf380d"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:dfcebb950aa7e667ec226a442722134539e77c575f6cfaa423f24371bb8d2e94"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:55a23dcd98c858c0db44fc5c04fc7ed81c4b4d33c653a7c45ddaebf6563a2f66"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:4241204e4b36ab5ae466ecec5c4c16527a054c69f99bba20f6f75232a6a534e2"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e574de99d735b3fc8364cba9912c2bec2da78775eba95cbb225ef7dda6acea24"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1302a54f87b5cd8528e4d6d1bf2133b6aa7c6122ff8e9dc5220fbc1e07bffebd"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8e81e4b55930e5ffab4a68db1af431629cf2e4066dbdbfef65348b8ab804ea8"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c99462ffc538717b3e60151dfaf91125f637e801f5ab008f81c402f1dff0cd0f"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e4cf2d5829f6963a5483ec01578ee76d329eb5caf330ecd05b3edd697e7d768a"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:cf10b7d58ae4a1f07fccbf4a0a956d705356fea05fb4c70608bb6fa81d103cda"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:399ac0891c284fa8eb998bcfa323f2234858f5d2efca3950ae58c8f88830f145"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c6a5c79b28003543db3ba67d1df336f253a87d3112dac3a51b94f7d48e4c0e1"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:599c87d79cab2a6a2a9df4aefe0455e61e7d2aeede2f8577c1b7c0aec643ee8e"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43e166ad47ba900f2542a80d83f9fc65fe99eb63ceec4debec160ae729824052"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3a0b5db001b98e1c649dd55afa928e75aa4087e587b9524a4992316fa23c9fba"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:747265448cb57a9f37572a488a57d873fd96bf51e5bb7edb52cfb37124516da4"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:7ebe3416785f65c28f4f9441e916bfc8a54179c8dea73c23023f7086fa601c5d"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:86c963186ca5e50d5c8287b1d1c9d3f8f024cbe343d048c5bd282aec2d8641f2"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:e0641b506486f0b4cd1500a2a65740243e8670a2549bb02bc4556a83af84ae03"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71d72ca5eaaa8d38c8df16b7deb1a2da4f650c41b58bb142f3fb75d5ad4a611f"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27e524624eace5c59af499cd97dc18bb201dc6a7a2da24bfc66ef151c69a5f2a"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3dde6cac75e0b0902778978d3b1646ca9f438654395a362cb21d9ad34b24acf"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:00646784f6cd993b1e1c0e7b0fdcbccc375d539db95555477771c27555e3c556"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:23598acb8ccaa3d1d875ef3b35cb6376535095e9405d91a3d57a8c7db5d29341"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7f41533d7e3cf9520065f610b41ac1c76bc2161415955fbcead4981b22c7611e"}, + {file = "pydantic_core-2.14.6.tar.gz", hash = "sha256:1fd0c1d395372843fba13a51c28e3bb9d59bd7aebfeb17358ffaaa1e4dbbe948"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "4d73e0c41cc21bec6acf9c2266ff3f64341d6cea591bab8e13b9dd201417c4c3" diff --git a/examples/joker-middleware-example/pyproject.toml b/examples/joker-middleware-example/pyproject.toml new file mode 100644 index 00000000..12505fdb --- /dev/null +++ b/examples/joker-middleware-example/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "joker-middleware-example" +version = "0.1.0" +description = "" +authors = ["Marcos Schroh "] +readme = "README.md" +packages = [{include = "joker_middleware_example"}] + +[tool.poetry.dependencies] +python = "^3.8" +kstreams = { path = "../../.", develop = true } + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +app = "joker_middleware_example.app:main" diff --git a/examples/simple-consumer-example/simple_consumer_example/app.py b/examples/simple-consumer-example/simple_consumer_example/app.py index d95ba8e9..4c60b004 100644 --- a/examples/simple-consumer-example/simple_consumer_example/app.py +++ b/examples/simple-consumer-example/simple_consumer_example/app.py @@ -1,5 +1,3 @@ -import signal - import aiorun import kstreams @@ -9,12 +7,7 @@ @stream_engine.stream(topics=["local--hello-world"], group_id="example-group") async def consume(cr: kstreams.ConsumerRecord): - print("Consumer started") - try: - print(f"Event consumed: headers: {cr.headers}, payload: {cr}") - finally: - # Terminate the program if something fails. (aiorun will cath this signal and properly shutdown this program.) - signal.alarm(signal.SIGTERM) + print(f"Event consumed: headers: {cr.headers}, payload: {cr}") async def start(): diff --git a/kstreams/__init__.py b/kstreams/__init__.py index 6715cd44..1db148a9 100644 --- a/kstreams/__init__.py +++ b/kstreams/__init__.py @@ -1,4 +1,4 @@ -from aiokafka.structs import ConsumerRecord, TopicPartition +from aiokafka.structs import ConsumerRecord, RecordMetadata, TopicPartition from .clients import Consumer, ConsumerType, Producer, ProducerType from .create import StreamEngine, create_engine @@ -27,6 +27,7 @@ "Stream", "stream", "ConsumerRecord", + "RecordMetadata", "TestStreamClient", "TopicPartition", "TopicPartitionOffset", diff --git a/kstreams/create.py b/kstreams/create.py index eda4b339..3be919b9 100644 --- a/kstreams/create.py +++ b/kstreams/create.py @@ -1,8 +1,9 @@ -from typing import Optional, Type +from typing import List, Optional, Type from .backends.kafka import Kafka from .clients import Consumer, ConsumerType, Producer, ProducerType from .engine import StreamEngine +from .middleware import Middleware from .prometheus.monitor import PrometheusMonitor from .serializers import Deserializer, Serializer @@ -15,6 +16,7 @@ def create_engine( serializer: Optional[Serializer] = None, deserializer: Optional[Deserializer] = None, monitor: Optional[PrometheusMonitor] = None, + middlewares: Optional[List[Middleware]] = None, ) -> StreamEngine: if monitor is None: monitor = PrometheusMonitor() @@ -22,6 +24,9 @@ def create_engine( if backend is None: backend = Kafka() + if middlewares is None: + middlewares = [] + return StreamEngine( backend=backend, title=title, @@ -30,4 +35,5 @@ def create_engine( serializer=serializer, deserializer=deserializer, monitor=monitor, + middlewares=middlewares, ) diff --git a/kstreams/engine.py b/kstreams/engine.py index bc6b1c21..d7b834e1 100644 --- a/kstreams/engine.py +++ b/kstreams/engine.py @@ -1,6 +1,6 @@ import inspect import logging -from typing import Any, Callable, Dict, List, Optional, Type, Union +import typing from aiokafka.structs import RecordMetadata @@ -9,12 +9,14 @@ from .backends.kafka import Kafka from .clients import ConsumerType, ProducerType from .exceptions import DuplicateStreamException, EngineNotStartedException +from .middleware import ExceptionMiddleware, Middleware from .prometheus.monitor import PrometheusMonitor from .rebalance_listener import MetricsRebalanceListener, RebalanceListener from .serializers import Deserializer, Serializer -from .streams import Stream, StreamFunc +from .streams import Stream, StreamFunc, UdfHandler from .streams import stream as stream_func -from .types import Headers +from .streams_utils import UDFType, inspect_udf +from .types import Headers, NextMiddlewareCall from .utils import encode_headers logger = logging.getLogger(__name__) @@ -59,12 +61,13 @@ def __init__( self, *, backend: Kafka, - consumer_class: Type[ConsumerType], - producer_class: Type[ProducerType], + consumer_class: typing.Type[ConsumerType], + producer_class: typing.Type[ProducerType], monitor: PrometheusMonitor, - title: Optional[str] = None, - deserializer: Optional[Deserializer] = None, - serializer: Optional[Serializer] = None, + middlewares: typing.List[Middleware], + title: typing.Optional[str] = None, + deserializer: typing.Optional[Deserializer] = None, + serializer: typing.Optional[Serializer] = None, ) -> None: self.title = title self.backend = backend @@ -73,19 +76,20 @@ def __init__( self.deserializer = deserializer self.serializer = serializer self.monitor = monitor - self._producer: Optional[Type[ProducerType]] = None - self._streams: List[Stream] = [] + self._producer: typing.Optional[typing.Type[ProducerType]] = None + self._streams: typing.List[Stream] = [] + self.middlewares = middlewares async def send( self, topic: str, - value: Any = None, - key: Any = None, - partition: Optional[int] = None, - timestamp_ms: Optional[int] = None, - headers: Optional[Headers] = None, - serializer: Optional[Serializer] = None, - serializer_kwargs: Optional[Dict] = None, + value: typing.Any = None, + key: typing.Any = None, + partition: typing.Optional[int] = None, + timestamp_ms: typing.Optional[int] = None, + headers: typing.Optional[Headers] = None, + serializer: typing.Optional[Serializer] = None, + serializer_kwargs: typing.Optional[typing.Dict] = None, ): """ Attributes: @@ -180,7 +184,7 @@ def exist_stream(self, name: str) -> bool: stream = self.get_stream(name) return True if stream is not None else False - def get_stream(self, name: str) -> Optional[Stream]: + def get_stream(self, name: str) -> typing.Optional[Stream]: stream = next((stream for stream in self._streams if stream.name == name), None) return stream @@ -201,6 +205,23 @@ def add_stream(self, stream: Stream) -> None: stream.rebalance_listener.stream = stream # type: ignore stream.rebalance_listener.engine = self # type: ignore + udf_type = inspect_udf(stream.func, Stream) + if udf_type != UDFType.NO_TYPING: + stream.func = self.build_stream_middleware_stack(stream) + + def build_stream_middleware_stack(self, stream: Stream) -> NextMiddlewareCall: + udf_handler = UdfHandler(handler=stream.func, stream=stream) + stream.middlewares = ( + [Middleware(ExceptionMiddleware)] + self.middlewares + stream.middlewares + ) + + next_call = udf_handler + for middleware, options in reversed(stream.middlewares): + next_call = middleware( + next_call=next_call, send=self.send, stream=stream, **options + ) + return next_call + async def remove_stream(self, stream: Stream) -> None: self._streams.remove(stream) await stream.stop() @@ -208,14 +229,15 @@ async def remove_stream(self, stream: Stream) -> None: def stream( self, - topics: Union[List[str], str], + topics: typing.Union[typing.List[str], str], *, - name: Optional[str] = None, - deserializer: Optional[Deserializer] = None, - initial_offsets: Optional[List[TopicPartitionOffset]] = None, - rebalance_listener: Optional[RebalanceListener] = None, + name: typing.Optional[str] = None, + deserializer: typing.Optional[Deserializer] = None, + initial_offsets: typing.Optional[typing.List[TopicPartitionOffset]] = None, + rebalance_listener: typing.Optional[RebalanceListener] = None, + middlewares: typing.Optional[typing.List[Middleware]] = None, **kwargs, - ) -> Callable[[StreamFunc], Stream]: + ) -> typing.Callable[[StreamFunc], Stream]: def decorator(func: StreamFunc) -> Stream: stream_from_func = stream_func( topics, @@ -223,6 +245,7 @@ def decorator(func: StreamFunc) -> Stream: deserializer=deserializer, initial_offsets=initial_offsets, rebalance_listener=rebalance_listener, + middlewares=middlewares, **kwargs, )(func) self.add_stream(stream_from_func) diff --git a/kstreams/middleware/__init__.py b/kstreams/middleware/__init__.py new file mode 100644 index 00000000..d3d23a81 --- /dev/null +++ b/kstreams/middleware/__init__.py @@ -0,0 +1,12 @@ +from .middleware import ( + BaseMiddleware, + ExceptionMiddleware, + Middleware, +) + +__all__ = [ + "BaseMiddleware", + "Middleware", + "Middleware", + "ExceptionMiddleware", +] diff --git a/kstreams/middleware/middleware.py b/kstreams/middleware/middleware.py new file mode 100644 index 00000000..a19dca59 --- /dev/null +++ b/kstreams/middleware/middleware.py @@ -0,0 +1,72 @@ +import logging +import typing + +from aiokafka import errors + +from kstreams import ConsumerRecord, types + +if typing.TYPE_CHECKING: + from kstreams import Stream # pragma: no cover + + +logger = logging.getLogger(__name__) + + +class MiddlewareProtocol(typing.Protocol): + def __init__( + self, + *, + next_call: types.NextMiddlewareCall, + send: types.Send, + stream: "Stream", + **kwargs: typing.Any, + ) -> None: + ... # pragma: no cover + + async def __call__(self, cr: ConsumerRecord) -> typing.Any: + ... # pragma: no cover + + +class Middleware: + def __init__( + self, middleware: typing.Type[MiddlewareProtocol], **kwargs: typing.Any + ) -> None: + self.middleware = middleware + self.kwargs = kwargs + + def __iter__(self) -> typing.Iterator: + return iter((self.middleware, self.kwargs)) + + +class BaseMiddleware: + def __init__( + self, + *, + next_call: types.NextMiddlewareCall, + send: types.Send, + stream: "Stream", + ) -> None: + self.next_call = next_call + self.send = send + self.stream = stream + + async def __call__(self, cr: ConsumerRecord) -> typing.Any: + raise NotImplementedError + + +class ExceptionMiddleware(BaseMiddleware): + async def __call__(self, cr: ConsumerRecord) -> typing.Any: + try: + return await self.next_call(cr) + except errors.ConsumerStoppedError as exc: + logger.exception( + f"Stream consuming from topics {self.stream.topics} has stopped!!! \n\n" + ) + await self.stream.stop() + raise exc + except Exception as exc: + logger.exception( + f"Stream consuming from topics {self.stream.topics} CRASHED!!! \n\n " + ) + await self.stream.stop() + raise exc diff --git a/kstreams/streams.py b/kstreams/streams.py index 8c2ea807..4e110512 100644 --- a/kstreams/streams.py +++ b/kstreams/streams.py @@ -1,19 +1,9 @@ import asyncio import inspect import logging +import typing import uuid from functools import update_wrapper -from typing import ( - Any, - Awaitable, - Callable, - Dict, - List, - Optional, - Set, - Type, - Union, -) from aiokafka import errors @@ -23,14 +13,14 @@ from .backends.kafka import Kafka from .clients import Consumer, ConsumerType +from .middleware import Middleware from .rebalance_listener import RebalanceListener from .serializers import Deserializer from .streams_utils import UDFType, inspect_udf +from .types import StreamFunc logger = logging.getLogger(__name__) -StreamFunc = Callable - class Stream: """ @@ -85,32 +75,32 @@ async def shutdown(loop): def __init__( self, - topics: Union[List[str], str], + topics: typing.Union[typing.List[str], str], *, func: StreamFunc, - backend: Optional[Kafka] = None, - consumer_class: Type[ConsumerType] = Consumer, - name: Optional[str] = None, - config: Optional[Dict] = None, - model: Optional[Any] = None, - deserializer: Optional[Deserializer] = None, - initial_offsets: Optional[List[TopicPartitionOffset]] = None, - rebalance_listener: Optional[RebalanceListener] = None, + backend: typing.Optional[Kafka] = None, + consumer_class: typing.Type[ConsumerType] = Consumer, + name: typing.Optional[str] = None, + config: typing.Optional[typing.Dict] = None, + deserializer: typing.Optional[Deserializer] = None, + initial_offsets: typing.Optional[typing.List[TopicPartitionOffset]] = None, + rebalance_listener: typing.Optional[RebalanceListener] = None, + middlewares: typing.Optional[typing.List[Middleware]] = None, ) -> None: self.func = func self.backend = backend self.consumer_class = consumer_class - self.consumer: Optional[ConsumerType] = None + self.consumer: typing.Optional[ConsumerType] = None self.config = config or {} - self._consumer_task: Optional[asyncio.Task] = None + self._consumer_task: typing.Optional[asyncio.Task] = None self.name = name or str(uuid.uuid4()) - self.model = model self.deserializer = deserializer self.running = False self.initial_offsets = initial_offsets self.seeked_initial_offsets = False self.rebalance_listener = rebalance_listener self.udf_type = inspect_udf(func, Stream) + self.middlewares = middlewares or [] # aiokafka expects topic names as arguments, meaning that # can receive N topics -> N arguments, @@ -146,7 +136,9 @@ async def _subscribe(self) -> None: ) self.running = True - async def commit(self, offsets: Optional[Dict[TopicPartition, int]] = None): + async def commit( + self, offsets: typing.Optional[typing.Dict[TopicPartition, int]] = None + ): await self.consumer.commit(offsets=offsets) # type: ignore async def getone(self) -> ConsumerRecord: @@ -161,10 +153,10 @@ async def getone(self) -> ConsumerRecord: async def getmany( self, - partitions: Optional[List[TopicPartition]] = None, + partitions: typing.Optional[typing.List[TopicPartition]] = None, timeout_ms: int = 0, - max_records: Optional[int] = None, - ) -> Dict[TopicPartition, List[ConsumerRecord]]: + max_records: typing.Optional[int] = None, + ) -> typing.Dict[TopicPartition, typing.List[ConsumerRecord]]: """ Get a batch of events from the assigned TopicPartition. @@ -227,7 +219,7 @@ async def start(self) -> None: ) return None - async def func_wrapper(self, func: Awaitable) -> None: + async def func_wrapper(self, func: typing.Awaitable) -> None: try: # await for the end user coroutine # we do this to show a better error message to the user @@ -237,25 +229,13 @@ async def func_wrapper(self, func: Awaitable) -> None: logger.exception(f"CRASHED Stream!!! Task {self._consumer_task} \n\n {e}") async def func_wrapper_with_typing(self) -> None: - try: - # await for the end user coroutine - # we do this to show a better error message to the user - # when the coroutine fails - while True: - cr = await self.getone() - if self.udf_type == UDFType.CR_ONLY_TYPING: - await self.func(cr) - else: - # typing with cr and stream - await self.func(cr, self) - except errors.ConsumerStoppedError: - return - except Exception as e: - logger.exception(f"CRASHED Stream!!! Task {self._consumer_task} \n\n {e}") + while True: + cr = await self.getone() + await self.func(cr) def seek_to_initial_offsets(self): if not self.seeked_initial_offsets: - assignments: Set[TopicPartition] = self.consumer.assignment() + assignments: typing.Set[TopicPartition] = self.consumer.assignment() if self.initial_offsets is not None: topicPartitionOffset: TopicPartitionOffset for topicPartitionOffset in self.initial_offsets: @@ -311,23 +291,68 @@ async def __anext__(self) -> ConsumerRecord: if self.udf_type == UDFType.NO_TYPING: return cr - elif self.udf_type == UDFType.CR_ONLY_TYPING: - return await anext(self.func(cr)) - else: - return await anext(self.func(cr, self)) + return await self.func(cr) except errors.ConsumerStoppedError: raise StopAsyncIteration # noqa: F821 +class UdfHandler: + def __init__( + self, + *, + handler: StreamFunc, + stream: Stream, + ) -> None: + self.handler = handler + self.stream = stream + self.udf_type = inspect_udf(self.handler, Stream) + + def get_udf_params(self, cr: ConsumerRecord) -> typing.Tuple: + if self.udf_type == UDFType.CR_ONLY_TYPING: + return (cr,) + return ( + cr, + self.stream, + ) + + async def __call__(self, cr: ConsumerRecord) -> typing.Any: + """ + Call the coroutine `async def my_function(...)` defined by the end user + in a proper way according to its parameters. The `handler` is the + coroutine defined by the user. + + Use cases: + 1. UDFType.CR_ONLY_TYPING: Only ConsumerRecord with typing + + @stream_engine.stream(topic, name="my-stream") + async def consume(cr: ConsumerRecord): + ... + + 2. UDFType.ALL_TYPING: ConsumerRecord and Stream with typing. + The order is important as they are arguments and not kwargs + + @stream_engine.stream(topic, name="my-stream") + async def consume(cr: ConsumerRecord, stream: Stream): + ... + + """ + params = self.get_udf_params(cr) + + if inspect.isasyncgenfunction(self.handler): + return await anext(self.handler(*params)) + return await self.handler(*params) + + def stream( - topics: Union[List[str], str], + topics: typing.Union[typing.List[str], str], *, - name: Optional[str] = None, - deserializer: Optional[Deserializer] = None, - initial_offsets: Optional[List[TopicPartitionOffset]] = None, - rebalance_listener: Optional[RebalanceListener] = None, + name: typing.Optional[str] = None, + deserializer: typing.Optional[Deserializer] = None, + initial_offsets: typing.Optional[typing.List[TopicPartitionOffset]] = None, + rebalance_listener: typing.Optional[RebalanceListener] = None, + middlewares: typing.Optional[typing.List[Middleware]] = None, **kwargs, -) -> Callable[[StreamFunc], Stream]: +) -> typing.Callable[[StreamFunc], Stream]: def decorator(func: StreamFunc) -> Stream: s = Stream( topics=topics, @@ -336,6 +361,7 @@ def decorator(func: StreamFunc) -> Stream: deserializer=deserializer, initial_offsets=initial_offsets, rebalance_listener=rebalance_listener, + middlewares=middlewares, config=kwargs, ) update_wrapper(s, func) diff --git a/kstreams/types.py b/kstreams/types.py index dda59b12..a4eb30e2 100644 --- a/kstreams/types.py +++ b/kstreams/types.py @@ -1,8 +1,24 @@ -from typing import ( - Dict, - Sequence, - Tuple, -) +import typing -Headers = Dict[str, str] -EncodedHeaders = Sequence[Tuple[str, bytes]] +from kstreams import ConsumerRecord, RecordMetadata + +if typing.TYPE_CHECKING: + from .serializers import Serializer # pragma: no cover + +Headers = typing.Dict[str, str] +EncodedHeaders = typing.Sequence[typing.Tuple[str, bytes]] +StreamFunc = typing.Callable +NextMiddlewareCall = typing.Callable[[ConsumerRecord], typing.Awaitable[None]] +Send = typing.Callable[ + [ + str, + typing.Any, + typing.Any, + typing.Optional[int], + typing.Optional[int], + typing.Optional[Headers], + typing.Optional["Serializer"], + typing.Optional[typing.Dict], + ], + typing.Awaitable[RecordMetadata], +] diff --git a/mkdocs.yml b/mkdocs.yml index 2d0c55e0..6cc79d87 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,12 +34,17 @@ nav: - Monitoring: 'monitoring.md' - Serialization: 'serialization.md' - Testing: 'test_client.md' + - Middleware: 'middleware.md' - Utils: 'utils.md' markdown_extensions: - pymdownx.highlight - pymdownx.inlinehilite - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.snippets - pymdownx.critic - pymdownx.caret diff --git a/poetry.lock b/poetry.lock index 8606bc7c..2938e402 100644 --- a/poetry.lock +++ b/poetry.lock @@ -51,6 +51,20 @@ lz4 = ["lz4 (>=3.1.3)"] snappy = ["cramjam"] zstd = ["cramjam"] +[[package]] +name = "aiorun" +version = "2023.7.2" +description = "Boilerplate for asyncio applications" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiorun-2023.7.2-py3-none-any.whl", hash = "sha256:61f839265d86d3f3b37921eb4a99ef42fa2ee026241b67c54315423d7dac29c9"}, + {file = "aiorun-2023.7.2.tar.gz", hash = "sha256:95e689dc1b263aaf4556ef8fa73ccae68f38ba0f1c1017fe196ef7da244031f9"}, +] + +[package.extras] +dev = ["pytest", "pytest-cov"] + [[package]] name = "annotated-types" version = "0.5.0" @@ -1454,4 +1468,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "4a49d3aab25a77e8bab8888605d9b3a5894e88330a98ec6a3b794dd9a6546390" +content-hash = "dbef19e19019e1a2bbcf3363c2ce8f92c84203db4acc29ac0c5273d6f8110161" diff --git a/pyproject.toml b/pyproject.toml index 4ff331f1..426ef8df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ prometheus-client = "<1.0" future = "^0.18.2" PyYAML = ">=5.4,<7.0.0" pydantic = "<3.0.0" +aiorun = "^2023.7.2" [tool.poetry.dev-dependencies] pytest = "^6.1" diff --git a/scripts/cluster/start b/scripts/cluster/start index 43b751da..0146abd4 100755 --- a/scripts/cluster/start +++ b/scripts/cluster/start @@ -6,5 +6,6 @@ scripts/cluster/topics/create "local--hello-world" scripts/cluster/topics/create "local--sse" scripts/cluster/topics/create "local--avro-user" scripts/cluster/topics/create "local--avro-address" +scripts/cluster/topics/create "kstreams--dlq-topic" scripts/cluster/logs diff --git a/tests/middleware/conftest.py b/tests/middleware/conftest.py new file mode 100644 index 00000000..df52dc95 --- /dev/null +++ b/tests/middleware/conftest.py @@ -0,0 +1,46 @@ +from collections import namedtuple +from unittest.mock import AsyncMock + +import pytest + +from kstreams import ConsumerRecord, middleware + +Middleware = namedtuple("Middleware", ["middleware", "call"]) + + +@pytest.fixture +def dlq_middleware(): + dlq = AsyncMock() + + class DLQMiddleware(middleware.BaseMiddleware): + async def __call__(self, cr: ConsumerRecord): + try: + return await self.next_call(cr) + except ValueError: + await dlq(cr.value) + + return Middleware(middleware=DLQMiddleware, call=dlq) + + +@pytest.fixture +def elastic_middleware(): + save_to_elastic = AsyncMock() + + class ElasticMiddleware(middleware.BaseMiddleware): + async def __call__(self, cr: ConsumerRecord): + await save_to_elastic(cr.value) + return await self.next_call(cr) + + return Middleware(middleware=ElasticMiddleware, call=save_to_elastic) + + +@pytest.fixture +def s3_middleware(): + backup_to_s3 = AsyncMock() + + class S3Middleware(middleware.BaseMiddleware): + async def __call__(self, cr: ConsumerRecord): + await backup_to_s3(cr.value) + return await self.next_call(cr) + + return Middleware(middleware=S3Middleware, call=backup_to_s3) diff --git a/tests/middleware/test_middleware.py b/tests/middleware/test_middleware.py new file mode 100644 index 00000000..08d2dee0 --- /dev/null +++ b/tests/middleware/test_middleware.py @@ -0,0 +1,318 @@ +from unittest.mock import AsyncMock, call + +import pytest +from aiokafka.errors import ConsumerStoppedError + +from kstreams import ConsumerRecord, Stream, StreamEngine, TestStreamClient, middleware + + +@pytest.mark.asyncio +async def test_middleware_stack_from_stream( + stream_engine: StreamEngine, dlq_middleware +): + stream_name = "my-stream" + stream_name_local = "my-stream-local" + + @stream_engine.stream( + "kstreams-topic", + name=stream_name, + middlewares=[middleware.Middleware(dlq_middleware.middleware)], + ) + async def consume(cr: ConsumerRecord): + ... + + @stream_engine.stream( + "kstreams-topic-local", + name=stream_name_local, + middlewares=[middleware.Middleware(dlq_middleware.middleware)], + ) + async def process(cr: ConsumerRecord, stream: Stream): + ... + + my_stream = stream_engine.get_stream(stream_name) + my_stream_local = stream_engine.get_stream(stream_name) + middlewares = [ + middleware_factory.middleware for middleware_factory in my_stream.middlewares + ] + middlewares_stream_local = [ + middleware_factory.middleware + for middleware_factory in my_stream_local.middlewares + ] + assert ( + middlewares + == middlewares_stream_local + == [middleware.ExceptionMiddleware, dlq_middleware.middleware] + ) + + +@pytest.mark.asyncio +async def test_middleware_stack_from_engine( + stream_engine: StreamEngine, dlq_middleware +): + stream_name = "my-stream" + stream_engine.middlewares = [middleware.Middleware(dlq_middleware.middleware)] + + @stream_engine.stream("kstreams-topic", name=stream_name) + async def consume(cr: ConsumerRecord): + ... + + my_stream = stream_engine.get_stream(stream_name) + middlewares = [ + middleware_factory.middleware for middleware_factory in my_stream.middlewares + ] + assert middlewares == [middleware.ExceptionMiddleware, dlq_middleware.middleware] + + +@pytest.mark.asyncio +async def test_middleware_stack_order( + stream_engine: StreamEngine, dlq_middleware, elastic_middleware +): + stream_name = "my-stream" + stream_engine.middlewares = [middleware.Middleware(elastic_middleware.middleware)] + + @stream_engine.stream( + "kstreams-topic", + name=stream_name, + middlewares=[middleware.Middleware(dlq_middleware.middleware)], + ) + async def consume(cr: ConsumerRecord): + ... + + my_stream = stream_engine.get_stream(stream_name) + middlewares = [ + middleware_factory.middleware for middleware_factory in my_stream.middlewares + ] + assert middlewares == [ + middleware.ExceptionMiddleware, + elastic_middleware.middleware, + dlq_middleware.middleware, + ] + + +@pytest.mark.asyncio +async def test_middleware_call_from_stream(stream_engine: StreamEngine, dlq_middleware): + topic = "local--hello-kpn" + value = b"joker" + save_to_db = AsyncMock() + client = TestStreamClient(stream_engine) + + middlewares = [middleware.Middleware(dlq_middleware.middleware)] + + @stream_engine.stream(topic, middlewares=middlewares) + async def my_stream(cr: ConsumerRecord): + if cr.value == value: + raise ValueError("Error from stream...") + await save_to_db(cr.value) + + async with client: + await client.send(topic, value=value) + + dlq_middleware.call.assert_awaited_once_with(value) + + # check that the call was not done + save_to_db.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_middleware_call_from_async_generator( + stream_engine: StreamEngine, elastic_middleware +): + topic = "local--hello-kpn" + event = b"batman" + save_to_db = AsyncMock() + client = TestStreamClient(stream_engine) + middlewares = [middleware.Middleware(elastic_middleware.middleware)] + + @stream_engine.stream(topic, middlewares=middlewares) + async def my_stream(cr: ConsumerRecord): + await save_to_db(cr.value) + yield cr + + async with client: + await client.send(topic, value=event) + + async with my_stream as stream_flow: + async for cr in stream_flow: + assert cr.value == event + break + + elastic_middleware.call.assert_awaited_once_with(event) + save_to_db.assert_awaited_once_with(event) + + +@pytest.mark.asyncio +async def test_middleware_call_from_stream_engine( + stream_engine: StreamEngine, dlq_middleware +): + topic = "local--hello-kpn" + value = b"joker" + save_to_db = AsyncMock() + + stream_engine.middlewares = [middleware.Middleware(dlq_middleware.middleware)] + client = TestStreamClient(stream_engine) + + @stream_engine.stream(topic) + async def consume(cr: ConsumerRecord): + if cr.value == value: + raise ValueError("Error from stream...") + await save_to_db(cr.value) + + async with client: + await client.send(topic, value=value) + + dlq_middleware.call.assert_awaited_once_with(value) + + # check that the call was not done + save_to_db.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_middleware_call_chain_from_stream( + stream_engine: StreamEngine, dlq_middleware, elastic_middleware, s3_middleware +): + topic = "local--hello-kpn" + event_1 = b"batman" + event_2 = b"joker" + save_to_db = AsyncMock() + client = TestStreamClient(stream_engine) + + middlewares = [ + middleware.Middleware(dlq_middleware.middleware), + middleware.Middleware(elastic_middleware.middleware), + middleware.Middleware(s3_middleware.middleware), + ] + + @stream_engine.stream(topic, middlewares=middlewares) + async def consume(cr: ConsumerRecord): + if cr.value == event_2: + raise ValueError("Error from stream...") + await save_to_db(cr.value) + + async with client: + await client.send(topic, value=event_1) + await client.send(topic, value=event_2) + + # called only when the whole chain has worked + save_to_db.assert_awaited_once_with(event_1) + + # called only when the whole chain is broken + dlq_middleware.call.assert_awaited_once_with(event_2) + + # called always + elastic_middleware.call.assert_has_awaits(calls=[call(event_1), call(event_2)]) + s3_middleware.call.assert_has_awaits(calls=[call(event_1), call(event_2)]) + + +@pytest.mark.asyncio +async def test_middleware_call_chain_from_stream_engine( + stream_engine: StreamEngine, dlq_middleware, elastic_middleware, s3_middleware +): + topic = "local--hello-kpn" + event_1 = b"batman" + event_2 = b"joker" + save_to_db = AsyncMock() + client = TestStreamClient(stream_engine) + + stream_engine.middlewares = [ + middleware.Middleware(dlq_middleware.middleware), + middleware.Middleware(elastic_middleware.middleware), + middleware.Middleware(s3_middleware.middleware), + ] + + @stream_engine.stream(topic) + async def consume(cr: ConsumerRecord): + if cr.value == event_2: + raise ValueError("Error from stream...") + await save_to_db(cr.value) + + async with client: + await client.send(topic, value=event_1) + await client.send(topic, value=event_2) + + # called only when the whole chain has worked + save_to_db.assert_awaited_once_with(event_1) + + # called only when the whole chain is broken + dlq_middleware.call.assert_awaited_once_with(event_2) + + # called always + elastic_middleware.call.assert_has_awaits(calls=[call(event_1), call(event_2)]) + s3_middleware.call.assert_has_awaits(calls=[call(event_1), call(event_2)]) + + +@pytest.mark.asyncio +async def test_combine_middleware_call_stream_and_engine( + stream_engine: StreamEngine, dlq_middleware, elastic_middleware, s3_middleware +): + topic = "local--hello-kpn" + event_1 = b"batman" + event_2 = b"joker" + save_to_db = AsyncMock() + client = TestStreamClient(stream_engine) + + stream_engine.middlewares = [ + middleware.Middleware(dlq_middleware.middleware), + ] + + @stream_engine.stream( + topic, + middlewares=[ + middleware.Middleware(elastic_middleware.middleware), + middleware.Middleware(s3_middleware.middleware), + ], + ) + async def consume(cr: ConsumerRecord): + if cr.value == event_2: + raise ValueError("Error from stream...") + await save_to_db(cr.value) + + async with client: + await client.send(topic, value=event_1) + await client.send(topic, value=event_2) + + # called only when the whole chain has worked + save_to_db.assert_awaited_once_with(event_1) + + # called only when the whole chain is broken + dlq_middleware.call.assert_awaited_once_with(event_2) + + # called always + elastic_middleware.call.assert_has_awaits(calls=[call(event_1), call(event_2)]) + s3_middleware.call.assert_has_awaits(calls=[call(event_1), call(event_2)]) + + +@pytest.mark.asyncio +async def test_base_middleware_exception(stream_engine: StreamEngine): + topic = "local--hello-kpn" + stream_name = "my_stream" + client = TestStreamClient(stream_engine) + stream_engine.middlewares = [middleware.Middleware(middleware.BaseMiddleware)] + + @stream_engine.stream(topic, name=stream_name) + async def consume(cr: ConsumerRecord): + ... + + my_stream = stream_engine.get_stream(stream_name) + + async with client: + await client.send(topic, value=b"test") + + assert not my_stream.running + + +@pytest.mark.asyncio +async def test_exception_middleware_consumer_stops(stream_engine: StreamEngine): + topic = "local--hello-kpn" + stream_name = "my_stream" + client = TestStreamClient(stream_engine) + + @stream_engine.stream(topic, name=stream_name) + async def consume(cr: ConsumerRecord): + raise ConsumerStoppedError + + my_stream = stream_engine.get_stream(stream_name) + + async with client: + await client.send(topic, value=b"test") + + assert not my_stream.running