pin dependencies

ch7/api-dockerization/README.md

@@ -0,0 +1,11 @@
+## Dockerization of indexing and query pipelines 
+
+Redis instance
+
+```bash
+docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
+```
+
+```bash
+python -m bytewax.run ./api-dockerization/indexing_dataflow:flow
+```

ch7/api-dockerization/indexing_dataflow.py

@@ -1,8 +1,8 @@
 from bytewax import operators as op
 from bytewax.dataflow import Dataflow
 from bytewax import operators as op
-from bytewax.connectors.stdio import StdOutSink
 from bytewax.connectors.files import FileSource
+from bytewax.bytewax_redis import RedisStreamSink
 
 from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter
 from haystack.components.embedders import OpenAIDocumentEmbedder
@@ -30,6 +30,11 @@
 load_dotenv(".env")
 open_ai_key = os.environ.get("OPENAI_API_KEY")
 
+REDIS_HOST = os.environ.get("REDIS_HOST", "localhost")
+REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379))
+REDIS_DB = int(os.environ.get("REDIS_DB", 0))
+REDIS_STREAM_NAME = os.environ.get("REDIS_STREAM_NAME", "indexing-stream")
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
@@ -156,8 +161,7 @@ def run(self, event: List[Union[str, Path, ByteStream]]):
         logger.info(f"Running BenzingaEmbeder with event: {event}")
         try:
             documents = self.pipeline.run({"get_news": {"sources": [event]}})
-            self.pipeline.draw("benzinga_pipeline.png")
-            logger.info("Pipeline executed successfully, drawing pipeline graph.")
+            logger.info("Pipeline executed successfully")
             return documents
         except Exception as e:
             logger.error(f"Error during pipeline execution: {e}")
@@ -177,4 +181,8 @@ def process_event(event):
 input_data = op.input("input", flow, FileSource("news_out.jsonl"))
 deserialize_data = op.map("deserialize", input_data, safe_deserialize)
 get_content = op.map("embed_content", deserialize_data, process_event)
-op.output("output", get_content, StdOutSink())
+op.output("output", get_content,
+          RedisStreamSink(REDIS_STREAM_NAME,
+                          REDIS_HOST,
+                          REDIS_PORT,
+                          REDIS_DB))

ch6/justfile → justfile

@@ -57,3 +57,36 @@ venv-compile-all:
     uv pip compile --generate-hashes -p 3.11 --all-extras pyproject.toml -o requirements/lib-py3.11.txt
     uv pip compile --generate-hashes -p 3.12 --all-extras pyproject.toml -o requirements/lib-py3.12.txt
 
+# Lint the code in the repo; runs in CI
+lint: _assert-venv
+    vermin --config-file vermin-lib.ini src/ pytests/
+    vermin --config-file vermin-dev.ini docs/ *.py
+    ruff check src/ pytests/ docs/
+    mypy -p bytewax.duckdb
+    mypy pytests/ docs/
+
+
+pytests := 'pytests/'
+
+# Run the Python tests; runs in CI
+test-py tests=pytests: _assert-venv
+    pytest {{tests}}
+
+# Run all the checks that will be run in CI locally
+ci-pre: lint test-py
+
+# Build the package
+build: _assert-venv
+    @echo 'Installing build tools'
+    uv pip install build
+    @echo 'Cleaning old build artifacts'
+    rm -rf dist/
+    @echo 'Building the package'
+    python -m build
+
+# Upload the package to PyPI
+upload: _assert-venv build
+    @echo 'Installing twine'
+    uv pip install twine
+    @echo 'Uploading the package to PyPI'
+    twine upload dist/*

ch6/pyproject.toml → pyproject.toml

@@ -4,15 +4,14 @@ version = "0.1"
 description = "Reproducible RAG system with Python"
 readme = "README.md"
 requires-python = ">=3.9"
-license = {file = "LICENSE.md"}
+license = {file = "LICENSE"}
 keywords = ["llm", "rag", "nlp", "pipelines"]
 authors = [
   {name = "Laura Funderburk"}
 ]
 
 classifiers = [
   "Development Status :: 1 - Beta",
-
   "Intended Audience :: Developers",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
@@ -40,6 +39,9 @@ dependencies = [
 requires = ["setuptools>=43.0.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools]
+packages = {find = {exclude = ["ch*", "venvs", "tests", "docs"]}}
+
 [tool.pytest.ini_options]
 addopts = "-v"
 testpaths = [
@@ -77,4 +79,4 @@ docstring-code-format = true
 
 [tool.ruff.lint.per-file-ignores]
 "examples/*.py" = ["D"]
-"test_*.py" = ["D"]
+"test_*.py" = ["D"]

requirements/dev.in

@@ -0,0 +1,9 @@
+-e .
+-r build.in
+ipython
+mypy
+pre-commit
+ruff
+sphinx-autobuild
+types-docutils
+vermin