clean up folder structure

ch7/api_dockerization/Dockerfile

@@ -0,0 +1,17 @@
+# Base image
+FROM python:3.10-slim
+
+# Set working directory
+WORKDIR /app
+
+# Copy application code
+COPY . /app
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Expose API port
+EXPOSE 8000
+
+# Run the application
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]

ch7/api-dockerization/README.md → ch7/api_dockerization/README.md

@@ -3,9 +3,10 @@
 Redis instance
 
 ```bash
-gunicorn -b 127.0.0.1:8000 myapp:app -k uvicorn.workers.UvicornWorker
+gunicorn -b 127.0.0.1:8000 app:app -k uvicorn.workers.UvicornWorker
 pip-compile requirements.in --output-file requirements.txt
-docker run --restart always -p 8000:8000 fastapi-bytewax-app
+docker build -t nlp-api .
+docker run -p 8000:8000 nlp-api
 
 ```
 

ch7/api-dockerization/app.py → ch7/api_dockerization/app.py

@@ -3,7 +3,6 @@
 from bytewax.dataflow import Dataflow
 from bytewax.run import cli_main
 from haystack.document_stores.in_memory import InMemoryDocumentStore
-from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
 import asyncio
 import logging
 import os

ch7/api-dockerization/indexing_dataflow.py → ch7/api_dockerization/indexing_dataflow.py

@@ -9,6 +9,7 @@
 from haystack.components.writers import DocumentWriter
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.utils import Secret
+from haystack.document_stores.in_memory import InMemoryDocumentStore
 
 
 from haystack import component, Document
@@ -25,8 +26,6 @@
 
 import logging
 
-
-
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
@@ -185,3 +184,11 @@ def process_event(event):
     embed_data = op.map("embed_data", filtered_data, process_event)
     op.output("output", embed_data, StdOutSink())
     return flow
+
+## sample usage
+# symbol = "AAPL"
+# load_dotenv(".env")
+# OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+
+# document_store = InMemoryDocumentStore()
+# flow = run_pipeline_with_symbol(symbol, document_store, OPENAI_API_KEY)

justfile

@@ -73,7 +73,7 @@ test-py tests=pytests: _assert-venv
     pytest {{tests}}
 
 # Run all the checks that will be run in CI locally
-ci-pre: lint test-py
+ci-pre: test-py
 
 # Build the package
 build: _assert-venv

pytests/ch7/test_dataflow.py

@@ -0,0 +1 @@
+from ch7.api_dockerization.indexing_dataflow import run_pipeline_with_symbol

repo_structure.txt

@@ -0,0 +1,110 @@
+.
+├── LICENSE
+├── README.md
+├── __init__.py
+├── ch2
+│   ├── README.md
+│   ├── __init__.py
+│   ├── jupyter-notebooks
+│   │   ├── images
+│   │   ├── interacting-with-llms-haystack2.0.ipynb
+│   │   ├── interacting-with-llms-hugging-face.ipynb
+│   │   ├── interacting-with-llms-openai.ipynb
+│   │   ├── llm-augmentation.ipynb
+│   │   ├── llm-overview.ipynb
+│   │   └── pipeline-diagrams
+│   └── scripts
+│       ├── __init__.py
+│       ├── huggingfaceendpoints.py
+│       └── pipelines.py
+├── ch3
+│   ├── README.md
+│   └── jupyter-notebooks
+│       ├── audio-files
+│       ├── components.ipynb
+│       ├── data-processing-components.ipynb
+│       ├── data_classes.ipynb
+│       ├── embedding-components.ipynb
+│       ├── images
+│       └── writing-components.ipynb
+├── ch4
+│   ├── README.md
+│   └── jupyter-notebooks
+│       ├── extraction-and-processing-pipelines.ipynb
+│       ├── images
+│       ├── indexing-pipelines.ipynb
+│       ├── markdown_pages
+│       ├── retriever-pipelines-with-llm.ipynb
+│       ├── retriever-pipelines.ipynb
+│       └── textfile-pages
+├── ch5
+│   ├── README.md
+│   └── jupyter-notebooks
+│       ├── custom-component-error-handling.ipynb
+│       ├── custom-component-logging-threading.ipynb
+│       ├── custom-component-logging.ipynb
+│       ├── custom-component-scalability.ipynb
+│       ├── custom-components.ipynb
+│       ├── images
+│       └── tests
+├── ch6
+│   ├── README.md
+│   ├── adding-evaluation
+│   │   ├── query_pipeline.py
+│   │   ├── query_pipeline_ragas.png
+│   │   └── wandb
+│   ├── adding-observability
+│   │   ├── indexing_pipeline.py
+│   │   ├── query_pipeline.png
+│   │   ├── query_pipeline.py
+│   │   └── wandb
+│   ├── benzinga_pipeline.png
+│   ├── case-I-q-and-a-dataset
+│   │   ├── benzinga_pipeline.png
+│   │   ├── indexingpipeline.py
+│   │   ├── query_pipeline.png
+│   │   └── querypipeline.py
+│   ├── case-II-q-and-a-complex
+│   │   ├── answer_generation_pipeline.png
+│   │   ├── indexing_complex_knowledge.png
+│   │   └── rag-tutorials.ipynb
+│   ├── docker-compose.yml
+│   ├── feedback-loop-guardrails
+│   │   └── query_pipeline.py
+│   ├── news_out.jsonl
+│   ├── query_pipeline.png
+│   └── query_pipeline_ragas.png
+├── ch7
+│   ├── api_dockerization
+│   │   ├── Dockerfile
+│   │   ├── README.md
+│   │   ├── app.py
+│   │   ├── benzinga_pipeline.png
+│   │   ├── config.yaml
+│   │   ├── dataflow-dag.png
+│   │   ├── indexing_dataflow.py
+│   │   ├── news_out.jsonl
+│   │   ├── querying.py
+│   │   └── requirements.txt
+│   ├── benzinga_pipeline.png
+│   └── pipeline-serialization
+├── just
+├── justfile
+├── mistral-7b-v1.0.yaml
+├── pyproject.toml
+├── repo_structure.txt
+├── requirements
+│   ├── dev.in
+│   ├── dev.txt
+│   ├── doc.txt
+│   ├── lib-py3.10.txt
+│   ├── lib-py3.11.txt
+│   ├── lib-py3.12.txt
+│   ├── lib-py3.9.txt
+│   ├── requirements.in
+│   └── requirements.txt
+├── tests
+└── venvs
+    └── dev
+
+34 directories, 74 files