Adds basic TQDM progress bar to KG creation pipeline

pyproject.toml

@@ -37,6 +37,7 @@ pinecone-client = {version = "^4.1.0", optional = true}
 types-mock = "^5.1.0.20240425"
 eval-type-backport = "^0.2.0"
 pypdf = "^4.3.1"
+tqdm = "^4.66.5"
 
 [tool.poetry.group.dev.dependencies]
 pylint = "^3.1.0"

src/neo4j_genai/experimental/pipeline/pipeline.py

@@ -22,6 +22,7 @@
 from typing import Any, AsyncGenerator, Awaitable, Callable, Optional
 
 from pydantic import BaseModel, Field
+from tqdm.asyncio import tqdm
 
 from neo4j_genai.experimental.pipeline.component import Component, DataModel
 from neo4j_genai.experimental.pipeline.exceptions import (
@@ -410,6 +411,7 @@ def on_task_complete(self, node: TaskPipelineNode, result: RunResult) -> None:
         if result.result:
             res_to_save = result.result.model_dump()
         self.add_result_for_component(node.name, res_to_save, is_final=node.is_leaf())
+        self.pbar.update(1)
 
     def add_result_for_component(
         self, name: str, result: dict[str, Any] | None, is_final: bool = False
@@ -471,12 +473,14 @@ def validate_inputs_config(self, data: dict[str, Any]) -> None:
             task.validate_inputs_config(data)
 
     async def run(self, data: dict[str, Any]) -> dict[str, Any]:
-        logging.info("Starting pipeline")
+        logger.debug("Starting pipeline")
+        self.pbar = tqdm(total=len(self._nodes), desc="Creating knowledge graph")
         start_time = default_timer()
         self.validate_inputs_config(data)
         self.reinitialize()
         orchestrator = Orchestrator(self)
         await orchestrator.run(data)
         end_time = default_timer()
-        logging.info(f"Pipeline finished in {end_time - start_time}s")
+        self.pbar.close()
+        logger.debug(f"Pipeline finished in {end_time - start_time}s")
         return self._final_results.all()