Skip to content

Commit

Permalink
Added a timer to the KG creation pipeline (neo4j#107)
Browse files Browse the repository at this point in the history
* Added a timer to the KG creation pipeline

* Reverted TaskPipelineNode logger call to debug

* Adds basic TQDM progress bar to KG creation pipeline

* Removed progress bar from KG pipeline
  • Loading branch information
alexthomas93 authored Aug 22, 2024
1 parent af27578 commit b89b932
Showing 1 changed file with 8 additions and 0 deletions.
8 changes: 8 additions & 0 deletions src/neo4j_genai/experimental/pipeline/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import enum
import logging
from datetime import datetime
from timeit import default_timer
from typing import Any, AsyncGenerator, Awaitable, Callable, Optional

from pydantic import BaseModel, Field
Expand Down Expand Up @@ -119,6 +120,7 @@ async def execute(self, **kwargs: Any) -> RunResult | None:
was unsuccessful.
"""
logger.debug(f"Running component {self.name} with {kwargs}")
start_time = default_timer()
try:
await self.set_status(RunStatus.RUNNING)
except PipelineStatusUpdateError:
Expand All @@ -130,6 +132,8 @@ async def execute(self, **kwargs: Any) -> RunResult | None:
status=self.status,
result=component_result,
)
end_time = default_timer()
logger.debug(f"Component {self.name} finished in {end_time - start_time}s")
return run_result

def validate_inputs_config(self, input_data: dict[str, Any]) -> None:
Expand Down Expand Up @@ -467,8 +471,12 @@ def validate_inputs_config(self, data: dict[str, Any]) -> None:
task.validate_inputs_config(data)

async def run(self, data: dict[str, Any]) -> dict[str, Any]:
logger.debug("Starting pipeline")
start_time = default_timer()
self.validate_inputs_config(data)
self.reinitialize()
orchestrator = Orchestrator(self)
await orchestrator.run(data)
end_time = default_timer()
logger.debug(f"Pipeline finished in {end_time - start_time}s")
return self._final_results.all()

0 comments on commit b89b932

Please sign in to comment.