fixes

deepsense-ai · Jan 9, 2025 · 18c9675 · 18c9675
1 parent 80173a4
commit 18c9675
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 41 deletions.
diff --git a/examples/evaluation/document-search/advanced/evaluate.py b/examples/evaluation/document-search/advanced/evaluate.py
@@ -8,9 +8,10 @@
 # ///
 import asyncio
 import logging
+from typing import cast
 
 import hydra
-from omegaconf import DictConfig
+from omegaconf import DictConfig, OmegaConf
 
 from ragbits.evaluate.evaluator import Evaluator
 from ragbits.evaluate.utils import log_to_file, log_to_neptune, setup_neptune
@@ -29,7 +30,7 @@ async def bench(config: DictConfig) -> None:
     """
     run = setup_neptune(config)
     log.info("Starting the experiment...")
-    results = await Evaluator.run_experiment_from_config(config=config)
+    results = await Evaluator.run_experiment_from_config(config=cast(dict, OmegaConf.to_container(config)))
     output_dir = log_to_file(results)
     if run:
         log_to_neptune(run, results, output_dir)

diff --git a/examples/evaluation/document-search/basic/basic_evaluate.py b/examples/evaluation/document-search/basic/basic_evaluate.py
@@ -12,8 +12,6 @@
 import uuid
 from pathlib import Path
 
-from omegaconf import OmegaConf
-
 from ragbits.evaluate.evaluator import Evaluator
 from ragbits.evaluate.utils import log_to_file
 
@@ -29,42 +27,38 @@ async def evaluate() -> dict:
     """
     log.info("Ingesting documents...")
 
-    config = OmegaConf.create(
-        {
-            "pipeline": {
-                "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline",
-                "ingest": False,
-                "search": True,
-                "vector_store": {
-                    "type": "ragbits.core.vector_stores.chroma:ChromaVectorStore",
-                    "config": {
-                        "client": {"type": "PersistentClient", "config": {"path": "chroma"}},
-                        "index_name": "default",
-                        "distance_method": "l2",
-                        "default_options": {"k": 3, "max_distance": 1.2},
-                    },
-                },
-                "providers": {
-                    "txt": {
-                        "type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"
-                    }
+    config = {
+        "pipeline": {
+            "type": "ragbits.evaluate.pipelines.document_search:DocumentSearchWithIngestionPipeline",
+            "ingest": False,
+            "search": True,
+            "vector_store": {
+                "type": "ragbits.core.vector_stores.chroma:ChromaVectorStore",
+                "config": {
+                    "client": {"type": "PersistentClient", "config": {"path": "chroma"}},
+                    "index_name": "default",
+                    "distance_method": "l2",
+                    "default_options": {"k": 3, "max_distance": 1.2},
                 },
             },
-            "data": {
-                "type": "ragbits.evaluate.loaders.hf:HFDataLoader",
-                "options": {"name": "hf-docs-retrieval", "path": "micpst/hf-docs-retrieval", "split": "train"},
+            "providers": {
+                "txt": {"type": "ragbits.document_search.ingestion.providers.unstructured:UnstructuredDefaultProvider"}
             },
-            "metrics": [
-                {
-                    "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1",
-                    "matching_strategy": "RougeChunkMatch",
-                    "options": {"threshold": 0.5},
-                }
-            ],
-            "neptune": {"project": "ragbits", "run": False},
-            "task": {"name": "default", "type": "document-search"},
-        }
-    )
+        },
+        "data": {
+            "type": "ragbits.evaluate.loaders.hf:HFDataLoader",
+            "options": {"name": "hf-docs-retrieval", "path": "micpst/hf-docs-retrieval", "split": "train"},
+        },
+        "metrics": [
+            {
+                "type": "ragbits.evaluate.metrics.document_search:DocumentSearchPrecisionRecallF1",
+                "matching_strategy": "RougeChunkMatch",
+                "options": {"threshold": 0.5},
+            }
+        ],
+        "neptune": {"project": "ragbits", "run": False},
+        "task": {"name": "default", "type": "document-search"},
+    }
 
     results = await Evaluator.run_experiment_from_config(config=config)
 

diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py b/packages/ragbits-evaluate/src/ragbits/evaluate/evaluator.py
@@ -49,20 +49,20 @@ async def compute(
         }
 
     @classmethod
-    async def run_experiment_from_config(cls, config: DictConfig) -> dict[str, Any]:
+    async def run_experiment_from_config(cls, config: dict) -> dict[str, Any]:
         """
         Runs the evaluation experiment basing on configuration
         Args:
             config: DictConfig - soe config
         Returns:
             dictionary of metrics with scores
         """
-        dataloader = dataloader_factory(cast(dict, OmegaConf.to_container(config.data)))
-        pipeline = pipeline_factory(cast(dict, OmegaConf.to_container(config.pipeline)))
+        dataloader = dataloader_factory(config["data"])
+        pipeline = pipeline_factory(config["pipeline"])
 
         metric_config = config.get("metrics", None)
         metrics = (
-            metric_set_factory(cast(list[dict], OmegaConf.to_container(metric_config)))
+            metric_set_factory(metric_config) # type: ignore
             if metric_config is not None
             else None
         )