Skip to content

Commit

Permalink
added parallel to text_embedder
Browse files Browse the repository at this point in the history
  • Loading branch information
nickprock committed Feb 13, 2024
1 parent ab42a3a commit f9263ad
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def __init__(
suffix: str = "",
batch_size: int = 256,
progress_bar: bool = True,
parallel: int = None,
):
"""
Create a FastembedTextEmbedder component.
Expand All @@ -53,6 +54,7 @@ def __init__(
self.suffix = suffix
self.batch_size = batch_size
self.progress_bar = progress_bar
self.parallel = parallel

def to_dict(self) -> Dict[str, Any]:
"""
Expand All @@ -65,6 +67,7 @@ def to_dict(self) -> Dict[str, Any]:
suffix=self.suffix,
batch_size=self.batch_size,
progress_bar=self.progress_bar,
parallel=self.parallel,
)

def warm_up(self):
Expand Down Expand Up @@ -93,6 +96,7 @@ def run(self, text: str):
text_to_embed,
batch_size=self.batch_size,
show_progress_bar=self.progress_bar,
parallel=self.parallel,
)[0]
)
return {"embedding": embedding}
10 changes: 10 additions & 0 deletions integrations/fastembed/tests/test_fastembed_text_embedder.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ def test_init_default(self):
assert embedder.suffix == ""
assert embedder.batch_size == 256
assert embedder.progress_bar is True
assert embedder.parallel is None

def test_init_with_parameters(self):
"""
Expand All @@ -30,12 +31,14 @@ def test_init_with_parameters(self):
suffix="suffix",
batch_size=64,
progress_bar=False,
parallel=1,
)
assert embedder.model_name == "BAAI/bge-small-en-v1.5"
assert embedder.prefix == "prefix"
assert embedder.suffix == "suffix"
assert embedder.batch_size == 64
assert embedder.progress_bar is False
assert embedder.parallel == 1

def test_to_dict(self):
"""
Expand All @@ -51,6 +54,7 @@ def test_to_dict(self):
"suffix": "",
"batch_size": 256,
"progress_bar": True,
"parallel": None,
},
}

Expand All @@ -64,6 +68,7 @@ def test_to_dict_with_custom_init_parameters(self):
suffix="suffix",
batch_size=64,
progress_bar=False,
parallel=1,
)
embedder_dict = embedder.to_dict()
assert embedder_dict == {
Expand All @@ -74,6 +79,7 @@ def test_to_dict_with_custom_init_parameters(self):
"suffix": "suffix",
"batch_size": 64,
"progress_bar": False,
"parallel": 1,
},
}

Expand All @@ -89,6 +95,7 @@ def test_from_dict(self):
"suffix": "",
"batch_size": 256,
"progress_bar": True,
"parallel": None,
},
}
embedder = default_from_dict(FastembedTextEmbedder, embedder_dict)
Expand All @@ -97,6 +104,7 @@ def test_from_dict(self):
assert embedder.suffix == ""
assert embedder.batch_size == 256
assert embedder.progress_bar is True
assert embedder.parallel is None

def test_from_dict_with_custom_init_parameters(self):
"""
Expand All @@ -110,6 +118,7 @@ def test_from_dict_with_custom_init_parameters(self):
"suffix": "suffix",
"batch_size": 64,
"progress_bar": False,
"parallel": 1,
},
}
embedder = default_from_dict(FastembedTextEmbedder, embedder_dict)
Expand All @@ -118,6 +127,7 @@ def test_from_dict_with_custom_init_parameters(self):
assert embedder.suffix == "suffix"
assert embedder.batch_size == 64
assert embedder.progress_bar is False
assert embedder.parallel == 1

@patch(
"haystack_integrations.components.embedders.fastembed.fastembed_text_embedder._FastembedEmbeddingBackendFactory"
Expand Down

0 comments on commit f9263ad

Please sign in to comment.