From fb82be9aebd92b8fd1928cdc6bd30a06d39d5207 Mon Sep 17 00:00:00 2001 From: Tibor Reiss Date: Thu, 12 Sep 2024 20:28:34 +0200 Subject: [PATCH] Make docs clearer and add min_cunk_size --- docs/docs/how_to/semantic-chunker.ipynb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/docs/how_to/semantic-chunker.ipynb b/docs/docs/how_to/semantic-chunker.ipynb index 35889ec53afd0..aa71aeafcc115 100644 --- a/docs/docs/how_to/semantic-chunker.ipynb +++ b/docs/docs/how_to/semantic-chunker.ipynb @@ -125,9 +125,11 @@ "\n", "There are a few ways to determine what that threshold is, which are controlled by the `breakpoint_threshold_type` kwarg.\n", "\n", + "Note: if the resulting chunk sizes are too small/big, the additional kwargs `breakpoint_threshold_amount` and `min_chunk_size` can be used for adjustments.\n", + "\n", "### Percentile\n", "\n", - "The default way to split is based on percentile. In this method, all differences between sentences are calculated, and then any difference greater than the X percentile is split." + "The default way to split is based on percentile. In this method, all differences between sentences are calculated, and then any difference greater than the X percentile is split. The default value for X is 95.0 and can be adjusted by the keyword argument `breakpoint_threshold_amount` which expects a number between 0.0 and 100.0." ] }, { @@ -186,7 +188,7 @@ "source": [ "### Standard Deviation\n", "\n", - "In this method, any difference greater than X standard deviations is split." + "In this method, any difference greater than X standard deviations is split. The default value for X is 3.0 and can be adjusted by the keyword argument `breakpoint_threshold_amount`." ] }, { @@ -245,7 +247,7 @@ "source": [ "### Interquartile\n", "\n", - "In this method, the interquartile distance is used to split chunks." + "In this method, the interquartile distance is used to split chunks. The interquartile range can be scaled by the keyword argument `breakpoint_threshold_amount`, the default value is 1.5." ] }, { @@ -306,8 +308,8 @@ "source": [ "### Gradient\n", "\n", - "In this method, the gradient of distance is used to split chunks along with the percentile method.\n", - "This method is useful when chunks are highly correlated with each other or specific to a domain e.g. legal or medical. The idea is to apply anomaly detection on gradient array so that the distribution become wider and easy to identify boundaries in highly semantic data." + "In this method, the gradient of distance is used to split chunks along with the percentile method. This method is useful when chunks are highly correlated with each other or specific to a domain e.g. legal or medical. The idea is to apply anomaly detection on gradient array so that the distribution become wider and easy to identify boundaries in highly semantic data.\n", + "Similar to the percentile method, the split can be adjusted by the keyword argument `breakpoint_threshold_amount` which expects a number between 0.0 and 100.0, the default value is 95.0." ] }, {