Use max-segment-size in tokens in step06+ to be consistent with step05

quarkusio · Feb 4, 2025 · 65375b9 · 65375b9
1 parent 01f3670
commit 65375b9
Show file tree

Hide file tree

Showing 6 changed files with 16 additions and 6 deletions.
diff --git a/docs/docs/step-06.md b/docs/docs/step-06.md
@@ -142,7 +142,7 @@ The `FileSystemDocumentLoader.loadDocumentsRecursively(documents)` method loads
 The `EmbeddingStoreIngestor` class is used to ingest the documents into the vector store.
 This is the cornerstone of the ingestion process.
 Configuring it correctly is crucial to the accuracy of the RAG pattern.
-Here, we use a recursive document splitter with a segment size of 100 and an overlap size of 25 (like we had in the previous step).
+Here, we use a recursive document splitter with a segment size of 100 tokens and an overlap size of 25 tokens (like we had in the previous step).
 
 !!! important
     The splitter, the segment size, and the overlap size are crucial to the accuracy of the RAG pattern.

diff --git a/step-06/src/main/java/dev/langchain4j/quarkus/workshop/RagIngestion.java b/step-06/src/main/java/dev/langchain4j/quarkus/workshop/RagIngestion.java
@@ -5,6 +5,7 @@
 import java.nio.file.Path;
 import java.util.List;
 
+import dev.langchain4j.model.embedding.onnx.HuggingFaceTokenizer;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.enterprise.event.Observes;
 
@@ -38,7 +39,8 @@ public void ingest(@Observes StartupEvent ev,
         EmbeddingStoreIngestor ingestor = EmbeddingStoreIngestor.builder()
                 .embeddingStore(store)
                 .embeddingModel(embeddingModel)
-                .documentSplitter(recursive(100, 25))
+                .documentSplitter(recursive(100, 25,
+                        new HuggingFaceTokenizer()))
                 .build();
         ingestor.ingest(list);
         Log.info("Documents ingested successfully");

diff --git a/step-07/src/main/java/dev/langchain4j/quarkus/workshop/RagIngestion.java b/step-07/src/main/java/dev/langchain4j/quarkus/workshop/RagIngestion.java
@@ -5,6 +5,7 @@
 import java.nio.file.Path;
 import java.util.List;
 
+import dev.langchain4j.model.embedding.onnx.HuggingFaceTokenizer;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.enterprise.event.Observes;
 
@@ -38,7 +39,8 @@ public void ingest(@Observes StartupEvent ev,
         EmbeddingStoreIngestor ingestor = EmbeddingStoreIngestor.builder()
                 .embeddingStore(store)
                 .embeddingModel(embeddingModel)
-                .documentSplitter(recursive(100, 25))
+                .documentSplitter(recursive(100, 25,
+                        new HuggingFaceTokenizer()))
                 .build();
         ingestor.ingest(list);
         Log.info("Documents ingested successfully");

diff --git a/step-08/src/main/java/dev/langchain4j/quarkus/workshop/RagIngestion.java b/step-08/src/main/java/dev/langchain4j/quarkus/workshop/RagIngestion.java
@@ -5,6 +5,7 @@
 import java.nio.file.Path;
 import java.util.List;
 
+import dev.langchain4j.model.embedding.onnx.HuggingFaceTokenizer;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.enterprise.event.Observes;
 
@@ -38,7 +39,8 @@ public void ingest(@Observes StartupEvent ev,
         EmbeddingStoreIngestor ingestor = EmbeddingStoreIngestor.builder()
                 .embeddingStore(store)
                 .embeddingModel(embeddingModel)
-                .documentSplitter(recursive(100, 25))
+                .documentSplitter(recursive(100, 25,
+                        new HuggingFaceTokenizer()))
                 .build();
         ingestor.ingest(list);
         Log.info("Documents ingested successfully");

diff --git a/step-09/src/main/java/dev/langchain4j/quarkus/workshop/RagIngestion.java b/step-09/src/main/java/dev/langchain4j/quarkus/workshop/RagIngestion.java
@@ -5,6 +5,7 @@
 import java.nio.file.Path;
 import java.util.List;
 
+import dev.langchain4j.model.embedding.onnx.HuggingFaceTokenizer;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.enterprise.event.Observes;
 
@@ -38,7 +39,8 @@ public void ingest(@Observes StartupEvent ev,
         EmbeddingStoreIngestor ingestor = EmbeddingStoreIngestor.builder()
                 .embeddingStore(store)
                 .embeddingModel(embeddingModel)
-                .documentSplitter(recursive(100, 25))
+                .documentSplitter(recursive(100, 25,
+                        new HuggingFaceTokenizer()))
                 .build();
         ingestor.ingest(list);
         Log.info("Documents ingested successfully");

diff --git a/step-10/src/main/java/dev/langchain4j/quarkus/workshop/RagIngestion.java b/step-10/src/main/java/dev/langchain4j/quarkus/workshop/RagIngestion.java
@@ -5,6 +5,7 @@
 import java.nio.file.Path;
 import java.util.List;
 
+import dev.langchain4j.model.embedding.onnx.HuggingFaceTokenizer;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.enterprise.event.Observes;
 
@@ -38,7 +39,8 @@ public void ingest(@Observes StartupEvent ev,
         EmbeddingStoreIngestor ingestor = EmbeddingStoreIngestor.builder()
                 .embeddingStore(store)
                 .embeddingModel(embeddingModel)
-                .documentSplitter(recursive(100, 25))
+                .documentSplitter(recursive(100, 25,
+                        new HuggingFaceTokenizer()))
                 .build();
         ingestor.ingest(list);
         Log.info("Documents ingested successfully");