From 6030d8363064ea8d293e3324bf6ac9ffe46cef5b Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 12 Sep 2024 14:31:09 +0200 Subject: [PATCH] MongoDB: Decrease default batch size to 100 --- cratedb_toolkit/io/mongodb/adapter.py | 2 +- doc/io/mongodb/loader.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cratedb_toolkit/io/mongodb/adapter.py b/cratedb_toolkit/io/mongodb/adapter.py index ef75a48..aeaa264 100644 --- a/cratedb_toolkit/io/mongodb/adapter.py +++ b/cratedb_toolkit/io/mongodb/adapter.py @@ -53,7 +53,7 @@ def __attrs_post_init__(self): @cached_property def batch_size(self) -> int: - return int(self.address.uri.query_params.get("batch-size", 500)) + return int(self.address.uri.query_params.get("batch-size", 100)) @cached_property def filter(self) -> t.Union[str, None]: diff --git a/doc/io/mongodb/loader.md b/doc/io/mongodb/loader.md index ef760a4..00324b2 100644 --- a/doc/io/mongodb/loader.md +++ b/doc/io/mongodb/loader.md @@ -87,8 +87,9 @@ see [](#file-import-tutorial). ## Options ### Batch Size -The default batch size is 500. You can adjust the value by appending the HTTP -URL query parameter `batch-size` to the source URL, like +The default batch size is 100, but for many datasets a much larger batch size +is applicable for most efficient data transfers. You can adjust the value by +appending the HTTP URL query parameter `batch-size` to the source URL, like `mongodb+srv://managed.mongodb.net/ticker/stocks?batch-size=5000`. ### Filter