From cfc46812c81a115a6a862c1d1743ded7a3a52813 Mon Sep 17 00:00:00 2001 From: rkuo-danswer Date: Wed, 13 Nov 2024 15:26:13 -0800 Subject: [PATCH] scale indexing sql pool based on concurrency (#3130) --- backend/danswer/background/celery/apps/indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/danswer/background/celery/apps/indexing.py b/backend/danswer/background/celery/apps/indexing.py index 01ec79e5c7b..533f9b96818 100644 --- a/backend/danswer/background/celery/apps/indexing.py +++ b/backend/danswer/background/celery/apps/indexing.py @@ -59,7 +59,7 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None: logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}") SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_APP_NAME) - SqlEngine.init_engine(pool_size=8, max_overflow=0) + SqlEngine.init_engine(pool_size=sender.concurrency, max_overflow=sender.concurrency) # Startup checks are not needed in multi-tenant case if MULTI_TENANT: