From 77cf9b353974fd3dab8888e276ca5134da286a7b Mon Sep 17 00:00:00 2001 From: rkuo-danswer Date: Mon, 25 Nov 2024 14:27:14 -0800 Subject: [PATCH] improve messaging and UI around cleanup of leftover index attempts (#3247) * improve messaging and UI around cleanup of leftover index attempts * add tag on init --- .../danswer/background/celery/apps/primary.py | 6 ++--- .../background/celery/tasks/indexing/tasks.py | 2 +- backend/danswer/db/enums.py | 2 ++ backend/danswer/db/index_attempt.py | 22 +++++++++++++++++++ .../configuration/search/UpgradingPage.tsx | 9 ++++---- .../[ccPairId]/IndexingAttemptsTable.tsx | 3 ++- web/src/components/Status.tsx | 8 ++++++- web/src/components/ui/badge.tsx | 2 ++ web/src/lib/types.ts | 1 + 9 files changed, 45 insertions(+), 10 deletions(-) diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py index f08e573723f..5efe8300670 100644 --- a/backend/danswer/background/celery/apps/primary.py +++ b/backend/danswer/background/celery/apps/primary.py @@ -24,7 +24,7 @@ from danswer.db.engine import get_session_with_default_tenant from danswer.db.engine import SqlEngine from danswer.db.index_attempt import get_index_attempt -from danswer.db.index_attempt import mark_attempt_failed +from danswer.db.index_attempt import mark_attempt_canceled from danswer.redis.redis_connector_credential_pair import RedisConnectorCredentialPair from danswer.redis.redis_connector_delete import RedisConnectorDelete from danswer.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync @@ -165,13 +165,13 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None: continue failure_reason = ( - f"Orphaned index attempt found on startup: " + f"Canceling leftover index attempt found on startup: " f"index_attempt={attempt.id} " f"cc_pair={attempt.connector_credential_pair_id} " f"search_settings={attempt.search_settings_id}" ) logger.warning(failure_reason) - mark_attempt_failed(attempt.id, db_session, failure_reason) + mark_attempt_canceled(attempt.id, db_session, failure_reason) @worker_ready.connect diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py index 14bf66cdc56..73b2b20a4e0 100644 --- a/backend/danswer/background/celery/tasks/indexing/tasks.py +++ b/backend/danswer/background/celery/tasks/indexing/tasks.py @@ -77,7 +77,7 @@ def __init__( self.started: datetime = datetime.now(timezone.utc) self.redis_lock.reacquire() - self.last_tag: str = "" + self.last_tag: str = "IndexingCallback.__init__" self.last_lock_reacquire: datetime = datetime.now(timezone.utc) def should_stop(self) -> bool: diff --git a/backend/danswer/db/enums.py b/backend/danswer/db/enums.py index 8d9515d387a..b1905d4e785 100644 --- a/backend/danswer/db/enums.py +++ b/backend/danswer/db/enums.py @@ -5,6 +5,7 @@ class IndexingStatus(str, PyEnum): NOT_STARTED = "not_started" IN_PROGRESS = "in_progress" SUCCESS = "success" + CANCELED = "canceled" FAILED = "failed" COMPLETED_WITH_ERRORS = "completed_with_errors" @@ -12,6 +13,7 @@ def is_terminal(self) -> bool: terminal_states = { IndexingStatus.SUCCESS, IndexingStatus.COMPLETED_WITH_ERRORS, + IndexingStatus.CANCELED, IndexingStatus.FAILED, } return self in terminal_states diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py index c0d28060ad5..06bbee10559 100644 --- a/backend/danswer/db/index_attempt.py +++ b/backend/danswer/db/index_attempt.py @@ -225,6 +225,28 @@ def mark_attempt_partially_succeeded( raise +def mark_attempt_canceled( + index_attempt_id: int, + db_session: Session, + reason: str = "Unknown", +) -> None: + try: + attempt = db_session.execute( + select(IndexAttempt) + .where(IndexAttempt.id == index_attempt_id) + .with_for_update() + ).scalar_one() + + if not attempt.time_started: + attempt.time_started = datetime.now(timezone.utc) + attempt.status = IndexingStatus.CANCELED + attempt.error_msg = reason + db_session.commit() + except Exception: + db_session.rollback() + raise + + def mark_attempt_failed( index_attempt_id: int, db_session: Session, diff --git a/web/src/app/admin/configuration/search/UpgradingPage.tsx b/web/src/app/admin/configuration/search/UpgradingPage.tsx index e48976c406f..ecd7f87316e 100644 --- a/web/src/app/admin/configuration/search/UpgradingPage.tsx +++ b/web/src/app/admin/configuration/search/UpgradingPage.tsx @@ -68,10 +68,11 @@ export default function UpgradingPage({ const statusOrder: Record = useMemo( () => ({ failed: 0, - completed_with_errors: 1, - not_started: 2, - in_progress: 3, - success: 4, + canceled: 1, + completed_with_errors: 2, + not_started: 3, + in_progress: 4, + success: 5, }), [] ); diff --git a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx index 78356310730..4b56d277bb9 100644 --- a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx +++ b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx @@ -322,7 +322,8 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) { )} - {indexAttempt.status === "failed" && + {(indexAttempt.status === "failed" || + indexAttempt.status === "canceled") && indexAttempt.error_msg && ( {indexAttempt.error_msg} diff --git a/web/src/components/Status.tsx b/web/src/components/Status.tsx index ce2cf134b43..0990968c551 100644 --- a/web/src/components/Status.tsx +++ b/web/src/components/Status.tsx @@ -52,7 +52,7 @@ export function IndexAttemptStatus({ popupContent={
The indexing attempt completed, but some errors were encountered - during the rrun. + during the run.

Click View Errors for more details. @@ -78,6 +78,12 @@ export function IndexAttemptStatus({ Scheduled ); + } else if (status === "canceled") { + badge = ( + + Canceled + + ); } else { badge = ( diff --git a/web/src/components/ui/badge.tsx b/web/src/components/ui/badge.tsx index b0f230c632f..fce4e1b54bb 100644 --- a/web/src/components/ui/badge.tsx +++ b/web/src/components/ui/badge.tsx @@ -8,6 +8,8 @@ const badgeVariants = cva( { variants: { variant: { + canceled: + "border-gray-200 bg-gray-50 text-gray-600 hover:bg-gray-75 dark:bg-gray-900 dark:text-neutral-50 dark:hover:bg-gray-850", orange: "border-orange-200 bg-orange-50 text-orange-600 hover:bg-orange-75 dark:bg-orange-900 dark:text-neutral-50 dark:hover:bg-orange-850", paused: diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 08d7f6f2cf6..8ea6047dd1a 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -72,6 +72,7 @@ export type ValidInputTypes = "load_state" | "poll" | "event"; export type ValidStatuses = | "success" | "completed_with_errors" + | "canceled" | "failed" | "in_progress" | "not_started";