Added counter for split builders

quickwit-oss · Mar 14, 2024 · 8c29018 · 8c29018
1 parent d1de0c0
commit 8c29018
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 1 deletion.
diff --git a/quickwit/quickwit-indexing/src/actors/indexer.rs b/quickwit/quickwit-indexing/src/actors/indexer.rs
@@ -225,6 +225,10 @@ impl IndexerState {
         let publish_lock = self.publish_lock.clone();
         let publish_token_opt = self.publish_token_opt.clone();
 
+        let mut split_builders_total_guard =
+            GaugeGuard::from_gauge(&crate::metrics::INDEXER_METRICS.split_builders_total);
+        split_builders_total_guard.add(1);
+
         let workbench = IndexingWorkbench {
             workbench_id,
             create_instant: Instant::now(),
@@ -242,6 +246,7 @@ impl IndexerState {
                     .in_flight_data
                     .index_writer,
             ),
+            split_builders_total_guard,
         };
         Ok(workbench)
     }
@@ -364,6 +369,7 @@ struct IndexingWorkbench {
     last_delete_opstamp: u64,
     // Number of bytes declared as used by tantivy.
     memory_usage: GaugeGuard,
+    split_builders_total_guard: GaugeGuard,
 }
 
 pub struct Indexer {
@@ -635,6 +641,7 @@ impl Indexer {
             batch_parent_span,
             indexing_permit,
             memory_usage,
+            split_builders_total_guard,
             ..
         }) = self.indexing_workbench_opt.take()
         else {
@@ -687,6 +694,7 @@ impl Indexer {
                 commit_trigger,
                 batch_parent_span,
                 memory_usage,
+                _split_builders_total_guard: split_builders_total_guard,
             },
         )
         .await?;

diff --git a/quickwit/quickwit-indexing/src/metrics.rs b/quickwit/quickwit-indexing/src/metrics.rs
@@ -27,6 +27,7 @@ pub struct IndexerMetrics {
     pub processed_bytes: IntCounterVec<2>,
     pub backpressure_micros: IntCounterVec<1>,
     pub available_concurrent_upload_permits: IntGaugeVec<1>,
+    pub split_builders_total: IntGauge,
     pub ongoing_merge_operations: IntGauge,
     pub pending_merge_operations: IntGauge,
     pub pending_merge_bytes: IntGauge,
@@ -66,6 +67,12 @@ impl Default for IndexerMetrics {
                 &[],
                 ["component"],
             ),
+            split_builders_total: new_gauge(
+                "split_builders_total",
+                "Number of existing index writer instance",
+                "indexing",
+                &[],
+            ),
             ongoing_merge_operations: new_gauge(
                 "ongoing_merge_operations",
                 "Number of ongoing merge operations",

diff --git a/quickwit/quickwit-indexing/src/models/indexed_split.rs b/quickwit/quickwit-indexing/src/models/indexed_split.rs
@@ -185,6 +185,7 @@ pub struct IndexedSplitBatchBuilder {
     pub commit_trigger: CommitTrigger,
     pub batch_parent_span: Span,
     pub memory_usage: GaugeGuard,
+    pub _split_builders_total_guard: GaugeGuard,
 }
 
 /// Sends notifications to the Publisher that the last batch of splits was emtpy.

diff --git a/quickwit/quickwit-ingest/src/ingest_v2/metrics.rs b/quickwit/quickwit-ingest/src/ingest_v2/metrics.rs
@@ -90,7 +90,8 @@ pub(super) fn report_wal_usage(wal_usage: ResourceUsage) {
     INGEST_V2_METRICS
         .wal_disk_used_bytes
         .set(wal_usage.disk_used_bytes as i64);
-    quickwit_common::metrics::MEMORY_METRICS.in_flight_data
+    quickwit_common::metrics::MEMORY_METRICS
+        .in_flight_data
         .wal
         .set(wal_usage.memory_allocated_bytes as i64);
     INGEST_V2_METRICS