Address review comments

quickwit-oss · Mar 21, 2024 · 4f9cde0 · 4f9cde0
1 parent 0dce794
commit 4f9cde0
Show file tree

Hide file tree

Showing 7 changed files with 537 additions and 522 deletions.
diff --git a/distribution/lambda/poetry.lock b/distribution/lambda/poetry.lock
diff --git a/distribution/lambda/pyproject.toml b/distribution/lambda/pyproject.toml
@@ -10,7 +10,7 @@ packages = [{include = "cdk"}]
 [tool.poetry.dependencies]
 python = ">=3.10,<4.0"
 aws-cdk-lib = "^2.95.1"
-cargo-lambda = "^0.21.0"
+cargo-lambda = "^1.1.0"
 constructs = ">=10.0.0,<11.0.0"
 pyyaml = "^6.0.1"
 black = "^24.3.0"

diff --git a/quickwit/quickwit-lambda/src/environment.rs b/quickwit/quickwit-lambda/src/environment.rs
@@ -24,6 +24,8 @@ use once_cell::sync::Lazy;
 pub static INDEX_ID: Lazy<String> =
     Lazy::new(|| var("QW_LAMBDA_INDEX_ID").expect("QW_LAMBDA_INDEX_ID must be set"));
 
+/// Configure the fmt tracing subscriber to log span boundaries. This is very verbose and is
+/// only used to generate advanced KPIs from Lambda runs (e.g for blogpost benchmarks)
 pub static LOG_SPAN_BOUNDARIES: Lazy<bool> =
     Lazy::new(|| var("QW_LAMBDA_LOG_SPAN_BOUNDARIES").is_ok_and(|v| v.as_str() == "true"));
 

diff --git a/quickwit/quickwit-lambda/src/indexer/ingest/helpers.rs b/quickwit/quickwit-lambda/src/indexer/ingest/helpers.rs
@@ -241,7 +241,7 @@ pub(super) async fn spawn_services(
                 SearchJobPlacer::default(),
                 storage_resolver,
                 event_broker,
-                true,
+                false,
             )
             .await?,
         )

diff --git a/quickwit/quickwit-lambda/src/indexer/ingest/mod.rs b/quickwit/quickwit-lambda/src/indexer/ingest/mod.rs
@@ -37,7 +37,7 @@ use quickwit_index_management::clear_cache_directory;
 use quickwit_indexing::models::IndexingStatistics;
 use tracing::{debug, info};
 
-use crate::indexer::environment::CONFIGURATION_TEMPLATE;
+use crate::indexer::environment::{CONFIGURATION_TEMPLATE, DISABLE_JANITOR};
 use crate::indexer::ingest::helpers::wait_for_merges;
 use crate::utils::load_node_config;
 
@@ -69,8 +69,11 @@ pub async fn ingest(args: IngestArgs) -> anyhow::Result<IndexingStatistics> {
     )
     .await?;
 
-    let services = [QuickwitService::Indexer, QuickwitService::Janitor];
-    let cluster = create_empty_cluster(&config, &services).await?;
+    let mut services = vec![QuickwitService::Indexer];
+    if !*DISABLE_JANITOR {
+        services.push(QuickwitService::Janitor);
+    }
+    let cluster = create_empty_cluster(&config, &services[..]).await?;
     let universe = Universe::new();
     let runtimes_config = RuntimesConfig::default();
 

diff --git a/quickwit/quickwit-lambda/src/searcher/handler.rs b/quickwit/quickwit-lambda/src/searcher/handler.rs
@@ -26,10 +26,10 @@ use quickwit_search::SearchResponseRest;
 use quickwit_serve::SearchRequestQueryString;
 use tracing::{debug_span, error, info_span, instrument, Instrument};
 
-use super::environment::DISABLE_SEARCH_CACHE;
 use super::search::{search, SearchArgs};
 use crate::environment::INDEX_ID;
 use crate::logger;
+use crate::searcher::environment::DISABLE_SEARCH_CACHE;
 use crate::utils::LambdaContainerContext;
 
 #[instrument(skip_all)]

diff --git a/quickwit/quickwit-lambda/src/searcher/search.rs b/quickwit/quickwit-lambda/src/searcher/search.rs
@@ -35,8 +35,8 @@ use quickwit_telemetry::payload::{QuickwitFeature, QuickwitTelemetryInfo, Teleme
 use tokio::sync::OnceCell;
 use tracing::debug;
 
-use super::environment::{CONFIGURATION_TEMPLATE, DISABLE_SEARCH_CACHE};
 use crate::environment::INDEX_ID;
+use crate::searcher::environment::{CONFIGURATION_TEMPLATE, DISABLE_SEARCH_CACHE};
 use crate::utils::load_node_config;
 
 static LAMBDA_SEARCH_CACHE: OnceCell<LambdaSearchCtx> = OnceCell::const_new();