diff --git a/distribution/lambda/README.md b/distribution/lambda/README.md index 48db36d878c..4ed5143831b 100644 --- a/distribution/lambda/README.md +++ b/distribution/lambda/README.md @@ -95,6 +95,12 @@ simplify the setup and avoid unstable deployments. [1]: https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html +> [!TIP] +> The Indexer Lambda's logging is quite verbose. To reduce the associated +> CloudWatch costs, you can disable some lower level logs by setting the +> `RUST_LOG` environment variable to `info,quickwit_actors=warn`, or disable +> INFO logs altogether by setting `RUST_LOG=warn`. + Indexer only: | Variable | Description | Default | |---|---|---| @@ -151,7 +157,13 @@ You can query and visualize the Quickwit Searcher Lambda from Grafana by using t #### Configure Grafana data source -You need to provide the following information. +If you don't have a Grafana instance running yet, you can start one with the Quickwit plugin installed using Docker: + +```bash +docker run -e GF_INSTALL_PLUGINS="quickwit-quickwit-datasource" -p 3000:3000 grafana/grafana +``` + +In the `Connections > Data sources` page, add a new Quickwit data source and configure the following settings: |Variable|Description|Example| |--|--|--| @@ -159,4 +171,4 @@ You need to provide the following information. |Custom HTTP Headers| If you configure API Gateway to require an API key, set `x-api-key` HTTP Header | Header: `x-api-key`
Value: API key value| |Index ID| Same as `QW_LAMBDA_INDEX_ID` | hdfs-logs | -After entering these values, click "Save & test" and you can now query your Quickwit Lambda from Grafana! +After entering these values, click "Save & test". You can now query your Quickwit Lambda from Grafana! diff --git a/distribution/lambda/cdk/cli.py b/distribution/lambda/cdk/cli.py index c18fd14f289..ecb3ffdb155 100644 --- a/distribution/lambda/cdk/cli.py +++ b/distribution/lambda/cdk/cli.py @@ -320,14 +320,16 @@ def _clean_s3_bucket(bucket_name: str, prefix: str = ""): print(f"Cleaning up bucket {bucket_name}/{prefix}...") s3 = session.resource("s3") bucket = s3.Bucket(bucket_name) - bucket.objects.filter(Prefix=prefix).delete() + try: + bucket.objects.filter(Prefix=prefix).delete() + except s3.meta.client.exceptions.NoSuchBucket: + print(f"Bucket {bucket_name} not found, skipping cleanup") def empty_hdfs_bucket(): bucket_name = _get_cloudformation_output_value( app.HDFS_STACK_NAME, hdfs_stack.INDEX_STORE_BUCKET_NAME_EXPORT_NAME ) - _clean_s3_bucket(bucket_name) diff --git a/distribution/lambda/cdk/stacks/examples/mock_data_stack.py b/distribution/lambda/cdk/stacks/examples/mock_data_stack.py index 027b8afb98c..8a4a5c9290b 100644 --- a/distribution/lambda/cdk/stacks/examples/mock_data_stack.py +++ b/distribution/lambda/cdk/stacks/examples/mock_data_stack.py @@ -165,7 +165,12 @@ def __init__( index_id=index_id, index_config_bucket=index_config.s3_bucket_name, index_config_key=index_config.s3_object_key, - indexer_environment=lambda_env, + indexer_environment={ + # the actor system is very verbose when the source is shutting + # down (each Lambda invocation) + "RUST_LOG": "info,quickwit_actors=warn", + **lambda_env, + }, searcher_environment=lambda_env, indexer_package_location=indexer_package_location, searcher_package_location=searcher_package_location, diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 0db9bbed0de..5551d880c3d 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -3517,9 +3517,9 @@ dependencies = [ [[package]] name = "lambda_runtime" -version = "0.11.3" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be8f0e7a5db270feb93a7a3593c22a4c5fb8e8f260f5f490e0c3a5ffeb009db" +checksum = "ed49669d6430292aead991e19bf13153135a884f916e68f32997c951af637ebe" dependencies = [ "async-stream", "base64 0.22.1", @@ -6140,7 +6140,7 @@ dependencies = [ "flate2", "http 0.2.12", "lambda_http", - "lambda_runtime 0.11.3", + "lambda_runtime 0.13.0", "mime_guess", "once_cell", "opentelemetry", diff --git a/quickwit/quickwit-lambda/Cargo.toml b/quickwit/quickwit-lambda/Cargo.toml index abb15cbcca0..009020df57f 100644 --- a/quickwit/quickwit-lambda/Cargo.toml +++ b/quickwit/quickwit-lambda/Cargo.toml @@ -26,7 +26,7 @@ chrono = { workspace = true } flate2 = { workspace = true } http = { workspace = true } lambda_http = "0.8.0" -lambda_runtime = "0.11.1" +lambda_runtime = "0.13.0" mime_guess = { workspace = true } once_cell = { workspace = true } opentelemetry = { workspace = true }