Lambda maintenance (#5277)

* Update lambda_runtime

* Decrease log verbosity

* Fix rare race condition on bucket cleanup

distribution/lambda/README.md

@@ -95,6 +95,12 @@ simplify the setup and avoid unstable deployments.
 [1]: https://rust-lang-nursery.github.io/rust-cookbook/development_tools/debugging/config_log.html
 
 
+> [!TIP]  
+> The Indexer Lambda's logging is quite verbose. To reduce the associated
+> CloudWatch costs, you can disable some lower level logs by setting the
+> `RUST_LOG` environment variable to `info,quickwit_actors=warn`, or disable
+> INFO logs altogether by setting `RUST_LOG=warn`.
+
 Indexer only:
 | Variable | Description | Default |
 |---|---|---|
@@ -151,12 +157,18 @@ You can query and visualize the Quickwit Searcher Lambda from Grafana by using t
 
 #### Configure Grafana data source
 
-You need to provide the following information.
+If you don't have a Grafana instance running yet, you can start one with the Quickwit plugin installed using Docker:
+
+```bash
+docker run -e GF_INSTALL_PLUGINS="quickwit-quickwit-datasource" -p 3000:3000 grafana/grafana
+```
+
+In the `Connections > Data sources` page, add a new Quickwit data source and configure the following settings:
 
 |Variable|Description|Example|
 |--|--|--|
 |HTTP URL| HTTP search endpoint for Quickwit Searcher Lambda | https://*******.execute-api.us-east-1.amazonaws.com/api/v1 |
 |Custom HTTP Headers| If you configure API Gateway to require an API key, set `x-api-key` HTTP Header | Header: `x-api-key` <br> Value: API key value|
 |Index ID| Same as `QW_LAMBDA_INDEX_ID` | hdfs-logs |
 
-After entering these values, click "Save & test" and you can now query your Quickwit Lambda from Grafana!
+After entering these values, click "Save & test". You can now query your Quickwit Lambda from Grafana!

distribution/lambda/cdk/cli.py

@@ -320,14 +320,16 @@ def _clean_s3_bucket(bucket_name: str, prefix: str = ""):
     print(f"Cleaning up bucket {bucket_name}/{prefix}...")
     s3 = session.resource("s3")
     bucket = s3.Bucket(bucket_name)
-    bucket.objects.filter(Prefix=prefix).delete()
+    try:
+        bucket.objects.filter(Prefix=prefix).delete()
+    except s3.meta.client.exceptions.NoSuchBucket:
+        print(f"Bucket {bucket_name} not found, skipping cleanup")
 
 
 def empty_hdfs_bucket():
     bucket_name = _get_cloudformation_output_value(
         app.HDFS_STACK_NAME, hdfs_stack.INDEX_STORE_BUCKET_NAME_EXPORT_NAME
     )
-
     _clean_s3_bucket(bucket_name)
 
 

distribution/lambda/cdk/stacks/examples/mock_data_stack.py

@@ -165,7 +165,12 @@ def __init__(
             index_id=index_id,
             index_config_bucket=index_config.s3_bucket_name,
             index_config_key=index_config.s3_object_key,
-            indexer_environment=lambda_env,
+            indexer_environment={
+                # the actor system is very verbose when the source is shutting
+                # down (each Lambda invocation)
+                "RUST_LOG": "info,quickwit_actors=warn",
+                **lambda_env,
+            },
             searcher_environment=lambda_env,
             indexer_package_location=indexer_package_location,
             searcher_package_location=searcher_package_location,

quickwit/quickwit-lambda/Cargo.toml

@@ -26,7 +26,7 @@ chrono = { workspace = true }
 flate2 = { workspace = true }
 http = { workspace = true }
 lambda_http = "0.8.0"
-lambda_runtime = "0.11.1"
+lambda_runtime = "0.13.0"
 mime_guess =  { workspace = true }
 once_cell = { workspace = true }
 opentelemetry = { workspace = true }