From 697a41379774befa114c3fd8b5ee0c9cd03f9a5d Mon Sep 17 00:00:00 2001 From: Ryan Ohnemus Date: Wed, 10 Jan 2024 14:42:30 -0600 Subject: [PATCH] out_stackdriver: add project_id_key override (#8209) Closes #8195 Signed-off-by: ryanohnemus --- plugins/out_stackdriver/stackdriver.c | 27 ++++- plugins/out_stackdriver/stackdriver.h | 2 + .../stackdriver/stackdriver_test_log_name.h | 12 ++ tests/runtime/out_stackdriver.c | 103 ++++++++++++++++++ 4 files changed, 142 insertions(+), 2 deletions(-) diff --git a/plugins/out_stackdriver/stackdriver.c b/plugins/out_stackdriver/stackdriver.c index adbe8381ea2..d10504e9ace 100644 --- a/plugins/out_stackdriver/stackdriver.c +++ b/plugins/out_stackdriver/stackdriver.c @@ -1513,6 +1513,7 @@ static int pack_json_payload(int insert_id_extracted, { monitored_resource_key, local_resource_id_key, + ctx->project_id_key, ctx->labels_key, ctx->severity_key, ctx->trace_key, @@ -1684,6 +1685,10 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, flb_sds_t out_buf; struct flb_mp_map_header mh; + /* Parameters for project_id_key */ + int project_id_extracted = FLB_FALSE; + flb_sds_t project_id_key; + /* Parameters for severity */ int severity_extracted = FLB_FALSE; severity_t severity; @@ -2208,6 +2213,13 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, entry_size += 1; } + /* Extract project id */ + project_id_extracted = FLB_FALSE; + if (ctx->project_id_key + && get_string(&project_id_key, obj, ctx->project_id_key) == 0) { + project_id_extracted = FLB_TRUE; + } + /* Extract log name */ log_name_extracted = FLB_FALSE; if (ctx->log_name_key @@ -2413,10 +2425,16 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, new_log_name = log_name; } - /* logName */ - len = snprintf(path, sizeof(path) - 1, + if (project_id_extracted == FLB_TRUE) { + len = snprintf(path, sizeof(path) - 1, + "projects/%s/logs/%s", project_id_key, new_log_name); + flb_sds_destroy(project_id_key); + } else { + len = snprintf(path, sizeof(path) - 1, "projects/%s/logs/%s", ctx->export_to_project_id, new_log_name); + } + /* logName */ if (log_name_extracted == FLB_TRUE) { flb_sds_destroy(log_name); } @@ -2978,6 +2996,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_stackdriver, export_to_project_id), "Export to project id" }, + { + FLB_CONFIG_MAP_STR, "project_id_key", DEFAULT_PROJECT_ID_KEY, + 0, FLB_TRUE, offsetof(struct flb_stackdriver, project_id_key), + "Set the gcp project id key" + }, { FLB_CONFIG_MAP_STR, "resource", FLB_SDS_RESOURCE_TYPE, 0, FLB_TRUE, offsetof(struct flb_stackdriver, resource), diff --git a/plugins/out_stackdriver/stackdriver.h b/plugins/out_stackdriver/stackdriver.h index fc2fa1f62c6..bfb6cfd9dd5 100644 --- a/plugins/out_stackdriver/stackdriver.h +++ b/plugins/out_stackdriver/stackdriver.h @@ -49,6 +49,7 @@ #define MONITORED_RESOURCE_KEY "logging.googleapis.com/monitored_resource" #define LOCAL_RESOURCE_ID_KEY "logging.googleapis.com/local_resource_id" #define DEFAULT_LABELS_KEY "logging.googleapis.com/labels" +#define DEFAULT_PROJECT_ID_KEY "logging.googleapis.com/projectId" #define DEFAULT_SEVERITY_KEY "logging.googleapis.com/severity" #define DEFAULT_TRACE_KEY "logging.googleapis.com/trace" #define DEFAULT_SPAN_ID_KEY "logging.googleapis.com/spanId" @@ -170,6 +171,7 @@ struct flb_stackdriver { /* other */ flb_sds_t export_to_project_id; + flb_sds_t project_id_key; flb_sds_t resource; flb_sds_t severity_key; flb_sds_t trace_key; diff --git a/tests/runtime/data/stackdriver/stackdriver_test_log_name.h b/tests/runtime/data/stackdriver/stackdriver_test_log_name.h index 4d39e242e46..cd8f5be6744 100644 --- a/tests/runtime/data/stackdriver/stackdriver_test_log_name.h +++ b/tests/runtime/data/stackdriver/stackdriver_test_log_name.h @@ -8,3 +8,15 @@ "1591111124," \ "{" \ "}]" + +#define LOG_NAME_PROJECT_ID_OVERRIDE "[" \ + "1591111124," \ + "{" \ + "\"test_project_key\": \"fluent-bit-test-project-2\"" \ + "}]" + +#define LOG_NAME_PROJECT_ID_NO_OVERRIDE "[" \ + "1591111124," \ + "{" \ + "\"logging.googleapis.com/projectId\": \"fluent-bit-test-project-2\"" \ + "}]" diff --git a/tests/runtime/out_stackdriver.c b/tests/runtime/out_stackdriver.c index f379aa2e6e7..643fc7d468a 100644 --- a/tests/runtime/out_stackdriver.c +++ b/tests/runtime/out_stackdriver.c @@ -841,6 +841,24 @@ static void cb_check_log_name_no_override(void *ctx, int ffd, flb_sds_destroy(res_data); } +static void cb_check_project_key_override(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + + /* logName in the entries is created using the value under log_name_key */ + ret = mp_kv_cmp( + res_data, res_size, "$entries[0]['logName']", "projects/fluent-bit-test-project-2/logs/test"); + TEST_CHECK(ret == FLB_TRUE); + + /* log_name_key has been removed from jsonPayload */ + ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['test_project_key']"); + TEST_CHECK(ret == FLB_FALSE); + + flb_sds_destroy(res_data); +} + static void cb_check_k8s_node_resource(void *ctx, int ffd, int res_ret, void *res_data, size_t res_size, void *data) @@ -2656,6 +2674,87 @@ void flb_test_set_metadata_server() flb_destroy(ctx); } +void flb_test_project_id_override() +{ + int ret; + int size = sizeof(LOG_NAME_PROJECT_ID_OVERRIDE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + "project_id_key", "test_project_key", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_project_key_override, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_PROJECT_ID_OVERRIDE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_project_id_no_override() +{ + int ret; + int size = sizeof(LOG_NAME_PROJECT_ID_NO_OVERRIDE) - 1; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", NULL); + + /* Lib input mode */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Stackdriver output */ + out_ffd = flb_output(ctx, (char *) "stackdriver", NULL); + flb_output_set(ctx, out_ffd, + "match", "test", + "resource", "gce_instance", + NULL); + + /* Enable test mode */ + ret = flb_output_set_test(ctx, out_ffd, "formatter", + cb_check_project_key_override, + NULL, NULL); + + /* Start */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Ingest data sample */ + flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_PROJECT_ID_NO_OVERRIDE, size); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + void flb_test_log_name_override() { int ret; @@ -6150,6 +6249,10 @@ TEST_LIST = { /* test metadata server */ {"set_metadata_server", flb_test_set_metadata_server}, + /* test project key */ + {"project_key_override", flb_test_project_id_override}, + {"project_key_no_override", flb_test_project_id_no_override}, + /* test log name */ {"log_name_override", flb_test_log_name_override}, {"log_name_no_override", flb_test_log_name_no_override},