Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

out_stackdriver: add project_id_key override to allow specifying gcp project id from the record #8209

Merged
merged 1 commit into from
Jan 10, 2024
Merged

out_stackdriver: add project_id_key override to allow specifying gcp project id from the record #8209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions plugins/out_stackdriver/stackdriver.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1509,6 +1509,7 @@ static int pack_json_payload(int insert_id_extracted,
{
monitored_resource_key,
local_resource_id_key,
ctx->project_id_key,
ctx->labels_key,
ctx->severity_key,
ctx->trace_key,
Expand Down Expand Up @@ -1680,6 +1681,10 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx,
flb_sds_t out_buf;
struct flb_mp_map_header mh;

/* Parameters for project_id_key */
int project_id_extracted = FLB_FALSE;
flb_sds_t project_id_key;

/* Parameters for severity */
int severity_extracted = FLB_FALSE;
severity_t severity;
Expand Down Expand Up @@ -2204,6 +2209,13 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx,
entry_size += 1;
}

/* Extract project id */
project_id_extracted = FLB_FALSE;
if (ctx->project_id_key
&& get_string(&project_id_key, obj, ctx->project_id_key) == 0) {
project_id_extracted = FLB_TRUE;
}

/* Extract log name */
log_name_extracted = FLB_FALSE;
if (ctx->log_name_key
Expand Down Expand Up @@ -2409,10 +2421,16 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx,
new_log_name = log_name;
}

/* logName */
len = snprintf(path, sizeof(path) - 1,
if (project_id_extracted == FLB_TRUE) {
len = snprintf(path, sizeof(path) - 1,
"projects/%s/logs/%s", project_id_key, new_log_name);
flb_sds_destroy(project_id_key);
} else {
len = snprintf(path, sizeof(path) - 1,
"projects/%s/logs/%s", ctx->export_to_project_id, new_log_name);
}

/* logName */
if (log_name_extracted == FLB_TRUE) {
flb_sds_destroy(log_name);
}
Expand Down Expand Up @@ -2968,6 +2986,11 @@ static struct flb_config_map config_map[] = {
0, FLB_TRUE, offsetof(struct flb_stackdriver, export_to_project_id),
"Export to project id"
},
{
FLB_CONFIG_MAP_STR, "project_id_key", DEFAULT_PROJECT_ID_KEY,
0, FLB_TRUE, offsetof(struct flb_stackdriver, project_id_key),
"Set the gcp project id key"
},
{
FLB_CONFIG_MAP_STR, "resource", FLB_SDS_RESOURCE_TYPE,
0, FLB_TRUE, offsetof(struct flb_stackdriver, resource),
Expand Down
2 changes: 2 additions & 0 deletions plugins/out_stackdriver/stackdriver.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
#define MONITORED_RESOURCE_KEY "logging.googleapis.com/monitored_resource"
#define LOCAL_RESOURCE_ID_KEY "logging.googleapis.com/local_resource_id"
#define DEFAULT_LABELS_KEY "logging.googleapis.com/labels"
#define DEFAULT_PROJECT_ID_KEY "logging.googleapis.com/projectId"
#define DEFAULT_SEVERITY_KEY "logging.googleapis.com/severity"
#define DEFAULT_TRACE_KEY "logging.googleapis.com/trace"
#define DEFAULT_SPAN_ID_KEY "logging.googleapis.com/spanId"
Expand Down Expand Up @@ -170,6 +171,7 @@ struct flb_stackdriver {

/* other */
flb_sds_t export_to_project_id;
flb_sds_t project_id_key;
flb_sds_t resource;
flb_sds_t severity_key;
flb_sds_t trace_key;
Expand Down
12 changes: 12 additions & 0 deletions tests/runtime/data/stackdriver/stackdriver_test_log_name.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,15 @@
"1591111124," \
"{" \
"}]"

#define LOG_NAME_PROJECT_ID_OVERRIDE "[" \
"1591111124," \
"{" \
"\"test_project_key\": \"fluent-bit-test-project-2\"" \
"}]"

#define LOG_NAME_PROJECT_ID_NO_OVERRIDE "[" \
"1591111124," \
"{" \
"\"logging.googleapis.com/projectId\": \"fluent-bit-test-project-2\"" \
"}]"
103 changes: 103 additions & 0 deletions tests/runtime/out_stackdriver.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -841,6 +841,24 @@ static void cb_check_log_name_no_override(void *ctx, int ffd,
flb_sds_destroy(res_data);
}

static void cb_check_project_key_override(void *ctx, int ffd,
int res_ret, void *res_data, size_t res_size,
void *data)
{
int ret;

/* logName in the entries is created using the value under log_name_key */
ret = mp_kv_cmp(
res_data, res_size, "$entries[0]['logName']", "projects/fluent-bit-test-project-2/logs/test");
TEST_CHECK(ret == FLB_TRUE);

/* log_name_key has been removed from jsonPayload */
ret = mp_kv_exists(res_data, res_size, "$entries[0]['jsonPayload']['test_project_key']");
TEST_CHECK(ret == FLB_FALSE);

flb_sds_destroy(res_data);
}

static void cb_check_k8s_node_resource(void *ctx, int ffd,
int res_ret, void *res_data, size_t res_size,
void *data)
Expand Down Expand Up @@ -2656,6 +2674,87 @@ void flb_test_set_metadata_server()
flb_destroy(ctx);
}

void flb_test_project_id_override()
{
int ret;
int size = sizeof(LOG_NAME_PROJECT_ID_OVERRIDE) - 1;
flb_ctx_t *ctx;
int in_ffd;
int out_ffd;

/* Create context, flush every second (some checks omitted here) */
ctx = flb_create();
flb_service_set(ctx, "flush", "1", "grace", "1", NULL);

/* Lib input mode */
in_ffd = flb_input(ctx, (char *) "lib", NULL);
flb_input_set(ctx, in_ffd, "tag", "test", NULL);

/* Stackdriver output */
out_ffd = flb_output(ctx, (char *) "stackdriver", NULL);
flb_output_set(ctx, out_ffd,
"match", "test",
"resource", "gce_instance",
"project_id_key", "test_project_key",
NULL);

/* Enable test mode */
ret = flb_output_set_test(ctx, out_ffd, "formatter",
cb_check_project_key_override,
NULL, NULL);

/* Start */
ret = flb_start(ctx);
TEST_CHECK(ret == 0);

/* Ingest data sample */
flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_PROJECT_ID_OVERRIDE, size);

sleep(2);
flb_stop(ctx);
flb_destroy(ctx);
}

void flb_test_project_id_no_override()
{
int ret;
int size = sizeof(LOG_NAME_PROJECT_ID_NO_OVERRIDE) - 1;
flb_ctx_t *ctx;
int in_ffd;
int out_ffd;

/* Create context, flush every second (some checks omitted here) */
ctx = flb_create();
flb_service_set(ctx, "flush", "1", "grace", "1", NULL);

/* Lib input mode */
in_ffd = flb_input(ctx, (char *) "lib", NULL);
flb_input_set(ctx, in_ffd, "tag", "test", NULL);

/* Stackdriver output */
out_ffd = flb_output(ctx, (char *) "stackdriver", NULL);
flb_output_set(ctx, out_ffd,
"match", "test",
"resource", "gce_instance",
NULL);

/* Enable test mode */
ret = flb_output_set_test(ctx, out_ffd, "formatter",
cb_check_project_key_override,
NULL, NULL);

/* Start */
ret = flb_start(ctx);
TEST_CHECK(ret == 0);

/* Ingest data sample */
flb_lib_push(ctx, in_ffd, (char *) LOG_NAME_PROJECT_ID_NO_OVERRIDE, size);

sleep(2);
flb_stop(ctx);
flb_destroy(ctx);
}

void flb_test_log_name_override()
{
int ret;
Expand Down Expand Up @@ -6150,6 +6249,10 @@ TEST_LIST = {
/* test metadata server */
{"set_metadata_server", flb_test_set_metadata_server},

/* test project key */
{"project_key_override", flb_test_project_id_override},
{"project_key_no_override", flb_test_project_id_no_override},

/* test log name */
{"log_name_override", flb_test_log_name_override},
{"log_name_no_override", flb_test_log_name_no_override},
Expand Down