From 80a424d4a92eda18d37c7891c010eed9bfa4b0c5 Mon Sep 17 00:00:00 2001 From: Tanmaya Panda Date: Tue, 9 Jan 2024 03:36:46 +0530 Subject: [PATCH 1/5] added default kusto ingest endpoint configs Signed-off-by: Tanmaya Panda --- plugins/out_azure_kusto/azure_kusto.c | 21 ++++++++++++++++++++ plugins/out_azure_kusto/azure_kusto.h | 7 +++++++ plugins/out_azure_kusto/azure_kusto_ingest.c | 5 +++++ 3 files changed, 33 insertions(+) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index 58ac559dbdc..d5e98fc90e3 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -126,9 +126,17 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs struct flb_http_client *c; flb_sds_t resp = NULL; + ctx->u->base.net.connect_timeout = ctx->ingestion_endpoint_connect_timeout; + ctx->u->base.net.keepalive_max_recycle = ctx->keep_alive_max_connection_recycle; + /* Get upstream connection */ u_conn = flb_upstream_conn_get(ctx->u); + if (!u_conn) { + flb_plg_error(ctx->ins, "[out_azure_kusto] cannot create upstream connection while connecting to kusto endpoint"); + FLB_OUTPUT_RETURN(FLB_RETRY); + } + if (u_conn) { token = get_azure_kusto_token(ctx); @@ -219,6 +227,13 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi return -1; } + const char *tmp = flb_output_get_property("ingestion_endpoint_connect_timeout", ins); + if (tmp != NULL) { + ctx->ingestion_endpoint_connect_timeout = strtol(tmp, NULL, 0); + } else { + ctx->ingestion_endpoint_connect_timeout = FLB_AZURE_KUSTO_INGEST_ENDPOINT_CONNECTION_TIMEOUT; + } + flb_output_set_context(ins, ctx); /* Network mode IPv6 */ @@ -462,6 +477,12 @@ static struct flb_config_map config_map[] = { offsetof(struct flb_azure_kusto, time_key), "The key name of the time. If 'include_time_key' is false, " "This property is ignored"}, + {FLB_CONFIG_MAP_TIME, "ingestion_endpoint_connect_timeout", FLB_AZURE_KUSTO_INGEST_ENDPOINT_CONNECTION_TIMEOUT, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, ingestion_endpoint_connect_timeout), + "Set the ingestion endpoint connection timeout in seconds"}, + {FLB_CONFIG_MAP_INT, "keep_alive_max_connection_recycle", FLB_AZURE_KUSTO_KEEP_ALIVE_MAX_RECYCLE, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, keep_alive_max_connection_recycle), + "Set the ingestion endpoint connection timeout in seconds"}, /* EOF */ {0}}; diff --git a/plugins/out_azure_kusto/azure_kusto.h b/plugins/out_azure_kusto/azure_kusto.h index b752e087e65..07ff4694c0c 100644 --- a/plugins/out_azure_kusto/azure_kusto.h +++ b/plugins/out_azure_kusto/azure_kusto.h @@ -51,6 +51,9 @@ #define FLB_AZURE_KUSTO_RESOURCES_LOAD_INTERVAL_SEC 3600 +#define FLB_AZURE_KUSTO_INGEST_ENDPOINT_CONNECTION_TIMEOUT "60" +#define FLB_AZURE_KUSTO_KEEP_ALIVE_MAX_RECYCLE "20" + struct flb_azure_kusto_resources { struct flb_upstream_ha *blob_ha; struct flb_upstream_ha *queue_ha; @@ -70,6 +73,10 @@ struct flb_azure_kusto { flb_sds_t table_name; flb_sds_t ingestion_mapping_reference; + /* connection configuration */ + int ingestion_endpoint_connect_timeout; + int keep_alive_max_connection_recycle; + /* records configuration */ flb_sds_t log_key; int include_tag_key; diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index 41c37308dc1..cc40621743c 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -149,6 +149,11 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t u_conn = flb_upstream_conn_get(u_node->u); + if (!u_conn) { + flb_plg_error(ctx->ins, "[out_azure_kusto] cannot create upstream connection while connecting to blob endpoint"); + FLB_OUTPUT_RETURN(FLB_RETRY); + } + if (u_conn) { uri = azure_kusto_create_blob_uri(ctx, u_node, blob_id); From 8584978e62ce9061e953840c6ed1c6979f9eabcf Mon Sep 17 00:00:00 2001 From: Tanmaya Panda Date: Tue, 13 Feb 2024 01:09:01 +0530 Subject: [PATCH 2/5] changed default kusto connection timeouts Signed-off-by: Tanmaya Panda --- plugins/out_azure_kusto/azure_kusto.c | 17 +++++++---------- plugins/out_azure_kusto/azure_kusto.h | 6 ++---- plugins/out_azure_kusto/azure_kusto_ingest.c | 6 ++++++ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index d5e98fc90e3..83968d78bad 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -126,8 +126,8 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs struct flb_http_client *c; flb_sds_t resp = NULL; - ctx->u->base.net.connect_timeout = ctx->ingestion_endpoint_connect_timeout; - ctx->u->base.net.keepalive_max_recycle = ctx->keep_alive_max_connection_recycle; + /* setting default connection timeout to kusto ingest endpoint */ + ctx->u->base.net.connect_timeout = ctx->kusto_endpoint_connect_timeout; /* Get upstream connection */ u_conn = flb_upstream_conn_get(ctx->u); @@ -227,11 +227,11 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi return -1; } - const char *tmp = flb_output_get_property("ingestion_endpoint_connect_timeout", ins); + const char *tmp = flb_output_get_property("kusto_endpoint_connect_timeout", ins); if (tmp != NULL) { - ctx->ingestion_endpoint_connect_timeout = strtol(tmp, NULL, 0); + ctx->kusto_endpoint_connect_timeout = strtol(tmp, NULL, 0); } else { - ctx->ingestion_endpoint_connect_timeout = FLB_AZURE_KUSTO_INGEST_ENDPOINT_CONNECTION_TIMEOUT; + ctx->kusto_endpoint_connect_timeout = FLB_AZURE_KUSTO_ENDPOINT_CONNECTION_TIMEOUT; } flb_output_set_context(ins, ctx); @@ -477,12 +477,9 @@ static struct flb_config_map config_map[] = { offsetof(struct flb_azure_kusto, time_key), "The key name of the time. If 'include_time_key' is false, " "This property is ignored"}, - {FLB_CONFIG_MAP_TIME, "ingestion_endpoint_connect_timeout", FLB_AZURE_KUSTO_INGEST_ENDPOINT_CONNECTION_TIMEOUT, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, ingestion_endpoint_connect_timeout), + {FLB_CONFIG_MAP_TIME, "kusto_endpoint_connect_timeout", FLB_AZURE_KUSTO_ENDPOINT_CONNECTION_TIMEOUT, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, kusto_endpoint_connect_timeout), "Set the ingestion endpoint connection timeout in seconds"}, - {FLB_CONFIG_MAP_INT, "keep_alive_max_connection_recycle", FLB_AZURE_KUSTO_KEEP_ALIVE_MAX_RECYCLE, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, keep_alive_max_connection_recycle), - "Set the ingestion endpoint connection timeout in seconds"}, /* EOF */ {0}}; diff --git a/plugins/out_azure_kusto/azure_kusto.h b/plugins/out_azure_kusto/azure_kusto.h index 07ff4694c0c..9e2e02fc398 100644 --- a/plugins/out_azure_kusto/azure_kusto.h +++ b/plugins/out_azure_kusto/azure_kusto.h @@ -51,8 +51,7 @@ #define FLB_AZURE_KUSTO_RESOURCES_LOAD_INTERVAL_SEC 3600 -#define FLB_AZURE_KUSTO_INGEST_ENDPOINT_CONNECTION_TIMEOUT "60" -#define FLB_AZURE_KUSTO_KEEP_ALIVE_MAX_RECYCLE "20" +#define FLB_AZURE_KUSTO_CONNECTION_ENDPOINT_CONNECTION_TIMEOUT "60" struct flb_azure_kusto_resources { struct flb_upstream_ha *blob_ha; @@ -74,8 +73,7 @@ struct flb_azure_kusto { flb_sds_t ingestion_mapping_reference; /* connection configuration */ - int ingestion_endpoint_connect_timeout; - int keep_alive_max_connection_recycle; + int kusto_endpoint_connect_timeout; /* records configuration */ flb_sds_t log_key; diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index cc40621743c..fbaeb7118a9 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -147,6 +147,9 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t return NULL; } + /* setting default connection timeout to kusto azure blob endpoint */ + ctx->u->base.net.connect_timeout = ctx->kusto_endpoint_connect_timeout; + u_conn = flb_upstream_conn_get(u_node->u); if (!u_conn) { @@ -353,6 +356,9 @@ static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t return -1; } + /* setting default connection timeout to kusto azure queue endpoint */ + ctx->u->base.net.connect_timeout = ctx->kusto_endpoint_connect_timeout; + u_conn = flb_upstream_conn_get(u_node->u); if (u_conn) { From e169558f88adc346245cf54d2b04c4e1db6aed80 Mon Sep 17 00:00:00 2001 From: Tanmaya Panda Date: Tue, 13 Feb 2024 01:13:56 +0530 Subject: [PATCH 3/5] removed unnecessary conditions Signed-off-by: Tanmaya Panda --- plugins/out_azure_kusto/azure_kusto.c | 5 ----- plugins/out_azure_kusto/azure_kusto_ingest.c | 5 ----- 2 files changed, 10 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index 83968d78bad..14ead84964f 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -132,11 +132,6 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs /* Get upstream connection */ u_conn = flb_upstream_conn_get(ctx->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "[out_azure_kusto] cannot create upstream connection while connecting to kusto endpoint"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - if (u_conn) { token = get_azure_kusto_token(ctx); diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index fbaeb7118a9..fe3fa6106d6 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -152,11 +152,6 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t u_conn = flb_upstream_conn_get(u_node->u); - if (!u_conn) { - flb_plg_error(ctx->ins, "[out_azure_kusto] cannot create upstream connection while connecting to blob endpoint"); - FLB_OUTPUT_RETURN(FLB_RETRY); - } - if (u_conn) { uri = azure_kusto_create_blob_uri(ctx, u_node, blob_id); From 61de75c90f45fb71ce750131ad2e1736ec11c0f4 Mon Sep 17 00:00:00 2001 From: Tanmaya Panda Date: Tue, 13 Feb 2024 01:17:45 +0530 Subject: [PATCH 4/5] renamed kusto endpoint connect timeout msg Signed-off-by: Tanmaya Panda --- plugins/out_azure_kusto/azure_kusto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index 14ead84964f..800ec1d3bae 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -474,7 +474,7 @@ static struct flb_config_map config_map[] = { "This property is ignored"}, {FLB_CONFIG_MAP_TIME, "kusto_endpoint_connect_timeout", FLB_AZURE_KUSTO_ENDPOINT_CONNECTION_TIMEOUT, 0, FLB_TRUE, offsetof(struct flb_azure_kusto, kusto_endpoint_connect_timeout), - "Set the ingestion endpoint connection timeout in seconds"}, + "Set the connection timeout of various kusto endpoints (kusto ingest endpoint, kusto ingestion blob endpoint, kusto ingestion blob endpoint) in seconds"}, /* EOF */ {0}}; From c9ab3afb033074f186f40a687488326f475d6a6c Mon Sep 17 00:00:00 2001 From: Tanmaya Panda Date: Thu, 14 Mar 2024 16:52:51 +0530 Subject: [PATCH 5/5] removed unnecessary code Signed-off-by: Tanmaya Panda --- plugins/out_azure_kusto/azure_kusto.c | 13 +++---------- plugins/out_azure_kusto/azure_kusto.h | 4 ++-- plugins/out_azure_kusto/azure_kusto_ingest.c | 4 ++-- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index 800ec1d3bae..c0f4987d580 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -127,7 +127,7 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs flb_sds_t resp = NULL; /* setting default connection timeout to kusto ingest endpoint */ - ctx->u->base.net.connect_timeout = ctx->kusto_endpoint_connect_timeout; + ctx->u->base.net.connect_timeout = ctx->kusto_endpoint_connection_timeout; /* Get upstream connection */ u_conn = flb_upstream_conn_get(ctx->u); @@ -222,13 +222,6 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi return -1; } - const char *tmp = flb_output_get_property("kusto_endpoint_connect_timeout", ins); - if (tmp != NULL) { - ctx->kusto_endpoint_connect_timeout = strtol(tmp, NULL, 0); - } else { - ctx->kusto_endpoint_connect_timeout = FLB_AZURE_KUSTO_ENDPOINT_CONNECTION_TIMEOUT; - } - flb_output_set_context(ins, ctx); /* Network mode IPv6 */ @@ -472,8 +465,8 @@ static struct flb_config_map config_map[] = { offsetof(struct flb_azure_kusto, time_key), "The key name of the time. If 'include_time_key' is false, " "This property is ignored"}, - {FLB_CONFIG_MAP_TIME, "kusto_endpoint_connect_timeout", FLB_AZURE_KUSTO_ENDPOINT_CONNECTION_TIMEOUT, 0, FLB_TRUE, - offsetof(struct flb_azure_kusto, kusto_endpoint_connect_timeout), + {FLB_CONFIG_MAP_TIME, "kusto_endpoint_connection_timeout", FLB_AZURE_KUSTO_INGEST_ENDPOINT_CONNECTION_TIMEOUT, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, kusto_endpoint_connection_timeout), "Set the connection timeout of various kusto endpoints (kusto ingest endpoint, kusto ingestion blob endpoint, kusto ingestion blob endpoint) in seconds"}, /* EOF */ {0}}; diff --git a/plugins/out_azure_kusto/azure_kusto.h b/plugins/out_azure_kusto/azure_kusto.h index 9e2e02fc398..996f59e4a73 100644 --- a/plugins/out_azure_kusto/azure_kusto.h +++ b/plugins/out_azure_kusto/azure_kusto.h @@ -51,7 +51,7 @@ #define FLB_AZURE_KUSTO_RESOURCES_LOAD_INTERVAL_SEC 3600 -#define FLB_AZURE_KUSTO_CONNECTION_ENDPOINT_CONNECTION_TIMEOUT "60" +#define FLB_AZURE_KUSTO_INGEST_ENDPOINT_CONNECTION_TIMEOUT "60" struct flb_azure_kusto_resources { struct flb_upstream_ha *blob_ha; @@ -73,7 +73,7 @@ struct flb_azure_kusto { flb_sds_t ingestion_mapping_reference; /* connection configuration */ - int kusto_endpoint_connect_timeout; + int kusto_endpoint_connection_timeout; /* records configuration */ flb_sds_t log_key; diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index fe3fa6106d6..6e901e5b6e0 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -148,7 +148,7 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t } /* setting default connection timeout to kusto azure blob endpoint */ - ctx->u->base.net.connect_timeout = ctx->kusto_endpoint_connect_timeout; + ctx->u->base.net.connect_timeout = ctx->kusto_endpoint_connection_timeout; u_conn = flb_upstream_conn_get(u_node->u); @@ -352,7 +352,7 @@ static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t } /* setting default connection timeout to kusto azure queue endpoint */ - ctx->u->base.net.connect_timeout = ctx->kusto_endpoint_connect_timeout; + ctx->u->base.net.connect_timeout = ctx->kusto_endpoint_connection_timeout; u_conn = flb_upstream_conn_get(u_node->u);