diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index 4b8ad9b82e0..ee47e326e2d 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -98,6 +98,10 @@ flb_sds_t get_azure_kusto_token(struct flb_azure_kusto *ctx) ctx->o->access_token); } + // if (output){ + // flb_sds_destroy(output); + // } + if (pthread_mutex_unlock(&ctx->token_mutex)) { flb_plg_error(ctx->ins, "error unlocking mutex"); if (output) { @@ -126,16 +130,36 @@ flb_sds_t execute_ingest_csl_command(struct flb_azure_kusto *ctx, const char *cs struct flb_http_client *c; flb_sds_t resp = NULL; + flb_plg_debug(ctx->ins, "before getting upstream connection"); + + //flb_upstream_init(); + + flb_plg_debug(ctx->ins, "Logging attributes of flb_azure_kusto_resources:"); + flb_plg_debug(ctx->ins, "blob_ha: %p", ctx->resources->blob_ha); + flb_plg_debug(ctx->ins, "queue_ha: %p", ctx->resources->queue_ha); + flb_plg_debug(ctx->ins, "identity_token: %s", ctx->resources->identity_token); + flb_plg_debug(ctx->ins, "load_time: %lu", ctx->resources->load_time); + + ctx->u->base.net.connect_timeout = ctx->ingestion_endpoint_connect_timeout ; + ctx->u->base.net.keepalive_max_recycle = ctx->keep_alive_max_connection_recycle; + /* Get upstream connection */ u_conn = flb_upstream_conn_get(ctx->u); + flb_plg_debug(ctx->ins, "inside execute ingest csl commnad :: after flb_upstream_conn_get"); + + if (!u_conn) { + FLB_OUTPUT_RETURN(FLB_RETRY); + } if (u_conn) { token = get_azure_kusto_token(ctx); + flb_plg_debug(ctx->ins, "after get azure kusto token"); if (token) { /* Compose request body */ body = flb_sds_create_size(sizeof(FLB_AZURE_KUSTO_MGMT_BODY_TEMPLATE) - 1 + strlen(csl)); + flb_plg_debug(ctx->ins, "after flb sds create size"); if (body) { flb_sds_snprintf(&body, flb_sds_alloc(body), @@ -212,6 +236,8 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi int io_flags = FLB_IO_TLS; struct flb_azure_kusto *ctx; + flb_plg_debug(ins, "inside azure kusto init"); + /* Create config context */ ctx = flb_azure_kusto_conf_create(ins, config); if (!ctx) { @@ -231,6 +257,7 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi */ pthread_mutex_init(&ctx->token_mutex, NULL); pthread_mutex_init(&ctx->resources_mutex, NULL); + pthread_mutex_init(&ctx->blob_mutex, NULL); /* * Create upstream context for Kusto Ingestion endpoint @@ -241,6 +268,8 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi return -1; } + // ctx->u->base.net.keepalive = FLB_FALSE; + /* Create oauth2 context */ ctx->o = flb_oauth2_create(ctx->config, ctx->oauth_url, FLB_AZURE_KUSTO_TOKEN_REFRESH); @@ -250,6 +279,8 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi } flb_output_upstream_set(ctx->u, ins); + flb_plg_debug(ctx->ins, "azure kusto init completed"); + return 0; } @@ -377,6 +408,7 @@ static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, /* Load or refresh ingestion resources */ ret = azure_kusto_load_ingestion_resources(ctx, config); + flb_plg_trace(ctx->ins, "after flushing bytes xxxx %d", ret); if (ret != 0) { flb_plg_error(ctx->ins, "cannot load ingestion resources"); FLB_OUTPUT_RETURN(FLB_RETRY); @@ -385,12 +417,14 @@ static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, /* Reformat msgpack to JSON payload */ ret = azure_kusto_format(ctx, event_chunk->tag, tag_len, event_chunk->data, event_chunk->size, (void **)&json, &json_size); + flb_plg_trace(ctx->ins, "after kusto format xxxx %d", ret); if (ret != 0) { flb_plg_error(ctx->ins, "cannot reformat data into json"); FLB_OUTPUT_RETURN(FLB_RETRY); } ret = azure_kusto_queued_ingestion(ctx, event_chunk->tag, tag_len, json, json_size); + flb_plg_trace(ctx->ins, "after kusto queued ingestion %d", ret); if (ret != 0) { flb_plg_error(ctx->ins, "cannot perform queued ingestion"); flb_sds_destroy(json); @@ -417,6 +451,10 @@ static int cb_azure_kusto_exit(void *data, struct flb_config *config) ctx->u = NULL; } + pthread_mutex_destroy(&ctx->resources_mutex); + pthread_mutex_destroy(&ctx->token_mutex); + pthread_mutex_destroy(&ctx->blob_mutex); + flb_azure_kusto_conf_destroy(ctx); return 0; @@ -462,6 +500,12 @@ static struct flb_config_map config_map[] = { offsetof(struct flb_azure_kusto, time_key), "The key name of the time. If 'include_time_key' is false, " "This property is ignored"}, + {FLB_CONFIG_MAP_TIME, "ingestion_endpoint_connect_timeout", FLB_AZURE_KUSTO_INGEST_ENDPOINT_CONNECTION_TIMEOUT, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, ingestion_endpoint_connect_timeout), + "Set the ingestion endpoint connection timeout in seconds"}, + {FLB_CONFIG_MAP_INT, "keep_alive_max_connection_recycle", FLB_AZURE_KUSTO_KEEP_ALIVE_MAX_RECYCLE, 0, FLB_TRUE, + offsetof(struct flb_azure_kusto, keep_alive_max_connection_recycle), + "Set the max connection recycle value of keep alive connections"}, /* EOF */ {0}}; diff --git a/plugins/out_azure_kusto/azure_kusto.h b/plugins/out_azure_kusto/azure_kusto.h index ac4eedfd0a8..cd0199ba0d1 100644 --- a/plugins/out_azure_kusto/azure_kusto.h +++ b/plugins/out_azure_kusto/azure_kusto.h @@ -51,6 +51,10 @@ #define FLB_AZURE_KUSTO_RESOURCES_LOAD_INTERVAL_SEC 3600 +#define FLB_AZURE_KUSTO_INGEST_ENDPOINT_CONNECTION_TIMEOUT "60" +#define FLB_AZURE_KUSTO_KEEP_ALIVE_MAX_RECYCLE "20" + + struct flb_azure_kusto_resources { struct flb_upstream_ha *blob_ha; struct flb_upstream_ha *queue_ha; @@ -70,6 +74,9 @@ struct flb_azure_kusto { flb_sds_t table_name; flb_sds_t ingestion_mapping_reference; + int ingestion_endpoint_connect_timeout; + int keep_alive_max_connection_recycle; + /* records configuration */ flb_sds_t log_key; int include_tag_key; @@ -94,6 +101,8 @@ struct flb_azure_kusto { /* mutex for loading reosurces */ pthread_mutex_t resources_mutex; + pthread_mutex_t blob_mutex; + /* Upstream connection to the backend server */ struct flb_upstream *u; diff --git a/plugins/out_azure_kusto/azure_kusto_conf.c b/plugins/out_azure_kusto/azure_kusto_conf.c index 5303fef672d..d1f9c0a48db 100644 --- a/plugins/out_azure_kusto/azure_kusto_conf.c +++ b/plugins/out_azure_kusto/azure_kusto_conf.c @@ -416,6 +416,8 @@ static flb_sds_t parse_ingestion_identity_token(struct flb_azure_kusto *ctx, return identity_token; } + + int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, struct flb_config *config) { @@ -426,11 +428,6 @@ int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, struct flb_upstream_ha *queue_ha = NULL; time_t now; - if (pthread_mutex_lock(&ctx->resources_mutex)) { - flb_plg_error(ctx->ins, "error locking mutex"); - return -1; - } - now = time(NULL); /* check if we have all resources and they are not stale */ @@ -451,9 +448,19 @@ int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, blob_ha = flb_upstream_ha_create("azure_kusto_blob_ha"); if (blob_ha) { + + if (pthread_mutex_lock(&ctx->resources_mutex)) { + flb_plg_error(ctx->ins, "error locking mutex"); + return -1; + } ret = parse_storage_resources(ctx, config, response, blob_ha, queue_ha); + if (pthread_mutex_unlock(&ctx->resources_mutex)) { + flb_plg_error(ctx->ins, "error unlocking mutex"); + return -1; + } + if (ret == 0) { flb_sds_destroy(response); response = NULL; @@ -461,26 +468,30 @@ int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, response = execute_ingest_csl_command(ctx, ".get kusto identity token"); + if (pthread_mutex_lock(&ctx->resources_mutex)) { + flb_plg_error(ctx->ins, "error locking mutex"); + return -1; + } if (response) { identity_token = parse_ingestion_identity_token(ctx, response); if (identity_token) { - ret = flb_azure_kusto_resources_clear(ctx->resources); + // ret = flb_azure_kusto_resources_clear(ctx->resources); - if (ret != -1) { + // if (ret != -1) { ctx->resources->blob_ha = blob_ha; ctx->resources->queue_ha = queue_ha; ctx->resources->identity_token = identity_token; ctx->resources->load_time = now; ret = 0; - } - else { - flb_plg_error( - ctx->ins, - "error destroying previous ingestion resources"); - } + //} + // else { + // flb_plg_error( + // ctx->ins, + // "error destroying previous ingestion resources"); + // } } else { flb_plg_error(ctx->ins, @@ -492,6 +503,11 @@ int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, flb_plg_error(ctx->ins, "error getting kusto identity token"); ret = -1; } + + if (pthread_mutex_unlock(&ctx->resources_mutex)) { + flb_plg_error(ctx->ins, "error unlocking mutex"); + return -1; + } } else { flb_plg_error(ctx->ins, @@ -525,11 +541,6 @@ int azure_kusto_load_ingestion_resources(struct flb_azure_kusto *ctx, } } - if (pthread_mutex_unlock(&ctx->resources_mutex)) { - flb_plg_error(ctx->ins, "error unlocking mutex"); - return -1; - } - return ret; } diff --git a/plugins/out_azure_kusto/azure_kusto_ingest.c b/plugins/out_azure_kusto/azure_kusto_ingest.c index d38d92e7fd8..028b5757d16 100644 --- a/plugins/out_azure_kusto/azure_kusto_ingest.c +++ b/plugins/out_azure_kusto/azure_kusto_ingest.c @@ -137,21 +137,66 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t char tmp[64]; int len; + struct timespec ts; + now = time(NULL); gmtime_r(&now, &tm); len = strftime(tmp, sizeof(tmp) - 1, "%a, %d %b %Y %H:%M:%S GMT", &tm); + + flb_plg_debug(ctx->ins,"inside blob before locking mutex"); + + + + // if (pthread_mutex_lock(&ctx->blob_mutex)) { + // flb_plg_error(ctx->ins, "error locking mutex"); + // return -1; + // } + + + flb_plg_debug(ctx->ins,"inside blob after locking mutex"); + u_node = flb_upstream_ha_node_get(ctx->resources->blob_ha); if (!u_node) { flb_plg_error(ctx->ins, "error getting blob upstream"); return NULL; } + + // if (pthread_mutex_unlock(&ctx->blob_mutex)) { + // flb_plg_error(ctx->ins, "error unlocking mutex"); + // return -1; + // } + + + flb_plg_debug(ctx->ins,"inside blob after upstream ha node get"); + u_node->u->base.net.connect_timeout = ctx->ingestion_endpoint_connect_timeout; + u_node->u->base.net.keepalive_max_recycle = ctx->keep_alive_max_connection_recycle; + u_conn = flb_upstream_conn_get(u_node->u); + + //if (pthread_mutex_unlock(&ctx->blob_mutex)) { + // flb_plg_error(ctx->ins, "error unlocking mutex"); + // return -1; + //} + + if (!u_conn) { + flb_plg_error(ctx->ins,"cannot create upstream connection for blob commit"); + return FLB_RETRY; + } + + + flb_plg_debug(ctx->ins,"inside blob after retrieving connection"); + if (u_conn) { + + flb_plg_debug(ctx->ins,"inside blob before create blob uri"); uri = azure_kusto_create_blob_uri(ctx, u_node, blob_id); + + flb_plg_debug(ctx->ins,"inside blob after create blob uri"); + if (uri) { flb_plg_debug(ctx->ins, "uploading payload to blob uri: %s", uri); c = flb_http_client(u_conn, FLB_HTTP_PUT, uri, payload, payload_size, NULL, 0, @@ -188,6 +233,7 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t } flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); } else { flb_plg_error(ctx->ins, @@ -209,6 +255,12 @@ static flb_sds_t azure_kusto_create_blob(struct flb_azure_kusto *ctx, flb_sds_t flb_plg_error(ctx->ins, "error getting blob container upstream connection"); } + + //if (pthread_mutex_unlock(&ctx->blob_mutex)) { + // flb_plg_error(ctx->ins, "error unlocking mutex"); + // return -1; + //} + return uri; } @@ -222,10 +274,23 @@ static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t size_t b64_len; size_t message_len; + + if (pthread_mutex_lock(&ctx->blob_mutex)) { + flb_plg_error(ctx->ins, "error unlocking mutex"); + return -1; + } + uuid = generate_uuid(); if (uuid) { message = flb_sds_create(NULL); + flb_plg_debug(ctx->ins,"uuid :: %s",uuid); + flb_plg_debug(ctx->ins,"blob uri :: %s",blob_uri); + flb_plg_debug(ctx->ins,"payload size :: %lu",payload_size); + flb_plg_debug(ctx->ins,"database_name :: %s",ctx->database_name); + flb_plg_debug(ctx->ins,"table name :: %s",ctx->table_name); + flb_plg_debug(ctx->ins,"identity token :: %s", ctx->resources->identity_token); + if (message) { message_len = flb_sds_snprintf(&message, 0, @@ -281,6 +346,12 @@ static flb_sds_t create_ingestion_message(struct flb_azure_kusto *ctx, flb_sds_t flb_plg_error(ctx->ins, "error generating unique ingestion UUID"); } + + if (pthread_mutex_unlock(&ctx->blob_mutex)) { + flb_plg_error(ctx->ins, "error unlocking mutex"); + return -1; + } + return message; } @@ -348,6 +419,11 @@ static int azure_kusto_enqueue_ingestion(struct flb_azure_kusto *ctx, flb_sds_t return -1; } + + u_node->u->base.net.connect_timeout = ctx->ingestion_endpoint_connect_timeout; + u_node->u->base.net.keepalive_max_recycle = ctx->keep_alive_max_connection_recycle; + + u_conn = flb_upstream_conn_get(u_node->u); if (u_conn) { @@ -466,9 +542,21 @@ int azure_kusto_queued_ingestion(struct flb_azure_kusto *ctx, flb_sds_t tag, flb_sds_t blob_id; flb_sds_t blob_uri; + + if (pthread_mutex_lock(&ctx->blob_mutex)) { + flb_plg_error(ctx->ins, "error unlocking mutex"); + return NULL; + } + /* flb________ */ blob_id = azure_kusto_create_blob_id(ctx, tag, tag_len); + + if (pthread_mutex_unlock(&ctx->blob_mutex)) { + flb_plg_error(ctx->ins, "error unlocking mutex"); + return NULL; + } + if (blob_id) { blob_uri = azure_kusto_create_blob(ctx, blob_id, payload, payload_size);