From 3cf8cbab5b1c6b2e2ee7c15cf848461937a0e85e Mon Sep 17 00:00:00 2001 From: Robin Lin <128118209+RobinLin666@users.noreply.github.com> Date: Sun, 14 Jan 2024 23:54:16 +0800 Subject: [PATCH] out_azure_blob: add support for Azure Blob SAS authentication (#8243) Sometimes user cannot use shared key authentication. So, this patch support authenticate Azure Blob Storage with shared access signatures. Here is the document about shared access signatures: https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview This patch add two configuration parameters for out_azure_blob plugin: auth_type and sas_token. - auth_type, default value is "key", it can be "key" or "sas". - sas_token, default value is NULL, it is the SAS token, it is required when auth_type is "sas". Signed-off-by: Yi Lin --- plugins/out_azure_blob/azure_blob.c | 23 ++++++++ plugins/out_azure_blob/azure_blob.h | 6 +++ .../out_azure_blob/azure_blob_appendblob.c | 4 ++ plugins/out_azure_blob/azure_blob_blockblob.c | 14 +++++ plugins/out_azure_blob/azure_blob_conf.c | 54 +++++++++++++++---- plugins/out_azure_blob/azure_blob_http.c | 22 ++++---- plugins/out_azure_blob/azure_blob_uri.c | 8 +++ 7 files changed, 111 insertions(+), 20 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index 8778b132556..f35667fb378 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -200,6 +200,12 @@ static int send_blob(struct flb_config *config, return FLB_OK; } else if (c->resp.status == 404) { + /* delete "&sig=..." in the c->uri for security */ + char *p = strstr(c->uri, "&sig="); + if (p) { + *p = '\0'; + } + flb_plg_info(ctx->ins, "blob not found: %s", c->uri); flb_http_client_destroy(c); return CREATE_BLOB; @@ -269,6 +275,11 @@ static int create_blob(struct flb_azure_blob *ctx, char *name) } if (c->resp.status == 201) { + /* delete "&sig=..." in the c->uri for security */ + char *p = strstr(c->uri, "&sig="); + if (p) { + *p = '\0'; + } flb_plg_info(ctx->ins, "blob created successfully: %s", c->uri); } else { @@ -572,6 +583,18 @@ static struct flb_config_map config_map[] = { "Name of the key that will have the record timestamp" }, + { + FLB_CONFIG_MAP_STR, "auth_type", "key", + 0, FLB_TRUE, offsetof(struct flb_azure_blob, auth_type), + "Set the auth type: key or sas" + }, + + { + FLB_CONFIG_MAP_STR, "sas_token", NULL, + 0, FLB_TRUE, offsetof(struct flb_azure_blob, sas_token), + "Azure Blob SAS token" + }, + /* EOF */ {0} }; diff --git a/plugins/out_azure_blob/azure_blob.h b/plugins/out_azure_blob/azure_blob.h index df9afddd9b5..b1812927adb 100644 --- a/plugins/out_azure_blob/azure_blob.h +++ b/plugins/out_azure_blob/azure_blob.h @@ -41,6 +41,9 @@ #define AZURE_BLOB_APPENDBLOB 0 #define AZURE_BLOB_BLOCKBLOB 1 +#define AZURE_BLOB_AUTH_KEY 0 +#define AZURE_BLOB_AUTH_SAS 1 + struct flb_azure_blob { int auto_create_container; int emulator_mode; @@ -53,11 +56,14 @@ struct flb_azure_blob { flb_sds_t endpoint; flb_sds_t path; flb_sds_t date_key; + flb_sds_t auth_type; + flb_sds_t sas_token; /* * Internal use */ int btype; /* blob type */ + int atype; /* auth type */ flb_sds_t real_endpoint; flb_sds_t base_uri; flb_sds_t shared_key_prefix; diff --git a/plugins/out_azure_blob/azure_blob_appendblob.c b/plugins/out_azure_blob/azure_blob_appendblob.c index bbc53b45c68..110e9eb5d4d 100644 --- a/plugins/out_azure_blob/azure_blob_appendblob.c +++ b/plugins/out_azure_blob/azure_blob_appendblob.c @@ -40,5 +40,9 @@ flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag) flb_sds_printf(&uri, "/%s?comp=appendblock", tag); } + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "&%s", ctx->sas_token); + } + return uri; } diff --git a/plugins/out_azure_blob/azure_blob_blockblob.c b/plugins/out_azure_blob/azure_blob_blockblob.c index 6741ea3ad5a..8a47a700967 100644 --- a/plugins/out_azure_blob/azure_blob_blockblob.c +++ b/plugins/out_azure_blob/azure_blob_blockblob.c @@ -65,6 +65,10 @@ flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *tag, tag, ms, ext, encoded_blockid); } + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "&%s", ctx->sas_token); + } + flb_sds_destroy(encoded_blockid); return uri; } @@ -95,6 +99,10 @@ flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx, flb_sds_printf(&uri, "/%s.%" PRIu64 "%s?comp=blocklist", tag, ms, ext); } + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "&%s", ctx->sas_token); + } + return uri; } @@ -214,6 +222,12 @@ int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, return FLB_OK; } else if (c->resp.status == 404) { + /* delete "&sig=..." in the c->uri for security */ + char *p = strstr(c->uri, "&sig="); + if (p) { + *p = '\0'; + } + flb_plg_info(ctx->ins, "blob not found: %s", c->uri); flb_http_client_destroy(c); flb_upstream_conn_release(u_conn); diff --git a/plugins/out_azure_blob/azure_blob_conf.c b/plugins/out_azure_blob/azure_blob_conf.c index 763f1f23dd6..f0f7d420714 100644 --- a/plugins/out_azure_blob/azure_blob_conf.c +++ b/plugins/out_azure_blob/azure_blob_conf.c @@ -85,8 +85,39 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in return NULL; } + /* Set Auth type */ + tmp = (char *) flb_output_get_property("auth_type", ins); + if (!tmp) { + ctx->atype = AZURE_BLOB_AUTH_KEY; + } + else { + if (strcasecmp(tmp, "key") == 0) { + ctx->atype = AZURE_BLOB_AUTH_KEY; + } + else if (strcasecmp(tmp, "sas") == 0) { + ctx->atype = AZURE_BLOB_AUTH_SAS; + } + else { + flb_plg_error(ctx->ins, "invalid auth_type value '%s'", tmp); + return NULL; + } + } + if (ctx->atype == AZURE_BLOB_AUTH_KEY && !ctx->shared_key) { + flb_plg_error(ctx->ins, "'shared_key' has not been set"); + return NULL; + } + if (ctx->atype == AZURE_BLOB_AUTH_SAS) { + if (!ctx->sas_token) { + flb_plg_error(ctx->ins, "'sas_token' has not been set"); + return NULL; + } + if (ctx->sas_token[0] == '?') { + ctx->sas_token++; + } + } + /* If the shared key is set decode it */ - if (ctx->shared_key) { + if (ctx->atype == AZURE_BLOB_AUTH_KEY && ctx->shared_key) { ret = set_shared_key(ctx); if (ret == -1) { return NULL; @@ -196,12 +227,14 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in } /* Prepare shared key buffer */ - ctx->shared_key_prefix = flb_sds_create_size(256); - if (!ctx->shared_key_prefix) { - flb_plg_error(ctx->ins, "cannot create shared key prefix"); - return NULL; + if (ctx->atype == AZURE_BLOB_AUTH_KEY) { + ctx->shared_key_prefix = flb_sds_create_size(256); + if (!ctx->shared_key_prefix) { + flb_plg_error(ctx->ins, "cannot create shared key prefix"); + return NULL; + } + flb_sds_printf(&ctx->shared_key_prefix, "SharedKey %s:", ctx->account_name); } - flb_sds_printf(&ctx->shared_key_prefix, "SharedKey %s:", ctx->account_name); /* Sanitize path: remove any ending slash */ if (ctx->path) { @@ -211,11 +244,12 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in } flb_plg_info(ctx->ins, - "account_name=%s, container_name=%s, blob_type=%s, emulator_mode=%s, endpoint=%s", + "account_name=%s, container_name=%s, blob_type=%s, emulator_mode=%s, endpoint=%s, auth_type=%s", ctx->account_name, ctx->container_name, - ctx->btype == AZURE_BLOB_APPENDBLOB ? "appendblob": "blockblob", - ctx->emulator_mode ? "yes": "no", - ctx->real_endpoint ? ctx->real_endpoint: "no"); + ctx->btype == AZURE_BLOB_APPENDBLOB ? "appendblob" : "blockblob", + ctx->emulator_mode ? "yes" : "no", + ctx->real_endpoint ? ctx->real_endpoint : "no", + ctx->atype == AZURE_BLOB_AUTH_KEY ? "key" : "sas"); return ctx; } diff --git a/plugins/out_azure_blob/azure_blob_http.c b/plugins/out_azure_blob/azure_blob_http.c index f458b87b80a..dd6f128a242 100644 --- a/plugins/out_azure_blob/azure_blob_http.c +++ b/plugins/out_azure_blob/azure_blob_http.c @@ -339,20 +339,22 @@ int azb_http_client_setup(struct flb_azure_blob *ctx, struct flb_http_client *c, /* Azure header: x-ms-version */ flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); - can_req = azb_http_canonical_request(ctx, c, content_length, content_type, - content_encoding); + if (ctx->atype == AZURE_BLOB_AUTH_KEY) { + can_req = azb_http_canonical_request(ctx, c, content_length, content_type, + content_encoding); - auth = flb_sds_create_size(64 + flb_sds_len(can_req)); + auth = flb_sds_create_size(64 + flb_sds_len(can_req)); - flb_sds_cat(auth, ctx->shared_key_prefix, flb_sds_len(ctx->shared_key_prefix)); - flb_sds_cat(auth, can_req, flb_sds_len(can_req)); + flb_sds_cat(auth, ctx->shared_key_prefix, flb_sds_len(ctx->shared_key_prefix)); + flb_sds_cat(auth, can_req, flb_sds_len(can_req)); - /* Azure header: authorization */ - flb_http_add_header(c, "Authorization", 13, auth, flb_sds_len(auth)); + /* Azure header: authorization */ + flb_http_add_header(c, "Authorization", 13, auth, flb_sds_len(auth)); - /* Release buffers */ - flb_sds_destroy(can_req); - flb_sds_destroy(auth); + /* Release buffers */ + flb_sds_destroy(can_req); + flb_sds_destroy(auth); + } /* Set callback context to the HTTP client context */ flb_http_set_callback_context(c, ctx->ins->callback); diff --git a/plugins/out_azure_blob/azure_blob_uri.c b/plugins/out_azure_blob/azure_blob_uri.c index 17124054075..75a643079aa 100644 --- a/plugins/out_azure_blob/azure_blob_uri.c +++ b/plugins/out_azure_blob/azure_blob_uri.c @@ -127,6 +127,10 @@ flb_sds_t azb_uri_ensure_or_create_container(struct flb_azure_blob *ctx) } flb_sds_printf(&uri, "?restype=container"); + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "&%s", ctx->sas_token); + } + return uri; } @@ -146,5 +150,9 @@ flb_sds_t azb_uri_create_blob(struct flb_azure_blob *ctx, char *tag) flb_sds_printf(&uri, "/%s", tag); } + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "?%s", ctx->sas_token); + } + return uri; }