From 2ff35bdaf38cf584ef51992acace23ef961278f4 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Thu, 16 May 2024 18:40:40 -0700 Subject: [PATCH 1/6] utils: add flb_utils_url_split_sds Signed-off-by: Wesley Pettit --- include/fluent-bit/flb_utils.h | 2 + src/flb_utils.c | 128 ++++++++++++++++++++++++++++++++- tests/internal/utils.c | 83 +++++++++++++++++++++ 3 files changed, 210 insertions(+), 3 deletions(-) diff --git a/include/fluent-bit/flb_utils.h b/include/fluent-bit/flb_utils.h index f1424d78842..e88fa9fd936 100644 --- a/include/fluent-bit/flb_utils.h +++ b/include/fluent-bit/flb_utils.h @@ -73,5 +73,7 @@ int flb_utils_get_machine_id(char **out_id, size_t *out_size); void flb_utils_set_plugin_string_property(const char *name, flb_sds_t *field_storage, flb_sds_t new_value); +int flb_utils_url_split_sds(const flb_sds_t in_url, flb_sds_t *out_protocol, + flb_sds_t *out_host, flb_sds_t *out_port, flb_sds_t *out_uri); #endif diff --git a/src/flb_utils.c b/src/flb_utils.c index 2f445980d84..180d690efb9 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -1017,13 +1017,27 @@ int flb_utils_write_str_buf(const char *str, size_t str_len, char **out, size_t static char *flb_copy_host(const char *string, int pos_init, int pos_end) { if (string[pos_init] == '[') { /* IPv6 */ - if (string[pos_end-1] != ']') + if (string[pos_end-1] != ']') { return NULL; - + } return mk_string_copy_substr(string, pos_init + 1, pos_end - 1); } - else + else { return mk_string_copy_substr(string, pos_init, pos_end); + } +} + +static char *flb_utils_copy_host_sds(const char *string, int pos_init, int pos_end) +{ + if (string[pos_init] == '[') { /* IPv6 */ + if (string[pos_end-1] != ']') { + return NULL; + } + return flb_sds_create_len(string + pos_init + 1, pos_end - 1); + } + else { + return flb_sds_create_len(string + pos_init, pos_end); + } } int flb_utils_url_split(const char *in_url, char **out_protocol, @@ -1121,6 +1135,114 @@ int flb_utils_url_split(const char *in_url, char **out_protocol, return -1; } +int flb_utils_url_split_sds(const flb_sds_t in_url, flb_sds_t *out_protocol, + flb_sds_t *out_host, flb_sds_t *out_port, flb_sds_t *out_uri) +{ + flb_sds_t protocol = NULL; + flb_sds_t host = NULL; + flb_sds_t port = NULL; + flb_sds_t uri = NULL; + char *p = NULL; + char *tmp = NULL; + char *sep = NULL; + + /* Protocol */ + p = strstr(in_url, "://"); + if (!p) { + return -1; + } + if (p == in_url) { + return -1; + } + + protocol = flb_sds_create_len(in_url, p - in_url); + if (!protocol) { + flb_errno(); + return -1; + } + + /* Advance position after protocol */ + p += 3; + + /* Check for first '/' */ + sep = strchr(p, '/'); + tmp = strchr(p, ':'); + + /* Validate port separator is found before the first slash */ + if (sep && tmp) { + if (tmp > sep) { + tmp = NULL; + } + } + + if (tmp) { + host = flb_utils_copy_host_sds(p, 0, tmp - p); + if (!host) { + flb_errno(); + goto error; + } + p = tmp + 1; + + /* Look for an optional URI */ + tmp = strchr(p, '/'); + if (tmp) { + port = flb_sds_create_len(p, tmp - p); + uri = flb_sds_create(tmp); + } + else { + port = flb_sds_create_len(p, strlen(p)); + uri = flb_sds_create("/"); + } + } + else { + tmp = strchr(p, '/'); + if (tmp) { + host = flb_utils_copy_host_sds(p, 0, tmp - p); + uri = flb_sds_create(tmp); + } + else { + host = flb_utils_copy_host_sds(p, 0, strlen(p)); + uri = flb_sds_create("/"); + } + } + + if (!port) { + if (strcmp(protocol, "http") == 0) { + port = flb_sds_create("80"); + } + else if (strcmp(protocol, "https") == 0) { + port = flb_sds_create("443"); + } + } + + if (!host) { + flb_errno(); + goto error; + } + if (!port) { + flb_errno(); + goto error; + } + if (!uri) { + flb_errno(); + goto error; + } + + *out_protocol = protocol; + *out_host = host; + *out_port = port; + *out_uri = uri; + + return 0; + + error: + if (protocol) { + flb_free(protocol); + } + + return -1; +} + /* * flb_utils_proxy_url_split parses a proxy's information from a http_proxy URL. diff --git a/tests/internal/utils.c b/tests/internal/utils.c index bcb9053d86f..410736a1b75 100644 --- a/tests/internal/utils.c +++ b/tests/internal/utils.c @@ -38,6 +38,88 @@ struct url_check url_checks[] = { {-1, "://", NULL, NULL, NULL, NULL}, }; +void test_url_split_sds() +{ + int i; + int ret; + int size; + flb_sds_t protocol; + flb_sds_t host; + flb_sds_t port; + flb_sds_t uri; + struct url_check *u; + + size = sizeof(url_checks) / sizeof(struct url_check); + for (i = 0; i < size; i ++) { + u = &url_checks[i]; + + protocol = NULL; + host = NULL; + port = NULL; + uri = NULL; + + ret = flb_utils_url_split_sds(u->url, &protocol, &host, &port, &uri); + TEST_CHECK(ret == u->ret); + if (ret == -1) { + continue; + } + + /* protocol */ + if (u->prot) { + TEST_CHECK(protocol != NULL); + + ret = strcmp(u->prot, protocol); + TEST_CHECK(ret == 0); + } + else { + TEST_CHECK(protocol == NULL); + } + + /* host */ + if (u->host) { + TEST_CHECK(host != NULL); + ret = strcmp(u->host, host); + TEST_CHECK(ret == 0); + } + else { + TEST_CHECK(host == NULL); + } + + /* port */ + if (u->port) { + TEST_CHECK(port != NULL); + ret = strcmp(u->port, port); + TEST_CHECK(ret == 0); + } + else { + TEST_CHECK(port == NULL); + } + + /* uri */ + if (u->uri) { + TEST_CHECK(uri != NULL); + ret = strcmp(u->uri, uri); + TEST_CHECK(ret == 0); + } + else { + TEST_CHECK(uri == NULL); + } + + if (protocol) { + flb_sds_destroy(protocol); + } + if (host) { + flb_sds_destroy(host); + } + if (port) { + flb_sds_destroy(port); + } + if (uri) { + flb_sds_destroy(uri); + } + } +} + void test_url_split() { int i; @@ -657,6 +739,7 @@ void test_size_to_bytes() TEST_LIST = { /* JSON maps iteration */ { "url_split", test_url_split }, + { "url_split_sds", test_url_split_sds }, { "write_str", test_write_str }, { "test_write_str_invalid_trailing_bytes", test_write_str_invalid_trailing_bytes }, { "test_write_str_invalid_leading_byte", test_write_str_invalid_leading_byte }, From 1159929fa4339188f0910373f0046221dc6a1bf2 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Thu, 16 May 2024 18:48:46 -0700 Subject: [PATCH 2/6] aws: http_credentials: support EKS Pod Identity This change brings the http credential provider in line with the latest spec and adds support for: - EKS Pod Identity - validate/support EKS credential link local IP 169.254.170.23 - Latest HTTP Provider spec: - AWS_CONTAINER_CREDENTIALS_RELATIVE_URI - AWS_CONTAINER_CREDENTIALS_FULL_URI - AWS_CONTAINER_AUTHORIZATION_TOKEN - AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE Signed-off-by: Wesley Pettit --- include/fluent-bit/flb_aws_credentials.h | 39 ++++- include/fluent-bit/flb_aws_util.h | 22 ++- src/aws/flb_aws_credentials.c | 2 +- src/aws/flb_aws_credentials_http.c | 195 +++++++++++++++++------ src/aws/flb_aws_util.c | 86 +++++++++- 5 files changed, 279 insertions(+), 65 deletions(-) diff --git a/include/fluent-bit/flb_aws_credentials.h b/include/fluent-bit/flb_aws_credentials.h index 5bcda676b5b..36652ead45a 100644 --- a/include/fluent-bit/flb_aws_credentials.h +++ b/include/fluent-bit/flb_aws_credentials.h @@ -256,19 +256,21 @@ struct flb_aws_provider *flb_aws_env_provider_create(); * Calling flb_aws_provider_destroy on this provider frees the memory * used by host and path. */ -struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, - flb_sds_t host, - flb_sds_t path, - struct - flb_aws_client_generator - *generator); +struct flb_aws_provider *flb_endpoint_provider_create(struct flb_config *config, + flb_sds_t host, + flb_sds_t path, + int port, + int insecure, + struct + flb_aws_client_generator + *generator); /* - * ECS Provider + * HTTP Provider for EKS and ECS * The ECS Provider is just a wrapper around the HTTP Provider * with the ECS credentials endpoint. */ -struct flb_aws_provider *flb_ecs_provider_create(struct flb_config *config, +struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, struct flb_aws_client_generator *generator); @@ -350,5 +352,26 @@ int try_lock_provider(struct flb_aws_provider *provider); void unlock_provider(struct flb_aws_provider *provider); +/* + * HTTP Credentials Provider - retrieve credentials from a local http server + * Used to implement the ECS Credentials provider. + * Equivalent to: + * https://github.com/aws/aws-sdk-go/tree/master/aws/credentials/endpointcreds + */ + +struct flb_aws_provider_http { + struct flb_aws_credentials *creds; + time_t next_refresh; + + struct flb_aws_client *client; + + /* Host and Path to request credentials */ + flb_sds_t host; + flb_sds_t path; + + flb_sds_t auth_token; /* optional */ +}; + + #endif #endif /* FLB_HAVE_AWS */ diff --git a/include/fluent-bit/flb_aws_util.h b/include/fluent-bit/flb_aws_util.h index ac978e6e23c..0de6223d986 100644 --- a/include/fluent-bit/flb_aws_util.h +++ b/include/fluent-bit/flb_aws_util.h @@ -104,6 +104,17 @@ struct flb_aws_client { int debug_only; }; +/* frees dynamic_headers */ +struct flb_http_client *flb_aws_client_request_basic_auth( + struct flb_aws_client *aws_client, + int method, const char *uri, + const char *body, size_t body_len, + struct flb_aws_header + *dynamic_headers, + size_t dynamic_headers_len, + char *header_name, + char* auth_token); + /* * Frees the aws_client, the internal flb_http_client, error_code, * and flb_upstream. @@ -144,13 +155,20 @@ flb_sds_t flb_aws_xml_error(char *response, size_t response_len); flb_sds_t flb_aws_error(char *response, size_t response_len); /* - * Similar to 'flb_aws_error', except it prints the JSON error type and message - * to the user in a error log. + * Similar to 'flb_aws_error', except it prints the JSON error __type and message + * field values to the user in a error log. * 'api' is the name of the API that was called; this is used in the error log. */ void flb_aws_print_error(char *response, size_t response_len, char *api, struct flb_output_instance *ins); +/* + * Error parsing for json APIs that respond with a + * Code and Message fields for error responses. + */ +void flb_aws_print_error_code(char *response, size_t response_len, + char *api); + /* Similar to 'flb_aws_print_error', but for APIs that return XML */ void flb_aws_print_xml_error(char *response, size_t response_len, char *api, struct flb_output_instance *ins); diff --git a/src/aws/flb_aws_credentials.c b/src/aws/flb_aws_credentials.c index 824a535d021..a84ee70d71c 100644 --- a/src/aws/flb_aws_credentials.c +++ b/src/aws/flb_aws_credentials.c @@ -581,7 +581,7 @@ static struct flb_aws_provider *standard_chain_create(struct flb_config } } - sub_provider = flb_ecs_provider_create(config, generator); + sub_provider = flb_http_provider_create(config, generator); if (sub_provider) { /* ECS Provider will fail creation if we are not running in ECS */ mk_list_add(&sub_provider->_head, &implementation->sub_providers); diff --git a/src/aws/flb_aws_credentials_http.c b/src/aws/flb_aws_credentials_http.c index d9095935ce6..0eab14bebdd 100644 --- a/src/aws/flb_aws_credentials_http.c +++ b/src/aws/flb_aws_credentials_http.c @@ -22,10 +22,12 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -36,32 +38,44 @@ #define ECS_CREDENTIALS_HOST "169.254.170.2" #define ECS_CREDENTIALS_HOST_LEN 13 -#define ECS_CREDENTIALS_PATH_ENV_VAR "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" +#define EKS_CREDENTIALS_HOST "169.254.170.23" +#define EKS_CREDENTIALS_HOST_LEN 14 +#define AWS_CREDENTIALS_PATH "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" +#define AWS_CREDENTIALS_FULL_URI "AWS_CONTAINER_CREDENTIALS_FULL_URI" + +#define AUTH_TOKEN_ENV_VAR "AWS_CONTAINER_AUTHORIZATION_TOKEN" +#define AUTH_TOKEN_FILE_ENV_VAR "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE" + /* Declarations */ -struct flb_aws_provider_http; static int http_credentials_request(struct flb_aws_provider_http *implementation); -/* - * HTTP Credentials Provider - retrieve credentials from a local http server - * Used to implement the ECS Credentials provider. - * Equivalent to: - * https://github.com/aws/aws-sdk-go/tree/master/aws/credentials/endpointcreds - */ - -struct flb_aws_provider_http { - struct flb_aws_credentials *creds; - time_t next_refresh; +/* +If the resolved URI’s scheme is HTTPS, its hostname may be used in the request. +Otherwise, implementations MUST fail to resolve when the URI hostname +does not satisfy any of the following conditions: - struct flb_aws_client *client; +is within the loopback CIDR (IPv4 127.0.0.0/8, IPv6 ::1/128) +is the ECS container host 169.254.170.2 +is the EKS container host (IPv4 169.254.170.23, IPv6 fd00:ec2::23)*/ +static int validate_http_credential_uri(flb_sds_t protocol, flb_sds_t host) +{ + if (strncmp(protocol, "https", 5) == 0) { + return 0; + } else if (strncmp(host, "127.", 4) == 0 || + strncmp(host, ECS_CREDENTIALS_HOST, ECS_CREDENTIALS_HOST_LEN) == 0 || + strncmp(host, EKS_CREDENTIALS_HOST, EKS_CREDENTIALS_HOST_LEN) == 0 || + strstr(host, "::1") != NULL || + strstr(host, "fd00:ec2::23") != NULL || + strstr(host, "fe80:") != NULL) { + return 0; + } - /* Host and Path to request credentials */ - flb_sds_t host; - flb_sds_t path; -}; + return -1; +} struct flb_aws_credentials *get_credentials_fn_http(struct flb_aws_provider @@ -83,6 +97,8 @@ struct flb_aws_credentials *get_credentials_fn_http(struct flb_aws_provider if (try_lock_provider(provider)) { http_credentials_request(implementation); unlock_provider(provider); + } else { + flb_error("try_lock_provider failed"); } } @@ -229,16 +245,19 @@ static struct flb_aws_provider_vtable http_provider_vtable = { .upstream_set = upstream_set_fn_http, }; -struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, - flb_sds_t host, - flb_sds_t path, - struct - flb_aws_client_generator - *generator) +struct flb_aws_provider *flb_endpoint_provider_create(struct flb_config *config, + flb_sds_t host, + flb_sds_t path, + int port, + int insecure, + struct + flb_aws_client_generator + *generator) { struct flb_aws_provider_http *implementation = NULL; struct flb_aws_provider *provider = NULL; struct flb_upstream *upstream = NULL; + int io_flags = insecure == FLB_TRUE ? FLB_IO_TCP : FLB_IO_TLS; flb_debug("[aws_credentials] Configuring HTTP provider with %s:80%s", host, path); @@ -266,7 +285,7 @@ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, implementation->host = host; implementation->path = path; - upstream = flb_upstream_create(config, host, 80, FLB_IO_TCP, NULL); + upstream = flb_upstream_create(config, host, port, io_flags, NULL); if (!upstream) { flb_aws_provider_destroy(provider); @@ -289,7 +308,7 @@ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, implementation->client->provider = NULL; implementation->client->region = NULL; implementation->client->service = NULL; - implementation->client->port = 80; + implementation->client->port = port; implementation->client->flags = 0; implementation->client->proxy = NULL; implementation->client->upstream = upstream; @@ -302,39 +321,85 @@ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, * The ECS Provider is just a wrapper around the HTTP Provider * with the ECS credentials endpoint. */ - - struct flb_aws_provider *flb_ecs_provider_create(struct flb_config *config, - struct - flb_aws_client_generator - *generator) +struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, + struct + flb_aws_client_generator + *generator) { - flb_sds_t host = NULL; flb_sds_t path = NULL; - char *path_var = NULL; + flb_sds_t protocol = NULL; + flb_sds_t host = NULL; + flb_sds_t port_sds = NULL; + int port = 80; + int insecure = FLB_TRUE; + char *relative_uri = NULL; + char *full_uri = NULL; + int ret; - host = flb_sds_create_len(ECS_CREDENTIALS_HOST, ECS_CREDENTIALS_HOST_LEN); - if (!host) { - flb_errno(); - return NULL; - } + relative_uri = getenv(AWS_CREDENTIALS_PATH); + full_uri = getenv(AWS_CREDENTIALS_FULL_URI); - path_var = getenv(ECS_CREDENTIALS_PATH_ENV_VAR); - if (path_var && strlen(path_var) > 0) { - path = flb_sds_create(path_var); + if (relative_uri && strlen(relative_uri) > 0) { + host = flb_sds_create_len(ECS_CREDENTIALS_HOST, ECS_CREDENTIALS_HOST_LEN); + if (!host) { + flb_errno(); + return NULL; + } + path = flb_sds_create(relative_uri); if (!path) { flb_errno(); flb_free(host); return NULL; } - - return flb_http_provider_create(config, host, path, generator); + } else if (full_uri && strlen(full_uri) > 0) { + ret = flb_utils_url_split_sds(full_uri, &protocol, &host, &port_sds, &path); + if (ret < 0) { + return NULL; + } + insecure = strncmp(protocol, "http", 4) == 0 ? FLB_TRUE : FLB_FALSE; + ret = validate_http_credential_uri(protocol, host); + if (ret < 0) { + flb_error("[aws credentials] %s must be set to an https:// address or a link local IP address." + " Found protocol=%s, host=%s, port=%s, path=%s", + AWS_CREDENTIALS_FULL_URI, protocol, host, port_sds, path); + flb_sds_destroy(protocol); + flb_sds_destroy(host); + flb_sds_destroy(port_sds); + flb_sds_destroy(path); + return NULL; + } } else { - flb_debug("[aws_credentials] Not initializing ECS Provider because" - " %s is not set", ECS_CREDENTIALS_PATH_ENV_VAR); - flb_sds_destroy(host); + flb_debug("[aws_credentials] Not initializing ECS/EKS HTTP Provider because" + " %s and %s is not set", AWS_CREDENTIALS_PATH, AWS_CREDENTIALS_FULL_URI); return NULL; } + if (port_sds != NULL) { + port = atoi(port_sds); + if (port == 0) { + flb_error("[aws credentials] invalid port: %s must be set to an https:// address or a link local IP address." + " Found protocol=%s, host=%s, port=%s, path=%s", + AWS_CREDENTIALS_FULL_URI, protocol, host, port_sds, path); + flb_sds_destroy(protocol); + flb_sds_destroy(host); + flb_sds_destroy(port_sds); + flb_sds_destroy(path); + return NULL; + } + } + + return flb_endpoint_provider_create(config, host, path, port, insecure, generator); + +} + +static void trim_newline(char *token) +{ + int i; + for (i = strlen(token) - 1; i > 0; i--) { + if (token[i] == '\r' || token[i] == '\n') { + token[i] = '\0'; + } + } } static int http_credentials_request(struct flb_aws_provider_http @@ -346,14 +411,43 @@ static int http_credentials_request(struct flb_aws_provider_http struct flb_aws_credentials *creds = NULL; struct flb_aws_client *client = implementation->client; struct flb_http_client *c = NULL; + int ret; + char *auth_token = NULL; + size_t auth_token_size = 0; + char *auth_token_path = NULL; + + auth_token_path = getenv(AUTH_TOKEN_FILE_ENV_VAR); + auth_token = getenv(AUTH_TOKEN_ENV_VAR); + if (auth_token_path != NULL && strlen(auth_token_path) > 0) { + flb_debug("[aws] reading authorization token from %s", auth_token_path); + ret = flb_read_file(auth_token_path, &auth_token, + &auth_token_size); + if (ret < 0) { + flb_error("[aws credentials] failed to read authorization token from %s", + auth_token_path); + return -1; + } + } - c = client->client_vtable->request(client, FLB_HTTP_GET, - implementation->path, NULL, 0, - NULL, 0); + if (auth_token != NULL && strlen(auth_token) > 0) { + trim_newline(auth_token); + c = flb_aws_client_request_basic_auth(client, FLB_HTTP_GET, implementation->path, + NULL, 0, NULL, 0, + "Authorization", + auth_token); + } else { + c = client->client_vtable->request(client, FLB_HTTP_GET, + implementation->path, NULL, 0, + NULL, 0); + } if (!c || c->resp.status != 200) { flb_debug("[aws_credentials] http credentials request failed"); if (c) { + if (c->resp.payload_size > 0) { + flb_aws_print_error_code(c->resp.payload, c->resp.payload_size, + "ContainerCredentialsLocalServer"); + } flb_http_client_destroy(c); } return -1; @@ -397,6 +491,7 @@ struct flb_aws_credentials *flb_parse_http_credentials(char *response, expiration); } +//TODO: error code handling struct flb_aws_credentials *flb_parse_json_credentials(char *response, size_t response_len, char* session_token_field, @@ -532,12 +627,12 @@ struct flb_aws_credentials *flb_parse_json_credentials(char *response, goto error; } *expiration = flb_aws_cred_expiration(tmp); - flb_sds_destroy(tmp); if (*expiration < 0) { flb_warn("[aws_credentials] '%s' was invalid or " "could not be parsed. Disabling auto-refresh of " - "credentials.", AWS_CREDENTIAL_RESPONSE_EXPIRATION); + "credentials.", tmp); } + flb_sds_destroy(tmp); } } diff --git a/src/aws/flb_aws_util.c b/src/aws/flb_aws_util.c index 6ddd75b82d9..56940c40a71 100644 --- a/src/aws/flb_aws_util.c +++ b/src/aws/flb_aws_util.c @@ -169,6 +169,7 @@ char *removeProtocol (char *endpoint, char *protocol) { return endpoint; } + struct flb_http_client *flb_aws_client_request(struct flb_aws_client *aws_client, int method, const char *uri, const char *body, size_t body_len, @@ -200,10 +201,8 @@ struct flb_http_client *flb_aws_client_request(struct flb_aws_client *aws_client if (flb_aws_is_auth_error(c->resp.payload, c->resp.payload_size) == FLB_TRUE) { flb_info("[aws_client] auth error, refreshing creds"); - aws_client->refresh_limit = time(NULL) - + FLB_AWS_CREDENTIAL_REFRESH_LIMIT; - aws_client->provider->provider_vtable-> - refresh(aws_client->provider); + aws_client->refresh_limit = time(NULL) + FLB_AWS_CREDENTIAL_REFRESH_LIMIT; + aws_client->provider->provider_vtable->refresh(aws_client->provider); } } } @@ -211,6 +210,50 @@ struct flb_http_client *flb_aws_client_request(struct flb_aws_client *aws_client return c; } +/* always frees dynamic_headers */ +struct flb_http_client *flb_aws_client_request_basic_auth( + struct flb_aws_client *aws_client, + int method, const char *uri, + const char *body, size_t body_len, + struct flb_aws_header + *dynamic_headers, + size_t dynamic_headers_len, + char *header_name, + char* auth_token) +{ + struct flb_http_client *c = NULL; + struct flb_aws_header *auth_header = NULL; + struct flb_aws_header *headers = NULL; + auth_header = flb_calloc(1, sizeof(struct flb_aws_header)); + if (!auth_header) { + flb_errno(); + return NULL; + } + + auth_header->key = header_name; + auth_header->key_len = strlen(header_name); + auth_header->val = auth_token; + auth_header->val_len = strlen(auth_token); + + if (dynamic_headers_len == 0) { + c = aws_client->client_vtable->request(aws_client, method, uri, body, body_len, + auth_header, 1); + } else { + headers = flb_realloc(dynamic_headers, (dynamic_headers_len + 1) * sizeof(struct flb_aws_header)); + if (!headers) { + flb_free(auth_header); + flb_errno(); + return NULL; + } + *(headers + dynamic_headers_len) = *auth_header; + c = aws_client->client_vtable->request(aws_client, method, uri, body, body_len, + headers, dynamic_headers_len + 1); + flb_free(headers); + } + flb_free(auth_header); + return c; +} + static struct flb_aws_client_vtable client_vtable = { .request = flb_aws_client_request, }; @@ -573,6 +616,10 @@ flb_sds_t flb_aws_xml_get_val(char *response, size_t response_len, char *tag, ch return val; } +/* + * Error parsing for json APIs that respond with an + * __type and message fields for error responses. + */ void flb_aws_print_error(char *response, size_t response_len, char *api, struct flb_output_instance *ins) { @@ -600,6 +647,37 @@ void flb_aws_print_error(char *response, size_t response_len, flb_sds_destroy(error); } +/* + * Error parsing for json APIs that respond with a + * Code and Message fields for error responses. + */ +void flb_aws_print_error_code(char *response, size_t response_len, + char *api) +{ + flb_sds_t error; + flb_sds_t message; + + error = flb_json_get_val(response, response_len, "Code"); + if (!error) { + /* error can not be parsed, print raw response */ + flb_warn("%s: Raw response: %s", api, response); + return; + } + + message = flb_json_get_val(response, response_len, "Message"); + if (!message) { + /* just print the error */ + flb_error("%s API responded with code='%s'", api, error); + } + else { + flb_error("%s API responded with code='%s', message='%s'", + api, error, message); + flb_sds_destroy(message); + } + + flb_sds_destroy(error); +} + /* parses AWS JSON API error responses and returns the value of the __type field */ flb_sds_t flb_aws_error(char *response, size_t response_len) { From fa7f66ebfbacba3a80f49163729fdb8031bacf80 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Thu, 16 May 2024 18:49:20 -0700 Subject: [PATCH 3/6] tests: internal: aws: EKS Pod identity tests Signed-off-by: Wesley Pettit --- tests/internal/aws_credentials_http.c | 454 +++++++++++++++++- .../container_authorization_token.txt | 1 + 2 files changed, 445 insertions(+), 10 deletions(-) create mode 100644 tests/internal/data/aws_credentials/container_authorization_token.txt diff --git a/tests/internal/aws_credentials_http.c b/tests/internal/aws_credentials_http.c index 55912da3a60..6e43b7f4d1e 100644 --- a/tests/internal/aws_credentials_http.c +++ b/tests/internal/aws_credentials_http.c @@ -12,6 +12,11 @@ #include "flb_tests_internal.h" +#include "../include/aws_client_mock.h" +#include "../include/aws_client_mock.c" + +#include "aws_credentials_test_internal.h" + #define ACCESS_KEY_HTTP "http_akid" #define SECRET_KEY_HTTP "http_skid" #define TOKEN_HTTP "http_token" @@ -24,6 +29,8 @@ \"Token\": \"http_token\"\n\ }" +#define TEST_AUTHORIZATION_TOKEN_FILE AWS_TEST_DATA_PATH("container_authorization_token.txt") + /* * Unexpected/invalid HTTP response. The goal of this is not to test anything * that might happen in production, but rather to test the error handling @@ -200,8 +207,7 @@ static void test_http_provider() return; } - provider = flb_http_provider_create(config, host, path, - generator_in_test()); + provider = flb_http_provider_create(config, generator_in_test()); if (!provider) { flb_errno(); @@ -279,8 +285,7 @@ static void test_http_provider_error_case() return; } - provider = flb_http_provider_create(config, host, path, - generator_in_test()); + provider = flb_http_provider_create(config, generator_in_test()); if (!provider) { flb_errno(); @@ -342,8 +347,7 @@ static void test_http_provider_malformed_response() return; } - provider = flb_http_provider_create(config, host, path, - generator_in_test()); + provider = flb_http_provider_create(config, generator_in_test()); if (!provider) { flb_errno(); @@ -373,10 +377,440 @@ static void test_http_provider_malformed_response() flb_config_exit(config); } +/* + * Setup test & Initialize test environment + */ +void setup_test(struct flb_aws_client_mock_request_chain *request_chain, + struct flb_aws_provider **out_provider, struct flb_config **out_config) { + struct flb_aws_provider *provider; + struct flb_config *config; + + /* Initialize test environment */ + config = flb_config_init(); + TEST_ASSERT(config != NULL); + + flb_aws_client_mock_configure_generator(request_chain); + + /* Init provider */ + provider = flb_http_provider_create(config, flb_aws_client_get_mock_generator()); + TEST_ASSERT(provider != NULL); + + *out_config = config; + *out_provider = provider; +} + +/* Test clean up */ +void cleanup_test(struct flb_aws_provider *provider, struct flb_config *config) { + flb_aws_client_mock_destroy_generator(); + if (provider != NULL) { + ((struct flb_aws_provider_http *) (provider->implementation))->client = NULL; + flb_aws_provider_destroy(provider); + provider = NULL; + } + if (config != NULL) { + flb_config_exit(config); + config = NULL; + } +} + +static void test_http_provider_ecs_case() +{ + struct flb_aws_provider *provider; + struct flb_aws_credentials *creds; + struct flb_config *config; + int ret; + + setenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/iam_credentials/pod1", 1); + + setup_test(FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/iam_credentials/pod1"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER_COUNT, 0), + set(STATUS, 200), + set(PAYLOAD, "{\n \"Code\" : \"Success\",\n \"LastUpdated\" : \"2021-09-16T18:29:09Z\",\n" + " \"Type\" : \"AWS-HMAC\",\n \"AccessKeyId\" : \"XACCESSEKSXXX\",\n \"SecretAccessKey\"" + " : \"XSECRETEKSXXXXXXXXXXXXXX\",\n \"Token\" : \"XTOKENEKSXXXXXXXXXXXXXXX==\",\n" + " \"Expiration\" : \"3021-09-17T00:41:00Z\"\n}"), + set(PAYLOAD_SIZE, 257) + ), + response( + expect(URI, "/iam_credentials/pod1"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "{\n \"Code\" : \"Success\",\n \"LastUpdated\" : \"2021-09-16T18:29:09Z\",\n" + " \"Type\" : \"AWS-HMAC\",\n \"AccessKeyId\" : \"YACCESSEKSXXX\",\n \"SecretAccessKey\"" + " : \"YSECRETEKSXXXXXXXXXXXXXX\",\n \"Token\" : \"YTOKENEKSXXXXXXXXXXXXXXX==\",\n" + " \"Expiration\" : \"3021-09-17T00:41:00Z\"\n}"), // Expires Year 3021 + set(PAYLOAD_SIZE, 257) + ) + ), &provider, &config); + + flb_time_msleep(1000); + + /* Repeated calls to get credentials should return the same set */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("XACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("XSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("XTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* Retrieve from cache */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("XACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("XSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("XTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* refresh should return 0 (success) */ + ret = provider->provider_vtable->refresh(provider); + TEST_CHECK(ret == 0); + + /* Retrieve refreshed credentials from cache */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("YACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("YSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("YTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* Check we have exhausted our response list */ + TEST_CHECK(flb_aws_client_mock_generator_count_unused_requests() == 0); + + cleanup_test(provider, config); +} + +static void test_http_provider_eks_with_token() +{ + struct flb_aws_provider *provider; + struct flb_aws_credentials *creds; + struct flb_config *config; + int ret; + + setenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/iam_credentials/pod1", 1); + setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "password", 1); + + setup_test(FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/iam_credentials/pod1"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER, "Authorization", "password"), + set(STATUS, 200), + set(PAYLOAD, "{\n \"Code\" : \"Success\",\n \"LastUpdated\" : \"2021-09-16T18:29:09Z\",\n" + " \"Type\" : \"AWS-HMAC\",\n \"AccessKeyId\" : \"XACCESSEKSXXX\",\n \"SecretAccessKey\"" + " : \"XSECRETEKSXXXXXXXXXXXXXX\",\n \"Token\" : \"XTOKENEKSXXXXXXXXXXXXXXX==\",\n" + " \"Expiration\" : \"3021-09-17T00:41:00Z\"\n}"), + set(PAYLOAD_SIZE, 257) + ), + response( + expect(URI, "/iam_credentials/pod1"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER, "Authorization", "password"), + set(STATUS, 200), + set(PAYLOAD, "{\n \"Code\" : \"Success\",\n \"LastUpdated\" : \"2021-09-16T18:29:09Z\",\n" + " \"Type\" : \"AWS-HMAC\",\n \"AccessKeyId\" : \"YACCESSEKSXXX\",\n \"SecretAccessKey\"" + " : \"YSECRETEKSXXXXXXXXXXXXXX\",\n \"Token\" : \"YTOKENEKSXXXXXXXXXXXXXXX==\",\n" + " \"Expiration\" : \"3021-09-17T00:41:00Z\"\n}"), + set(PAYLOAD_SIZE, 257) + ) + ), &provider, &config); + + flb_time_msleep(1000); + + /* Repeated calls to get credentials should return the same set */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("XACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("XSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("XTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* Retrieve from cache */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("XACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("XSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("XTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* refresh should return 0 (success) */ + ret = provider->provider_vtable->refresh(provider); + TEST_CHECK(ret == 0); + + /* Retrieve refreshed credentials from cache */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("YACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("YSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("YTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* Check we have exhausted our response list */ + TEST_CHECK(flb_aws_client_mock_generator_count_unused_requests() == 0); + + cleanup_test(provider, config); +} + +static void test_http_provider_eks_with_token_file() +{ + struct flb_aws_provider *provider; + struct flb_aws_credentials *creds; + struct flb_config *config; + int ret; + + /* tests validation of valid non-default local loopback IP */ + setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://127.0.0.7:80/iam_credentials/pod1", 1); + setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", TEST_AUTHORIZATION_TOKEN_FILE, 1); + + setup_test(FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/iam_credentials/pod1"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER, "Authorization", "local-http-credential-server-authorization-token"), + set(STATUS, 200), + set(PAYLOAD, "{\n \"Code\" : \"Success\",\n \"LastUpdated\" : \"2021-09-16T18:29:09Z\",\n" + " \"Type\" : \"AWS-HMAC\",\n \"AccessKeyId\" : \"XACCESSEKSXXX\",\n \"SecretAccessKey\"" + " : \"XSECRETEKSXXXXXXXXXXXXXX\",\n \"Token\" : \"XTOKENEKSXXXXXXXXXXXXXXX==\",\n" + " \"Expiration\" : \"3021-09-17T00:41:00Z\"\n}"), + set(PAYLOAD_SIZE, 257) + ), + response( + expect(URI, "/iam_credentials/pod1"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER, "Authorization", "local-http-credential-server-authorization-token"), + set(STATUS, 200), + set(PAYLOAD, "{\n \"Code\" : \"Success\",\n \"LastUpdated\" : \"2021-09-16T18:29:09Z\",\n" + " \"Type\" : \"AWS-HMAC\",\n \"AccessKeyId\" : \"YACCESSEKSXXX\",\n \"SecretAccessKey\"" + " : \"YSECRETEKSXXXXXXXXXXXXXX\",\n \"Token\" : \"YTOKENEKSXXXXXXXXXXXXXXX==\",\n" + " \"Expiration\" : \"3021-09-17T00:41:00Z\"\n}"), + set(PAYLOAD_SIZE, 257) + ) + ), &provider, &config); + + flb_time_msleep(1000); + + /* Repeated calls to get credentials should return the same set */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("XACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("XSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("XTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* Retrieve from cache */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("XACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("XSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("XTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* refresh should return 0 (success) */ + ret = provider->provider_vtable->refresh(provider); + TEST_CHECK(ret == 0); + + /* Retrieve refreshed credentials from cache */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("YACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("YSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("YTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* Check we have exhausted our response list */ + TEST_CHECK(flb_aws_client_mock_generator_count_unused_requests() == 0); + + cleanup_test(provider, config); +} + + +static void test_http_provider_https_endpoint() +{ + struct flb_aws_provider *provider; + struct flb_aws_credentials *creds; + struct flb_config *config; + int ret; + + setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", "https://customers-vpc-credential-vending-server/iam_credentials/pod1", 1); + setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", TEST_AUTHORIZATION_TOKEN_FILE, 1); + + setup_test(FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/iam_credentials/pod1"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER, "Authorization", "local-http-credential-server-authorization-token"), + set(STATUS, 200), + set(PAYLOAD, "{\n \"Code\" : \"Success\",\n \"LastUpdated\" : \"2021-09-16T18:29:09Z\",\n" + " \"Type\" : \"AWS-HMAC\",\n \"AccessKeyId\" : \"XACCESSEKSXXX\",\n \"SecretAccessKey\"" + " : \"XSECRETEKSXXXXXXXXXXXXXX\",\n \"Token\" : \"XTOKENEKSXXXXXXXXXXXXXXX==\",\n" + " \"Expiration\" : \"3021-09-17T00:41:00Z\"\n}"), + set(PAYLOAD_SIZE, 257) + ), + response( + expect(URI, "/iam_credentials/pod1"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER, "Authorization", "local-http-credential-server-authorization-token"), + set(STATUS, 200), + set(PAYLOAD, "{\n \"Code\" : \"Success\",\n \"LastUpdated\" : \"2021-09-16T18:29:09Z\",\n" + " \"Type\" : \"AWS-HMAC\",\n \"AccessKeyId\" : \"YACCESSEKSXXX\",\n \"SecretAccessKey\"" + " : \"YSECRETEKSXXXXXXXXXXXXXX\",\n \"Token\" : \"YTOKENEKSXXXXXXXXXXXXXXX==\",\n" + " \"Expiration\" : \"3021-09-17T00:41:00Z\"\n}"), + set(PAYLOAD_SIZE, 257) + ) + ), &provider, &config); + + flb_time_msleep(1000); + + /* Repeated calls to get credentials should return the same set */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("XACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("XSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("XTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* Retrieve from cache */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("XACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("XSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("XTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* refresh should return 0 (success) */ + ret = provider->provider_vtable->refresh(provider); + TEST_CHECK(ret == 0); + + /* Retrieve refreshed credentials from cache */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds != NULL); + TEST_CHECK(strcmp("YACCESSEKSXXX", creds->access_key_id) == 0); + TEST_CHECK(strcmp("YSECRETEKSXXXXXXXXXXXXXX", creds->secret_access_key) == 0); + TEST_CHECK(strcmp("YTOKENEKSXXXXXXXXXXXXXXX==", creds->session_token) == 0); + + flb_aws_credentials_destroy(creds); + + /* Check we have exhausted our response list */ + TEST_CHECK(flb_aws_client_mock_generator_count_unused_requests() == 0); + + cleanup_test(provider, config); +} + +static void test_http_provider_server_failure() +{ + struct flb_aws_provider *provider; + struct flb_aws_credentials *creds; + struct flb_config *config; + int ret; + + setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", "https://customers-vpc-credential-vending-server/iam_credentials/pod1", 1); + setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", TEST_AUTHORIZATION_TOKEN_FILE, 1); + + setup_test(FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/iam_credentials/pod1"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER, "Authorization", "local-http-credential-server-authorization-token"), + set(STATUS, 400), + set(PAYLOAD, "{\"Message\": \"Invalid Authorization token\",\"Code\": \"ClientError\"}"), + set(PAYLOAD_SIZE, 64) + ), + response( + expect(URI, "/iam_credentials/pod1"), + expect(METHOD, FLB_HTTP_GET), + expect(HEADER, "Authorization", "local-http-credential-server-authorization-token"), + set(STATUS, 500), + set(PAYLOAD, "{\"Message\": \"Internal Server Error\",\"Code\": \"ServerError\"}"), + set(PAYLOAD_SIZE, 58) + ) + ), &provider, &config); + + flb_time_msleep(1000); + + /* Endpoint failure, no creds returnd */ + creds = provider->provider_vtable->get_credentials(provider); + TEST_ASSERT(creds == NULL); + + /* refresh should return 0 (success) */ + ret = provider->provider_vtable->refresh(provider); + TEST_CHECK(ret != 0); + + /* Check we have exhausted our response list */ + TEST_CHECK(flb_aws_client_mock_generator_count_unused_requests() == 0); + + cleanup_test(provider, config); +} + +static void test_http_validator_invalid_host() +{ + struct flb_aws_provider *provider; + struct flb_config *config; + + setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://104.156.107.142:80/iam_credentials/pod1", 1); + setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "password", 1); + + flb_aws_client_mock_configure_generator(NULL); + + config = flb_calloc(1, sizeof(struct flb_config)); + TEST_ASSERT(config != NULL); + mk_list_init(&config->upstreams); + + /* provider creation will fail with error message indicating host was invalid */ + provider = flb_http_provider_create(config, flb_aws_client_get_mock_generator()); + TEST_ASSERT(provider == NULL); + + flb_aws_client_mock_destroy_generator(); + flb_free(config); + +} + +static void test_http_validator_invalid_port() +{ + struct flb_aws_provider *provider; + struct flb_config *config; + + setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://104.156.107.142:AA/iam_credentials/pod1", 1); + setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "password", 1); + + flb_aws_client_mock_configure_generator(NULL); + + config = flb_calloc(1, sizeof(struct flb_config)); + TEST_ASSERT(config != NULL); + mk_list_init(&config->upstreams); + + /* provider creation will fail with error message indicating port was invalid */ + provider = flb_http_provider_create(config, flb_aws_client_get_mock_generator()); + TEST_ASSERT(provider == NULL); + + flb_aws_client_mock_destroy_generator(); + flb_free(config); + +} + TEST_LIST = { - { "test_http_provider" , test_http_provider}, - { "test_http_provider_error_case" , test_http_provider_error_case}, - { "test_http_provider_malformed_response" , - test_http_provider_malformed_response}, + { "test_http_provider", test_http_provider}, + { "test_http_provider_error_case", test_http_provider_error_case}, + { "test_http_provider_malformed_response", test_http_provider_malformed_response}, + { "test_http_provider", test_http_provider}, + { "test_http_provider_ecs_case", test_http_provider_ecs_case}, + { "test_http_provider_eks_with_token", test_http_provider_eks_with_token}, + { "test_http_provider_eks_with_token_file", test_http_provider_eks_with_token_file}, + { "test_http_provider_https_endpoint", test_http_provider_https_endpoint}, + { "test_http_provider_server_failure", test_http_provider_server_failure}, + { "test_http_validator_invalid_host", test_http_validator_invalid_host}, + { "test_http_validator_invalid_port", test_http_validator_invalid_port}, { 0 } }; diff --git a/tests/internal/data/aws_credentials/container_authorization_token.txt b/tests/internal/data/aws_credentials/container_authorization_token.txt new file mode 100644 index 00000000000..05f55515d2b --- /dev/null +++ b/tests/internal/data/aws_credentials/container_authorization_token.txt @@ -0,0 +1 @@ +local-http-credential-server-authorization-token \ No newline at end of file From d7a76688c8454736b08b488794dc4c8a69f7c181 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Aug 2024 13:54:39 -0600 Subject: [PATCH 4/6] aws: fix leaks on new EKS Pod identities Signed-off-by: Eduardo Silva --- src/aws/flb_aws_credentials_http.c | 58 +++++++++++++++++++++++------- src/aws/flb_aws_util.c | 6 ++-- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/aws/flb_aws_credentials_http.c b/src/aws/flb_aws_credentials_http.c index 0eab14bebdd..edd33d906a3 100644 --- a/src/aws/flb_aws_credentials_http.c +++ b/src/aws/flb_aws_credentials_http.c @@ -53,9 +53,9 @@ static int http_credentials_request(struct flb_aws_provider_http *implementation); -/* -If the resolved URI’s scheme is HTTPS, its hostname may be used in the request. -Otherwise, implementations MUST fail to resolve when the URI hostname +/* +If the resolved URI’s scheme is HTTPS, its hostname may be used in the request. +Otherwise, implementations MUST fail to resolve when the URI hostname does not satisfy any of the following conditions: is within the loopback CIDR (IPv4 127.0.0.0/8, IPv6 ::1/128) @@ -67,7 +67,7 @@ static int validate_http_credential_uri(flb_sds_t protocol, flb_sds_t host) return 0; } else if (strncmp(host, "127.", 4) == 0 || strncmp(host, ECS_CREDENTIALS_HOST, ECS_CREDENTIALS_HOST_LEN) == 0 || - strncmp(host, EKS_CREDENTIALS_HOST, EKS_CREDENTIALS_HOST_LEN) == 0 || + strncmp(host, EKS_CREDENTIALS_HOST, EKS_CREDENTIALS_HOST_LEN) == 0 || strstr(host, "::1") != NULL || strstr(host, "fd00:ec2::23") != NULL || strstr(host, "fe80:") != NULL) { @@ -263,7 +263,6 @@ struct flb_aws_provider *flb_endpoint_provider_create(struct flb_config *config, host, path); provider = flb_calloc(1, sizeof(struct flb_aws_provider)); - if (!provider) { flb_errno(); return NULL; @@ -322,9 +321,7 @@ struct flb_aws_provider *flb_endpoint_provider_create(struct flb_config *config, * with the ECS credentials endpoint. */ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, - struct - flb_aws_client_generator - *generator) + struct flb_aws_client_generator *generator) { flb_sds_t path = NULL; flb_sds_t protocol = NULL; @@ -351,16 +348,18 @@ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, flb_free(host); return NULL; } - } else if (full_uri && strlen(full_uri) > 0) { + } + else if (full_uri && strlen(full_uri) > 0) { ret = flb_utils_url_split_sds(full_uri, &protocol, &host, &port_sds, &path); if (ret < 0) { return NULL; } + insecure = strncmp(protocol, "http", 4) == 0 ? FLB_TRUE : FLB_FALSE; ret = validate_http_credential_uri(protocol, host); if (ret < 0) { flb_error("[aws credentials] %s must be set to an https:// address or a link local IP address." - " Found protocol=%s, host=%s, port=%s, path=%s", + " Found protocol=%s, host=%s, port=%s, path=%s", AWS_CREDENTIALS_FULL_URI, protocol, host, port_sds, path); flb_sds_destroy(protocol); flb_sds_destroy(host); @@ -368,7 +367,8 @@ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, flb_sds_destroy(path); return NULL; } - } else { + } + else { flb_debug("[aws_credentials] Not initializing ECS/EKS HTTP Provider because" " %s and %s is not set", AWS_CREDENTIALS_PATH, AWS_CREDENTIALS_FULL_URI); return NULL; @@ -378,7 +378,7 @@ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, port = atoi(port_sds); if (port == 0) { flb_error("[aws credentials] invalid port: %s must be set to an https:// address or a link local IP address." - " Found protocol=%s, host=%s, port=%s, path=%s", + " Found protocol=%s, host=%s, port=%s, path=%s", AWS_CREDENTIALS_FULL_URI, protocol, host, port_sds, path); flb_sds_destroy(protocol); flb_sds_destroy(host); @@ -388,6 +388,9 @@ struct flb_aws_provider *flb_http_provider_create(struct flb_config *config, } } + flb_sds_destroy(port_sds); + flb_sds_destroy(protocol); + return flb_endpoint_provider_create(config, host, path, port, insecure, generator); } @@ -412,14 +415,30 @@ static int http_credentials_request(struct flb_aws_provider_http struct flb_aws_client *client = implementation->client; struct flb_http_client *c = NULL; int ret; + char *tmp; char *auth_token = NULL; size_t auth_token_size = 0; char *auth_token_path = NULL; auth_token_path = getenv(AUTH_TOKEN_FILE_ENV_VAR); - auth_token = getenv(AUTH_TOKEN_ENV_VAR); + tmp = getenv(AUTH_TOKEN_ENV_VAR); + if (tmp) { + auth_token = flb_malloc(strlen(tmp) + 1); + if (!auth_token) { + flb_errno(); + return -1; + } + strcpy(auth_token, tmp); + } + if (auth_token_path != NULL && strlen(auth_token_path) > 0) { flb_debug("[aws] reading authorization token from %s", auth_token_path); + + if (auth_token) { + flb_free(auth_token); + auth_token = NULL; + } + ret = flb_read_file(auth_token_path, &auth_token, &auth_token_size); if (ret < 0) { @@ -441,6 +460,11 @@ static int http_credentials_request(struct flb_aws_provider_http NULL, 0); } + if (auth_token) { + flb_free(auth_token); + auth_token = NULL; + } + if (!c || c->resp.status != 200) { flb_debug("[aws_credentials] http credentials request failed"); if (c) { @@ -450,15 +474,22 @@ static int http_credentials_request(struct flb_aws_provider_http } flb_http_client_destroy(c); } + if (auth_token) { + flb_free(auth_token); + } return -1; } + response = c->resp.payload; response_len = c->resp.payload_size; creds = flb_parse_http_credentials(response, response_len, &expiration); if (!creds) { flb_http_client_destroy(c); + if (auth_token) { + flb_free(auth_token); + } return -1; } @@ -469,6 +500,7 @@ static int http_credentials_request(struct flb_aws_provider_http implementation->creds = creds; implementation->next_refresh = expiration - FLB_AWS_REFRESH_WINDOW; flb_http_client_destroy(c); + return 0; } diff --git a/src/aws/flb_aws_util.c b/src/aws/flb_aws_util.c index 56940c40a71..37584d89f3a 100644 --- a/src/aws/flb_aws_util.c +++ b/src/aws/flb_aws_util.c @@ -215,8 +215,7 @@ struct flb_http_client *flb_aws_client_request_basic_auth( struct flb_aws_client *aws_client, int method, const char *uri, const char *body, size_t body_len, - struct flb_aws_header - *dynamic_headers, + struct flb_aws_header *dynamic_headers, size_t dynamic_headers_len, char *header_name, char* auth_token) @@ -224,12 +223,13 @@ struct flb_http_client *flb_aws_client_request_basic_auth( struct flb_http_client *c = NULL; struct flb_aws_header *auth_header = NULL; struct flb_aws_header *headers = NULL; + auth_header = flb_calloc(1, sizeof(struct flb_aws_header)); if (!auth_header) { flb_errno(); return NULL; } - + auth_header->key = header_name; auth_header->key_len = strlen(header_name); auth_header->val = auth_token; From f0e6f2bc13ee4f9e7e02d332fcdca24b44d3db03 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Aug 2024 13:55:16 -0600 Subject: [PATCH 5/6] utils: validate sds port Signed-off-by: Eduardo Silva --- src/flb_utils.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/flb_utils.c b/src/flb_utils.c index 180d690efb9..bd73bce7d90 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -1138,6 +1138,7 @@ int flb_utils_url_split(const char *in_url, char **out_protocol, int flb_utils_url_split_sds(const flb_sds_t in_url, flb_sds_t *out_protocol, flb_sds_t *out_host, flb_sds_t *out_port, flb_sds_t *out_uri) { + int i; flb_sds_t protocol = NULL; flb_sds_t host = NULL; flb_sds_t port = NULL; @@ -1219,10 +1220,21 @@ int flb_utils_url_split_sds(const flb_sds_t in_url, flb_sds_t *out_protocol, flb_errno(); goto error; } + if (!port) { flb_errno(); goto error; } + else { + /* check that port is a number */ + for (i = 0; i < flb_sds_len(port); i++) { + if (!isdigit(port[i])) { + goto error; + } + } + + } + if (!uri) { flb_errno(); goto error; @@ -1237,7 +1249,16 @@ int flb_utils_url_split_sds(const flb_sds_t in_url, flb_sds_t *out_protocol, error: if (protocol) { - flb_free(protocol); + flb_sds_destroy(protocol); + } + if (host) { + flb_sds_destroy(host); + } + if (port) { + flb_sds_destroy(port); + } + if (uri) { + flb_sds_destroy(uri); } return -1; From ec2209c948ab0d825c5f1da357d84c96d7f4b079 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Aug 2024 13:55:57 -0600 Subject: [PATCH 6/6] tests: internal: aws_credentials_http: fix leaks Signed-off-by: Eduardo Silva --- tests/internal/aws_credentials_http.c | 51 ++------------------------- 1 file changed, 3 insertions(+), 48 deletions(-) diff --git a/tests/internal/aws_credentials_http.c b/tests/internal/aws_credentials_http.c index 6e43b7f4d1e..45bf6912673 100644 --- a/tests/internal/aws_credentials_http.c +++ b/tests/internal/aws_credentials_http.c @@ -183,8 +183,6 @@ static void test_http_provider() struct flb_aws_credentials *creds; int ret; struct flb_config *config; - flb_sds_t host; - flb_sds_t path; g_request_count = 0; @@ -194,19 +192,6 @@ static void test_http_provider() return; } - host = flb_sds_create("127.0.0.1"); - if (!host) { - flb_errno(); - flb_config_exit(config); - return; - } - path = flb_sds_create("/happy-case"); - if (!path) { - flb_errno(); - flb_config_exit(config); - return; - } - provider = flb_http_provider_create(config, generator_in_test()); if (!provider) { @@ -261,8 +246,6 @@ static void test_http_provider_error_case() struct flb_aws_credentials *creds; int ret; struct flb_config *config; - flb_sds_t host; - flb_sds_t path; g_request_count = 0; @@ -272,21 +255,7 @@ static void test_http_provider_error_case() return; } - host = flb_sds_create("127.0.0.1"); - if (!host) { - flb_errno(); - flb_config_exit(config); - return; - } - path = flb_sds_create("/error-case"); - if (!path) { - flb_errno(); - flb_config_exit(config); - return; - } - provider = flb_http_provider_create(config, generator_in_test()); - if (!provider) { flb_errno(); flb_config_exit(config); @@ -311,6 +280,7 @@ static void test_http_provider_error_case() */ TEST_CHECK(g_request_count == 3); + flb_aws_provider_destroy(provider); flb_config_exit(config); } @@ -321,8 +291,6 @@ static void test_http_provider_malformed_response() struct flb_aws_credentials *creds; int ret; struct flb_config *config; - flb_sds_t host; - flb_sds_t path; g_request_count = 0; @@ -334,19 +302,6 @@ static void test_http_provider_malformed_response() mk_list_init(&config->upstreams); - host = flb_sds_create("127.0.0.1"); - if (!host) { - flb_errno(); - flb_config_exit(config); - return; - } - path = flb_sds_create("/malformed"); - if (!path) { - flb_errno(); - flb_config_exit(config); - return; - } - provider = flb_http_provider_create(config, generator_in_test()); if (!provider) { @@ -774,7 +729,6 @@ static void test_http_validator_invalid_host() flb_aws_client_mock_destroy_generator(); flb_free(config); - } static void test_http_validator_invalid_port() @@ -789,6 +743,8 @@ static void test_http_validator_invalid_port() config = flb_calloc(1, sizeof(struct flb_config)); TEST_ASSERT(config != NULL); + TEST_ASSERT(config != NULL); + mk_list_init(&config->upstreams); /* provider creation will fail with error message indicating port was invalid */ @@ -797,7 +753,6 @@ static void test_http_validator_invalid_port() flb_aws_client_mock_destroy_generator(); flb_free(config); - } TEST_LIST = {