From 398a519a8740e909493a4434ca6f9fb88ab450a5 Mon Sep 17 00:00:00 2001 From: commiterate <111539270+commiterate@users.noreply.github.com> Date: Sun, 15 Dec 2024 18:35:58 -0500 Subject: [PATCH] filter_aws: add more IMDS categories Signed-off-by: commiterate <111539270+commiterate@users.noreply.github.com> --- include/fluent-bit/aws/flb_aws_imds.h | 29 +- plugins/filter_aws/aws.c | 495 +++++++++++++- plugins/filter_aws/aws.h | 63 ++ tests/runtime/filter_aws.c | 912 +++++++++++++++++++++++++- 4 files changed, 1482 insertions(+), 17 deletions(-) diff --git a/include/fluent-bit/aws/flb_aws_imds.h b/include/fluent-bit/aws/flb_aws_imds.h index d135d473fad..d328ba06a43 100644 --- a/include/fluent-bit/aws/flb_aws_imds.h +++ b/include/fluent-bit/aws/flb_aws_imds.h @@ -31,16 +31,25 @@ /* The following metadata paths can be evaluated with flb_aws_imds_request * to retrieve specific metadata members */ -#define FLB_AWS_IMDS_INSTANCE_ID_PATH "/latest/meta-data/instance-id/" -#define FLB_AWS_IMDS_AZ_PATH "/latest/meta-data/placement/availability-zone/" -#define FLB_AWS_IMDS_INSTANCE_TYPE_PATH "/latest/meta-data/instance-type/" -#define FLB_AWS_IMDS_PRIVATE_IP_PATH "/latest/meta-data/local-ipv4/" -#define FLB_AWS_IMDS_VPC_ID_PATH_PREFIX "/latest/meta-data/network/interfaces/macs/" -#define FLB_AWS_IMDS_AMI_ID_PATH "/latest/meta-data/ami-id/" -#define FLB_AWS_IMDS_ACCOUNT_ID_PATH "/latest/dynamic/instance-identity/document/" -#define FLB_AWS_IMDS_HOSTNAME_PATH "/latest/meta-data/hostname/" -#define FLB_AWS_IMDS_MAC_PATH "/latest/meta-data/mac/" -#define FLB_AWS_IMDS_INSTANCE_TAG "/latest/meta-data/tags/instance" +#define FLB_AWS_IMDS_INSTANCE_ID_PATH "/latest/meta-data/instance-id/" +#define FLB_AWS_IMDS_PARTITION_PATH "/latest/meta-data/services/partition/" +#define FLB_AWS_IMDS_DOMAIN_PATH "/latest/meta-data/services/domain/" +#define FLB_AWS_IMDS_REGION_PATH "/latest/meta-data/placement/region/" +#define FLB_AWS_IMDS_AZ_PATH "/latest/meta-data/placement/availability-zone/" +#define FLB_AWS_IMDS_AZ_ID_PATH "/latest/meta-data/placement/availability-zone-id/" +#define FLB_AWS_IMDS_PLACEMENT_GROUP_PATH "/latest/meta-data/placement/group-name/" +#define FLB_AWS_IMDS_PARTITION_NUMBER_PATH "/latest/meta-data/placement/partition-number/" +#define FLB_AWS_IMDS_HOST_ID_PATH "/latest/meta-data/placement/host-id/" +#define FLB_AWS_IMDS_INSTANCE_TYPE_PATH "/latest/meta-data/instance-type/" +#define FLB_AWS_IMDS_PRIVATE_IP_PATH "/latest/meta-data/local-ipv4/" +#define FLB_AWS_IMDS_PUBLIC_IP_PATH "/latest/meta-data/public-ipv4/" +#define FLB_AWS_IMDS_IPV6_PATH "/latest/meta-data/ipv6/" +#define FLB_AWS_IMDS_VPC_ID_PATH_PREFIX "/latest/meta-data/network/interfaces/macs/" +#define FLB_AWS_IMDS_AMI_ID_PATH "/latest/meta-data/ami-id/" +#define FLB_AWS_IMDS_ACCOUNT_ID_PATH "/latest/dynamic/instance-identity/document/" +#define FLB_AWS_IMDS_HOSTNAME_PATH "/latest/meta-data/hostname/" +#define FLB_AWS_IMDS_MAC_PATH "/latest/meta-data/mac/" +#define FLB_AWS_IMDS_INSTANCE_TAG "/latest/meta-data/tags/instance" #include #include diff --git a/plugins/filter_aws/aws.c b/plugins/filter_aws/aws.c index c117ffe16ba..4214189e138 100644 --- a/plugins/filter_aws/aws.c +++ b/plugins/filter_aws/aws.c @@ -54,6 +54,36 @@ static void expose_aws_meta(struct flb_filter_aws *ctx) flb_env_set(env, "aws", "enabled"); + if (ctx->group_partition.done && + !ctx->group_partition.exposed) { + if (ctx->partition_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_PARTITION_KEY, + ctx->partition); + } + ctx->group_partition.exposed = FLB_TRUE; + } + + if (ctx->group_domain.done && + !ctx->group_domain.exposed) { + if (ctx->domain_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_DOMAIN_KEY, + ctx->domain); + } + ctx->group_domain.exposed = FLB_TRUE; + } + + if (ctx->group_region.done && + !ctx->group_region.exposed) { + if (ctx->region_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_REGION_KEY, + ctx->region); + } + ctx->group_region.exposed = FLB_TRUE; + } + if (ctx->group_az.done && !ctx->group_az.exposed) { if (ctx->availability_zone_include) { @@ -64,6 +94,46 @@ static void expose_aws_meta(struct flb_filter_aws *ctx) ctx->group_az.exposed = FLB_TRUE; } + if (ctx->group_az_id.done && + !ctx->group_az_id.exposed) { + if (ctx->availability_zone_id_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_AVAILABILITY_ZONE_ID_KEY, + ctx->availability_zone_id); + } + ctx->group_az_id.exposed = FLB_TRUE; + } + + if (ctx->group_placement_group.done && + !ctx->group_placement_group.exposed) { + if (ctx->placement_group_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_PLACEMENT_GROUP_KEY, + ctx->placement_group); + } + ctx->group_placement_group.exposed = FLB_TRUE; + } + + if (ctx->group_partition_number.done && + !ctx->group_partition_number.exposed) { + if (ctx->partition_number_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_PARTITION_NUMBER_KEY, + ctx->partition_number); + } + ctx->group_partition_number.exposed = FLB_TRUE; + } + + if (ctx->group_host_id.done && + !ctx->group_host_id.exposed) { + if (ctx->host_id_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_HOST_ID_KEY, + ctx->host_id); + } + ctx->group_host_id.exposed = FLB_TRUE; + } + if (ctx->group_instance_id.done && !ctx->group_instance_id.exposed) { if (ctx->instance_id_include) { @@ -94,6 +164,26 @@ static void expose_aws_meta(struct flb_filter_aws *ctx) ctx->group_private_ip.exposed = FLB_TRUE; } + if (ctx->group_public_ip.done && + !ctx->group_public_ip.exposed) { + if (ctx->public_ip_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_PUBLIC_IP_KEY, + ctx->public_ip); + } + ctx->group_public_ip.exposed = FLB_TRUE; + } + + if (ctx->group_ipv6.done && + !ctx->group_ipv6.exposed) { + if (ctx->ipv6_include) { + flb_env_set(env, + "aws." FLB_FILTER_AWS_IPV6_KEY, + ctx->ipv6); + } + ctx->group_ipv6.exposed = FLB_TRUE; + } + if (ctx->group_vpc_id.done && !ctx->group_vpc_id.exposed) { if (ctx->vpc_id_include) { @@ -687,6 +777,40 @@ static int get_ec2_metadata_private_ip(struct flb_filter_aws *ctx) return 0; } +static int get_ec2_metadata_public_ip(struct flb_filter_aws *ctx) +{ + int ret; + + if (ctx->public_ip_include && !ctx->public_ip) { + ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_PUBLIC_IP_PATH, + &ctx->public_ip, &ctx->public_ip_len); + + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to get instance public IP"); + return -1; + } + } + + return 0; +} + +static int get_ec2_metadata_ipv6(struct flb_filter_aws *ctx) +{ + int ret; + + if (ctx->ipv6_include && !ctx->ipv6) { + ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_IPV6_PATH, + &ctx->ipv6, &ctx->ipv6_len); + + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to get instance IPv6"); + return -1; + } + } + + return 0; +} + static int get_ec2_metadata_vpc_id(struct flb_filter_aws *ctx) { int ret; @@ -756,6 +880,60 @@ static int get_ec2_metadata_hostname(struct flb_filter_aws *ctx) return 0; } +static int get_ec2_metadata_partition(struct flb_filter_aws *ctx) +{ + int ret; + + if (ctx->partition_include && !ctx->partition) { + ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_PARTITION_PATH, + &ctx->partition, + &ctx->partition_len); + + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to get instance partition"); + return -1; + } + } + + return 0; +} + +static int get_ec2_metadata_domain(struct flb_filter_aws *ctx) +{ + int ret; + + if (ctx->domain_include && !ctx->domain) { + ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_DOMAIN_PATH, + &ctx->domain, + &ctx->domain_len); + + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to get instance domain"); + return -1; + } + } + + return 0; +} + +static int get_ec2_metadata_region(struct flb_filter_aws *ctx) +{ + int ret; + + if (ctx->region_include && !ctx->region) { + ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_REGION_PATH, + &ctx->region, + &ctx->region_len); + + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to get instance region"); + return -1; + } + } + + return 0; +} + static int get_ec2_metadata_az(struct flb_filter_aws *ctx) { int ret; @@ -774,6 +952,77 @@ static int get_ec2_metadata_az(struct flb_filter_aws *ctx) return 0; } +static int get_ec2_metadata_az_id(struct flb_filter_aws *ctx) +{ + int ret; + + if (ctx->availability_zone_id_include && !ctx->availability_zone_id) { + ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_AZ_ID_PATH, + &ctx->availability_zone_id, + &ctx->availability_zone_id_len); + + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to get instance AZ ID"); + return -1; + } + } + + return 0; +} + +static int get_ec2_metadata_placement_group(struct flb_filter_aws *ctx) +{ + int ret; + + if (ctx->placement_group_include && !ctx->placement_group) { + ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_PLACEMENT_GROUP_PATH, + &ctx->placement_group, + &ctx->placement_group_len); + + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to get instance placement group"); + return -1; + } + } + + return 0; +} + +static int get_ec2_metadata_partition_number(struct flb_filter_aws *ctx) +{ + int ret; + + if (ctx->partition_number_include && !ctx->partition_number) { + ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_PARTITION_NUMBER_PATH, + &ctx->partition_number, + &ctx->partition_number_len); + + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to get instance partition number"); + return -1; + } + } + + return 0; +} + +static int get_ec2_metadata_host_id(struct flb_filter_aws *ctx) +{ + int ret; + + if (ctx->host_id_include && !ctx->host_id) { + ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_HOST_ID_PATH, + &ctx->host_id, + &ctx->host_id_len); + + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to get instance host ID"); + return -1; + } + } + + return 0; +} static int get_ec2_metadata_tags(struct flb_filter_aws *ctx) { @@ -804,7 +1053,7 @@ static int ec2_metadata_group_should_fetch(struct flb_filter_aws *ctx, interval = now - group->last_fetch_attempt; - if (group->last_fetch_attempt > 0 && + if (group->last_fetch_attempt > 0 && interval < required_interval) { return FLB_FALSE; } @@ -882,6 +1131,23 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) metadata_fetched = FLB_FALSE; } + ret = get_ec2_metadata_group(ctx, &ctx->group_public_ip, + get_ec2_metadata_public_ip); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_ipv6, get_ec2_metadata_ipv6); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + ret = get_ec2_metadata_group(ctx, &ctx->group_vpc_id, get_ec2_metadata_vpc_id); if (ret < 0) { if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { @@ -906,6 +1172,30 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) metadata_fetched = FLB_FALSE; } + ret = get_ec2_metadata_group(ctx, &ctx->group_partition, get_ec2_metadata_partition); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_domain, get_ec2_metadata_domain); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_region, get_ec2_metadata_region); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + ret = get_ec2_metadata_group(ctx, &ctx->group_az, get_ec2_metadata_az); if (ret < 0) { if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { @@ -914,6 +1204,38 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx) metadata_fetched = FLB_FALSE; } + ret = get_ec2_metadata_group(ctx, &ctx->group_az_id, get_ec2_metadata_az_id); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_placement_group, get_ec2_metadata_placement_group); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_partition_number, get_ec2_metadata_partition_number); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + + ret = get_ec2_metadata_group(ctx, &ctx->group_host_id, get_ec2_metadata_host_id); + if (ret < 0) { + if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { + return ret; + } + metadata_fetched = FLB_FALSE; + } + ret = get_ec2_metadata_group(ctx, &ctx->group_tag, get_ec2_metadata_tags); if (ret < 0) { if (ret == FLB_FILTER_AWS_CONFIGURATION_ERROR) { @@ -1008,6 +1330,36 @@ static int cb_aws_filter(const void *data, size_t bytes, } /* append new keys */ + if (ctx->partition_include && + ctx->group_partition.done && + ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_values( + &log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_PARTITION_KEY), + FLB_LOG_EVENT_STRING_VALUE(ctx->partition, + ctx->partition_len)); + } + + if (ctx->domain_include && + ctx->group_domain.done && + ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_values( + &log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_DOMAIN_KEY), + FLB_LOG_EVENT_STRING_VALUE(ctx->domain, + ctx->domain_len)); + } + + if (ctx->region_include && + ctx->group_region.done && + ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_values( + &log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_REGION_KEY), + FLB_LOG_EVENT_STRING_VALUE(ctx->region, + ctx->region_len)); + } + if (ctx->availability_zone_include && ctx->group_az.done && ret == FLB_EVENT_ENCODER_SUCCESS) { @@ -1018,6 +1370,46 @@ static int cb_aws_filter(const void *data, size_t bytes, ctx->availability_zone_len)); } + if (ctx->availability_zone_id_include && + ctx->group_az_id.done && + ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_values( + &log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_AVAILABILITY_ZONE_ID_KEY), + FLB_LOG_EVENT_STRING_VALUE(ctx->availability_zone_id, + ctx->availability_zone_id_len)); + } + + if (ctx->placement_group_include && + ctx->group_placement_group.done && + ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_values( + &log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_PLACEMENT_GROUP_KEY), + FLB_LOG_EVENT_STRING_VALUE(ctx->placement_group, + ctx->placement_group_len)); + } + + if (ctx->partition_number_include && + ctx->group_partition_number.done && + ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_values( + &log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_PARTITION_NUMBER_KEY), + FLB_LOG_EVENT_STRING_VALUE(ctx->partition_number, + ctx->partition_number_len)); + } + + if (ctx->host_id_include && + ctx->group_host_id.done && + ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_values( + &log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_HOST_ID_KEY), + FLB_LOG_EVENT_STRING_VALUE(ctx->host_id, + ctx->host_id_len)); + } + if (ctx->instance_id_include && ctx->group_instance_id.done && ret == FLB_EVENT_ENCODER_SUCCESS) { @@ -1048,6 +1440,26 @@ static int cb_aws_filter(const void *data, size_t bytes, ctx->private_ip_len)); } + if (ctx->public_ip_include && + ctx->group_public_ip.done && + ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_values( + &log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_PUBLIC_IP_KEY), + FLB_LOG_EVENT_STRING_VALUE(ctx->public_ip, + ctx->public_ip_len)); + } + + if (ctx->ipv6_include && + ctx->group_ipv6.done && + ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_values( + &log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_IPV6_KEY), + FLB_LOG_EVENT_STRING_VALUE(ctx->ipv6, + ctx->ipv6_len)); + } + if (ctx->vpc_id_include && ctx->group_vpc_id.done && ret == FLB_EVENT_ENCODER_SUCCESS) { @@ -1150,10 +1562,38 @@ static void flb_filter_aws_destroy(struct flb_filter_aws *ctx) flb_aws_imds_destroy(ctx->client_imds); } + if (ctx->partition) { + flb_sds_destroy(ctx->partition); + } + + if (ctx->domain) { + flb_sds_destroy(ctx->domain); + } + + if (ctx->region) { + flb_sds_destroy(ctx->region); + } + if (ctx->availability_zone) { flb_sds_destroy(ctx->availability_zone); } + if (ctx->availability_zone_id) { + flb_sds_destroy(ctx->availability_zone_id); + } + + if (ctx->placement_group) { + flb_sds_destroy(ctx->placement_group); + } + + if (ctx->partition_number) { + flb_sds_destroy(ctx->partition_number); + } + + if (ctx->host_id) { + flb_sds_destroy(ctx->host_id); + } + if (ctx->instance_id) { flb_sds_destroy(ctx->instance_id); } @@ -1166,6 +1606,14 @@ static void flb_filter_aws_destroy(struct flb_filter_aws *ctx) flb_sds_destroy(ctx->private_ip); } + if (ctx->public_ip) { + flb_sds_destroy(ctx->public_ip); + } + + if (ctx->ipv6) { + flb_sds_destroy(ctx->ipv6); + } + if (ctx->vpc_id) { flb_sds_destroy(ctx->vpc_id); } @@ -1206,11 +1654,46 @@ static struct flb_config_map config_map[] = { " will be used: 'v1' or 'v2'. 'v2' may not work" " if you run Fluent Bit in a container." }, + { + FLB_CONFIG_MAP_BOOL, "partition", "false", + 0, FLB_TRUE, offsetof(struct flb_filter_aws, partition_include), + "Enable EC2 instance partition" + }, + { + FLB_CONFIG_MAP_BOOL, "domain", "false", + 0, FLB_TRUE, offsetof(struct flb_filter_aws, domain_include), + "Enable EC2 instance domain" + }, + { + FLB_CONFIG_MAP_BOOL, "region", "false", + 0, FLB_TRUE, offsetof(struct flb_filter_aws, region_include), + "Enable EC2 instance region" + }, { FLB_CONFIG_MAP_BOOL, "az", "true", 0, FLB_TRUE, offsetof(struct flb_filter_aws, availability_zone_include), "Enable EC2 instance availability zone" }, + { + FLB_CONFIG_MAP_BOOL, "az_id", "false", + 0, FLB_TRUE, offsetof(struct flb_filter_aws, availability_zone_id_include), + "Enable EC2 instance availability zone ID" + }, + { + FLB_CONFIG_MAP_BOOL, "placement_group", "false", + 0, FLB_TRUE, offsetof(struct flb_filter_aws, placement_group_include), + "Enable EC2 instance placement group" + }, + { + FLB_CONFIG_MAP_BOOL, "partition_number", "false", + 0, FLB_TRUE, offsetof(struct flb_filter_aws, partition_number_include), + "Enable EC2 instance partition number" + }, + { + FLB_CONFIG_MAP_BOOL, "host_id", "false", + 0, FLB_TRUE, offsetof(struct flb_filter_aws, host_id_include), + "Enable EC2 instance host ID" + }, { FLB_CONFIG_MAP_BOOL, "ec2_instance_id", "true", 0, FLB_TRUE, offsetof(struct flb_filter_aws, instance_id_include), @@ -1226,6 +1709,16 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_filter_aws, private_ip_include), "Enable EC2 instance private IP" }, + { + FLB_CONFIG_MAP_BOOL, "public_ip", "false", + 0, FLB_TRUE, offsetof(struct flb_filter_aws, public_ip_include), + "Enable EC2 instance public IP" + }, + { + FLB_CONFIG_MAP_BOOL, "ipv6", "false", + 0, FLB_TRUE, offsetof(struct flb_filter_aws, ipv6_include), + "Enable EC2 instance IPv6" + }, { FLB_CONFIG_MAP_BOOL, "vpc_id", "false", 0, FLB_TRUE, offsetof(struct flb_filter_aws, vpc_id_include), diff --git a/plugins/filter_aws/aws.h b/plugins/filter_aws/aws.h index f369f0bbcc7..57f56f964ff 100644 --- a/plugins/filter_aws/aws.h +++ b/plugins/filter_aws/aws.h @@ -23,14 +23,32 @@ #include #include +#define FLB_FILTER_AWS_PARTITION_KEY "partition" +#define FLB_FILTER_AWS_PARTITION_KEY_LEN 9 +#define FLB_FILTER_AWS_DOMAIN_KEY "domain" +#define FLB_FILTER_AWS_DOMAIN_KEY_LEN 6 +#define FLB_FILTER_AWS_REGION_KEY "region" +#define FLB_FILTER_AWS_REGION_KEY_LEN 6 #define FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY "az" #define FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY_LEN 2 +#define FLB_FILTER_AWS_AVAILABILITY_ZONE_ID_KEY "az_id" +#define FLB_FILTER_AWS_AVAILABILITY_ZONE_ID_KEY_LEN 5 +#define FLB_FILTER_AWS_PLACEMENT_GROUP_KEY "placement_group" +#define FLB_FILTER_AWS_PLACEMENT_GROUP_KEY_LEN 15 +#define FLB_FILTER_AWS_PARTITION_NUMBER_KEY "partition_number" +#define FLB_FILTER_AWS_PARTITION_NUMBER_KEY_LEN 16 +#define FLB_FILTER_AWS_HOST_ID_KEY "host_id" +#define FLB_FILTER_AWS_HOST_ID_KEY_LEN 7 #define FLB_FILTER_AWS_INSTANCE_ID_KEY "ec2_instance_id" #define FLB_FILTER_AWS_INSTANCE_ID_KEY_LEN 15 #define FLB_FILTER_AWS_INSTANCE_TYPE_KEY "ec2_instance_type" #define FLB_FILTER_AWS_INSTANCE_TYPE_KEY_LEN 17 #define FLB_FILTER_AWS_PRIVATE_IP_KEY "private_ip" #define FLB_FILTER_AWS_PRIVATE_IP_KEY_LEN 10 +#define FLB_FILTER_AWS_PUBLIC_IP_KEY "public_ip" +#define FLB_FILTER_AWS_PUBLIC_IP_KEY_LEN 9 +#define FLB_FILTER_AWS_IPV6_KEY "ipv6" +#define FLB_FILTER_AWS_IPV6_KEY_LEN 4 #define FLB_FILTER_AWS_VPC_ID_KEY "vpc_id" #define FLB_FILTER_AWS_VPC_ID_KEY_LEN 6 #define FLB_FILTER_AWS_AMI_ID_KEY "ami_id" @@ -75,10 +93,38 @@ struct flb_filter_aws { /* Metadata fields * These are queried only once; ec2 metadata is assumed to be immutable */ + flb_sds_t partition; + size_t partition_len; + int partition_include; + + flb_sds_t domain; + size_t domain_len; + int domain_include; + + flb_sds_t region; + size_t region_len; + int region_include; + flb_sds_t availability_zone; size_t availability_zone_len; int availability_zone_include; + flb_sds_t availability_zone_id; + size_t availability_zone_id_len; + int availability_zone_id_include; + + flb_sds_t placement_group; + size_t placement_group_len; + int placement_group_include; + + flb_sds_t partition_number; + size_t partition_number_len; + int partition_number_include; + + flb_sds_t host_id; + size_t host_id_len; + int host_id_include; + flb_sds_t instance_id; size_t instance_id_len; int instance_id_include; @@ -91,6 +137,14 @@ struct flb_filter_aws { size_t private_ip_len; int private_ip_include; + flb_sds_t public_ip; + size_t public_ip_len; + int public_ip_include; + + flb_sds_t ipv6; + size_t ipv6_len; + int ipv6_include; + flb_sds_t vpc_id; size_t vpc_id_len; int vpc_id_include; @@ -132,10 +186,19 @@ struct flb_filter_aws { /* metadata group contains information for potential retries and * if group was already fetched successfully */ + struct flb_filter_aws_metadata_group group_partition; + struct flb_filter_aws_metadata_group group_domain; + struct flb_filter_aws_metadata_group group_region; struct flb_filter_aws_metadata_group group_az; + struct flb_filter_aws_metadata_group group_az_id; + struct flb_filter_aws_metadata_group group_placement_group; + struct flb_filter_aws_metadata_group group_partition_number; + struct flb_filter_aws_metadata_group group_host_id; struct flb_filter_aws_metadata_group group_instance_id; struct flb_filter_aws_metadata_group group_instance_type; struct flb_filter_aws_metadata_group group_private_ip; + struct flb_filter_aws_metadata_group group_public_ip; + struct flb_filter_aws_metadata_group group_ipv6; struct flb_filter_aws_metadata_group group_vpc_id; struct flb_filter_aws_metadata_group group_ami_id; struct flb_filter_aws_metadata_group group_account_id; diff --git a/tests/runtime/filter_aws.c b/tests/runtime/filter_aws.c index f25430b1bda..080d90b54ea 100644 --- a/tests/runtime/filter_aws.c +++ b/tests/runtime/filter_aws.c @@ -337,6 +337,204 @@ void flb_test_aws_private_ip() { set_output(NULL); } +void flb_test_aws_public_ip() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/public-ipv4/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "10.158.112.84"), + set(PAYLOAD_SIZE, 13) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "public_ip", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"public_ip\":\"10.158.112.84\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_ipv6() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/ipv6/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "2001:0db8:85a3:0000:0000:8a2e:0370:7334"), + set(PAYLOAD_SIZE, 39) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ipv6", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"ipv6\":\"2001:0db8:85a3:0000:0000:8a2e:0370:7334\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + void flb_test_aws_vpc_id() { int ret; int bytes; @@ -740,7 +938,7 @@ void flb_test_aws_hostname() { set_output(NULL); } -void flb_test_aws_az() { +void flb_test_aws_partition() { int ret; int bytes; char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; @@ -757,11 +955,11 @@ void flb_test_aws_az() { request_chain = FLB_AWS_CLIENT_MOCK( response( - expect(URI, "/latest/meta-data/placement/availability-zone/"), + expect(URI, "/latest/meta-data/services/partition/"), expect(METHOD, FLB_HTTP_GET), set(STATUS, 200), - set(PAYLOAD, "us-east-1a"), - set(PAYLOAD_SIZE, 10) + set(PAYLOAD, "aws"), + set(PAYLOAD_SIZE, 3) ) ); flb_aws_client_mock_configure_generator(request_chain); @@ -800,7 +998,9 @@ void flb_test_aws_az() { TEST_CHECK(ret == 0); ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); TEST_CHECK(ret == 0); - ret = flb_filter_set(ctx, filter_ffd, "az", "true", NULL); + ret = flb_filter_set(ctx, filter_ffd, "partition", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); TEST_CHECK(ret == 0); ret = flb_start(ctx); @@ -815,7 +1015,698 @@ void flb_test_aws_az() { output = get_output(); if (output) { - result = strstr(output, "\"az\":\"us-east-1a\""); + result = strstr(output, "\"partition\":\"aws\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_domain() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/services/domain/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "amazonaws.com"), + set(PAYLOAD_SIZE, 13) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "domain", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"domain\":\"amazonaws.com\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_region() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/placement/region/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "us-east-1"), + set(PAYLOAD_SIZE, 9) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "region", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"region\":\"us-east-1\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_az() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/placement/availability-zone/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "us-east-1a"), + set(PAYLOAD_SIZE, 10) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "true", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"az\":\"us-east-1a\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_az_id() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/placement/availability-zone-id/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "use1-az1"), + set(PAYLOAD_SIZE, 8) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az_id", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"az_id\":\"use1-az1\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_placement_group() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/placement/group-name/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "my_placement_group"), + set(PAYLOAD_SIZE, 18) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "placement_group", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"placement_group\":\"my_placement_group\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_partition_number() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/placement/partition-number/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "1"), + set(PAYLOAD_SIZE, 1) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "partition_number", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"partition_number\":\"1\""); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + result = strstr(output, "hello, from my ec2 instance"); + if (!TEST_CHECK(result != NULL)) { + TEST_MSG("output:%s\n", output); + } + } + else { + TEST_CHECK(false); + TEST_MSG("output is empty\n"); + } + + flb_stop(ctx); + flb_aws_client_mock_destroy_generator(); + flb_destroy(ctx); + flb_free(ops); + + set_output(NULL); +} + +void flb_test_aws_host_id() { + int ret; + int bytes; + char *p = "[0, {\"log\": \"hello, from my ec2 instance\"}]"; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + struct flb_aws_client_generator *client_generator; + struct flb_filter_aws_init_options *ops; + struct flb_aws_client_mock_request_chain *request_chain; + char *output = NULL; + char *result; + + request_chain = FLB_AWS_CLIENT_MOCK( + response( + expect(URI, "/latest/meta-data/placement/host-id/"), + expect(METHOD, FLB_HTTP_GET), + set(STATUS, 200), + set(PAYLOAD, "h-012a3456b7890cdef"), + set(PAYLOAD_SIZE, 19) + ) + ); + flb_aws_client_mock_configure_generator(request_chain); + + client_generator = flb_aws_client_get_mock_generator(); + ops = flb_calloc(1, sizeof(struct flb_filter_aws_init_options)); + if (ops == NULL) { + TEST_MSG("calloc for aws plugin options failed\n"); + TEST_CHECK(false); + return; + } + ops->client_generator = client_generator; + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + + filter_ffd = flb_filter(ctx, (char *) "aws", ops); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, "match", "*", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "ec2_instance_id", "false", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "host_id", "true", NULL); + TEST_CHECK(ret == 0); + ret = flb_filter_set(ctx, filter_ffd, "az", "false", NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + if (!TEST_CHECK(bytes > 0)) { + TEST_MSG("zero bytes were pushed\n"); + } + + flb_time_msleep(1500); /* waiting flush */ + + output = get_output(); + if (output) { + result = strstr(output, "\"host_id\":\"h-012a3456b7890cdef\""); if (!TEST_CHECK(result != NULL)) { TEST_MSG("output:%s\n", output); } @@ -1646,11 +2537,20 @@ TEST_LIST = { {"aws_instance_id", flb_test_aws_instance_id}, {"aws_instance_type", flb_test_aws_instance_type}, {"aws_private_ip", flb_test_aws_private_ip}, + {"aws_public_ip", flb_test_aws_public_ip}, + {"aws_ipv6", flb_test_aws_ipv6}, {"aws_vpc_id", flb_test_aws_vpc_id}, {"aws_ami_id", flb_test_aws_ami_id}, {"aws_account_id", flb_test_aws_account_id}, {"aws_hostname", flb_test_aws_hostname}, + {"aws_partition", flb_test_aws_partition}, + {"aws_domain", flb_test_aws_domain}, + {"aws_region", flb_test_aws_region}, {"aws_az", flb_test_aws_az}, + {"aws_az_id", flb_test_aws_az_id}, + {"aws_placement_group", flb_test_aws_placement_group}, + {"aws_partition_number", flb_test_aws_partition_number}, + {"aws_host_id", flb_test_aws_host_id}, {"aws_ec2_tags_present", flb_test_aws_ec2_tags_present}, {"aws_ec2_tags_404", flb_test_aws_ec2_tags_404}, {"aws_ec2_tags_list_500", flb_test_aws_ec2_tags_list_500},