Add option to use custom metadata with shipper (coralogix#145)

* add option to use custom metadata

* fix changlog

* add option to use custom metadata

* sync the branch with master and add support for custom_csv_header

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## v1.0.95
+### 🚀 New components 🚀
+#### **coralogix-aws-shipper**
+- Add new variables custom_csv_header and custom_metadata
+
 ## v1.0.94
 #### **coralogix-aws-shipper**
 ### 🧰 Bug fixes 🧰
@@ -31,7 +36,7 @@
 
 ## v1.0.90
 ### 💡 Enhancements 💡
-#### **coralogix-aws-shipper**
+
 - [cds-1050] add support for x86 to template
 
 ## v1.0.89

examples/coralogix-aws-shipper/variables.tf

@@ -76,6 +76,12 @@ variable "cs_delimiter" {
   default     = ","
 }
 
+variable "custom_csv_header" {
+  type        = string
+  description = "List separated by cs delimiter of a new headers for your csv, the variable must be with the same delimiter as the cs_delimiter, for example if the cs_delimiter is \";\" then the value of the variable should be name;country;age so the new headers will be name, country, age"
+  default     = null
+}
+
 # cloudwatch variables
 
 variable "log_groups" {
@@ -119,7 +125,7 @@ variable "cpu_arch" {
   validation {
     condition     = contains(["arm64", "x86_64"], var.cpu_arch)
     error_message = "The CPU architecture must be one of these values: [arm64, x86_64]."
-  } 
+  }
 }
 
 # Integration Generic Config (Optional)
@@ -168,7 +174,7 @@ variable "integration_type" {
   description = "the aws service that send the data to the s3"
   type        = string
   validation {
-    condition     = contains(["CloudWatch", "CloudTrail", "VpcFlow", "S3", "S3Csv", "Sns", "Sqs", "Kinesis", "CloudFront", "MSK", "Kafka","EcrScan", ""], var.integration_type)
+    condition     = contains(["CloudWatch", "CloudTrail", "VpcFlow", "S3", "S3Csv", "Sns", "Sqs", "Kinesis", "CloudFront", "MSK", "Kafka", "EcrScan", ""], var.integration_type)
     error_message = "The integration type must be: [CloudWatch, CloudTrail, VpcFlow, S3, S3Csv, Sns, Sqs, Kinesis, CloudFront, MSK, Kafka, EcrScan]."
   }
   default = ""
@@ -204,6 +210,12 @@ variable "add_metadata" {
   type        = string
 }
 
+variable "custom_metadata" {
+  default     = null
+  description = "Add custom metadata to the log message. Expects comma separated values. Options are key1=value1,key2=value2 "
+  type        = string
+}
+
 variable "integration_info" {
   description = "Values of s3 integraion in case that you want to deploy more than one integration"
   type = map(object({

modules/coralogix-aws-shipper/README.md

@@ -57,6 +57,7 @@ If you want to avoid this issue, you can deploy in other ways:
 | <a name="input_s3_key_prefix"></a> [s3\_key\_prefix](#input\_s3\_key\_prefix) | The S3 path prefix to watch. | `string` |  n/a | no |
 | <a name="input_s3_key_suffix"></a> [s3\_key\_suffix](#input\_s3\_key\_suffix) | The S3 path suffix to watch. | `string` |  n/a | no |
 | <a name="input_csv_delimiter"></a> [csv_delimiter](#input\_csv\_delimiter) | Specify a single character to be used as a delimiter when ingesting a CSV file with a header line. This value is applicable when the S3Csv integration type is selected, for example, “,” or ” “.  | `string` |  n/a | no |
+| <a name="input_custom_csv_header"></a> [custom\_csv\_header](#input\_custom\_csv\_header) | List seperated by cs delimiter of a new headers for your csv, the variable must be with the same delimiter as the cs_delimiter, for example if the cs_delimiter is \";\" then the value of the variable should be name;country;age and the new headers that you will see in corlaogix will be name, country, age | `string` | n/a | no |
 | <a name="input_newline_pattern"></a> [newline\_pattern](#input\_newline\_pattern) | nter a regular expression to detect a new log line for multiline logs, e.g., \n(?=\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d{3}). | `string` | n/a | no |
 | [integration_info](#additional-parameters-for-integration_info) | A map of integration information. Use this when you want to deploy more then one integration using the same s3 bucket. [Parameters are here.](#integration_info)| `mapping` | n/a | no |
 
@@ -117,6 +118,7 @@ If you want to avoid this issue, you can deploy in other ways:
 
 | Name | Description | Type | Default | Required | 
 |------|-------------|------|---------|:--------:|
+| <a name="input_custom_metadata"></a> [custom\_metadata](#input\_custom\_metadata) | Add custom metadata to the log message. Expects comma separated values. Options are `key1=value1,key2=value2` | `string` | n/a | no |
 | <a name="input_add_metadata"></a> [add\_metadata](#input\_add\_metadata) | Add metadata to the log message. Expects comma separated values. Options for S3 are `bucket_name`,`key_name`. For CloudWatch `stream_name` | `string` | n/a | no |
 | <a name="input_lambda_name"></a> [lambda\_name](#input\_lambda\_name) | Name the Lambda function that you want to create. | `string` | n/a | no |
 | <a name="input_blocking_pattern"></a> [blocking\_pattern](#input\_blocking\_pattern) | Enter a regular expression to identify lines excluded from being sent to Coralogix. For example, use `MainActivity.java:\d{3}` to match log lines with MainActivity followed by exactly three digits. | `string` | n/a | no |

modules/coralogix-aws-shipper/main.tf

@@ -57,6 +57,8 @@ module "lambda" {
     BLOCKING_PATTERN   = var.blocking_pattern
     SAMPLING           = tostring(var.sampling_rate)
     ADD_METADATA       = var.add_metadata
+    CUSTOM_METADATA    = var.custom_metadata
+    CUSTOM_CSV_HEADER  = var.custom_csv_header
   }
   s3_existing_package = {
     bucket = var.custom_s3_bucket == "" ? "coralogix-serverless-repo-${data.aws_region.this.name}" : var.custom_s3_bucket

modules/coralogix-aws-shipper/variables.tf

@@ -76,6 +76,12 @@ variable "cs_delimiter" {
   default     = ","
 }
 
+variable "custom_csv_header" {
+  type        = string
+  description = "List separated by cs delimiter of a new headers for your csv, the variable must be with the same delimiter as the cs_delimiter, for example if the cs_delimiter is \";\" then the value of the variable should be name;country;age so the new headers will be name, country, age"
+  default     = null
+}
+
 # cloudwatch variables
 
 variable "log_groups" {
@@ -204,6 +210,12 @@ variable "add_metadata" {
   type        = string
 }
 
+variable "custom_metadata" {
+  default     = null
+  description = "Add custom metadata to the log message. Expects comma separated values. Options are key1=value1,key2=value2 "
+  type        = string
+}
+
 variable "integration_info" {
   description = "Values of s3 integraion in case that you want to deploy more than one integration"
   type = map(object({