From 5abd119339aaba375f7572fbd3cf74bc22b18efe Mon Sep 17 00:00:00 2001 From: "Lindsley, Chris" Date: Wed, 6 Mar 2024 07:49:58 -0500 Subject: [PATCH 1/8] only create egress_lambda_log_group if it doesn't already exist --- cumulus/thin-egress.tf | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/cumulus/thin-egress.tf b/cumulus/thin-egress.tf index 31ff93d..9d8638b 100644 --- a/cumulus/thin-egress.tf +++ b/cumulus/thin-egress.tf @@ -64,16 +64,22 @@ resource "aws_cloudwatch_log_subscription_filter" "egress_api_gateway_log_subscr } # Egress Lambda Log Group +# does it already exist +data "aws_cloudwatch_log_group" "egress_lambda_log_group" { + name = "/aws/lambda/${module.thin_egress_app.egress_lambda_name}" +} + resource "aws_cloudwatch_log_group" "egress_lambda_log_group" { - count = (var.log_destination_arn != null) ? 1 : 0 + count = (var.log_destination_arn != null && data.aws_cloudwatch_log_group.egress_lambda_log_group == null) ? 1 : 0 name = "/aws/lambda/${module.thin_egress_app.egress_lambda_name}" retention_in_days = var.egress_lambda_log_retention_days tags = local.default_tags } # Egress Lambda Log Group Filter -resource "aws_cloudwatch_log_subscription_filter" "egress_lambda_log_subscription_filter" { - count = (var.log_destination_arn != null) ? 1 : 0 +# if log group just created +resource "aws_cloudwatch_log_subscription_filter" "egress_lambda_log_subscription_filter_new" { + count = (var.log_destination_arn != null && data.aws_cloudwatch_log_group.egress_lambda_log_group == null) ? 1 : 0 depends_on = [aws_cloudwatch_log_group.egress_lambda_log_group] name = "${local.prefix}-EgressLambdaLogSubscriptionToSharedDestination" destination_arn = var.log_destination_arn @@ -81,3 +87,15 @@ resource "aws_cloudwatch_log_subscription_filter" "egress_lambda_log_subscriptio filter_pattern = "" log_group_name = aws_cloudwatch_log_group.egress_lambda_log_group[0].name } + +# Egress Lambda Log Group Filter +# if log group already exists +resource "aws_cloudwatch_log_subscription_filter" "egress_lambda_log_subscription_filter_update" { + count = (var.log_destination_arn != null && data.aws_cloudwatch_log_group.egress_lambda_log_group != null) ? 1 : 0 + depends_on = [data.aws_cloudwatch_log_group.egress_lambda_log_group] + name = "${local.prefix}-EgressLambdaLogSubscriptionToSharedDestination" + destination_arn = var.log_destination_arn + distribution = "ByLogStream" + filter_pattern = "" + log_group_name = data.aws_cloudwatch_log_group.egress_lambda_log_group.name +} From ebfee34e513959cc7a26c47a54c99eacd3e7f40f Mon Sep 17 00:00:00 2001 From: "Lindsley, Chris" Date: Fri, 15 Mar 2024 10:27:11 -0400 Subject: [PATCH 2/8] Revert "only create egress_lambda_log_group if it doesn't already exist" This reverts commit 5abd119339aaba375f7572fbd3cf74bc22b18efe. --- cumulus/thin-egress.tf | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/cumulus/thin-egress.tf b/cumulus/thin-egress.tf index 9d8638b..31ff93d 100644 --- a/cumulus/thin-egress.tf +++ b/cumulus/thin-egress.tf @@ -64,22 +64,16 @@ resource "aws_cloudwatch_log_subscription_filter" "egress_api_gateway_log_subscr } # Egress Lambda Log Group -# does it already exist -data "aws_cloudwatch_log_group" "egress_lambda_log_group" { - name = "/aws/lambda/${module.thin_egress_app.egress_lambda_name}" -} - resource "aws_cloudwatch_log_group" "egress_lambda_log_group" { - count = (var.log_destination_arn != null && data.aws_cloudwatch_log_group.egress_lambda_log_group == null) ? 1 : 0 + count = (var.log_destination_arn != null) ? 1 : 0 name = "/aws/lambda/${module.thin_egress_app.egress_lambda_name}" retention_in_days = var.egress_lambda_log_retention_days tags = local.default_tags } # Egress Lambda Log Group Filter -# if log group just created -resource "aws_cloudwatch_log_subscription_filter" "egress_lambda_log_subscription_filter_new" { - count = (var.log_destination_arn != null && data.aws_cloudwatch_log_group.egress_lambda_log_group == null) ? 1 : 0 +resource "aws_cloudwatch_log_subscription_filter" "egress_lambda_log_subscription_filter" { + count = (var.log_destination_arn != null) ? 1 : 0 depends_on = [aws_cloudwatch_log_group.egress_lambda_log_group] name = "${local.prefix}-EgressLambdaLogSubscriptionToSharedDestination" destination_arn = var.log_destination_arn @@ -87,15 +81,3 @@ resource "aws_cloudwatch_log_subscription_filter" "egress_lambda_log_subscriptio filter_pattern = "" log_group_name = aws_cloudwatch_log_group.egress_lambda_log_group[0].name } - -# Egress Lambda Log Group Filter -# if log group already exists -resource "aws_cloudwatch_log_subscription_filter" "egress_lambda_log_subscription_filter_update" { - count = (var.log_destination_arn != null && data.aws_cloudwatch_log_group.egress_lambda_log_group != null) ? 1 : 0 - depends_on = [data.aws_cloudwatch_log_group.egress_lambda_log_group] - name = "${local.prefix}-EgressLambdaLogSubscriptionToSharedDestination" - destination_arn = var.log_destination_arn - distribution = "ByLogStream" - filter_pattern = "" - log_group_name = data.aws_cloudwatch_log_group.egress_lambda_log_group.name -} From 75672bad26832ad6e8d10c7185be3ca754920606 Mon Sep 17 00:00:00 2001 From: "Lindsley, Chris" Date: Fri, 15 Mar 2024 10:28:15 -0400 Subject: [PATCH 3/8] add example tf file to import thin egress cloudwatch group --- cumulus/tea-cloudwatch.tf.example | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 cumulus/tea-cloudwatch.tf.example diff --git a/cumulus/tea-cloudwatch.tf.example b/cumulus/tea-cloudwatch.tf.example new file mode 100644 index 0000000..9287f37 --- /dev/null +++ b/cumulus/tea-cloudwatch.tf.example @@ -0,0 +1,39 @@ + + +# If you run into this issue when deploying Cumulus +# +# Error: creating CloudWatch Logs Log Group +# (/aws/lambda/DEPLOY_NAME-cumulus-MATURITY-thin-egress-app-EgressLambda): operation +# error CloudWatch Logs: CreateLogGroup, https response error StatusCode: 400, +# RequestID: XXXX, ResourceAlreadyExistsException: The specified log group already +# exists +# +# with aws_cloudwatch_log_group.egress_lambda_log_group[0], +# on thin-egress.tf line 67, in resource "aws_cloudwatch_log_group" "egress_lambda_log_group": +# 67: resource "aws_cloudwatch_log_group" "egress_lambda_log_group" { +# +# You can rename this file from tea-cloudwatch.tf.example to tea-cloudwatch.tf and then +# fill in the 'id' for your cloudwatch group. +# +# Terraform 1.5.x requires the 'id' of the import statement to be a string, it can't +# even be a variable that is a string. +# +# https://developer.hashicorp.com/terraform/language/v1.5.x/import +# +# Later versions of Terraform do allow a variable, but it can't a calculated value +# +# https://developer.hashicorp.com/terraform/language/v1.6.x/import +# +# The thin egress cloudwatch group follows this pattern and should have been reported +# in the error message +# +# /aws/lambda/DEPLOY_NAME-cumulus-MATURITY-thin-egress-app-EgressLambda + +import { + to = aws_cloudwatch_log_group.egress_lambda_log_group[0] + id = "" +} + +output "imported_tea_log_group" { + value = aws_cloudwatch_log_group.egress_lambda_log_group +} From 668abd2a7e303d8e3384a8ba0a2265ef488f82cd Mon Sep 17 00:00:00 2001 From: "Lindsley, Chris" Date: Fri, 15 Mar 2024 10:32:23 -0400 Subject: [PATCH 4/8] clean up example file --- cumulus/tea-cloudwatch.tf.example | 2 -- 1 file changed, 2 deletions(-) diff --git a/cumulus/tea-cloudwatch.tf.example b/cumulus/tea-cloudwatch.tf.example index 9287f37..a71f982 100644 --- a/cumulus/tea-cloudwatch.tf.example +++ b/cumulus/tea-cloudwatch.tf.example @@ -1,5 +1,3 @@ - - # If you run into this issue when deploying Cumulus # # Error: creating CloudWatch Logs Log Group From 0addd0fdfbc8be3206e2278a55978e4524f298a8 Mon Sep 17 00:00:00 2001 From: "Lindsley, Chris" Date: Tue, 19 Mar 2024 12:28:15 -0400 Subject: [PATCH 5/8] update Makefile to do the import instead --- Makefile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Makefile b/Makefile index fb299fd..c7e8f52 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ PYTHON_VER ?= python3 CIRRUS_CORE_VERSION := $(or $(shell git tag --points-at HEAD | head -n1),$(shell git rev-parse --short HEAD)) CIRRUS_DAAC_VERSION := $(or $(shell git -C $(DAAC_DIR) tag --points-at HEAD | head -n1),$(shell git -C $(DAAC_DIR) rev-parse --short HEAD)) +THIN_EGRESS_LOG_EXIST := "0" # --------------------------- SELF_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) @@ -246,6 +247,36 @@ plan-cumulus: cumulus-init eval $$TF_CMD fi +# --------------------------- +.PHONY: import-thin-egress-log +import-thin-egress-log: cumulus-init + $(banner) + if [ -f "${DAAC_DIR}/cumulus/secrets/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export SECRETS_OPT="-var-file=${DAAC_DIR}/cumulus/secrets/${MATURITY}.tfvars" + echo "Found maturity-specific secrets: $$SECRETS_OPT" + echo "***************************************************************" + fi + cd cumulus + if [ -f "${DAAC_DIR}/cumulus/variables/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export VARIABLES_OPT="-var-file=${DAAC_DIR}/cumulus/variables/${MATURITY}.tfvars" + echo "Found maturity-specific variables: $$VARIABLES_OPT" + echo "***************************************************************" + fi + export TF_CMD="terraform import \ + -var-file=${DAAC_DIR}/cumulus/terraform.tfvars \ + $$VARIABLES_OPT \ + $$SECRETS_OPT \ + -input=false \ + -no-color \ + aws_cloudwatch_log_group.egress_lambda_log_group[0] \ + ${DEPLOY_NAME}-cumulus-${MATURITY}-thin-egress-app-EgressLambda" + eval $$TF_CMD + +# --------------------------- .PHONY: destroy-cumulus destroy-cumulus: cumulus-init $(banner) From 249573139f07cfc8814223374bef60f5008198ad Mon Sep 17 00:00:00 2001 From: "Lindsley, Chris" Date: Fri, 22 Mar 2024 16:16:17 -0400 Subject: [PATCH 6/8] fix log group name in new import target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c7e8f52..ac8d50f 100644 --- a/Makefile +++ b/Makefile @@ -273,7 +273,7 @@ import-thin-egress-log: cumulus-init -input=false \ -no-color \ aws_cloudwatch_log_group.egress_lambda_log_group[0] \ - ${DEPLOY_NAME}-cumulus-${MATURITY}-thin-egress-app-EgressLambda" + /aws/lambda/${DEPLOY_NAME}-cumulus-${MATURITY}-thin-egress-app-EgressLambda" eval $$TF_CMD # --------------------------- From 8aae14c6f5219ffce5733bae39b65079865f235c Mon Sep 17 00:00:00 2001 From: "Lindsley, Chris" Date: Fri, 22 Mar 2024 16:16:57 -0400 Subject: [PATCH 7/8] add .gitconfig file to prevent dubious ownership warning --- .gitconfig | 3 +++ Dockerfile | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 .gitconfig diff --git a/.gitconfig b/.gitconfig new file mode 100644 index 0000000..daa3f9f --- /dev/null +++ b/.gitconfig @@ -0,0 +1,3 @@ +[safe] + directory = /CIRRUS-core + directory = /CIRRUS-DAAC \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 961d190..7e852cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,8 @@ ARG USER RUN \ echo "user:x:${USER}:0:root:/:/bin/bash" >> /etc/passwd +COPY .gitconfig /.gitconfig + WORKDIR /CIRRUS-core # Python38 target From 81db6b72eb7828ddaa2042c5f7c39fad4a8bef44 Mon Sep 17 00:00:00 2001 From: "Lindsley, Chris" Date: Thu, 18 Apr 2024 10:52:20 -0400 Subject: [PATCH 8/8] update CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe154e1..710f358 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG +## unreleased +* add a Makefile target to import tea lambda cloudwatch group if getting an "The +specified log group already exists" error: `make import-thin-egress-log` +* add .gitconfig file to Docker image to mark /CIRRUS-core and /CIRRUS-DAAC as safe + ## v18.2.0.0 * Upgrade to [Cumulus v18.2.0](https://github.com/nasa/cumulus/releases/tag/v18.2.0)