From 0f911197fad06e761ee9c00ccaabcc11e5e89109 Mon Sep 17 00:00:00 2001 From: mihai-satmarean <4729542+mihai-satmarean@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:14:48 +0100 Subject: [PATCH 1/7] Create main.tf adds the idea of configuring an s3 bucket as the holder of the state file https://github.com/mihai-satmarean/sciitdevops/issues/49 --- terraform/terraform-modules/state-bucket/main.tf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 terraform/terraform-modules/state-bucket/main.tf diff --git a/terraform/terraform-modules/state-bucket/main.tf b/terraform/terraform-modules/state-bucket/main.tf new file mode 100644 index 0000000..8c911fc --- /dev/null +++ b/terraform/terraform-modules/state-bucket/main.tf @@ -0,0 +1,16 @@ +provider "aws" { + region = "eu-central-1" +} + +resource "aws_s3_bucket" "terraform_state" { + bucket = "your-terraform-state-bucket" + acl = "private" + + versioning { + enabled = true + } + + tags = { + Name = "Terraform State Bucket" + } +} From 2c694905a6602ef24883361635ed0492eb7a3f6c Mon Sep 17 00:00:00 2001 From: mihai-satmarean <4729542+mihai-satmarean@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:27:21 +0100 Subject: [PATCH 2/7] Update main.tf improved handling --- .../terraform-modules/state-bucket/main.tf | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/terraform/terraform-modules/state-bucket/main.tf b/terraform/terraform-modules/state-bucket/main.tf index 8c911fc..72d2d70 100644 --- a/terraform/terraform-modules/state-bucket/main.tf +++ b/terraform/terraform-modules/state-bucket/main.tf @@ -1,16 +1,42 @@ +# main.tf provider "aws" { region = "eu-central-1" } -resource "aws_s3_bucket" "terraform_state" { - bucket = "your-terraform-state-bucket" - acl = "private" - - versioning { - enabled = true +# Local backend for initial setup +terraform { + backend "local" { + path = "terraform.tfstate" } +} + +resource "aws_s3_bucket" "terraform_state_bucket" { + bucket = "terraform-state-${terraform.workspace}" + acl = "private" + force_destroy = true # For simplicity in training scenarios; remove in production. tags = { Name = "Terraform State Bucket" } } + +resource "aws_s3_bucket_versioning" "state_bucket_versioning" { + bucket = aws_s3_bucket.terraform_state_bucket.id + + versioning_configuration { + status = "Enabled" + } +} + +# Terraform backend configuration post-initialization +output "backend_config" { + value = < Date: Thu, 5 Dec 2024 11:29:03 +0100 Subject: [PATCH 3/7] Update terraform-apply.yml try some magic with statefiles and buckets --- .github/workflows/terraform-apply.yml | 29 ++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/workflows/terraform-apply.yml b/.github/workflows/terraform-apply.yml index 74d8d1f..e3dd862 100644 --- a/.github/workflows/terraform-apply.yml +++ b/.github/workflows/terraform-apply.yml @@ -1,23 +1,42 @@ -name: Terraform apply +name: Terraform Apply with Remote State run-name: ${{ github.actor }} is deploying on AWS 🚀 on: push: branches: - - feature/aws-s3-module + - mihai-satmarean-patch-1 jobs: Deploy-AWS-Ec2: runs-on: ubuntu-latest steps: + # Step 1: Configure AWS credentials - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_MARIUS }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_MARIUS }} aws-region: eu-central-1 + + # Step 2: Checkout the repository code - name: Checkout Repository uses: actions/checkout@v4 - - name: Deploy Infrastructure + + # Step 3: Ensure Terraform state S3 bucket exists + - name: Create Terraform State Bucket run: | - cd terraform/terraform-modules/tf-ec2-module/ + cd terraform/terraform-modules/state-bucket terraform init - terraform destroy -auto-approve + terraform apply -auto-approve + + # Step 4: Reconfigure Backend to Use S3 + - name: Reconfigure Backend to S3 + run: | + cd terraform/terraform-modules/tf-ec2-module/ + BUCKET_NAME=$(terraform output -raw backend_config | grep bucket | cut -d '=' -f2 | tr -d ' ') + terraform init -backend-config="bucket=${BUCKET_NAME}" \ + -backend-config="key=state/${GITHUB_REF#refs/heads/}/terraform.tfstate" \ + -backend-config="region=eu-central-1" + # Step 5: Apply Terraform with S3 backend + - name: Finalize Infrastructure Deployment + run: | + cd terraform/terraform-modules/tf-ec2-module/ + terraform apply -auto-approve From 6bc14fd49d2ff886b81f92e57909e1c2b721a449 Mon Sep 17 00:00:00 2001 From: mihai-satmarean <4729542+mihai-satmarean@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:39:59 +0100 Subject: [PATCH 4/7] Update terraform-apply.yml Add aws cli verification for stating the objects were created, and destroy at the end. Dont error if the state bucket exists --- .github/workflows/terraform-apply.yml | 32 ++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/terraform-apply.yml b/.github/workflows/terraform-apply.yml index e3dd862..d70cb00 100644 --- a/.github/workflows/terraform-apply.yml +++ b/.github/workflows/terraform-apply.yml @@ -25,7 +25,7 @@ jobs: run: | cd terraform/terraform-modules/state-bucket terraform init - terraform apply -auto-approve + terraform apply -auto-approve || echo "Bucket already exists, continuing..." # Step 4: Reconfigure Backend to Use S3 - name: Reconfigure Backend to S3 @@ -40,3 +40,33 @@ jobs: run: | cd terraform/terraform-modules/tf-ec2-module/ terraform apply -auto-approve + # Step 6: Validate Resources with AWS CLI + - name: List Resources Created by Terraform + run: | + # Capture Terraform output variables + VPC_ID=$(terraform output -raw vpc_id) + PUBLIC_SUBNET_ID=$(terraform output -raw public_subnet_id) + PRIVATE_SUBNET_ID=$(terraform output -raw private_subnet_id) + SECURITY_GROUP_ID=$(terraform output -raw security_group_id) + + # List VPC + echo "Listing VPC with ID: $VPC_ID" + aws ec2 describe-vpcs --vpc-ids $VPC_ID + + # List Public Subnet + echo "Listing Public Subnet with ID: $PUBLIC_SUBNET_ID" + aws ec2 describe-subnets --subnet-ids $PUBLIC_SUBNET_ID + + # List Private Subnet + echo "Listing Private Subnet with ID: $PRIVATE_SUBNET_ID" + aws ec2 describe-subnets --subnet-ids $PRIVATE_SUBNET_ID + + # List Security Group + echo "Listing Security Group with ID: $SECURITY_GROUP_ID" + aws ec2 describe-security-groups --group-ids $SECURITY_GROUP_ID + + # Step 7: Destroy Infrastructure + - name: Destroy Infrastructure + run: | + cd terraform/terraform-modules/tf-ec2-module/ + terraform destroy -auto-approve From 545ddc9f3dfa3dae80e13f72d7bbeb4f5e0a59d3 Mon Sep 17 00:00:00 2001 From: mihai-satmarean <4729542+mihai-satmarean@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:54:41 +0100 Subject: [PATCH 5/7] Update terraform-apply.yml trying to capture theoutputs and ise aws cli to verify --- .github/workflows/terraform-apply.yml | 45 +++++++++++++++++++-------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/.github/workflows/terraform-apply.yml b/.github/workflows/terraform-apply.yml index d70cb00..86b34cb 100644 --- a/.github/workflows/terraform-apply.yml +++ b/.github/workflows/terraform-apply.yml @@ -26,21 +26,27 @@ jobs: cd terraform/terraform-modules/state-bucket terraform init terraform apply -auto-approve || echo "Bucket already exists, continuing..." - # Step 4: Reconfigure Backend to Use S3 - name: Reconfigure Backend to S3 run: | cd terraform/terraform-modules/tf-ec2-module/ - BUCKET_NAME=$(terraform output -raw backend_config | grep bucket | cut -d '=' -f2 | tr -d ' ') - terraform init -backend-config="bucket=${BUCKET_NAME}" \ + terraform init -backend-config="bucket=terraform_state_bucket" \ -backend-config="key=state/${GITHUB_REF#refs/heads/}/terraform.tfstate" \ -backend-config="region=eu-central-1" + # Step 5: Apply Terraform with S3 backend - name: Finalize Infrastructure Deployment run: | cd terraform/terraform-modules/tf-ec2-module/ terraform apply -auto-approve - # Step 6: Validate Resources with AWS CLI + + # Step 6: Refresh Terraform State to ensure it's up to date with AWS + - name: Refresh Terraform State + run: | + cd terraform/terraform-modules/tf-ec2-module/ + terraform refresh + + # Step 7: Validate Resources with AWS CLI - name: List Resources Created by Terraform run: | # Capture Terraform output variables @@ -49,24 +55,37 @@ jobs: PRIVATE_SUBNET_ID=$(terraform output -raw private_subnet_id) SECURITY_GROUP_ID=$(terraform output -raw security_group_id) - # List VPC + # Print and List VPC echo "Listing VPC with ID: $VPC_ID" - aws ec2 describe-vpcs --vpc-ids $VPC_ID + aws ec2 describe-vpcs --vpc-ids $VPC_ID || echo "Failed to list VPC with ID: $VPC_ID" - # List Public Subnet + # Print and List Public Subnet echo "Listing Public Subnet with ID: $PUBLIC_SUBNET_ID" - aws ec2 describe-subnets --subnet-ids $PUBLIC_SUBNET_ID + aws ec2 describe-subnets --subnet-ids $PUBLIC_SUBNET_ID || echo "Failed to list Public Subnet with ID: $PUBLIC_SUBNET_ID" - # List Private Subnet + # Print and List Private Subnet echo "Listing Private Subnet with ID: $PRIVATE_SUBNET_ID" - aws ec2 describe-subnets --subnet-ids $PRIVATE_SUBNET_ID + aws ec2 describe-subnets --subnet-ids $PRIVATE_SUBNET_ID || echo "Failed to list Private Subnet with ID: $PRIVATE_SUBNET_ID" - # List Security Group + # Print and List Security Group echo "Listing Security Group with ID: $SECURITY_GROUP_ID" - aws ec2 describe-security-groups --group-ids $SECURITY_GROUP_ID + aws ec2 describe-security-groups --group-ids $SECURITY_GROUP_ID || echo "Failed to list Security Group with ID: $SECURITY_GROUP_ID" - # Step 7: Destroy Infrastructure + # Step 8: Destroy Infrastructure - name: Destroy Infrastructure run: | cd terraform/terraform-modules/tf-ec2-module/ terraform destroy -auto-approve + + # Step 9: Verify Resources are Destroyed (Optional) + - name: Verify Resources are Destroyed + run: | + # Verify that resources were destroyed by listing them again + echo "Verifying VPC Destruction..." + aws ec2 describe-vpcs --vpc-ids $VPC_ID || echo "VPC with ID $VPC_ID does not exist." + echo "Verifying Public Subnet Destruction..." + aws ec2 describe-subnets --subnet-ids $PUBLIC_SUBNET_ID || echo "Public Subnet with ID $PUBLIC_SUBNET_ID does not exist." + echo "Verifying Private Subnet Destruction..." + aws ec2 describe-subnets --subnet-ids $PRIVATE_SUBNET_ID || echo "Private Subnet with ID $PRIVATE_SUBNET_ID does not exist." + echo "Verifying Security Group Destruction..." + aws ec2 describe-security-groups --group-ids $SECURITY_GROUP_ID || echo "Security Group with ID $SECURITY_GROUP_ID does not exist." From 2552eedc4c662cd65a25d0c196b63747eeb7166d Mon Sep 17 00:00:00 2001 From: mihai-satmarean <4729542+mihai-satmarean@users.noreply.github.com> Date: Thu, 5 Dec 2024 12:01:03 +0100 Subject: [PATCH 6/7] Update terraform-apply.yml try changing folder to module path --- .github/workflows/terraform-apply.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/terraform-apply.yml b/.github/workflows/terraform-apply.yml index 86b34cb..17efa4e 100644 --- a/.github/workflows/terraform-apply.yml +++ b/.github/workflows/terraform-apply.yml @@ -49,6 +49,7 @@ jobs: # Step 7: Validate Resources with AWS CLI - name: List Resources Created by Terraform run: | + cd terraform/terraform-modules/tf-ec2-module/ # Capture Terraform output variables VPC_ID=$(terraform output -raw vpc_id) PUBLIC_SUBNET_ID=$(terraform output -raw public_subnet_id) From 978c3accaf8edb299d08b02939c6cdac64a7fd2c Mon Sep 17 00:00:00 2001 From: mihai-satmarean <4729542+mihai-satmarean@users.noreply.github.com> Date: Thu, 5 Dec 2024 12:11:34 +0100 Subject: [PATCH 7/7] Update terraform-apply.yml we need to make sure that the correct IDs are passed on to subsequent calls --- .github/workflows/terraform-apply.yml | 34 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/terraform-apply.yml b/.github/workflows/terraform-apply.yml index 17efa4e..82307bb 100644 --- a/.github/workflows/terraform-apply.yml +++ b/.github/workflows/terraform-apply.yml @@ -45,40 +45,46 @@ jobs: run: | cd terraform/terraform-modules/tf-ec2-module/ terraform refresh - - # Step 7: Validate Resources with AWS CLI - - name: List Resources Created by Terraform + # Step 7: Capture Terraform Outputs to Variables + - name: Capture Terraform Outputs + id: terraform_outputs run: | cd terraform/terraform-modules/tf-ec2-module/ - # Capture Terraform output variables - VPC_ID=$(terraform output -raw vpc_id) - PUBLIC_SUBNET_ID=$(terraform output -raw public_subnet_id) - PRIVATE_SUBNET_ID=$(terraform output -raw private_subnet_id) - SECURITY_GROUP_ID=$(terraform output -raw security_group_id) + export VPC_ID=$(terraform output -raw vpc_id) + export PUBLIC_SUBNET_ID=$(terraform output -raw public_subnet_id) + export PRIVATE_SUBNET_ID=$(terraform output -raw private_subnet_id) + export SECURITY_GROUP_ID=$(terraform output -raw security_group_id) + echo "VPC_ID=$VPC_ID" >> $GITHUB_ENV + echo "PUBLIC_SUBNET_ID=$PUBLIC_SUBNET_ID" >> $GITHUB_ENV + echo "PRIVATE_SUBNET_ID=$PRIVATE_SUBNET_ID" >> $GITHUB_ENV + echo "SECURITY_GROUP_ID=$SECURITY_GROUP_ID" >> $GITHUB_ENV - # Print and List VPC + # Step 8: Validate Resources with AWS CLI + - name: List Resources Created by Terraform + run: | + # List VPC echo "Listing VPC with ID: $VPC_ID" aws ec2 describe-vpcs --vpc-ids $VPC_ID || echo "Failed to list VPC with ID: $VPC_ID" - # Print and List Public Subnet + # List Public Subnet echo "Listing Public Subnet with ID: $PUBLIC_SUBNET_ID" aws ec2 describe-subnets --subnet-ids $PUBLIC_SUBNET_ID || echo "Failed to list Public Subnet with ID: $PUBLIC_SUBNET_ID" - # Print and List Private Subnet + # List Private Subnet echo "Listing Private Subnet with ID: $PRIVATE_SUBNET_ID" aws ec2 describe-subnets --subnet-ids $PRIVATE_SUBNET_ID || echo "Failed to list Private Subnet with ID: $PRIVATE_SUBNET_ID" - # Print and List Security Group + # List Security Group echo "Listing Security Group with ID: $SECURITY_GROUP_ID" aws ec2 describe-security-groups --group-ids $SECURITY_GROUP_ID || echo "Failed to list Security Group with ID: $SECURITY_GROUP_ID" - # Step 8: Destroy Infrastructure + # Step 9: Destroy Infrastructure - name: Destroy Infrastructure run: | cd terraform/terraform-modules/tf-ec2-module/ terraform destroy -auto-approve - # Step 9: Verify Resources are Destroyed (Optional) + # Step 10: Verify Resources are Destroyed - name: Verify Resources are Destroyed run: | # Verify that resources were destroyed by listing them again