From 3f7e85c82a321b38fc335bcac1cd4b4775ca534d Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Mon, 6 Mar 2023 11:12:57 -0500 Subject: [PATCH 01/12] initial commit - debug sample --- .gitignore | 1 + rds-flyway-migrations/cloudformation.yml | 346 ++++++++++++++++++ rds-flyway-migrations/deploy.sh | 2 + .../V2023.02.08__001.config-db-options.sql | 1 + .../V2023.02.08__002.create-user-table 1.sql | 11 + .../V2023.02.08__003.create-role-tables.sql | 17 + 6 files changed, 378 insertions(+) create mode 100644 .gitignore create mode 100644 rds-flyway-migrations/cloudformation.yml create mode 100755 rds-flyway-migrations/deploy.sh create mode 100644 rds-flyway-migrations/flyway/migrations/V2023.02.08__001.config-db-options.sql create mode 100644 rds-flyway-migrations/flyway/migrations/V2023.02.08__002.create-user-table 1.sql create mode 100644 rds-flyway-migrations/flyway/migrations/V2023.02.08__003.create-role-tables.sql diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/rds-flyway-migrations/cloudformation.yml b/rds-flyway-migrations/cloudformation.yml new file mode 100644 index 0000000..427cbde --- /dev/null +++ b/rds-flyway-migrations/cloudformation.yml @@ -0,0 +1,346 @@ +## +## Creates a VPC with two public and two private subnets, each of which supports 4096 IP +## addresses (ex reserved addresses), and with NATs for use by the private subnets. +## +## The VPC ID and Subnet IDs are exported, for use by other stacks. +## + +## +## Due to the need to populate the database, this template must be +## loaded twice: the first pass creates all resources other than the +## ECS cluster, while the second (which is run after the database is +## populated) creates the ECS Service. +## + +AWSTemplateFormatVersion: "2010-09-09" +Description: "VPC, CodeBuild and RDS for Flyway Demo" + +Parameters: + BaseCIDR: + Description: "The first two bytes of a /16 CIDR that will be used for the VPC" + Type: "String" + Default: "10.10" + + GitHubRepoUrl: + Description: "The URL of the project to clone" + Type: "String" + Default: "https://github.com/chariotsolutions/ews-examples.git" + +Resources: + + VPC: + Type: "AWS::EC2::VPC" + Properties: + CidrBlock: !Sub "${BaseCIDR}.0.0/16" + EnableDnsSupport: true + EnableDnsHostnames: true + Tags: + - Key: "Name" + Value: "rds-flyway-migrations-sample" + + PublicSubnet01: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: !Ref VPC + CidrBlock: !Sub "${BaseCIDR}.0.0/20" + AvailabilityZone: !Sub "${AWS::Region}a" + MapPublicIpOnLaunch: true + + PublicSubnet02: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: !Ref VPC + CidrBlock: !Sub "${BaseCIDR}.16.0/20" + AvailabilityZone: !Sub "${AWS::Region}b" + MapPublicIpOnLaunch: true + + InternetGateway: + Type: "AWS::EC2::InternetGateway" + + + InternetGatewayAttachment: + Type: "AWS::EC2::VPCGatewayAttachment" + Properties: + VpcId: !Ref VPC + InternetGatewayId: !Ref InternetGateway + + PublicSubnet01Routes: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PublicSubnet01 + RouteTableId: !Ref PublicRouteTable + + PublicSubnet02Routes: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PublicSubnet02 + RouteTableId: !Ref PublicRouteTable + + PublicRouteTable: + Type: "AWS::EC2::RouteTable" + Properties: + VpcId: !Ref VPC + + ## + ## Private Subnets -- each needs its own route table to support NAT + ## + + PrivateSubnet01: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: !Ref VPC + CidrBlock: !Sub "${BaseCIDR}.128.0/20" + AvailabilityZone: !Sub "${AWS::Region}a" + MapPublicIpOnLaunch: false + + PrivateSubnet02: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: !Ref VPC + CidrBlock: !Sub "${BaseCIDR}.144.0/20" + AvailabilityZone: !Sub "${AWS::Region}b" + MapPublicIpOnLaunch: false + + PrivateRouteTable: + Type: "AWS::EC2::RouteTable" + Properties: + VpcId: !Ref VPC + + PrivateSubnet01Routes: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PrivateSubnet01 + RouteTableId: !Ref PrivateRouteTable + + PrivateSubnet02Routes: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PrivateSubnet02 + RouteTableId: !Ref PrivateRouteTable + + ## + ## NATs + ## + + ElasticIP01: + Type: "AWS::EC2::EIP" + Properties: + Domain: "vpc" + + ElasticIP02: + Type: "AWS::EC2::EIP" + Properties: + Domain: "vpc" + + NAT: + Type: "AWS::EC2::NatGateway" + Properties: + SubnetId: !Ref PublicSubnet01 + AllocationId: !GetAtt ElasticIP01.AllocationId + + PrivateRouteNAT: + Type: "AWS::EC2::Route" + Properties: + RouteTableId: !Ref PrivateRouteTable + DestinationCidrBlock: "0.0.0.0/0" + NatGatewayId: !Ref NAT + + CodeBuildSecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupName: !Sub "${AWS::StackName}-CodeBuildSG" + GroupDescription: "Security Group for CodeBuild" + VpcId: !Ref VPC + + DBSecrets: + Type: AWS::SecretsManager::Secret + Properties: + Description: "Secrets for the demo application" + GenerateSecretString: + SecretStringTemplate: | + { + "dbDatabaseName": "demodb", + "dbUserName": "demouser" + } + GenerateStringKey: "dbPassword" + ExcludePunctuation: true # easier to select for those times that you need to copy/paste + PasswordLength: 20 + + DatabaseSecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupName: !Sub "${AWS::StackName}-dbSG" + GroupDescription: "Allows access to the database (nominally from application)" + VpcId: !Ref VPC + + RDSParameterGroup: + Type: "AWS::RDS::DBClusterParameterGroup" + Properties: + DBClusterParameterGroupName: !Sub "${AWS::StackName}-dbcluster-param-group" + Description: !Sub "Managed by CloudFormation: ${AWS::StackName}" + Family: "aurora-postgresql11" + Parameters: # cluster parameter groups must have parameters; this is mostly harmless, customize as you wish + client_encoding: "UTF8" + + RDSSubnetGroup: + Type: "AWS::RDS::DBSubnetGroup" + Properties: + DBSubnetGroupName: !Sub "${AWS::StackName}-db-subnet-group" + DBSubnetGroupDescription: !Sub "Managed by CloudFormation: ${AWS::StackName}" + SubnetIds: + - !Ref PrivateSubnet01 + - !Ref PrivateSubnet02 + + DatabaseCluster: + Type: "AWS::RDS::DBCluster" + Properties: + DBClusterIdentifier: !Sub "${AWS::StackName}-PostgresCluster" + Engine: "aurora-postgresql" + # n.b. this query tells you the provisioned -vs- serverless versions available + # aws rds describe-db-engine-versions --engine aurora-postgresql --query 'DBEngineVersions[].[EngineVersion, SupportedEngineModes]' + # Only 11.13 was available for the serverless mode + EngineVersion: "11.13" + EngineMode: "serverless" + # NEVER LEAVE THIS ON IN PRODUCTION + EnableHttpEndpoint: true + ScalingConfiguration: + MinCapacity: 2 + MaxCapacity: 4 + AutoPause: true + SecondsUntilAutoPause: 3600 + # You can add logging with EnableCloudwatchLogsExports: + # - postgresql + DBClusterParameterGroupName: !Ref RDSParameterGroup + MasterUsername: !Sub "{{resolve:secretsmanager:${DBSecrets}:SecretString:dbUserName}}" + MasterUserPassword: !Sub "{{resolve:secretsmanager:${DBSecrets}:SecretString:dbPassword}}" + DatabaseName: !Sub "{{resolve:secretsmanager:${DBSecrets}:SecretString:dbDatabaseName}}" + Port: 5432 # note, default is 3306, even for Postgres + DBSubnetGroupName: !Ref RDSSubnetGroup + VpcSecurityGroupIds: + - !Ref DatabaseSecurityGroup + BackupRetentionPeriod: 7 + StorageEncrypted: true + + + CodeBuildExecutionRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: !Sub "${AWS::StackName}-cbExecRole" + Path: "/builder/" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "codebuild.amazonaws.com" + Action: + - "sts:AssumeRole" + ManagedPolicyArns: + - "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + - "arn:aws:iam::aws:policy/AWSCloudFormationReadOnlyAccess" + Policies: + - PolicyName: CodeBuildLoggingPolicy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + Resource: "*" + - PolicyName: CodeBuildECSTaskExecution + PolicyDocument: + Version: "2012-10-17" + Statement: + - # Secrets manager + Effect: "Allow" + Action: + - "secretsmanager:Describe*" + - "secretsmanager:Get*" + - "secretsmanager:List*" + Resource: !Ref DBSecrets + - PolicyName: CodeBuildEC2NetworkingAccess + PolicyDocument: + Version: "2012-10-17" + Statement: + - # Access to networking resources + Effect: Allow + Action: + - "ec2:CreateNetworkInterface" + - "ec2:DescribeDhcpOptions" + - "ec2:DescribeNetworkInterfaces" + - "ec2:DeleteNetworkInterface" + - "ec2:DescribeSubnets" + - "ec2:DescribeSecurityGroups" + - "ec2:DescribeVpcs" + - "ec2:CreateNetworkInterfacePermission" + Resource: "*" + + MigrationsProject: + Type: AWS::CodeBuild::Project + Properties: + Name: !Sub "${AWS::StackName}-cbMigrations" + Description: Run DB Migrations for the platform + ServiceRole: !GetAtt CodeBuildExecutionRole.Arn + Artifacts: + Type: NO_ARTIFACTS + VpcConfig: + VpcId: !Ref VPC + SecurityGroupIds: [ !Ref CodeBuildSecurityGroup ] + Subnets: [ !Ref PrivateSubnet01, !Ref PrivateSubnet02 ] + LogsConfig: + CloudWatchLogs: + GroupName: "/codebuild/migration" + Status: ENABLED + StreamName: "codebuild-log" + Environment: + Type: LINUX_CONTAINER + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/amazonlinux2-x86_64-standard:4.0 + PrivilegedMode: true + EnvironmentVariables: + - Name: "DB_DATABASE" + Value: !Join ["", ["{{resolve:secretsmanager:", !Ref "DBSecrets", ":SecretString:dbDatabaseName}}"]] + + - Name: "DB_HOSTNAME" + Value: !GetAtt DatabaseCluster.Endpoint.Address + + - Name: "DB_PORT" + Value: !GetAtt DatabaseCluster.Endpoint.Port + + - Name: "DB_USERNAME" + Value: !Join ["", ["{{resolve:secretsmanager:", !Ref "DBSecrets", ":SecretString:dbUserName}}"]] + + - Name: "DB_PASSWORD" + Value: !Join ["", ["{{resolve:secretsmanager:", !Ref "DBSecrets", ":SecretString:dbPassword}}"]] + + - Name: "AWS_REGION" + Value: !Sub "${AWS::Region}" + + Source: + Auth: + Type: OAUTH + Location: !Ref GitHubRepoUrl + Type: GITHUB + GitCloneDepth: 1 + BuildSpec: | + version: 0.2 + phases: + install: + runtime-versions: + java: corretto17 + build: + commands: + - wget -qO- https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/9.11.0/flyway-commandline-9.11.0-linux-x64.tar.gz | tar xvz && ln -s `pwd`/flyway-9.11.0/flyway /usr/local/bin + - cd rds-flyway-migrations/flyway + - flyway migrate -locations=filesystem:migrations/**/*.sql -password=$DB_PASSWORD -user=$DB_USERNAME -url=jdbc:postgresql://$DB_HOSTNAME:$DB_PORT/$DB_DATABASE -connectRetries=300 -X + TimeoutInMinutes: 10 + +Outputs: + DatabaseEndpointAddress: + Value: !GetAtt DatabaseCluster.Endpoint.Address + + DatabaseEndpointPort: + Value: !GetAtt DatabaseCluster.Endpoint.Port diff --git a/rds-flyway-migrations/deploy.sh b/rds-flyway-migrations/deploy.sh new file mode 100755 index 0000000..be4befb --- /dev/null +++ b/rds-flyway-migrations/deploy.sh @@ -0,0 +1,2 @@ +# Use with: ./deploy.sh stackname region +aws cloudformation deploy --stack-name $1 --region $2 --capabilities CAPABILITY_NAMED_IAM --template-file cloudformation.yml diff --git a/rds-flyway-migrations/flyway/migrations/V2023.02.08__001.config-db-options.sql b/rds-flyway-migrations/flyway/migrations/V2023.02.08__001.config-db-options.sql new file mode 100644 index 0000000..24a6d91 --- /dev/null +++ b/rds-flyway-migrations/flyway/migrations/V2023.02.08__001.config-db-options.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS pgcrypto; \ No newline at end of file diff --git a/rds-flyway-migrations/flyway/migrations/V2023.02.08__002.create-user-table 1.sql b/rds-flyway-migrations/flyway/migrations/V2023.02.08__002.create-user-table 1.sql new file mode 100644 index 0000000..088f8d6 --- /dev/null +++ b/rds-flyway-migrations/flyway/migrations/V2023.02.08__002.create-user-table 1.sql @@ -0,0 +1,11 @@ +/* New table for application users */ +create table user_accounts ( + id uuid primary key default gen_random_uuid(), + cognito_id varchar not null, + email_address varchar not null, + first_name varchar not null, + last_name varchar not null, + create_date date not null, + unique(cognito_id) +); + diff --git a/rds-flyway-migrations/flyway/migrations/V2023.02.08__003.create-role-tables.sql b/rds-flyway-migrations/flyway/migrations/V2023.02.08__003.create-role-tables.sql new file mode 100644 index 0000000..ed1ccdb --- /dev/null +++ b/rds-flyway-migrations/flyway/migrations/V2023.02.08__003.create-role-tables.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS app_roles ( + role_id uuid primary key default gen_random_uuid(), + role_name varchar not null +); + +CREATE TABLE IF NOT EXISTS user_app_roles ( + user_id uuid references user_accounts, + role_id uuid references app_roles, + active boolean not null default true, + primary key (user_id, role_id) +); + +insert into app_roles(role_name) values ('admin'); +insert into app_roles(role_name) values ('user'); + +select * from app_roles; + From 12ecf013e8cc12eb432e17731373049291ecb87f Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Mon, 6 Mar 2023 11:31:43 -0500 Subject: [PATCH 02/12] fat fingered the github repo name --- rds-flyway-migrations/cloudformation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds-flyway-migrations/cloudformation.yml b/rds-flyway-migrations/cloudformation.yml index 427cbde..eeefbf1 100644 --- a/rds-flyway-migrations/cloudformation.yml +++ b/rds-flyway-migrations/cloudformation.yml @@ -24,7 +24,7 @@ Parameters: GitHubRepoUrl: Description: "The URL of the project to clone" Type: "String" - Default: "https://github.com/chariotsolutions/ews-examples.git" + Default: "https://github.com/chariotsolutions/aws-examples.git" Resources: From 1d9df0b3e86efcf28437295bd0d7457b67abe1b2 Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Mon, 6 Mar 2023 13:02:24 -0500 Subject: [PATCH 03/12] fixed missing internet gateway route --- rds-flyway-migrations/cloudformation.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rds-flyway-migrations/cloudformation.yml b/rds-flyway-migrations/cloudformation.yml index eeefbf1..ebd8e3c 100644 --- a/rds-flyway-migrations/cloudformation.yml +++ b/rds-flyway-migrations/cloudformation.yml @@ -57,7 +57,6 @@ Resources: InternetGateway: Type: "AWS::EC2::InternetGateway" - InternetGatewayAttachment: Type: "AWS::EC2::VPCGatewayAttachment" Properties: @@ -76,6 +75,13 @@ Resources: SubnetId: !Ref PublicSubnet02 RouteTableId: !Ref PublicRouteTable + PublicRouteTableIGW: + Type: "AWS::EC2::Route" + Properties: + RouteTableId: !Ref PublicRouteTable + DestinationCidrBlock: "0.0.0.0/0" + GatewayId: !Ref InternetGateway + PublicRouteTable: Type: "AWS::EC2::RouteTable" Properties: From bac8c3623435b0d0f59c0cb6ecae079915e72fba Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Mon, 6 Mar 2023 13:09:35 -0500 Subject: [PATCH 04/12] add ingress rule --- rds-flyway-migrations/cloudformation.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rds-flyway-migrations/cloudformation.yml b/rds-flyway-migrations/cloudformation.yml index ebd8e3c..0168e43 100644 --- a/rds-flyway-migrations/cloudformation.yml +++ b/rds-flyway-migrations/cloudformation.yml @@ -179,6 +179,18 @@ Resources: GroupDescription: "Allows access to the database (nominally from application)" VpcId: !Ref VPC + + # Add ingress rule for codebuild to talk to RDS + DatabaseSecurityGroupCBtoDB: + Type: "AWS::EC2::SecurityGroupIngress" + Properties: + GroupId: !Ref DatabaseSecurityGroup + Description: "Access from codebuild container" + IpProtocol: "tcp" + FromPort: 5432 + ToPort: 5432 + SourceSecurityGroupId: !GetAtt CodeBuildSecurityGroup.GroupId + RDSParameterGroup: Type: "AWS::RDS::DBClusterParameterGroup" Properties: From e08672e3769421a900be7e5c3375269487f7409e Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Mon, 6 Mar 2023 14:06:53 -0500 Subject: [PATCH 05/12] fixed file names and added encryption to variables --- rds-flyway-migrations/cloudformation.yml | 17 ++++++++--------- ...l => V2023.02.08.001__config-db-options.sql} | 0 ...ql => V2023.02.08.002_create-user-table.sql} | 0 ... => V2023.02.08.003__create-role-tables.sql} | 0 4 files changed, 8 insertions(+), 9 deletions(-) rename rds-flyway-migrations/flyway/migrations/{V2023.02.08__001.config-db-options.sql => V2023.02.08.001__config-db-options.sql} (100%) rename rds-flyway-migrations/flyway/migrations/{V2023.02.08__002.create-user-table 1.sql => V2023.02.08.002_create-user-table.sql} (100%) rename rds-flyway-migrations/flyway/migrations/{V2023.02.08__003.create-role-tables.sql => V2023.02.08.003__create-role-tables.sql} (100%) diff --git a/rds-flyway-migrations/cloudformation.yml b/rds-flyway-migrations/cloudformation.yml index 0168e43..207b9ba 100644 --- a/rds-flyway-migrations/cloudformation.yml +++ b/rds-flyway-migrations/cloudformation.yml @@ -319,24 +319,18 @@ Resources: Image: aws/codebuild/amazonlinux2-x86_64-standard:4.0 PrivilegedMode: true EnvironmentVariables: - - Name: "DB_DATABASE" - Value: !Join ["", ["{{resolve:secretsmanager:", !Ref "DBSecrets", ":SecretString:dbDatabaseName}}"]] - - Name: "DB_HOSTNAME" Value: !GetAtt DatabaseCluster.Endpoint.Address - Name: "DB_PORT" Value: !GetAtt DatabaseCluster.Endpoint.Port - - Name: "DB_USERNAME" - Value: !Join ["", ["{{resolve:secretsmanager:", !Ref "DBSecrets", ":SecretString:dbUserName}}"]] - - - Name: "DB_PASSWORD" - Value: !Join ["", ["{{resolve:secretsmanager:", !Ref "DBSecrets", ":SecretString:dbPassword}}"]] - - Name: "AWS_REGION" Value: !Sub "${AWS::Region}" + - Name: "DB_SECRETS_ARN" + Value: !Ref DBSecrets + Source: Auth: Type: OAUTH @@ -345,6 +339,11 @@ Resources: GitCloneDepth: 1 BuildSpec: | version: 0.2 + env: + secrets-manager: + DB_DATABASE: ${DB_SECRETS_ARN}:dbDatabaseName + DB_USERNAME: ${DB_SECRETS_ARN}:dbUserName + DB_PASSWORD: ${DB_SECRETS_ARN}:dbPassword phases: install: runtime-versions: diff --git a/rds-flyway-migrations/flyway/migrations/V2023.02.08__001.config-db-options.sql b/rds-flyway-migrations/flyway/migrations/V2023.02.08.001__config-db-options.sql similarity index 100% rename from rds-flyway-migrations/flyway/migrations/V2023.02.08__001.config-db-options.sql rename to rds-flyway-migrations/flyway/migrations/V2023.02.08.001__config-db-options.sql diff --git a/rds-flyway-migrations/flyway/migrations/V2023.02.08__002.create-user-table 1.sql b/rds-flyway-migrations/flyway/migrations/V2023.02.08.002_create-user-table.sql similarity index 100% rename from rds-flyway-migrations/flyway/migrations/V2023.02.08__002.create-user-table 1.sql rename to rds-flyway-migrations/flyway/migrations/V2023.02.08.002_create-user-table.sql diff --git a/rds-flyway-migrations/flyway/migrations/V2023.02.08__003.create-role-tables.sql b/rds-flyway-migrations/flyway/migrations/V2023.02.08.003__create-role-tables.sql similarity index 100% rename from rds-flyway-migrations/flyway/migrations/V2023.02.08__003.create-role-tables.sql rename to rds-flyway-migrations/flyway/migrations/V2023.02.08.003__create-role-tables.sql From 781faa850eee77491b3d0bb8be92d2f28e52fea9 Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Mon, 6 Mar 2023 14:09:40 -0500 Subject: [PATCH 06/12] you need two underscores in a filename of a migration before the label --- ...eate-user-table.sql => V2023.02.08.002__create-user-table.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename rds-flyway-migrations/flyway/migrations/{V2023.02.08.002_create-user-table.sql => V2023.02.08.002__create-user-table.sql} (100%) diff --git a/rds-flyway-migrations/flyway/migrations/V2023.02.08.002_create-user-table.sql b/rds-flyway-migrations/flyway/migrations/V2023.02.08.002__create-user-table.sql similarity index 100% rename from rds-flyway-migrations/flyway/migrations/V2023.02.08.002_create-user-table.sql rename to rds-flyway-migrations/flyway/migrations/V2023.02.08.002__create-user-table.sql From 3360eddd30ae1591f867fa46f399dd851246147a Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Mon, 6 Mar 2023 15:03:35 -0500 Subject: [PATCH 07/12] added README --- rds-flyway-migrations/README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 rds-flyway-migrations/README.md diff --git a/rds-flyway-migrations/README.md b/rds-flyway-migrations/README.md new file mode 100644 index 0000000..a3265bf --- /dev/null +++ b/rds-flyway-migrations/README.md @@ -0,0 +1,21 @@ +# RDS Flyway Migrations Sample + +This sample project creates an RDS instance within the private subnet of a VPC, and installs a CodeBuild project to run Flyway database migrations to provision the database. + +Setup steps: + +1. Install AWS CLI +2. Access an AWS account with rights to create an RDS instance, VPC, and create and run CodeBuilds. +3. Run the deploy.sh script - two positional parameters are + - the name of the stack to create + - the region to create your stack within + + - Example: + ```bash + ./deploy.sh rds-flyway-migrations us-east-2 + ``` +4. Run the project created within CodeBuild to perform the migrations +5. Observe the logs to see that the environment variables for the username, database name and password are not exposed to the logs +6. You may also use the RDS control panel to perform queries against the database + + From 687d675be15cfc3b16f23ebb5f40105f50066b48 Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Tue, 7 Mar 2023 10:29:56 -0500 Subject: [PATCH 08/12] cleaned up order and added better comments to cfn --- rds-flyway-migrations/cloudformation.yml | 55 +++++++++++++----------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/rds-flyway-migrations/cloudformation.yml b/rds-flyway-migrations/cloudformation.yml index 207b9ba..4b63b67 100644 --- a/rds-flyway-migrations/cloudformation.yml +++ b/rds-flyway-migrations/cloudformation.yml @@ -1,15 +1,5 @@ ## -## Creates a VPC with two public and two private subnets, each of which supports 4096 IP -## addresses (ex reserved addresses), and with NATs for use by the private subnets. -## -## The VPC ID and Subnet IDs are exported, for use by other stacks. -## - -## -## Due to the need to populate the database, this template must be -## loaded twice: the first pass creates all resources other than the -## ECS cluster, while the second (which is run after the database is -## populated) creates the ECS Service. +## Sample RDS configuration with Flyway migrations ## AWSTemplateFormatVersion: "2010-09-09" @@ -88,7 +78,7 @@ Resources: VpcId: !Ref VPC ## - ## Private Subnets -- each needs its own route table to support NAT + ## Private Subnets - two are configured to support RDS (which requires two subnets for a serverless implementation) ## PrivateSubnet01: @@ -125,7 +115,8 @@ Resources: RouteTableId: !Ref PrivateRouteTable ## - ## NATs + ## NAT configuration - this is a demonstration sample, so we are only configuring a single NAT. + ## For production, each AZ would have its own NAT. ## ElasticIP01: @@ -133,11 +124,6 @@ Resources: Properties: Domain: "vpc" - ElasticIP02: - Type: "AWS::EC2::EIP" - Properties: - Domain: "vpc" - NAT: Type: "AWS::EC2::NatGateway" Properties: @@ -151,12 +137,10 @@ Resources: DestinationCidrBlock: "0.0.0.0/0" NatGatewayId: !Ref NAT - CodeBuildSecurityGroup: - Type: "AWS::EC2::SecurityGroup" - Properties: - GroupName: !Sub "${AWS::StackName}-CodeBuildSG" - GroupDescription: "Security Group for CodeBuild" - VpcId: !Ref VPC + ## + ## Configure a SecretsManager Secret to hold the database name, user name and password + ## in an encrypted store + ## DBSecrets: Type: AWS::SecretsManager::Secret @@ -172,6 +156,9 @@ Resources: ExcludePunctuation: true # easier to select for those times that you need to copy/paste PasswordLength: 20 + ## + ## Define network visibility rules via a security group and ingress rule + ## DatabaseSecurityGroup: Type: "AWS::EC2::SecurityGroup" Properties: @@ -179,8 +166,6 @@ Resources: GroupDescription: "Allows access to the database (nominally from application)" VpcId: !Ref VPC - - # Add ingress rule for codebuild to talk to RDS DatabaseSecurityGroupCBtoDB: Type: "AWS::EC2::SecurityGroupIngress" Properties: @@ -191,6 +176,10 @@ Resources: ToPort: 5432 SourceSecurityGroupId: !GetAtt CodeBuildSecurityGroup.GroupId + ## + ## RDS Configuration - Parameter Group, Subnet Group, and Cluster + ## + RDSParameterGroup: Type: "AWS::RDS::DBClusterParameterGroup" Properties: @@ -209,6 +198,9 @@ Resources: - !Ref PrivateSubnet01 - !Ref PrivateSubnet02 + ## + ## This is a Serverless Aurora V1 RDS Cluster supporting Postgres 11 + ## DatabaseCluster: Type: "AWS::RDS::DBCluster" Properties: @@ -239,6 +231,17 @@ Resources: BackupRetentionPeriod: 7 StorageEncrypted: true + ## + ## CodeBuild - set up security group, execution role and CodeBuild project + ## + + CodeBuildSecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupName: !Sub "${AWS::StackName}-CodeBuildSG" + GroupDescription: "Security Group for CodeBuild" + VpcId: !Ref VPC + CodeBuildExecutionRole: Type: "AWS::IAM::Role" From c51d6bb39f88420193be8f407b89e19308629dc3 Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Wed, 8 Mar 2023 10:22:03 -0500 Subject: [PATCH 09/12] upgraded to serverless v2 --- rds-flyway-migrations/cloudformation.yml | 36 ++++++++++-------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/rds-flyway-migrations/cloudformation.yml b/rds-flyway-migrations/cloudformation.yml index 4b63b67..84a9e18 100644 --- a/rds-flyway-migrations/cloudformation.yml +++ b/rds-flyway-migrations/cloudformation.yml @@ -177,48 +177,39 @@ Resources: SourceSecurityGroupId: !GetAtt CodeBuildSecurityGroup.GroupId ## - ## RDS Configuration - Parameter Group, Subnet Group, and Cluster + ## Database - Aurora Serverless V2 ## RDSParameterGroup: Type: "AWS::RDS::DBClusterParameterGroup" Properties: - DBClusterParameterGroupName: !Sub "${AWS::StackName}-dbcluster-param-group" Description: !Sub "Managed by CloudFormation: ${AWS::StackName}" - Family: "aurora-postgresql11" + Family: "aurora-postgresql14" Parameters: # cluster parameter groups must have parameters; this is mostly harmless, customize as you wish client_encoding: "UTF8" RDSSubnetGroup: Type: "AWS::RDS::DBSubnetGroup" Properties: - DBSubnetGroupName: !Sub "${AWS::StackName}-db-subnet-group" + DBSubnetGroupName: !Sub "${AWS::StackName}-react-nodejs-sample" DBSubnetGroupDescription: !Sub "Managed by CloudFormation: ${AWS::StackName}" SubnetIds: - - !Ref PrivateSubnet01 - - !Ref PrivateSubnet02 + - !Ref PrivateSubnet01 + - !Ref PrivateSubnet02 - ## - ## This is a Serverless Aurora V1 RDS Cluster supporting Postgres 11 - ## DatabaseCluster: Type: "AWS::RDS::DBCluster" Properties: - DBClusterIdentifier: !Sub "${AWS::StackName}-PostgresCluster" + DBClusterIdentifier: !Sub "${AWS::StackName}-postgres-cluster" Engine: "aurora-postgresql" - # n.b. this query tells you the provisioned -vs- serverless versions available - # aws rds describe-db-engine-versions --engine aurora-postgresql --query 'DBEngineVersions[].[EngineVersion, SupportedEngineModes]' - # Only 11.13 was available for the serverless mode - EngineVersion: "11.13" - EngineMode: "serverless" + EngineMode: "provisioned" + EngineVersion: "14.5" # NEVER LEAVE THIS ON IN PRODUCTION EnableHttpEndpoint: true - ScalingConfiguration: - MinCapacity: 2 - MaxCapacity: 4 - AutoPause: true - SecondsUntilAutoPause: 3600 - # You can add logging with EnableCloudwatchLogsExports: + ServerlessV2ScalingConfiguration: + MinCapacity: 0.5 + MaxCapacity: 2 + # TODO - enable logs? EnableCloudwatchLogsExports: # - postgresql DBClusterParameterGroupName: !Ref RDSParameterGroup MasterUsername: !Sub "{{resolve:secretsmanager:${DBSecrets}:SecretString:dbUserName}}" @@ -228,6 +219,9 @@ Resources: DBSubnetGroupName: !Ref RDSSubnetGroup VpcSecurityGroupIds: - !Ref DatabaseSecurityGroup + # TODO - define when backups and maintenance run + # PreferredMaintenanceWindow: "Sun:06:00-Sun:06:59" + # PreferredBackupWindow: "05:00-05:30" BackupRetentionPeriod: 7 StorageEncrypted: true From 851b8fab34eaec8740babaa9ddcf1ab971c79aa5 Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Mon, 13 Mar 2023 12:45:26 -0400 Subject: [PATCH 10/12] Proper settings for Serverless V2 --- rds-flyway-migrations/cloudformation.yml | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/rds-flyway-migrations/cloudformation.yml b/rds-flyway-migrations/cloudformation.yml index 84a9e18..6e6427c 100644 --- a/rds-flyway-migrations/cloudformation.yml +++ b/rds-flyway-migrations/cloudformation.yml @@ -17,7 +17,6 @@ Parameters: Default: "https://github.com/chariotsolutions/aws-examples.git" Resources: - VPC: Type: "AWS::EC2::VPC" Properties: @@ -176,7 +175,7 @@ Resources: ToPort: 5432 SourceSecurityGroupId: !GetAtt CodeBuildSecurityGroup.GroupId - ## + ## ## Database - Aurora Serverless V2 ## @@ -185,27 +184,21 @@ Resources: Properties: Description: !Sub "Managed by CloudFormation: ${AWS::StackName}" Family: "aurora-postgresql14" - Parameters: # cluster parameter groups must have parameters; this is mostly harmless, customize as you wish + Parameters: client_encoding: "UTF8" RDSSubnetGroup: Type: "AWS::RDS::DBSubnetGroup" Properties: - DBSubnetGroupName: !Sub "${AWS::StackName}-react-nodejs-sample" + DBSubnetGroupName: !Sub "${AWS::StackName}-rds-subnet-group" DBSubnetGroupDescription: !Sub "Managed by CloudFormation: ${AWS::StackName}" - SubnetIds: - - !Ref PrivateSubnet01 - - !Ref PrivateSubnet02 + SubnetIds: [ !Ref PrivateSubnet01, !Ref PrivateSubnet02 ] DatabaseCluster: Type: "AWS::RDS::DBCluster" Properties: - DBClusterIdentifier: !Sub "${AWS::StackName}-postgres-cluster" Engine: "aurora-postgresql" - EngineMode: "provisioned" - EngineVersion: "14.5" - # NEVER LEAVE THIS ON IN PRODUCTION - EnableHttpEndpoint: true + EngineVersion: "14.6" ServerlessV2ScalingConfiguration: MinCapacity: 0.5 MaxCapacity: 2 @@ -225,6 +218,14 @@ Resources: BackupRetentionPeriod: 7 StorageEncrypted: true + ServerlessInstanceRedundantMuch: + Type: "AWS::RDS::DBInstance" + Properties: + DBClusterIdentifier: !Ref DatabaseCluster + DBInstanceClass: db.serverless + Engine: aurora-postgresql + + ## ## CodeBuild - set up security group, execution role and CodeBuild project ## From ee2697be410d55995cac210a16461e94ab506bef Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Mon, 13 Mar 2023 12:46:21 -0400 Subject: [PATCH 11/12] clean up comments --- rds-flyway-migrations/cloudformation.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rds-flyway-migrations/cloudformation.yml b/rds-flyway-migrations/cloudformation.yml index 6e6427c..57b7af8 100644 --- a/rds-flyway-migrations/cloudformation.yml +++ b/rds-flyway-migrations/cloudformation.yml @@ -202,7 +202,8 @@ Resources: ServerlessV2ScalingConfiguration: MinCapacity: 0.5 MaxCapacity: 2 - # TODO - enable logs? EnableCloudwatchLogsExports: + # enable logs? + # EnableCloudwatchLogsExports: # - postgresql DBClusterParameterGroupName: !Ref RDSParameterGroup MasterUsername: !Sub "{{resolve:secretsmanager:${DBSecrets}:SecretString:dbUserName}}" @@ -212,7 +213,7 @@ Resources: DBSubnetGroupName: !Ref RDSSubnetGroup VpcSecurityGroupIds: - !Ref DatabaseSecurityGroup - # TODO - define when backups and maintenance run + # can define when backups and maintenance run # PreferredMaintenanceWindow: "Sun:06:00-Sun:06:59" # PreferredBackupWindow: "05:00-05:30" BackupRetentionPeriod: 7 From 54798df8a9647935cb33ed09829b68eb9addde99 Mon Sep 17 00:00:00 2001 From: Ken Rimple Date: Mon, 13 Mar 2023 12:52:29 -0400 Subject: [PATCH 12/12] renamed the dbinstance in cloudformation --- rds-flyway-migrations/cloudformation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds-flyway-migrations/cloudformation.yml b/rds-flyway-migrations/cloudformation.yml index 57b7af8..a8f80e3 100644 --- a/rds-flyway-migrations/cloudformation.yml +++ b/rds-flyway-migrations/cloudformation.yml @@ -219,7 +219,7 @@ Resources: BackupRetentionPeriod: 7 StorageEncrypted: true - ServerlessInstanceRedundantMuch: + ServerlessInstance: Type: "AWS::RDS::DBInstance" Properties: DBClusterIdentifier: !Ref DatabaseCluster