From 3dc5aa7ec8a5184f9b476d33cbe3d5a0d260f6ae Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Mon, 3 Oct 2022 11:06:53 +0530 Subject: [PATCH 01/98] Added cloud formation template to deploy dashboard + mongo --- templates/akto-quick-setup.yaml | 286 ++++++++++++++++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 templates/akto-quick-setup.yaml diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml new file mode 100644 index 0000000..0509408 --- /dev/null +++ b/templates/akto-quick-setup.yaml @@ -0,0 +1,286 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: + This template does a simple setup for all Akto modules. It sets up all + modules on a single instance. If you want a scalable and flexible setup, please + contact support@akto.io. +Parameters: + SubnetId: + Description: + Select existing subnet for Akto EC2 instance. If you are choosing + private subnet, then make sure private subnet is configured to access public + internet + Type: AWS::EC2::Subnet::Id + ConstraintDescription: must be an existing subnet + KeyPair: + Description: Select the key pair to connect to Akto EC2 instance + Type: AWS::EC2::KeyPair::KeyName + VpcId: + Description: Select vpc + Type: AWS::EC2::VPC::Id + VpcCidrBlock: + Type: String + Description: Enter cidr block for selected vpc + PublicSubnetIds: + Type: List + Description: + The list of public subnets in your VPC. Select the 1 public subnet + from the same availability zone as SubnetId + ConstraintDescription: + must be a list of at least two existing subnets associated + with at least two different availability zones. They should be residing in the + selected Virtual Private Cloud. +Mappings: + RegionMap: + "af-south-1": + AMI: "ami-0adee70ff4394e3d5" + "eu-north-1": + AMI: "ami-04e8b0e36ed3403dc" + "ap-south-1": + AMI: "ami-09de362f44ba0a166" + "eu-west-3": + AMI: "ami-0614433a16ab15878" + "eu-west-2": + AMI: "ami-030770b178fa9d374" + "eu-south-1": + AMI: "ami-0432f14b68c3e0273" + "eu-west-1": + AMI: "ami-0bba0a4cb75835f71" + "ap-northeast-3": + AMI: "ami-0253beba286f3e848" + "ap-northeast-2": + AMI: "ami-0e1d09d8b7c751816" + "me-south-1": + AMI: "ami-07a68e42e669daed0" + "ap-northeast-1": + AMI: "ami-06ce6680729711877" + "sa-east-1": + AMI: "ami-0656df2cc0dfd150a" + "ca-central-1": + AMI: "ami-04c12937e87474def" + "ap-east-1": + AMI: "ami-0b751f901b93720a5" + "ap-southeast-1": + AMI: "ami-0adf622550366ea53" + "ap-southeast-2": + AMI: "ami-03b836d87d294e89e" + "eu-central-1": + AMI: "ami-094c442a8e9a67935" + "ap-southeast-3": + AMI: "ami-0483d92a8124da6c9" + "us-east-1": + AMI: "ami-065efef2c739d613b" + "us-east-2": + AMI: "ami-07251f912d2a831a3" + "us-west-1": + AMI: "ami-09b2f6d85764ec71b" + "us-west-2": + AMI: "ami-0d08ef957f0e4722b" +Resources: + ## Setting up mongo + AktoMongoSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: !Ref VpcId + GroupDescription: Enable the ports Mongo requires (22, 27017) + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCidrBlock + - IpProtocol: tcp + FromPort: 27017 + ToPort: 27017 + CidrIp: !Ref VpcCidrBlock + SecurityGroupEgress: [] + AktoMongoLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateData: + ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] + InstanceType: m5.xlarge + KeyName: + Ref: KeyPair + NetworkInterfaces: + - SubnetId: !Ref SubnetId + AssociatePublicIpAddress: "false" + DeviceIndex: 0 + Groups: [!Ref AktoMongoSecurityGroup] + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + VolumeType: gp2 + DeleteOnTermination: "true" + VolumeSize: "50" + Encrypted: true + MetadataOptions: + HttpTokens: required + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-mongo.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + AktoMongoInstance: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: Akto Mongo instance + LaunchTemplate: + LaunchTemplateId: !Ref AktoMongoLaunchTemplate + Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + ## Setting up Dashboard + AktoDashboardTargetTrackingNetworkPolicy: + Type: AWS::AutoScaling::ScalingPolicy + Properties: + PolicyType: TargetTrackingScaling + AutoScalingGroupName: + Ref: AktoDashboardAutoScalingGroup + EstimatedInstanceWarmup: 30 + TargetTrackingConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ASGAverageCPUUtilization + TargetValue: 60 + AktoDashboardSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: !Ref VpcId + GroupDescription: Enable the ports Akto requires (22, 8080) + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCidrBlock + - IpProtocol: tcp + FromPort: 8080 + ToPort: 8080 + CidrIp: !Ref VpcCidrBlock + SecurityGroupEgress: [] + AktoDashboardTargetGroup2: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: "8080" + Protocol: HTTP + HealthCheckEnabled: "true" + HealthCheckIntervalSeconds: 10 + HealthCheckPath: "/metrics" + HealthCheckPort: "8080" + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 6 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 2 + TargetType: instance + VpcId: !Ref VpcId + Targets: [] + Name: AktoDashboardTargetGroup2 + AktoDashboardAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + AutoScalingGroupName: AktoDashboardAutoScalingGroup + VPCZoneIdentifier: + - Ref: SubnetId + LaunchConfigurationName: + Ref: AktoDashboardASGLaunchConfiguration + TargetGroupARNs: + - Ref: AktoDashboardTargetGroup2 + MaxSize: "10" + MinSize: "1" + + AktoDashboardASGLaunchConfiguration: + Type: AWS::AutoScaling::LaunchConfiguration + DependsOn: + - AktoMongoInstance + #- AktoNLB + Properties: + InstanceType: m5.xlarge + ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] + AssociatePublicIpAddress: "false" + KeyName: + Ref: KeyPair + SecurityGroups: + - Ref: AktoDashboardSecurityGroup + MetadataOptions: + HttpTokens: required + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + VolumeType: gp2 + DeleteOnTermination: "true" + VolumeSize: "20" + Encrypted: true + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - Fn::Sub: export AKTO_MONGO_CONN='${AktoMongoInstance.PrivateIp}' + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - export COMPOSE_FILE=docker-compose-dashboard.yml + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - sudo echo >> ~/akto/infra/docker-dashboard.env + - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_CONN:27017/admini >> + ~/akto/infra/docker-dashboard.env + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + AktoDashboardListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + LoadBalancerArn: + Ref: AktoLBDashboard + Port: "80" + Protocol: HTTP + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: AktoDashboardTargetGroup2 + AktoLBSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: !Ref VpcId + GroupDescription: Enable the port AktoLB requires + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIp: !Ref VpcCidrBlock + SecurityGroupEgress: [] + AktoLBDashboard: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Type: application + IpAddressType: ipv4 + Subnets: + Ref: PublicSubnetIds + Name: AktoLBDashboard + SecurityGroups: [!Ref AktoLBSecurityGroup] +Outputs: + AktoLBDashboard: + Value: + Ref: AktoLBDashboard + Description: Arn of Akto Dashboard LB From 375bc2d69340f0a73e5f1d134dbe0ba5651defc9 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Fri, 14 Oct 2022 10:47:25 +0530 Subject: [PATCH 02/98] Added target group and listener --- templates/akto-quick-setup.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 0509408..d6fdb1b 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -279,6 +279,34 @@ Resources: Ref: PublicSubnetIds Name: AktoLBDashboard SecurityGroups: [!Ref AktoLBSecurityGroup] + AktoDashboardTargetGroup2: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: "8080" + Protocol: HTTP + HealthCheckEnabled: "true" + HealthCheckIntervalSeconds: 10 + HealthCheckPath: "/metrics" + HealthCheckPort: "8080" + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 6 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 2 + TargetType: instance + VpcId: !Ref VpcId + Targets: [] + Name: AktoDashboardTargetGroup2 + AktoDashboardListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + LoadBalancerArn: + Ref: AktoLBDashboard + Port: "80" + Protocol: HTTP + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: AktoDashboardTargetGroup2 Outputs: AktoLBDashboard: Value: From 7e2ac79b285df2817aeff76ca2d7b7f7ea6315ed Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Fri, 14 Oct 2022 11:32:11 +0530 Subject: [PATCH 03/98] Added changes to export keypair and subnetid as env vars in dashboard --- templates/akto-quick-setup.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index d6fdb1b..89f97cf 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -231,6 +231,8 @@ Resources: - touch /tmp/hello.txt - touch ~/hello.txt - Fn::Sub: export AKTO_MONGO_CONN='${AktoMongoInstance.PrivateIp}' + - Fn::Sub: export EC2_KEY_PAIR='${KeyPair}' + - Fn::Sub: export EC2_SUBNET_ID='${SubnetId}' - sudo yum update -y - sudo yum install -y python python-setuptools - sudo yum install -y docker From 06e2137a86bfb9060ef10941f2377ba88522d957 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Thu, 20 Oct 2022 00:22:04 +0530 Subject: [PATCH 04/98] [To be reverted] For testing --- docker-compose-dashboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 901d3ba..8931ff4 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -2,7 +2,7 @@ version: '2' services: akto-api-security-dashboard: - image: aktosecurity/akto-api-security-dashboard:latest + image: aktosecurity/akto-api-security-dashboard:qs env_file: ./docker-dashboard.env restart: always ports: From bcd5929de9a2c32aa953a1cd0927668838861940 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Thu, 20 Oct 2022 11:29:31 +0530 Subject: [PATCH 05/98] [To be reverted] For testing --- templates/akto-quick-setup.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 89f97cf..2495593 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -129,10 +129,10 @@ Resources: - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" AktoMongoInstance: @@ -241,13 +241,13 @@ Resources: - export COMPOSE_FILE=docker-compose-dashboard.yml - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - sudo echo >> ~/akto/infra/docker-dashboard.env - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_CONN:27017/admini >> ~/akto/infra/docker-dashboard.env - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" AktoDashboardListener: From f276a42f1b910d567f9c7570316b46bcc9ef5f00 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Thu, 20 Oct 2022 15:20:08 +0530 Subject: [PATCH 06/98] [To be reverted] For testing --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 71dc4ff..663e3f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,7 +60,7 @@ services: - "27017:27017" akto-api-security-dashboard: - image: aktosecurity/akto-api-security-dashboard:latest + image: aktosecurity/akto-api-security-dashboard:qs env_file: ./docker.env restart: always mem_limit: 4g From a1c0804b19b2fd978c2cc31242c49d14e0d99233 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Fri, 21 Oct 2022 00:31:37 +0530 Subject: [PATCH 07/98] Added missing env vars --- templates/akto-quick-setup.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 2495593..ae186e2 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -247,6 +247,8 @@ Resources: - sudo echo >> ~/akto/infra/docker-dashboard.env - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_CONN:27017/admini >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" From f2624ff09bd2289959a6bd0cd66167b603ca0549 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Fri, 21 Oct 2022 15:39:00 +0530 Subject: [PATCH 08/98] [To be reverted] For testing --- cf-deploy-akto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 94a87ea..452a158 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/segregation_2.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-segregation_2 infra + mv infra-feature-quick-setup infra fi # NB: this is to make onprem containers to all get named the same. From d236280e25b408e81d3d593b1b69726f56238a97 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Wed, 26 Oct 2022 21:10:43 +0530 Subject: [PATCH 09/98] Fixed quick setup template --- templates/akto-quick-setup.yaml | 51 +++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index ae186e2..6f8fb86 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -15,7 +15,7 @@ Parameters: Description: Select the key pair to connect to Akto EC2 instance Type: AWS::EC2::KeyPair::KeyName VpcId: - Description: Select vpc + Description: Select a vpc where your resources are deployed, Akto will be deployed in this VPC only. Type: AWS::EC2::VPC::Id VpcCidrBlock: Type: String @@ -114,6 +114,7 @@ Resources: Encrypted: true MetadataOptions: HttpTokens: required + HttpPutResponseHopLimit: 2 UserData: Fn::Base64: Fn::Join: @@ -200,22 +201,40 @@ Resources: - Ref: AktoDashboardTargetGroup2 MaxSize: "10" MinSize: "1" - + AktoDashboardRole: + Type: AWS::IAM::Role + Properties: + RoleName: AktoDashboardRole + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + IamInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: "/" + Roles: + - Ref: AktoDashboardRole AktoDashboardASGLaunchConfiguration: Type: AWS::AutoScaling::LaunchConfiguration DependsOn: - AktoMongoInstance - #- AktoNLB Properties: InstanceType: m5.xlarge ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] AssociatePublicIpAddress: "false" KeyName: Ref: KeyPair + IamInstanceProfile: + Ref: IamInstanceProfile SecurityGroups: - Ref: AktoDashboardSecurityGroup MetadataOptions: HttpTokens: required + HttpPutResponseHopLimit: 2 BlockDeviceMappings: - DeviceName: "/dev/xvda" Ebs: @@ -231,8 +250,6 @@ Resources: - touch /tmp/hello.txt - touch ~/hello.txt - Fn::Sub: export AKTO_MONGO_CONN='${AktoMongoInstance.PrivateIp}' - - Fn::Sub: export EC2_KEY_PAIR='${KeyPair}' - - Fn::Sub: export EC2_SUBNET_ID='${SubnetId}' - sudo yum update -y - sudo yum install -y python python-setuptools - sudo yum install -y docker @@ -247,6 +264,8 @@ Resources: - sudo echo >> ~/akto/infra/docker-dashboard.env - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_CONN:27017/admini >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo EC2_SUBNET_ID='${SubnetId}' >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start @@ -300,19 +319,13 @@ Resources: VpcId: !Ref VpcId Targets: [] Name: AktoDashboardTargetGroup2 - AktoDashboardListener: - Type: AWS::ElasticLoadBalancingV2::Listener - Properties: - LoadBalancerArn: - Ref: AktoLBDashboard - Port: "80" - Protocol: HTTP - DefaultActions: - - Type: forward - TargetGroupArn: - Ref: AktoDashboardTargetGroup2 Outputs: AktoLBDashboard: - Value: - Ref: AktoLBDashboard - Description: Arn of Akto Dashboard LB + Value: !Join + - '' + - - 'http://' + - !GetAtt + - AktoLBDashboard + - DNSName + - ':80' + Description: Url of Akto Dashboard LB From 82c08222149250e05629338f35ac1df60667f0fa Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Mon, 31 Oct 2022 00:13:39 +0530 Subject: [PATCH 10/98] Updated CFT --- templates/akto-quick-setup.yaml | 125 +++++++++++++++++++++++++++----- 1 file changed, 108 insertions(+), 17 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 6f8fb86..767afbd 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -14,12 +14,6 @@ Parameters: KeyPair: Description: Select the key pair to connect to Akto EC2 instance Type: AWS::EC2::KeyPair::KeyName - VpcId: - Description: Select a vpc where your resources are deployed, Akto will be deployed in this VPC only. - Type: AWS::EC2::VPC::Id - VpcCidrBlock: - Type: String - Description: Enter cidr block for selected vpc PublicSubnetIds: Type: List Description: @@ -76,21 +70,99 @@ Mappings: "us-west-2": AMI: "ami-0d08ef957f0e4722b" Resources: + ## Setting up VpcDetailsLambda + GetVpcDetailsLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ec2:DescribeVpcs + - ec2:DescribeSubnets + Resource: "*" + GetVpcDetailsLambda: + Type: AWS::Lambda::Function + Properties: + Description: Look up info from a VPC + Handler: index.handler + Runtime: nodejs12.x + Timeout: 30 + Role: + Fn::GetAtt: + - GetVpcDetailsLambdaRole + - Arn + Environment: + Variables: + SUBNET_ID: + Ref: SubnetId + Code: + ZipFile: > + var SUBNET_ID = process.env.SUBNET_ID; + var aws = require('aws-sdk'); + var response = require('cfn-response'); + var ec2 = new aws.EC2(); + exports.handler = async function(event, context) { + if (event.RequestType == 'Delete') { + await response.send(event, context, 'SUCCESS'); + return; + } + var params = { + SubnetIds: [SUBNET_ID] + }; + var subnets = await ec2.describeSubnets(params).promise().catch(err => { + console.error(err); + }); + var vpcId = subnets['Subnets'][0]['VpcId']; + var vpcs = await ec2.describeVpcs({VpcIds: [vpcId]}).promise().catch(err => { + console.error(err); + }); + await response.send(event, context, 'SUCCESS', {CidrBlock: vpcs['Vpcs'][0]['CidrBlock'], VpcId: vpcId}) + }; + CustomSourceGetVpcDetails: + Type: AWS::CloudFormation::CustomResource + Properties: + ServiceToken: + Fn::GetAtt: + - GetVpcDetailsLambda + - Arn ## Setting up mongo AktoMongoSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: - VpcId: !Ref VpcId + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId GroupDescription: Enable the ports Mongo requires (22, 27017) SecurityGroupIngress: - IpProtocol: tcp FromPort: 22 ToPort: 22 - CidrIp: !Ref VpcCidrBlock + CidrIp: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - CidrBlock - IpProtocol: tcp FromPort: 27017 ToPort: 27017 - CidrIp: !Ref VpcCidrBlock + CidrIp: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - CidrBlock SecurityGroupEgress: [] AktoMongoLaunchTemplate: Type: AWS::EC2::LaunchTemplate @@ -160,17 +232,26 @@ Resources: AktoDashboardSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: - VpcId: !Ref VpcId + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId GroupDescription: Enable the ports Akto requires (22, 8080) SecurityGroupIngress: - IpProtocol: tcp FromPort: 22 ToPort: 22 - CidrIp: !Ref VpcCidrBlock + CidrIp: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - CidrBlock - IpProtocol: tcp FromPort: 8080 ToPort: 8080 - CidrIp: !Ref VpcCidrBlock + CidrIp: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - CidrBlock SecurityGroupEgress: [] AktoDashboardTargetGroup2: Type: AWS::ElasticLoadBalancingV2::TargetGroup @@ -186,7 +267,10 @@ Resources: HealthyThresholdCount: 2 UnhealthyThresholdCount: 2 TargetType: instance - VpcId: !Ref VpcId + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId Targets: [] Name: AktoDashboardTargetGroup2 AktoDashboardAutoScalingGroup: @@ -266,7 +350,8 @@ Resources: ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo EC2_SUBNET_ID='${SubnetId}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start @@ -285,13 +370,16 @@ Resources: AktoLBSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: - VpcId: !Ref VpcId + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId GroupDescription: Enable the port AktoLB requires SecurityGroupIngress: - IpProtocol: tcp FromPort: 80 ToPort: 80 - CidrIp: !Ref VpcCidrBlock + CidrIp: 0.0.0.0/0 SecurityGroupEgress: [] AktoLBDashboard: Type: AWS::ElasticLoadBalancingV2::LoadBalancer @@ -316,7 +404,10 @@ Resources: HealthyThresholdCount: 2 UnhealthyThresholdCount: 2 TargetType: instance - VpcId: !Ref VpcId + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId Targets: [] Name: AktoDashboardTargetGroup2 Outputs: From 1be3768bdd6ff73fab078c194c49182083e8fe22 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Thu, 3 Nov 2022 11:13:44 +0530 Subject: [PATCH 11/98] Revert "[To be reverted] For testing" This reverts commit 06e2137a86bfb9060ef10941f2377ba88522d957. --- docker-compose-dashboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 8931ff4..901d3ba 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -2,7 +2,7 @@ version: '2' services: akto-api-security-dashboard: - image: aktosecurity/akto-api-security-dashboard:qs + image: aktosecurity/akto-api-security-dashboard:latest env_file: ./docker-dashboard.env restart: always ports: From 352595abf2ca1dd8d6f05229f4be07b13ecb7543 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Thu, 3 Nov 2022 11:14:11 +0530 Subject: [PATCH 12/98] Revert "[To be reverted] For testing" This reverts commit f276a42f1b910d567f9c7570316b46bcc9ef5f00. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 663e3f5..71dc4ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,7 +60,7 @@ services: - "27017:27017" akto-api-security-dashboard: - image: aktosecurity/akto-api-security-dashboard:qs + image: aktosecurity/akto-api-security-dashboard:latest env_file: ./docker.env restart: always mem_limit: 4g From 3155add1e48f0163b6ce0b5adfa7cd7f196cc5a6 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Thu, 3 Nov 2022 14:05:47 +0530 Subject: [PATCH 13/98] Updated instance type to m5a.xlarge --- templates/akto-quick-setup.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 767afbd..ff0e065 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -169,7 +169,7 @@ Resources: Properties: LaunchTemplateData: ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] - InstanceType: m5.xlarge + InstanceType: m5a.xlarge KeyName: Ref: KeyPair NetworkInterfaces: @@ -307,7 +307,7 @@ Resources: DependsOn: - AktoMongoInstance Properties: - InstanceType: m5.xlarge + InstanceType: m5a.xlarge ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] AssociatePublicIpAddress: "false" KeyName: From 10b6fae53004765642a9ff9538fac5acb56e9a1d Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Thu, 3 Nov 2022 14:35:12 +0530 Subject: [PATCH 14/98] Updated description --- templates/akto-quick-setup.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index ff0e065..2a71d7d 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -1,7 +1,6 @@ AWSTemplateFormatVersion: "2010-09-09" Description: - This template does a simple setup for all Akto modules. It sets up all - modules on a single instance. If you want a scalable and flexible setup, please + This template does a quick setup of Akto modules. It sets up Akto dashboard and mongo on seperate ec2 instances. If you want a flexible setup, please contact support@akto.io. Parameters: SubnetId: From 6a09fe2c89f6c5438d575bd37d3016419b38e1e3 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Wed, 9 Nov 2022 15:34:54 +0530 Subject: [PATCH 15/98] Demonize docker-compose up --- cf-deploy-akto | 4 ++-- cf-deploy-akto-start | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 452a158..19ebfc0 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/akto-experience.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-quick-setup infra + mv infra-feature-akto-experience infra fi # NB: this is to make onprem containers to all get named the same. diff --git a/cf-deploy-akto-start b/cf-deploy-akto-start index 8b3f85c..358dc82 100644 --- a/cf-deploy-akto-start +++ b/cf-deploy-akto-start @@ -29,7 +29,7 @@ fi cd ~/akto/infra -$MAYBE_SUDO docker-compose up +$MAYBE_SUDO docker-compose up -d echo "" echo " -- ${GREEN}${BOLD}!! AKTO IS BOOTING !!${NORMAL} --" From 4d85ff39cad91f2b600cd72c72f4bb8ada17f437 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Wed, 9 Nov 2022 18:21:37 +0530 Subject: [PATCH 16/98] Added changes to fix 502 bad gateway --- templates/akto-quick-setup.yaml | 143 ++++++++++++++++---------------- 1 file changed, 70 insertions(+), 73 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 2a71d7d..cde9525 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -252,38 +252,24 @@ Resources: - CustomSourceGetVpcDetails - CidrBlock SecurityGroupEgress: [] - AktoDashboardTargetGroup2: - Type: AWS::ElasticLoadBalancingV2::TargetGroup - Properties: - Port: "8080" - Protocol: HTTP - HealthCheckEnabled: "true" - HealthCheckIntervalSeconds: 10 - HealthCheckPath: "/metrics" - HealthCheckPort: "8080" - HealthCheckProtocol: HTTP - HealthCheckTimeoutSeconds: 6 - HealthyThresholdCount: 2 - UnhealthyThresholdCount: 2 - TargetType: instance - VpcId: - Fn::GetAtt: - - CustomSourceGetVpcDetails - - VpcId - Targets: [] - Name: AktoDashboardTargetGroup2 AktoDashboardAutoScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup Properties: AutoScalingGroupName: AktoDashboardAutoScalingGroup VPCZoneIdentifier: - Ref: SubnetId - LaunchConfigurationName: - Ref: AktoDashboardASGLaunchConfiguration + LaunchTemplate: + LaunchTemplateId: !Ref AktoDashboardLaunchTemplate + Version: !GetAtt AktoDashboardLaunchTemplate.LatestVersionNumber TargetGroupARNs: - Ref: AktoDashboardTargetGroup2 MaxSize: "10" MinSize: "1" + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 10 + ResourceSignal: + Timeout: PT5M AktoDashboardRole: Type: AWS::IAM::Role Properties: @@ -301,60 +287,71 @@ Resources: Path: "/" Roles: - Ref: AktoDashboardRole - AktoDashboardASGLaunchConfiguration: - Type: AWS::AutoScaling::LaunchConfiguration + AktoDashboardLaunchTemplate: + Type: AWS::EC2::LaunchTemplate DependsOn: - AktoMongoInstance Properties: - InstanceType: m5a.xlarge - ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] - AssociatePublicIpAddress: "false" - KeyName: - Ref: KeyPair - IamInstanceProfile: - Ref: IamInstanceProfile - SecurityGroups: - - Ref: AktoDashboardSecurityGroup - MetadataOptions: - HttpTokens: required - HttpPutResponseHopLimit: 2 - BlockDeviceMappings: - - DeviceName: "/dev/xvda" - Ebs: - VolumeType: gp2 - DeleteOnTermination: "true" - VolumeSize: "20" - Encrypted: true - UserData: - Fn::Base64: - Fn::Join: - - "\n" - - - "#!/bin/bash -xe" - - touch /tmp/hello.txt - - touch ~/hello.txt - - Fn::Sub: export AKTO_MONGO_CONN='${AktoMongoInstance.PrivateIp}' - - sudo yum update -y - - sudo yum install -y python python-setuptools - - sudo yum install -y docker - - sudo dockerd& - - sudo mkdir -p /opt/aws/bin - - export COMPOSE_FILE=docker-compose-dashboard.yml - - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - - sudo chmod 700 cf-deploy-akto - - "./cf-deploy-akto < <(echo 'test')" - - sudo echo >> ~/akto/infra/docker-dashboard.env - - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_CONN:27017/admini >> - ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo EC2_SUBNET_ID='${SubnetId}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - - sudo chmod 700 cf-deploy-akto-start - - "./cf-deploy-akto-start < <(echo 'test')" + LaunchTemplateData: + ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] + InstanceType: m5a.xlarge + KeyName: + Ref: KeyPair + NetworkInterfaces: + - SubnetId: !Ref SubnetId + AssociatePublicIpAddress: "false" + DeviceIndex: 0 + Groups: [!Ref AktoDashboardSecurityGroup] + IamInstanceProfile: + Arn: !GetAtt IamInstanceProfile.Arn + MetadataOptions: + HttpTokens: required + HttpPutResponseHopLimit: 2 + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + VolumeType: gp2 + DeleteOnTermination: "true" + VolumeSize: "20" + Encrypted: true + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - Fn::Sub: export AKTO_MONGO_CONN='${AktoMongoInstance.PrivateIp}' + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - export COMPOSE_FILE=docker-compose-dashboard.yml + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - sudo echo >> ~/akto/infra/docker-dashboard.env + - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_CONN:27017/admini >> + ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo EC2_SUBNET_ID='${SubnetId}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoDashboardAutoScalingGroup" + - " --region " + - Ref: AWS::Region AktoDashboardListener: Type: AWS::ElasticLoadBalancingV2::Listener Properties: From 060a17843bd8c4b32388fd73e1118f65b200e004 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Wed, 9 Nov 2022 18:24:32 +0530 Subject: [PATCH 17/98] Reverted branch --- cf-deploy-akto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 19ebfc0..452a158 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/akto-experience.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-akto-experience infra + mv infra-feature-quick-setup infra fi # NB: this is to make onprem containers to all get named the same. From eacc145899d9566e18d18afdcf7205ca5ccb1253 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Sun, 27 Nov 2022 00:09:13 +0530 Subject: [PATCH 18/98] Updated template to track stack deployment --- templates/akto-quick-setup.yaml | 93 ++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 25 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index cde9525..acf7c58 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -22,6 +22,8 @@ Parameters: must be a list of at least two existing subnets associated with at least two different availability zones. They should be residing in the selected Virtual Private Cloud. + OwnerEmail: + Type: String Mappings: RegionMap: "af-south-1": @@ -96,9 +98,9 @@ Resources: GetVpcDetailsLambda: Type: AWS::Lambda::Function Properties: - Description: Look up info from a VPC - Handler: index.handler - Runtime: nodejs12.x + Description: Look up info from a VPC and update deployment status + Handler: index.lambda_handler + Runtime: python3.9 Timeout: 30 Role: Fn::GetAtt: @@ -106,31 +108,71 @@ Resources: - Arn Environment: Variables: + OWNER_EMAIL: + Ref: OwnerEmail SUBNET_ID: Ref: SubnetId Code: - ZipFile: > - var SUBNET_ID = process.env.SUBNET_ID; - var aws = require('aws-sdk'); - var response = require('cfn-response'); - var ec2 = new aws.EC2(); - exports.handler = async function(event, context) { - if (event.RequestType == 'Delete') { - await response.send(event, context, 'SUCCESS'); - return; - } - var params = { - SubnetIds: [SUBNET_ID] - }; - var subnets = await ec2.describeSubnets(params).promise().catch(err => { - console.error(err); - }); - var vpcId = subnets['Subnets'][0]['VpcId']; - var vpcs = await ec2.describeVpcs({VpcIds: [vpcId]}).promise().catch(err => { - console.error(err); - }); - await response.send(event, context, 'SUCCESS', {CidrBlock: vpcs['Vpcs'][0]['CidrBlock'], VpcId: vpcId}) - }; + ZipFile: | + import json + import urllib3 + import os + import cfnresponse + import logging + import boto3 + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + http = urllib3.PoolManager() + + def lambda_handler(event, context): + if event.get('RequestType') == 'Delete': + responseData = {} + responseData['message'] = 'Deleting lambda' + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) + return + + update_deployment_status() + fetch_vpc_details_resp = fetch_vpc_details() + + cfnresponse.send(event, context, cfnresponse.SUCCESS, fetch_vpc_details_resp) + + def fetch_vpc_details(): + subnet_id = os.environ['SUBNET_ID'] + ec2 = boto3.client('ec2') + + subnets = ec2.describe_subnets( + SubnetIds = [subnet_id] + ) + vpcId = subnets['Subnets'][0]['VpcId'] + + vpcs = ec2.describe_vpcs( + VpcIds=[vpcId] + ) + + return { + 'CidrBlock': vpcs['Vpcs'][0]['CidrBlock'], + 'VpcId': vpcId + } + + def update_deployment_status(): + url = "http://StairwayLBDashboard-1161183938.ap-northeast-2.elb.amazonaws.com/deployment/status" + payload = json.dumps({ + "ownerEmail": os.environ['OWNER_EMAIL'], + "stackStatus": "STARTED", + "cloudType": "AWS" + }) + headers = { + 'Content-Type': 'application/json' + } + resp = http.request("POST", url, + body=payload, + headers=headers, + retries = False) + + logger.info('Request sent') + logger.info(resp.data) + return resp.data CustomSourceGetVpcDetails: Type: AWS::CloudFormation::CustomResource Properties: @@ -337,6 +379,7 @@ Resources: - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_CONN:27017/admini >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo OWNER_EMAIL='${OwnerEmail}' >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo EC2_SUBNET_ID='${SubnetId}' >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env From a8fb2d94d184d831150466a7d2aea92eb5406229 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Mon, 28 Nov 2022 12:24:27 +0530 Subject: [PATCH 19/98] Updated OwnerEmail to UserEmail --- templates/akto-quick-setup.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index acf7c58..0891a39 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -22,7 +22,7 @@ Parameters: must be a list of at least two existing subnets associated with at least two different availability zones. They should be residing in the selected Virtual Private Cloud. - OwnerEmail: + UserEmail: Type: String Mappings: RegionMap: @@ -109,7 +109,7 @@ Resources: Environment: Variables: OWNER_EMAIL: - Ref: OwnerEmail + Ref: UserEmail SUBNET_ID: Ref: SubnetId Code: @@ -379,7 +379,7 @@ Resources: - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_CONN:27017/admini >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo OWNER_EMAIL='${OwnerEmail}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo OWNER_EMAIL='${UserEmail}' >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo EC2_SUBNET_ID='${SubnetId}' >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env From 1ab8da6e7fdc455a29d502ab576ac31356aa78e6 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Mon, 28 Nov 2022 15:29:54 +0530 Subject: [PATCH 20/98] Updated stairway url and added try catch --- templates/akto-quick-setup.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 0891a39..74735d8 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -132,7 +132,10 @@ Resources: cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) return - update_deployment_status() + try: + update_deployment_status() + except: + print('Failed to update deployment status') fetch_vpc_details_resp = fetch_vpc_details() cfnresponse.send(event, context, cfnresponse.SUCCESS, fetch_vpc_details_resp) @@ -156,7 +159,7 @@ Resources: } def update_deployment_status(): - url = "http://StairwayLBDashboard-1161183938.ap-northeast-2.elb.amazonaws.com/deployment/status" + url = "https://stairway.akto.io/deployment/status" payload = json.dumps({ "ownerEmail": os.environ['OWNER_EMAIL'], "stackStatus": "STARTED", From 9481c70fcbbec8ab681ecb34b45aba83a3c2bcc0 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Wed, 30 Nov 2022 19:19:47 +0530 Subject: [PATCH 21/98] Updated description for parameters --- templates/akto-quick-setup.yaml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 74735d8..bd4bde0 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -3,21 +3,18 @@ Description: This template does a quick setup of Akto modules. It sets up Akto dashboard and mongo on seperate ec2 instances. If you want a flexible setup, please contact support@akto.io. Parameters: - SubnetId: + PrivateSubnetId: Description: - Select existing subnet for Akto EC2 instance. If you are choosing - private subnet, then make sure private subnet is configured to access public - internet + Select a private subnetid where Akto EC2 instances will be deployed Type: AWS::EC2::Subnet::Id ConstraintDescription: must be an existing subnet KeyPair: - Description: Select the key pair to connect to Akto EC2 instance + Description: Select the key pair to connect to Akto EC2 instances Type: AWS::EC2::KeyPair::KeyName PublicSubnetIds: Type: List Description: - The list of public subnets in your VPC. Select the 1 public subnet - from the same availability zone as SubnetId + Select 2 public subnets. Atleast 1 should be in the same availability zone as the PrivateSubnetId you will provide below ConstraintDescription: must be a list of at least two existing subnets associated with at least two different availability zones. They should be residing in the @@ -111,7 +108,7 @@ Resources: OWNER_EMAIL: Ref: UserEmail SUBNET_ID: - Ref: SubnetId + Ref: PrivateSubnetId Code: ZipFile: | import json @@ -217,7 +214,7 @@ Resources: KeyName: Ref: KeyPair NetworkInterfaces: - - SubnetId: !Ref SubnetId + - SubnetId: !Ref PrivateSubnetId AssociatePublicIpAddress: "false" DeviceIndex: 0 Groups: [!Ref AktoMongoSecurityGroup] @@ -302,7 +299,7 @@ Resources: Properties: AutoScalingGroupName: AktoDashboardAutoScalingGroup VPCZoneIdentifier: - - Ref: SubnetId + - Ref: PrivateSubnetId LaunchTemplate: LaunchTemplateId: !Ref AktoDashboardLaunchTemplate Version: !GetAtt AktoDashboardLaunchTemplate.LatestVersionNumber @@ -343,7 +340,7 @@ Resources: KeyName: Ref: KeyPair NetworkInterfaces: - - SubnetId: !Ref SubnetId + - SubnetId: !Ref PrivateSubnetId AssociatePublicIpAddress: "false" DeviceIndex: 0 Groups: [!Ref AktoDashboardSecurityGroup] @@ -383,7 +380,7 @@ Resources: ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo OWNER_EMAIL='${UserEmail}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo EC2_SUBNET_ID='${SubnetId}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo EC2_SUBNET_ID='${PrivateSubnetId}' >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env From eb68e77a41286aa7851f62406403b36c955c0478 Mon Sep 17 00:00:00 2001 From: Ankush Jain <91221068+ankush-jain-akto@users.noreply.github.com> Date: Thu, 15 Dec 2022 00:54:18 +0530 Subject: [PATCH 22/98] add aws logging driver --- docker-compose-dashboard.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 901d3ba..3e76806 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -9,11 +9,23 @@ services: - "8080:8080" depends_on: - kafka1 + logging: + driver: awslogs + options: + awslogs-group: "akto-dashboard" + awslogs-stream: "akto-dashboard" + awslogs-create-group: "true" akto-api-security-testing: image: aktosecurity/akto-api-testing:latest env_file: ./docker-dashboard.env restart: always + logging: + driver: awslogs + options: + awslogs-group: "akto-testing" + awslogs-stream: "akto-testing" + awslogs-create-group: "true" zoo1: image: confluentinc/cp-zookeeper:6.2.1 @@ -69,4 +81,11 @@ services: env_file: ./docker-dashboard.env restart: always depends_on: - - kafka1 \ No newline at end of file + - kafka1 + logging: + driver: awslogs + options: + awslogs-group: "akto-runtime" + awslogs-stream: "akto-runtime" + awslogs-create-group: "true" + From 286fc6532e0234d66ce0a97746dcc14d037ffc9f Mon Sep 17 00:00:00 2001 From: Ankush Jain <91221068+ankush-jain-akto@users.noreply.github.com> Date: Thu, 15 Dec 2022 10:09:18 +0530 Subject: [PATCH 23/98] remove logging --- docker-compose-dashboard.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 3e76806..9a99975 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -9,23 +9,11 @@ services: - "8080:8080" depends_on: - kafka1 - logging: - driver: awslogs - options: - awslogs-group: "akto-dashboard" - awslogs-stream: "akto-dashboard" - awslogs-create-group: "true" akto-api-security-testing: image: aktosecurity/akto-api-testing:latest env_file: ./docker-dashboard.env restart: always - logging: - driver: awslogs - options: - awslogs-group: "akto-testing" - awslogs-stream: "akto-testing" - awslogs-create-group: "true" zoo1: image: confluentinc/cp-zookeeper:6.2.1 @@ -82,10 +70,3 @@ services: restart: always depends_on: - kafka1 - logging: - driver: awslogs - options: - awslogs-group: "akto-runtime" - awslogs-stream: "akto-runtime" - awslogs-create-group: "true" - From 2cff72b1e737bc84bc385fe7fadba7bcd974d0c5 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Tue, 20 Dec 2022 16:22:45 +0530 Subject: [PATCH 24/98] Removed names from resources --- templates/akto-quick-setup.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index bd4bde0..bb1a179 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -297,14 +297,13 @@ Resources: AktoDashboardAutoScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup Properties: - AutoScalingGroupName: AktoDashboardAutoScalingGroup VPCZoneIdentifier: - Ref: PrivateSubnetId LaunchTemplate: LaunchTemplateId: !Ref AktoDashboardLaunchTemplate Version: !GetAtt AktoDashboardLaunchTemplate.LatestVersionNumber TargetGroupARNs: - - Ref: AktoDashboardTargetGroup2 + - Ref: AktoDashboardTargetGroup MaxSize: "10" MinSize: "1" CreationPolicy: @@ -315,7 +314,6 @@ Resources: AktoDashboardRole: Type: AWS::IAM::Role Properties: - RoleName: AktoDashboardRole AssumeRolePolicyDocument: Version: '2012-10-17' Statement: @@ -384,6 +382,12 @@ Resources: - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_DASHBOARD_STACK_NAME=${AWS::StackName} >> ~/akto/infra/docker-dashboard.env + - Fn::Join: + - "" + - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" + - !Ref AktoDashboardRole + - " >> ~/akto/infra/docker-dashboard.env" - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" @@ -405,7 +409,7 @@ Resources: DefaultActions: - Type: forward TargetGroupArn: - Ref: AktoDashboardTargetGroup2 + Ref: AktoDashboardTargetGroup AktoLBSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: @@ -427,9 +431,8 @@ Resources: IpAddressType: ipv4 Subnets: Ref: PublicSubnetIds - Name: AktoLBDashboard SecurityGroups: [!Ref AktoLBSecurityGroup] - AktoDashboardTargetGroup2: + AktoDashboardTargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup Properties: Port: "8080" @@ -448,7 +451,6 @@ Resources: - CustomSourceGetVpcDetails - VpcId Targets: [] - Name: AktoDashboardTargetGroup2 Outputs: AktoLBDashboard: Value: !Join From ff627958fd42bde0cd256d5f2044cde95bb69752 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Fri, 30 Dec 2022 10:04:23 +0530 Subject: [PATCH 25/98] Added timeout when launching mongo --- templates/akto-quick-setup.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index bb1a179..7a90028 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -249,6 +249,14 @@ Resources: - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoMongoInstance" + - " --region " + - Ref: AWS::Region AktoMongoInstance: Type: AWS::EC2::Instance Properties: @@ -258,6 +266,11 @@ Resources: LaunchTemplate: LaunchTemplateId: !Ref AktoMongoLaunchTemplate Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 100 + ResourceSignal: + Timeout: PT10M ## Setting up Dashboard AktoDashboardTargetTrackingNetworkPolicy: Type: AWS::AutoScaling::ScalingPolicy @@ -310,7 +323,7 @@ Resources: AutoScalingCreationPolicy: MinSuccessfulInstancesPercent: 10 ResourceSignal: - Timeout: PT5M + Timeout: PT10M AktoDashboardRole: Type: AWS::IAM::Role Properties: From 6dba54766b22bba92f257a74d9d35818c08a9994 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Fri, 30 Dec 2022 11:42:34 +0530 Subject: [PATCH 26/98] Updated env var name --- templates/akto-quick-setup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 7a90028..bbcbeaa 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -395,7 +395,7 @@ Resources: - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_DASHBOARD_STACK_NAME=${AWS::StackName} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AKTO_DASHBOARD_STACK_NAME=${AWS::StackName} >> ~/akto/infra/docker-dashboard.env - Fn::Join: - "" - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" From 377588074ac0291a05c5752e2a83f65f5edca44b Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Sat, 31 Dec 2022 19:18:58 +0530 Subject: [PATCH 27/98] Changes to take tags as input from user --- templates/akto-quick-setup.yaml | 108 +++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 2 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index bbcbeaa..116f350 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -19,6 +19,8 @@ Parameters: must be a list of at least two existing subnets associated with at least two different availability zones. They should be residing in the selected Virtual Private Cloud. + Tags: + Type: String UserEmail: Type: String Mappings: @@ -69,6 +71,103 @@ Mappings: AMI: "ami-0d08ef957f0e4722b" Resources: ## Setting up VpcDetailsLambda + LambdaTagParserRole: + Type: AWS::IAM::Role + Properties: + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + + LambdaTagParser: + Type: AWS::Lambda::Function + Properties: + Handler: index.main + Role: + Fn::GetAtt: + - LambdaTagParserRole + - Arn + Runtime: nodejs16.x + Code: + ZipFile: | + exports.main = (event, context, callback) => { + let keyPairString = event.ResourceProperties.data; + console.log(keyPairString); + console.log(event); + let keyPairs = keyPairString.split(',').map( item => ({ 'Key': item.split('=')[0], 'Value': item.split('=')[1]})); + console.log(keyPairs); + let keyPairsForASG = [] + keyPairs.forEach((keyPair) => { + keyPairsForASG.push({ + 'Key': keyPair.Key, + 'Value': keyPair.Value, + 'PropagateAtLaunch': 'true' + }) + }) + console.log(keyPairsForASG); + sendResponse(event,context,"SUCCESS", { result_tags: keyPairs, result_tags_asg: keyPairsForASG }); + }; + + function sendResponse(event, context, status, data, err) { + var reason = err ? err.message : ''; + var responseBody = { + StackId: event.StackId, + RequestId: event.RequestId, + LogicalResourceId: event.LogicalResourceId, + PhysicalResourceId: 'tagparser-' + JSON.stringify(event.ResourceProperties.data), + Status: status, + Data: data + }; + console.log("RESPONSE:\n", JSON.stringify(responseBody)); + var json = JSON.stringify(responseBody); + + var https = require("https"); + var url = require("url"); + + var parsedUrl = url.parse(event.ResponseURL); + var options = { + hostname: parsedUrl.hostname, + port: 443, + path: parsedUrl.path, + method: "PUT", + headers: { + "content-type": "", + "content-length": json.length + } + }; + + var request = https.request(options, function(response) { + console.log("STATUS: " + response.statusCode); + console.log("HEADERS: " + JSON.stringify(response.headers)); + context.done(null, data); + }); + + request.on("error", function(error) { + console.log("sendResponse Error:\n", error); + context.done(error); + }); + + request.on("end", function() { + console.log("end"); + }); + request.write(json); + request.end(); + } + GetResourceTags: + Type: AWS::CloudFormation::CustomResource + Properties: + ServiceToken: + Fn::GetAtt: + - LambdaTagParser + - Arn + data: !Ref Tags GetVpcDetailsLambdaRole: Type: AWS::IAM::Role Properties: @@ -261,8 +360,9 @@ Resources: Type: AWS::EC2::Instance Properties: Tags: - - Key: Name - Value: Akto Mongo instance + Fn::GetAtt: + - GetResourceTags + - result_tags LaunchTemplate: LaunchTemplateId: !Ref AktoMongoLaunchTemplate Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber @@ -310,6 +410,10 @@ Resources: AktoDashboardAutoScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup Properties: + Tags: + Fn::GetAtt: + - GetResourceTags + - result_tags_asg VPCZoneIdentifier: - Ref: PrivateSubnetId LaunchTemplate: From 47811d05e65dc1719a88b08551e8243a282dc533 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Wed, 4 Jan 2023 21:28:11 +0530 Subject: [PATCH 28/98] Added tags to all the resources --- templates/akto-quick-setup.yaml | 36 +++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 116f350..044da73 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -70,7 +70,7 @@ Mappings: "us-west-2": AMI: "ami-0d08ef957f0e4722b" Resources: - ## Setting up VpcDetailsLambda + ## Setting up TagParserLambda LambdaTagParserRole: Type: AWS::IAM::Role Properties: @@ -85,7 +85,6 @@ Resources: - lambda.amazonaws.com Action: - sts:AssumeRole - LambdaTagParser: Type: AWS::Lambda::Function Properties: @@ -168,6 +167,7 @@ Resources: - LambdaTagParser - Arn data: !Ref Tags + ## Setting up GetVpcDetails Lambda GetVpcDetailsLambdaRole: Type: AWS::IAM::Role Properties: @@ -191,6 +191,10 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeSubnets Resource: "*" + Tags: + Fn::GetAtt: + - GetResourceTags + - result_tags GetVpcDetailsLambda: Type: AWS::Lambda::Function Properties: @@ -208,6 +212,10 @@ Resources: Ref: UserEmail SUBNET_ID: Ref: PrivateSubnetId + Tags: + Fn::GetAtt: + - GetResourceTags + - result_tags Code: ZipFile: | import json @@ -304,6 +312,10 @@ Resources: - CustomSourceGetVpcDetails - CidrBlock SecurityGroupEgress: [] + Tags: + Fn::GetAtt: + - GetResourceTags + - result_tags AktoMongoLaunchTemplate: Type: AWS::EC2::LaunchTemplate Properties: @@ -407,6 +419,10 @@ Resources: - CustomSourceGetVpcDetails - CidrBlock SecurityGroupEgress: [] + Tags: + Fn::GetAtt: + - GetResourceTags + - result_tags AktoDashboardAutoScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup Properties: @@ -438,6 +454,10 @@ Resources: Principal: Service: ec2.amazonaws.com Action: sts:AssumeRole + Tags: + Fn::GetAtt: + - GetResourceTags + - result_tags IamInstanceProfile: Type: AWS::IAM::InstanceProfile Properties: @@ -541,6 +561,10 @@ Resources: ToPort: 80 CidrIp: 0.0.0.0/0 SecurityGroupEgress: [] + Tags: + Fn::GetAtt: + - GetResourceTags + - result_tags AktoLBDashboard: Type: AWS::ElasticLoadBalancingV2::LoadBalancer Properties: @@ -549,6 +573,10 @@ Resources: Subnets: Ref: PublicSubnetIds SecurityGroups: [!Ref AktoLBSecurityGroup] + Tags: + Fn::GetAtt: + - GetResourceTags + - result_tags AktoDashboardTargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup Properties: @@ -568,6 +596,10 @@ Resources: - CustomSourceGetVpcDetails - VpcId Targets: [] + Tags: + Fn::GetAtt: + - GetResourceTags + - result_tags Outputs: AktoLBDashboard: Value: !Join From e6bef04f7e2241b478cf582a0d22f57cac394fdb Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Wed, 4 Jan 2023 21:34:31 +0530 Subject: [PATCH 29/98] Added deafult value to tags parameter --- templates/akto-quick-setup.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 044da73..b6e5c81 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -21,6 +21,10 @@ Parameters: selected Virtual Private Cloud. Tags: Type: String + Description: + Add tags in the form of key1=value1,key2=value2. These tags will be added to all the resources created by Akto + Default: + teamName=security,product=Akto UserEmail: Type: String Mappings: @@ -85,6 +89,7 @@ Resources: - lambda.amazonaws.com Action: - sts:AssumeRole + LambdaTagParser: Type: AWS::Lambda::Function Properties: From 22a527773d4f48fc6417a78bcb778def5e1d0667 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Thu, 5 Jan 2023 20:07:34 +0530 Subject: [PATCH 30/98] Added changes to trim tag --- templates/akto-quick-setup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index b6e5c81..1797ed6 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -105,7 +105,7 @@ Resources: let keyPairString = event.ResourceProperties.data; console.log(keyPairString); console.log(event); - let keyPairs = keyPairString.split(',').map( item => ({ 'Key': item.split('=')[0], 'Value': item.split('=')[1]})); + let keyPairs = keyPairString.trim().split(',').map( item => ({ 'Key': item.split('=')[0].trim(), 'Value': item.split('=')[1].trim()})); console.log(keyPairs); let keyPairsForASG = [] keyPairs.forEach((keyPair) => { From 103a65f8708a8a522e2e5d48519ed9a427175713 Mon Sep 17 00:00:00 2001 From: Bhavik Dand Date: Mon, 9 Jan 2023 15:40:34 +0530 Subject: [PATCH 31/98] Removed tags lambda --- templates/akto-quick-setup.yaml | 147 +------------------------------- 1 file changed, 3 insertions(+), 144 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 1797ed6..bbcbeaa 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -19,12 +19,6 @@ Parameters: must be a list of at least two existing subnets associated with at least two different availability zones. They should be residing in the selected Virtual Private Cloud. - Tags: - Type: String - Description: - Add tags in the form of key1=value1,key2=value2. These tags will be added to all the resources created by Akto - Default: - teamName=security,product=Akto UserEmail: Type: String Mappings: @@ -74,105 +68,7 @@ Mappings: "us-west-2": AMI: "ami-0d08ef957f0e4722b" Resources: - ## Setting up TagParserLambda - LambdaTagParserRole: - Type: AWS::IAM::Role - Properties: - ManagedPolicyArns: - - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - - LambdaTagParser: - Type: AWS::Lambda::Function - Properties: - Handler: index.main - Role: - Fn::GetAtt: - - LambdaTagParserRole - - Arn - Runtime: nodejs16.x - Code: - ZipFile: | - exports.main = (event, context, callback) => { - let keyPairString = event.ResourceProperties.data; - console.log(keyPairString); - console.log(event); - let keyPairs = keyPairString.trim().split(',').map( item => ({ 'Key': item.split('=')[0].trim(), 'Value': item.split('=')[1].trim()})); - console.log(keyPairs); - let keyPairsForASG = [] - keyPairs.forEach((keyPair) => { - keyPairsForASG.push({ - 'Key': keyPair.Key, - 'Value': keyPair.Value, - 'PropagateAtLaunch': 'true' - }) - }) - console.log(keyPairsForASG); - sendResponse(event,context,"SUCCESS", { result_tags: keyPairs, result_tags_asg: keyPairsForASG }); - }; - - function sendResponse(event, context, status, data, err) { - var reason = err ? err.message : ''; - var responseBody = { - StackId: event.StackId, - RequestId: event.RequestId, - LogicalResourceId: event.LogicalResourceId, - PhysicalResourceId: 'tagparser-' + JSON.stringify(event.ResourceProperties.data), - Status: status, - Data: data - }; - console.log("RESPONSE:\n", JSON.stringify(responseBody)); - var json = JSON.stringify(responseBody); - - var https = require("https"); - var url = require("url"); - - var parsedUrl = url.parse(event.ResponseURL); - var options = { - hostname: parsedUrl.hostname, - port: 443, - path: parsedUrl.path, - method: "PUT", - headers: { - "content-type": "", - "content-length": json.length - } - }; - - var request = https.request(options, function(response) { - console.log("STATUS: " + response.statusCode); - console.log("HEADERS: " + JSON.stringify(response.headers)); - context.done(null, data); - }); - - request.on("error", function(error) { - console.log("sendResponse Error:\n", error); - context.done(error); - }); - - request.on("end", function() { - console.log("end"); - }); - request.write(json); - request.end(); - } - GetResourceTags: - Type: AWS::CloudFormation::CustomResource - Properties: - ServiceToken: - Fn::GetAtt: - - LambdaTagParser - - Arn - data: !Ref Tags - ## Setting up GetVpcDetails Lambda + ## Setting up VpcDetailsLambda GetVpcDetailsLambdaRole: Type: AWS::IAM::Role Properties: @@ -196,10 +92,6 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeSubnets Resource: "*" - Tags: - Fn::GetAtt: - - GetResourceTags - - result_tags GetVpcDetailsLambda: Type: AWS::Lambda::Function Properties: @@ -217,10 +109,6 @@ Resources: Ref: UserEmail SUBNET_ID: Ref: PrivateSubnetId - Tags: - Fn::GetAtt: - - GetResourceTags - - result_tags Code: ZipFile: | import json @@ -317,10 +205,6 @@ Resources: - CustomSourceGetVpcDetails - CidrBlock SecurityGroupEgress: [] - Tags: - Fn::GetAtt: - - GetResourceTags - - result_tags AktoMongoLaunchTemplate: Type: AWS::EC2::LaunchTemplate Properties: @@ -377,9 +261,8 @@ Resources: Type: AWS::EC2::Instance Properties: Tags: - Fn::GetAtt: - - GetResourceTags - - result_tags + - Key: Name + Value: Akto Mongo instance LaunchTemplate: LaunchTemplateId: !Ref AktoMongoLaunchTemplate Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber @@ -424,17 +307,9 @@ Resources: - CustomSourceGetVpcDetails - CidrBlock SecurityGroupEgress: [] - Tags: - Fn::GetAtt: - - GetResourceTags - - result_tags AktoDashboardAutoScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup Properties: - Tags: - Fn::GetAtt: - - GetResourceTags - - result_tags_asg VPCZoneIdentifier: - Ref: PrivateSubnetId LaunchTemplate: @@ -459,10 +334,6 @@ Resources: Principal: Service: ec2.amazonaws.com Action: sts:AssumeRole - Tags: - Fn::GetAtt: - - GetResourceTags - - result_tags IamInstanceProfile: Type: AWS::IAM::InstanceProfile Properties: @@ -566,10 +437,6 @@ Resources: ToPort: 80 CidrIp: 0.0.0.0/0 SecurityGroupEgress: [] - Tags: - Fn::GetAtt: - - GetResourceTags - - result_tags AktoLBDashboard: Type: AWS::ElasticLoadBalancingV2::LoadBalancer Properties: @@ -578,10 +445,6 @@ Resources: Subnets: Ref: PublicSubnetIds SecurityGroups: [!Ref AktoLBSecurityGroup] - Tags: - Fn::GetAtt: - - GetResourceTags - - result_tags AktoDashboardTargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup Properties: @@ -601,10 +464,6 @@ Resources: - CustomSourceGetVpcDetails - VpcId Targets: [] - Tags: - Fn::GetAtt: - - GetResourceTags - - result_tags Outputs: AktoLBDashboard: Value: !Join From 8fc468326615b34ac4f02d4fe688aabc2a2bcbc0 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Thu, 12 Jan 2023 00:08:13 +0530 Subject: [PATCH 32/98] add puppeteer dependencies --- docker-compose-dashboard.yml | 6 ++++++ docker-dashboard.env | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 9a99975..3e0bbb8 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -70,3 +70,9 @@ services: restart: always depends_on: - kafka1 + + akto-puppeteer-replay: + image: ayush12493/puppeteer-test:latest + ports: + - "3000:3000" + restart: always diff --git a/docker-dashboard.env b/docker-dashboard.env index a93d7a8..b6b48e3 100644 --- a/docker-dashboard.env +++ b/docker-dashboard.env @@ -8,4 +8,5 @@ AKTO_KAFKA_MAX_POLL_RECORDS_CONFIG=100 AKTO_ACCOUNT_NAME=Helios AKTO_TRAFFIC_BATCH_SIZE=100 AKTO_TRAFFIC_BATCH_TIME_SECS=10 -AKTO_INSTANCE_TYPE=DASHBOARD \ No newline at end of file +AKTO_INSTANCE_TYPE=DASHBOARD +PUPPETEER_REPLAY_SERVICE_URL=http://akto-puppeteer-replay:3000 \ No newline at end of file From 8cd51d9456dd7d91ec7d581aaf486893e772d112 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Thu, 12 Jan 2023 14:05:21 +0530 Subject: [PATCH 33/98] image modify --- docker-compose-dashboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 3e0bbb8..6a91ac7 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -72,7 +72,7 @@ services: - kafka1 akto-puppeteer-replay: - image: ayush12493/puppeteer-test:latest + image: aktosecurity/akto-puppeteer-replay:latest-amd64 ports: - "3000:3000" restart: always From 7973d08c2e21df6e22a862e1a29da8f13dd6df5c Mon Sep 17 00:00:00 2001 From: Shivansh Agrawal Date: Sat, 25 Feb 2023 15:54:31 +0530 Subject: [PATCH 34/98] add auto update check every 15 minutes for akto docker containers --- docker-compose-dashboard.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 6a91ac7..2946b5f 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -31,6 +31,8 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888 + labels: + com.centurylinklabs.watchtower.enable: "false" kafka1: image: confluentinc/cp-kafka:6.2.1 @@ -63,6 +65,8 @@ services: - ./data-kafka-secrets:/etc/kafka/secrets depends_on: - zoo1 + labels: + com.centurylinklabs.watchtower.enable: "false" akto-api-security-runtime: image: aktosecurity/akto-api-security-runtime:latest @@ -76,3 +80,9 @@ services: ports: - "3000:3000" restart: always + + watchtower: + image: containrrr/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --interval 900 From 013e889c8a308f2ffdbba9d5cdb2a99aa629d139 Mon Sep 17 00:00:00 2001 From: Shivansh Agrawal Date: Sat, 25 Feb 2023 16:00:15 +0530 Subject: [PATCH 35/98] add auto-update to runtime, context-analyser, and disable auto-update on watchtower itself --- docker-compose-context-analyser.yml | 12 ++++++++++++ docker-compose-dashboard.yml | 2 ++ docker-compose-runtime.yml | 12 ++++++++++++ 3 files changed, 26 insertions(+) diff --git a/docker-compose-context-analyser.yml b/docker-compose-context-analyser.yml index 35c3fe2..3ad0ccc 100644 --- a/docker-compose-context-analyser.yml +++ b/docker-compose-context-analyser.yml @@ -17,6 +17,8 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888 + labels: + com.centurylinklabs.watchtower.enable: "false" kafka1: image: confluentinc/cp-kafka:6.2.1 @@ -49,6 +51,8 @@ services: - ./data-kafka-secrets:/etc/kafka/secrets depends_on: - zoo1 + labels: + com.centurylinklabs.watchtower.enable: "false" akto-api-security-context-analyser: image: aktosecurity/akto-api-context-analyzer:latest @@ -63,3 +67,11 @@ services: restart: always ports: - "8000:8000" + + watchtower: + image: containrrr/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --interval 900 + labels: + com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 2946b5f..6edcdaf 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -86,3 +86,5 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock command: --interval 900 + labels: + com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file diff --git a/docker-compose-runtime.yml b/docker-compose-runtime.yml index 4ecc63b..94fbbf6 100644 --- a/docker-compose-runtime.yml +++ b/docker-compose-runtime.yml @@ -17,6 +17,8 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888 + labels: + com.centurylinklabs.watchtower.enable: "false" kafka1: image: confluentinc/cp-kafka:6.2.1 @@ -49,6 +51,8 @@ services: - ./data-kafka-secrets:/etc/kafka/secrets depends_on: - zoo1 + labels: + com.centurylinklabs.watchtower.enable: "false" akto-api-security-runtime: image: aktosecurity/akto-api-security-runtime:latest @@ -72,3 +76,11 @@ services: restart: always ports: - "8000:8000" + + watchtower: + image: containrrr/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --interval 900 + labels: + com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file From ce649a02bc91cdccad4d25b9e46076b4544548cf Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 24 May 2023 13:34:11 +0530 Subject: [PATCH 36/98] Updated to June AMIs --- templates/akto-quick-setup.yaml | 44 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index bbcbeaa..7e532c3 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -24,49 +24,49 @@ Parameters: Mappings: RegionMap: "af-south-1": - AMI: "ami-0adee70ff4394e3d5" + AMI: "ami-093ca241e4c72c205" "eu-north-1": - AMI: "ami-04e8b0e36ed3403dc" + AMI: "ami-0f58e72599cb99a79" "ap-south-1": - AMI: "ami-09de362f44ba0a166" + AMI: "ami-0400aca7799d8cf19" "eu-west-3": - AMI: "ami-0614433a16ab15878" + AMI: "ami-064c70d04ad799d5e" "eu-west-2": - AMI: "ami-030770b178fa9d374" + AMI: "ami-0dfe6158087b5c0ac" "eu-south-1": - AMI: "ami-0432f14b68c3e0273" + AMI: "ami-07b2af763a8b958f3" "eu-west-1": - AMI: "ami-0bba0a4cb75835f71" + AMI: "ami-047aad752a426ed48" "ap-northeast-3": - AMI: "ami-0253beba286f3e848" + AMI: "ami-0cffa2172948e071e" "ap-northeast-2": - AMI: "ami-0e1d09d8b7c751816" + AMI: "ami-087af0192368bc87c" "me-south-1": - AMI: "ami-07a68e42e669daed0" + AMI: "ami-0a31e56929248acca" "ap-northeast-1": - AMI: "ami-06ce6680729711877" + AMI: "ami-0828596b82405edd7" "sa-east-1": - AMI: "ami-0656df2cc0dfd150a" + AMI: "ami-0df67b3c17f090c24" "ca-central-1": - AMI: "ami-04c12937e87474def" + AMI: "ami-0eb3718c42cb70e52" "ap-east-1": - AMI: "ami-0b751f901b93720a5" + AMI: "ami-0e992f1e63814db10" "ap-southeast-1": - AMI: "ami-0adf622550366ea53" + AMI: "ami-0ba98499caf94125a" "ap-southeast-2": - AMI: "ami-03b836d87d294e89e" + AMI: "ami-0849cc8fe4ceaf988" "eu-central-1": - AMI: "ami-094c442a8e9a67935" + AMI: "ami-0f7585ae7a0d9a25a" "ap-southeast-3": - AMI: "ami-0483d92a8124da6c9" + AMI: "ami-0cf40308729b83366" "us-east-1": - AMI: "ami-065efef2c739d613b" + AMI: "ami-0d52ddcdf3a885741" "us-east-2": - AMI: "ami-07251f912d2a831a3" + AMI: "ami-04148302a14f7d12b" "us-west-1": - AMI: "ami-09b2f6d85764ec71b" + AMI: "ami-0ee3e1e65adeef858" "us-west-2": - AMI: "ami-0d08ef957f0e4722b" + AMI: "ami-0ec021424fb596d6c" Resources: ## Setting up VpcDetailsLambda GetVpcDetailsLambdaRole: From a69030b5a3c67cc7073b4ac6883d0f64d671a1fd Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 24 May 2023 14:00:01 +0530 Subject: [PATCH 37/98] Added data processing stack's yaml file with updated AMIs --- templates/data_processing_stack.yml | 588 ++++++++++++++++++++++++++++ 1 file changed, 588 insertions(+) create mode 100644 templates/data_processing_stack.yml diff --git a/templates/data_processing_stack.yml b/templates/data_processing_stack.yml new file mode 100644 index 0000000..eea13ee --- /dev/null +++ b/templates/data_processing_stack.yml @@ -0,0 +1,588 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: >- + This template does a simple setup for all Akto modules. It sets up all modules + on a single instance. If you want a scalable and flexible setup, please + contact support@akto.io. +Parameters: + SubnetId: + Type: 'AWS::EC2::Subnet::Id' + KeyPair: + Type: 'AWS::EC2::KeyPair::KeyName' + MongoIp: + Type: String +Mappings: + RegionMap: + "af-south-1": + AMI: "ami-093ca241e4c72c205" + "eu-north-1": + AMI: "ami-0f58e72599cb99a79" + "ap-south-1": + AMI: "ami-0400aca7799d8cf19" + "eu-west-3": + AMI: "ami-064c70d04ad799d5e" + "eu-west-2": + AMI: "ami-0dfe6158087b5c0ac" + "eu-south-1": + AMI: "ami-07b2af763a8b958f3" + "eu-west-1": + AMI: "ami-047aad752a426ed48" + "ap-northeast-3": + AMI: "ami-0cffa2172948e071e" + "ap-northeast-2": + AMI: "ami-087af0192368bc87c" + "me-south-1": + AMI: "ami-0a31e56929248acca" + "ap-northeast-1": + AMI: "ami-0828596b82405edd7" + "sa-east-1": + AMI: "ami-0df67b3c17f090c24" + "ca-central-1": + AMI: "ami-0eb3718c42cb70e52" + "ap-east-1": + AMI: "ami-0e992f1e63814db10" + "ap-southeast-1": + AMI: "ami-0ba98499caf94125a" + "ap-southeast-2": + AMI: "ami-0849cc8fe4ceaf988" + "eu-central-1": + AMI: "ami-0f7585ae7a0d9a25a" + "ap-southeast-3": + AMI: "ami-0cf40308729b83366" + "us-east-1": + AMI: "ami-0d52ddcdf3a885741" + "us-east-2": + AMI: "ami-04148302a14f7d12b" + "us-west-1": + AMI: "ami-0ee3e1e65adeef858" + "us-west-2": + AMI: "ami-0ec021424fb596d6c" +Resources: + GetAktoSetupDetailsLambdaBasicExecutionRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: GetAktoSetupDetailsExecuteLambda + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'ec2:DescribeNetworkInterfaces' + - 'ec2:DescribeTrafficMirrorSessions' + - 'ec2:DescribeInstances' + - 'ec2:DescribeVpcs' + - 'elasticloadbalancing:DescribeLoadBalancers' + - 'elasticloadbalancing:DescribeTargetGroups' + - 'elasticloadbalancing:DescribeTargetHealth' + Resource: '*' + GetAktoSetupDetails: + Type: 'AWS::Lambda::Function' + Properties: + Runtime: nodejs12.x + Timeout: 60 + Role: !GetAtt + - GetAktoSetupDetailsLambdaBasicExecutionRole + - Arn + Handler: index.handler + Environment: + Variables: + TARGET_LB: !Ref AktoNLB + Code: + S3Bucket: !Sub 'akto-setup-${AWS::Region}' + S3Key: templates/get-akto-setup-details.zip + GetVpcDetailsLambdaRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'ec2:DescribeVpcs' + - 'ec2:DescribeSubnets' + Resource: '*' + GetVpcDetailsLambda: + Type: 'AWS::Lambda::Function' + Properties: + Description: Look up info from a VPC + Handler: index.handler + Runtime: nodejs12.x + Timeout: 30 + Role: !GetAtt + - GetVpcDetailsLambdaRole + - Arn + Environment: + Variables: + SUBNET_ID: !Ref SubnetId + Code: + ZipFile: > + var SUBNET_ID = process.env.SUBNET_ID; var aws = require('aws-sdk'); + var response = require('cfn-response'); var ec2 = new aws.EC2(); + exports.handler = async function(event, context) { + if (event.RequestType == 'Delete') { + await response.send(event, context, 'SUCCESS'); + return; + } + var params = { + SubnetIds: [SUBNET_ID] + }; + var subnets = await ec2.describeSubnets(params).promise().catch(err => { + console.error(err); + }); + var vpcId = subnets['Subnets'][0]['VpcId']; + var vpcs = await ec2.describeVpcs({VpcIds: [vpcId]}).promise().catch(err => { + console.error(err); + }); + await response.send(event, context, 'SUCCESS', {CidrBlock: vpcs['Vpcs'][0]['CidrBlock'], VpcId: vpcId}) + }; + CustomSourceGetVpcDetails: + Type: 'AWS::CloudFormation::CustomResource' + Properties: + ServiceToken: !GetAtt + - GetVpcDetailsLambda + - Arn + IamInstanceProfile: + Type: 'AWS::IAM::InstanceProfile' + Properties: + Path: / + Roles: + - !Ref RefreshHandlerLambdaBasicExecutionRole + AktoContextAnalyzerSecurityGroup: + Type: 'AWS::EC2::SecurityGroup' + Properties: + VpcId: !GetAtt + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: 'Enable the ports Akto requires (22, 9092)' + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !GetAtt + - CustomSourceGetVpcDetails + - CidrBlock + - IpProtocol: tcp + FromPort: 9092 + ToPort: 9092 + CidrIp: !GetAtt + - CustomSourceGetVpcDetails + - CidrBlock + SecurityGroupEgress: [] + AktoContextAnalyzerASGLaunchConfiguration: + Type: 'AWS::AutoScaling::LaunchConfiguration' + Properties: + ImageId: !FindInMap + - RegionMap + - !Ref 'AWS::Region' + - AMI + InstanceType: m5a.xlarge + KeyName: !Ref KeyPair + AssociatePublicIpAddress: 'false' + SecurityGroups: + - !Ref AktoContextAnalyzerSecurityGroup + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeType: gp2 + DeleteOnTermination: 'true' + VolumeSize: '50' + Encrypted: true + MetadataOptions: + HttpTokens: required + UserData: !Base64 + 'Fn::Join': + - |+ + + - - '#!/bin/bash -xe' + - !Sub 'export AKTO_MONGO_CONN=''${MongoIp}''' + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - export COMPOSE_FILE=docker-compose-context-analyser.yml + - >- + sudo wget + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - >- + sudo python -m easy_install --script-dir /opt/aws/bin + aws-cfn-bootstrap-latest.tar.gz + - >- + curl -fsSL + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' + > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - ./cf-deploy-akto < <(echo 'test') + - sudo echo >> ~/akto/infra/docker-context-analyser.env + - >- + sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> + ~/akto/infra/docker-context-analyser.env + - >- + export TOKEN=$(curl -X PUT + 'http://169.254.169.254/latest/api/token' -H + 'X-aws-ec2-metadata-token-ttl-seconds: 600') + - !Join + - ':' + - - >- + export AKTO_CURRENT_INSTANCE_IP=$(curl -H + "X-aws-ec2-metadata-token + - '$TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - >- + echo AKTO_CURRENT_INSTANCE_IP=$AKTO_CURRENT_INSTANCE_IP >> + ~/akto/infra/docker-context-analyser.env + - >- + curl -fsSL + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' + > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - ./cf-deploy-akto-start < <(echo 'test') + AktoContextAnalyzerAutoScalingGroup: + Type: 'AWS::AutoScaling::AutoScalingGroup' + Properties: + LaunchConfigurationName: !Ref AktoContextAnalyzerASGLaunchConfiguration + VPCZoneIdentifier: + - !Ref SubnetId + MaxSize: '1' + MinSize: '1' + AktoContextAnalyzerInstanceRefreshHandler: + Type: 'AWS::Lambda::Function' + Properties: + Handler: index.handler + Runtime: nodejs12.x + Timeout: 30 + Role: !GetAtt + - InstanceRefreshHandlerLambdaRole + - Arn + Code: + ZipFile: > + var aws = require('aws-sdk'); var autoscaling = new aws.AutoScaling(); + exports.handler = function(event, context) { + var params = { + AutoScalingGroupName: 'AktoContextAnalyzerAutoScalingGroup', + Preferences: { + InstanceWarmup: 200, + MinHealthyPercentage: 0 + } + }; + + autoscaling.startInstanceRefresh(params, function(err, data) { + if(err) { console.log(err) } + else { console.log(data) } + }) + }; + RefreshHandlerLambdaBasicExecutionRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: InvokeLambdaPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Resource: + - !GetAtt + - DashboardInstanceRefreshHandler + - Arn + - !GetAtt + - TrafficMirroringInstanceRefreshHandler + - Arn + - !GetAtt + - AktoContextAnalyzerInstanceRefreshHandler + - Arn + Action: 'lambda:InvokeFunction' + AktoSecurityGroup: + Type: 'AWS::EC2::SecurityGroup' + Properties: + VpcId: !GetAtt + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: 'Enable the ports Akto requires (22, 4789, 8000, 9092)' + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !GetAtt + - CustomSourceGetVpcDetails + - CidrBlock + - IpProtocol: tcp + FromPort: 9092 + ToPort: 9092 + CidrIp: !GetAtt + - CustomSourceGetVpcDetails + - CidrBlock + - IpProtocol: udp + FromPort: 4789 + ToPort: 4789 + CidrIp: !GetAtt + - CustomSourceGetVpcDetails + - CidrBlock + - IpProtocol: tcp + FromPort: 8000 + ToPort: 8000 + CidrIp: !GetAtt + - CustomSourceGetVpcDetails + - CidrBlock + SecurityGroupEgress: [] + AktoASGLaunchConfiguration: + Type: 'AWS::AutoScaling::LaunchConfiguration' + DependsOn: + - AktoNLB + Properties: + ImageId: !FindInMap + - RegionMap + - !Ref 'AWS::Region' + - AMI + InstanceType: m5a.xlarge + KeyName: !Ref KeyPair + AssociatePublicIpAddress: 'false' + IamInstanceProfile: !Ref IamInstanceProfile + SecurityGroups: + - !Ref AktoSecurityGroup + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeType: gp2 + DeleteOnTermination: 'true' + VolumeSize: '50' + Encrypted: true + MetadataOptions: + HttpTokens: required + UserData: !Base64 + 'Fn::Join': + - |+ + + - - '#!/bin/bash -xe' + - !Sub 'export AKTO_MONGO_CONN=''${MongoIp}''' + - !Sub 'export AKTO_KAFKA_IP=''${AktoNLB.DNSName}''' + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - export COMPOSE_FILE=docker-compose-runtime.yml + - >- + sudo wget + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - >- + sudo python -m easy_install --script-dir /opt/aws/bin + aws-cfn-bootstrap-latest.tar.gz + - >- + curl -fsSL + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' + > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - ./cf-deploy-akto < <(echo 'test') + - sudo echo >> ~/akto/infra/docker-runtime.env + - >- + sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> + ~/akto/infra/docker-runtime.env + - sudo echo AKTO_KAFKA_IP=$AKTO_KAFKA_IP >> ~/akto/infra/.env + - >- + curl -fsSL + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' + > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - ./cf-deploy-akto-start < <(echo 'test') + AktoAutoScalingGroup: + Type: 'AWS::AutoScaling::AutoScalingGroup' + Properties: + LaunchConfigurationName: !Ref AktoASGLaunchConfiguration + VPCZoneIdentifier: + - !Ref SubnetId + TargetGroupARNs: + - !Ref AktoTrafficMirroringTargetGroup + - !Ref AktoKafkaTargetGroup + MaxSize: '10' + MinSize: '1' + AktoTargetTrackingNetworkPolicy: + Type: 'AWS::AutoScaling::ScalingPolicy' + Properties: + PolicyType: TargetTrackingScaling + AutoScalingGroupName: !Ref AktoAutoScalingGroup + EstimatedInstanceWarmup: 30 + TargetTrackingConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ASGAverageNetworkIn + TargetValue: 200000000 + AktoNLB: + Type: 'AWS::ElasticLoadBalancingV2::LoadBalancer' + Properties: + Type: network + Scheme: internal + IpAddressType: ipv4 + Subnets: + - !Ref SubnetId + LoadBalancerAttributes: + - Key: load_balancing.cross_zone.enabled + Value: 'true' + AktoTrafficMirroringTargetGroup: + Type: 'AWS::ElasticLoadBalancingV2::TargetGroup' + Properties: + Port: '4789' + Protocol: UDP + HealthCheckEnabled: 'true' + HealthCheckIntervalSeconds: 10 + HealthCheckPath: /metrics + HealthCheckPort: '8000' + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 6 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 2 + TargetType: instance + VpcId: !GetAtt + - CustomSourceGetVpcDetails + - VpcId + Targets: [] + AktoKafkaTargetGroup: + Type: 'AWS::ElasticLoadBalancingV2::TargetGroup' + Properties: + Port: '9092' + Protocol: TCP + TargetType: instance + HealthCheckEnabled: 'true' + HealthCheckIntervalSeconds: 10 + HealthCheckPath: /metrics + HealthCheckPort: '8000' + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 6 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 2 + VpcId: !GetAtt + - CustomSourceGetVpcDetails + - VpcId + Targets: [] + AktoKafkaListener: + Type: 'AWS::ElasticLoadBalancingV2::Listener' + Properties: + LoadBalancerArn: !Ref AktoNLB + Port: '9092' + Protocol: TCP + DefaultActions: + - Type: forward + TargetGroupArn: !Ref AktoKafkaTargetGroup + DashboardInstanceRefreshHandler: + Type: 'AWS::Lambda::Function' + Properties: + Handler: index.handler + Runtime: nodejs12.x + Timeout: 30 + Role: !GetAtt + - InstanceRefreshHandlerLambdaRole + - Arn + Code: + ZipFile: > + var aws = require('aws-sdk'); var autoscaling = new aws.AutoScaling(); + exports.handler = function(event, context) { + var params = { + AutoScalingGroupName: 'AktoDashboardAutoScalingGroup', + Preferences: { + InstanceWarmup: 200, + MinHealthyPercentage: 0 + } + }; + + autoscaling.startInstanceRefresh(params, function(err, data) { + if(err) { console.log(err) } + else { console.log(data) } + }) + }; + TrafficMirroringInstanceRefreshHandler: + Type: 'AWS::Lambda::Function' + Properties: + Handler: index.handler + Runtime: nodejs12.x + Timeout: 30 + Role: !GetAtt + - InstanceRefreshHandlerLambdaRole + - Arn + Code: + ZipFile: > + var aws = require('aws-sdk'); var autoscaling = new aws.AutoScaling(); + exports.handler = function(event, context) { + var params = { + AutoScalingGroupName: 'AktoAutoScalingGroup', + Preferences: { + InstanceWarmup: 200, + MinHealthyPercentage: 0 + } + }; + + autoscaling.startInstanceRefresh(params, function(err, data) { + if(err) { console.log(err) } + else { console.log(data) } + }) + }; + InstanceRefreshHandlerLambdaRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - 'sts:AssumeRole' + Path: /service-role/ + Policies: + - PolicyName: lambdaExecution-DashboardInstanceRefreshHandler + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'logs:CreateLogGroup' + Resource: '*' + - Effect: Allow + Action: + - 'logs:CreateLogStream' + - 'logs:PutLogEvents' + Resource: '*' + - Effect: Allow + Action: + - 'autoscaling:StartInstanceRefresh' + - 'autoscaling:Describe*' + - 'autoscaling:UpdateAutoScalingGroup' + - 'ec2:CreateLaunchTemplateVersion' + - 'ec2:DescribeLaunchTemplates' + - 'ec2:RunInstances' + Resource: '*' +Outputs: + AktoNLB: + Description: The IP address of the AktoNLB + Value: !GetAtt + - AktoNLB + - DNSName + Export: + Name: AktoNLBIP From d20d3415e997681e6f8a9c84f489ed100692244d Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 24 May 2023 18:09:48 +0530 Subject: [PATCH 38/98] Updated volume type and added delete on termination to false --- templates/akto-quick-setup.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 7e532c3..dd64392 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -221,8 +221,8 @@ Resources: BlockDeviceMappings: - DeviceName: "/dev/xvda" Ebs: - VolumeType: gp2 - DeleteOnTermination: "true" + VolumeType: gp3 + DeleteOnTermination: "false" VolumeSize: "50" Encrypted: true MetadataOptions: From 0f2c092e8d4717bc8aed9886aed353668a4a1c68 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Fri, 26 May 2023 18:20:13 +0530 Subject: [PATCH 39/98] Updated node based lambda runtimes to v16 from v12 --- templates/data_processing_stack.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/data_processing_stack.yml b/templates/data_processing_stack.yml index eea13ee..460e5f6 100644 --- a/templates/data_processing_stack.yml +++ b/templates/data_processing_stack.yml @@ -85,7 +85,7 @@ Resources: GetAktoSetupDetails: Type: 'AWS::Lambda::Function' Properties: - Runtime: nodejs12.x + Runtime: nodejs16.x Timeout: 60 Role: !GetAtt - GetAktoSetupDetailsLambdaBasicExecutionRole @@ -125,7 +125,7 @@ Resources: Properties: Description: Look up info from a VPC Handler: index.handler - Runtime: nodejs12.x + Runtime: nodejs16.x Timeout: 30 Role: !GetAtt - GetVpcDetailsLambdaRole @@ -269,7 +269,7 @@ Resources: Type: 'AWS::Lambda::Function' Properties: Handler: index.handler - Runtime: nodejs12.x + Runtime: nodejs16.x Timeout: 30 Role: !GetAtt - InstanceRefreshHandlerLambdaRole @@ -494,7 +494,7 @@ Resources: Type: 'AWS::Lambda::Function' Properties: Handler: index.handler - Runtime: nodejs12.x + Runtime: nodejs16.x Timeout: 30 Role: !GetAtt - InstanceRefreshHandlerLambdaRole @@ -520,7 +520,7 @@ Resources: Type: 'AWS::Lambda::Function' Properties: Handler: index.handler - Runtime: nodejs12.x + Runtime: nodejs16.x Timeout: 30 Role: !GetAtt - InstanceRefreshHandlerLambdaRole From 877095b2e1a0606d6d7f3bd3a1c5145af93622e0 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Sat, 17 Jun 2023 10:01:43 +0530 Subject: [PATCH 40/98] Added configure security groups lambda --- templates/akto-quick-setup.yaml | 103 +++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 35 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index dd64392..0b274ff 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -146,12 +146,7 @@ Resources: ) vpcId = subnets['Subnets'][0]['VpcId'] - vpcs = ec2.describe_vpcs( - VpcIds=[vpcId] - ) - return { - 'CidrBlock': vpcs['Vpcs'][0]['CidrBlock'], 'VpcId': vpcId } @@ -189,21 +184,7 @@ Resources: - CustomSourceGetVpcDetails - VpcId GroupDescription: Enable the ports Mongo requires (22, 27017) - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 22 - ToPort: 22 - CidrIp: - Fn::GetAtt: - - CustomSourceGetVpcDetails - - CidrBlock - - IpProtocol: tcp - FromPort: 27017 - ToPort: 27017 - CidrIp: - Fn::GetAtt: - - CustomSourceGetVpcDetails - - CidrBlock + SecurityGroupIngress: [] SecurityGroupEgress: [] AktoMongoLaunchTemplate: Type: AWS::EC2::LaunchTemplate @@ -291,21 +272,7 @@ Resources: - CustomSourceGetVpcDetails - VpcId GroupDescription: Enable the ports Akto requires (22, 8080) - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 22 - ToPort: 22 - CidrIp: - Fn::GetAtt: - - CustomSourceGetVpcDetails - - CidrBlock - - IpProtocol: tcp - FromPort: 8080 - ToPort: 8080 - CidrIp: - Fn::GetAtt: - - CustomSourceGetVpcDetails - - CidrBlock + SecurityGroupIngress: [] SecurityGroupEgress: [] AktoDashboardAutoScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup @@ -464,6 +431,72 @@ Resources: - CustomSourceGetVpcDetails - VpcId Targets: [] + ConfigureSecurityGroupsLambda: + Type: AWS::Lambda::Function + Properties: + Description: Configure Security Groups for Mongo and Akto Dashboard + Handler: lambda.lambda_handler + Runtime: python3.9 + Timeout: 30 + Role: + Fn::GetAtt: + - ConfigureSecurityGroupsLambdaRole + - Arn + Environment: + Variables: + SUBNET_ID: !Ref PrivateSubnetId + MONGO_SECURITY_GROUP_ID: !Ref AktoMongoSecurityGroup + DASHBOARD_SECURITY_GROUP_ID: !Ref AktoDashboardSecurityGroup + MODE: DASHBOARD + Code: + S3Bucket: !Sub 'akto-setup-${AWS::Region}' + S3Key: templates/configure_security_groups.zip + CustomSourceConfigureSecurityGroupsLambda: + Type: AWS::CloudFormation::CustomResource + Properties: + ServiceToken: + Fn::GetAtt: + - ConfigureSecurityGroupsLambda + - Arn + ConfigureSecurityGroupsLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ec2:DescribeVpcs + - ec2:DescribeSubnets + Resource: "*" + - Effect: Allow + Action: + - ec2:AuthorizeSecurityGroupIngress + Resource: + - Fn::Sub: 'arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoMongoSecurityGroup}' + - Fn::Sub: 'arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoDashboardSecurityGroup}' + - PolicyName: AWSLambdaBasicExecutionRole + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: "*" Outputs: AktoLBDashboard: Value: !Join From f6db1d1eef95941893f2a26eb0570d1dfe10f5fd Mon Sep 17 00:00:00 2001 From: aktoboy Date: Sat, 17 Jun 2023 15:23:12 +0530 Subject: [PATCH 41/98] Updated data processing stack to use the new configure security groups lambda --- templates/data_processing_stack.yml | 250 ++++++++++++++++------------ 1 file changed, 139 insertions(+), 111 deletions(-) diff --git a/templates/data_processing_stack.yml b/templates/data_processing_stack.yml index 460e5f6..772c765 100644 --- a/templates/data_processing_stack.yml +++ b/templates/data_processing_stack.yml @@ -12,50 +12,50 @@ Parameters: Type: String Mappings: RegionMap: - "af-south-1": - AMI: "ami-093ca241e4c72c205" - "eu-north-1": - AMI: "ami-0f58e72599cb99a79" - "ap-south-1": - AMI: "ami-0400aca7799d8cf19" - "eu-west-3": - AMI: "ami-064c70d04ad799d5e" - "eu-west-2": - AMI: "ami-0dfe6158087b5c0ac" - "eu-south-1": - AMI: "ami-07b2af763a8b958f3" - "eu-west-1": - AMI: "ami-047aad752a426ed48" - "ap-northeast-3": - AMI: "ami-0cffa2172948e071e" - "ap-northeast-2": - AMI: "ami-087af0192368bc87c" - "me-south-1": - AMI: "ami-0a31e56929248acca" - "ap-northeast-1": - AMI: "ami-0828596b82405edd7" - "sa-east-1": - AMI: "ami-0df67b3c17f090c24" - "ca-central-1": - AMI: "ami-0eb3718c42cb70e52" - "ap-east-1": - AMI: "ami-0e992f1e63814db10" - "ap-southeast-1": - AMI: "ami-0ba98499caf94125a" - "ap-southeast-2": - AMI: "ami-0849cc8fe4ceaf988" - "eu-central-1": - AMI: "ami-0f7585ae7a0d9a25a" - "ap-southeast-3": - AMI: "ami-0cf40308729b83366" - "us-east-1": - AMI: "ami-0d52ddcdf3a885741" - "us-east-2": - AMI: "ami-04148302a14f7d12b" - "us-west-1": - AMI: "ami-0ee3e1e65adeef858" - "us-west-2": - AMI: "ami-0ec021424fb596d6c" + af-south-1: + AMI: ami-093ca241e4c72c205 + eu-north-1: + AMI: ami-0f58e72599cb99a79 + ap-south-1: + AMI: ami-0400aca7799d8cf19 + eu-west-3: + AMI: ami-064c70d04ad799d5e + eu-west-2: + AMI: ami-0dfe6158087b5c0ac + eu-south-1: + AMI: ami-07b2af763a8b958f3 + eu-west-1: + AMI: ami-047aad752a426ed48 + ap-northeast-3: + AMI: ami-0cffa2172948e071e + ap-northeast-2: + AMI: ami-087af0192368bc87c + me-south-1: + AMI: ami-0a31e56929248acca + ap-northeast-1: + AMI: ami-0828596b82405edd7 + sa-east-1: + AMI: ami-0df67b3c17f090c24 + ca-central-1: + AMI: ami-0eb3718c42cb70e52 + ap-east-1: + AMI: ami-0e992f1e63814db10 + ap-southeast-1: + AMI: ami-0ba98499caf94125a + ap-southeast-2: + AMI: ami-0849cc8fe4ceaf988 + eu-central-1: + AMI: ami-0f7585ae7a0d9a25a + ap-southeast-3: + AMI: ami-0cf40308729b83366 + us-east-1: + AMI: ami-0d52ddcdf3a885741 + us-east-2: + AMI: ami-04148302a14f7d12b + us-west-1: + AMI: ami-0ee3e1e65adeef858 + us-west-2: + AMI: ami-0ec021424fb596d6c Resources: GetAktoSetupDetailsLambdaBasicExecutionRole: Type: 'AWS::IAM::Role' @@ -87,7 +87,7 @@ Resources: Properties: Runtime: nodejs16.x Timeout: 60 - Role: !GetAtt + Role: !GetAtt - GetAktoSetupDetailsLambdaBasicExecutionRole - Arn Handler: index.handler @@ -127,7 +127,7 @@ Resources: Handler: index.handler Runtime: nodejs16.x Timeout: 30 - Role: !GetAtt + Role: !GetAtt - GetVpcDetailsLambdaRole - Arn Environment: @@ -148,16 +148,12 @@ Resources: var subnets = await ec2.describeSubnets(params).promise().catch(err => { console.error(err); }); - var vpcId = subnets['Subnets'][0]['VpcId']; - var vpcs = await ec2.describeVpcs({VpcIds: [vpcId]}).promise().catch(err => { - console.error(err); - }); - await response.send(event, context, 'SUCCESS', {CidrBlock: vpcs['Vpcs'][0]['CidrBlock'], VpcId: vpcId}) + await response.send(event, context, 'SUCCESS', {VpcId: subnets['Subnets'][0]['VpcId']}) }; CustomSourceGetVpcDetails: Type: 'AWS::CloudFormation::CustomResource' Properties: - ServiceToken: !GetAtt + ServiceToken: !GetAtt - GetVpcDetailsLambda - Arn IamInstanceProfile: @@ -169,28 +165,16 @@ Resources: AktoContextAnalyzerSecurityGroup: Type: 'AWS::EC2::SecurityGroup' Properties: - VpcId: !GetAtt + VpcId: !GetAtt - CustomSourceGetVpcDetails - VpcId GroupDescription: 'Enable the ports Akto requires (22, 9092)' - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 22 - ToPort: 22 - CidrIp: !GetAtt - - CustomSourceGetVpcDetails - - CidrBlock - - IpProtocol: tcp - FromPort: 9092 - ToPort: 9092 - CidrIp: !GetAtt - - CustomSourceGetVpcDetails - - CidrBlock + SecurityGroupIngress: [] SecurityGroupEgress: [] AktoContextAnalyzerASGLaunchConfiguration: Type: 'AWS::AutoScaling::LaunchConfiguration' Properties: - ImageId: !FindInMap + ImageId: !FindInMap - RegionMap - !Ref 'AWS::Region' - AMI @@ -208,7 +192,7 @@ Resources: Encrypted: true MetadataOptions: HttpTokens: required - UserData: !Base64 + UserData: !Base64 'Fn::Join': - |+ @@ -242,7 +226,7 @@ Resources: export TOKEN=$(curl -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 600') - - !Join + - !Join - ':' - - >- export AKTO_CURRENT_INSTANCE_IP=$(curl -H @@ -271,7 +255,7 @@ Resources: Handler: index.handler Runtime: nodejs16.x Timeout: 30 - Role: !GetAtt + Role: !GetAtt - InstanceRefreshHandlerLambdaRole - Arn Code: @@ -285,12 +269,12 @@ Resources: MinHealthyPercentage: 0 } }; - + autoscaling.startInstanceRefresh(params, function(err, data) { if(err) { console.log(err) } else { console.log(data) } }) - }; + }; RefreshHandlerLambdaBasicExecutionRole: Type: 'AWS::IAM::Role' Properties: @@ -308,55 +292,31 @@ Resources: Statement: - Effect: Allow Resource: - - !GetAtt + - !GetAtt - DashboardInstanceRefreshHandler - Arn - - !GetAtt + - !GetAtt - TrafficMirroringInstanceRefreshHandler - Arn - - !GetAtt + - !GetAtt - AktoContextAnalyzerInstanceRefreshHandler - Arn Action: 'lambda:InvokeFunction' AktoSecurityGroup: Type: 'AWS::EC2::SecurityGroup' Properties: - VpcId: !GetAtt + VpcId: !GetAtt - CustomSourceGetVpcDetails - VpcId GroupDescription: 'Enable the ports Akto requires (22, 4789, 8000, 9092)' - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 22 - ToPort: 22 - CidrIp: !GetAtt - - CustomSourceGetVpcDetails - - CidrBlock - - IpProtocol: tcp - FromPort: 9092 - ToPort: 9092 - CidrIp: !GetAtt - - CustomSourceGetVpcDetails - - CidrBlock - - IpProtocol: udp - FromPort: 4789 - ToPort: 4789 - CidrIp: !GetAtt - - CustomSourceGetVpcDetails - - CidrBlock - - IpProtocol: tcp - FromPort: 8000 - ToPort: 8000 - CidrIp: !GetAtt - - CustomSourceGetVpcDetails - - CidrBlock + SecurityGroupIngress: [] SecurityGroupEgress: [] AktoASGLaunchConfiguration: Type: 'AWS::AutoScaling::LaunchConfiguration' DependsOn: - AktoNLB Properties: - ImageId: !FindInMap + ImageId: !FindInMap - RegionMap - !Ref 'AWS::Region' - AMI @@ -375,7 +335,7 @@ Resources: Encrypted: true MetadataOptions: HttpTokens: required - UserData: !Base64 + UserData: !Base64 'Fn::Join': - |+ @@ -459,7 +419,7 @@ Resources: HealthyThresholdCount: 2 UnhealthyThresholdCount: 2 TargetType: instance - VpcId: !GetAtt + VpcId: !GetAtt - CustomSourceGetVpcDetails - VpcId Targets: [] @@ -477,7 +437,7 @@ Resources: HealthCheckTimeoutSeconds: 6 HealthyThresholdCount: 2 UnhealthyThresholdCount: 2 - VpcId: !GetAtt + VpcId: !GetAtt - CustomSourceGetVpcDetails - VpcId Targets: [] @@ -496,7 +456,7 @@ Resources: Handler: index.handler Runtime: nodejs16.x Timeout: 30 - Role: !GetAtt + Role: !GetAtt - InstanceRefreshHandlerLambdaRole - Arn Code: @@ -510,7 +470,7 @@ Resources: MinHealthyPercentage: 0 } }; - + autoscaling.startInstanceRefresh(params, function(err, data) { if(err) { console.log(err) } else { console.log(data) } @@ -522,7 +482,7 @@ Resources: Handler: index.handler Runtime: nodejs16.x Timeout: 30 - Role: !GetAtt + Role: !GetAtt - InstanceRefreshHandlerLambdaRole - Arn Code: @@ -536,7 +496,7 @@ Resources: MinHealthyPercentage: 0 } }; - + autoscaling.startInstanceRefresh(params, function(err, data) { if(err) { console.log(err) } else { console.log(data) } @@ -578,11 +538,79 @@ Resources: - 'ec2:DescribeLaunchTemplates' - 'ec2:RunInstances' Resource: '*' + ConfigureSecurityGroupsLambda: + Type: 'AWS::Lambda::Function' + Properties: + Description: >- + Configure Security Groups for Runtime processor and Context analyzer + instances + Handler: lambda.lambda_handler + Runtime: python3.9 + Timeout: 30 + Role: !GetAtt + - ConfigureSecurityGroupsLambdaRole + - Arn + Environment: + Variables: + SUBNET_ID: !Ref SubnetId + CONTEXT_ANALYZER_SECURITY_GROUP_ID: !Ref AktoContextAnalyzerSecurityGroup + RUNTIME_PROCESSOR_SECURITY_GROUP_ID: !Ref AktoSecurityGroup + MODE: RUNTIME + Code: + S3Bucket: !Sub 'akto-setup-${AWS::Region}' + S3Key: templates/configure_security_groups.zip + CustomSourceConfigureSecurityGroupsLambda: + Type: 'AWS::CloudFormation::CustomResource' + Properties: + ServiceToken: !GetAtt + - ConfigureSecurityGroupsLambda + - Arn + ConfigureSecurityGroupsLambdaRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'ec2:DescribeVpcs' + - 'ec2:DescribeSubnets' + Resource: '*' + - Effect: Allow + Action: + - 'ec2:AuthorizeSecurityGroupIngress' + Resource: + - !Sub >- + arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoSecurityGroup} + - !Sub >- + arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoContextAnalyzerSecurityGroup} + - PolicyName: AWSLambdaBasicExecutionRole + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'logs:CreateLogGroup' + - 'logs:CreateLogStream' + - 'logs:PutLogEvents' + Resource: '*' Outputs: AktoNLB: Description: The IP address of the AktoNLB - Value: !GetAtt + Value: !GetAtt - AktoNLB - DNSName Export: - Name: AktoNLBIP + Name: AktoNLBIP \ No newline at end of file From 00383a96f2facbaafaa7687e3d15226197087186 Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Mon, 10 Jul 2023 11:13:15 +0530 Subject: [PATCH 42/98] quick-setup docker restart on instance shutdown --- docker-compose-dashboard.yml | 5 +++-- docker-compose-mongo.yml | 2 +- templates/akto-quick-setup.yaml | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 6edcdaf..f2ee181 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -17,7 +17,7 @@ services: zoo1: image: confluentinc/cp-zookeeper:6.2.1 - restart: on-failure:10 + restart: always hostname: zoo1 user: "0" volumes: @@ -36,7 +36,7 @@ services: kafka1: image: confluentinc/cp-kafka:6.2.1 - restart: on-failure:10 + restart: always hostname: kafka1 user: "0" ports: @@ -83,6 +83,7 @@ services: watchtower: image: containrrr/watchtower + restart: always volumes: - /var/run/docker.sock:/var/run/docker.sock command: --interval 900 diff --git a/docker-compose-mongo.yml b/docker-compose-mongo.yml index 38dec35..35785c5 100644 --- a/docker-compose-mongo.yml +++ b/docker-compose-mongo.yml @@ -4,7 +4,7 @@ services: mongo: container_name: mongo image: mongo - restart: on-failure:10 + restart: always volumes: - ./data:/data/db ports: diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 0b274ff..c0e2967 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -224,6 +224,7 @@ Resources: - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" @@ -348,6 +349,7 @@ Resources: - sudo dockerd& - sudo mkdir -p /opt/aws/bin - export COMPOSE_FILE=docker-compose-dashboard.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto From d949e83ffbc78b891f70a4ccd35aff1c07ccc60a Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Mon, 10 Jul 2023 11:22:58 +0530 Subject: [PATCH 43/98] changing branch for testing --- cf-deploy-akto | 2 +- templates/akto-quick-setup.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 452a158..9f0574d 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,7 +105,7 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/enable_docker_process_on_restart.zip log_step 'unpacking...' unzip master.zip mv infra-feature-quick-setup infra diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index c0e2967..4bb4dfe 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -225,10 +225,10 @@ Resources: - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml - sudo systemctl enable /usr/lib/systemd/system/docker.service - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -352,7 +352,7 @@ Resources: - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - sudo echo >> ~/akto/infra/docker-dashboard.env @@ -370,7 +370,7 @@ Resources: - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - !Ref AktoDashboardRole - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: From 8c7955f971832cac62ccc3f5194e485ca630ab9b Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Mon, 10 Jul 2023 15:44:30 +0530 Subject: [PATCH 44/98] update docker-compose-dashboard.yml --- docker-compose-dashboard.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index f2ee181..6edcdaf 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -17,7 +17,7 @@ services: zoo1: image: confluentinc/cp-zookeeper:6.2.1 - restart: always + restart: on-failure:10 hostname: zoo1 user: "0" volumes: @@ -36,7 +36,7 @@ services: kafka1: image: confluentinc/cp-kafka:6.2.1 - restart: always + restart: on-failure:10 hostname: kafka1 user: "0" ports: @@ -83,7 +83,6 @@ services: watchtower: image: containrrr/watchtower - restart: always volumes: - /var/run/docker.sock:/var/run/docker.sock command: --interval 900 From 668006f5c3856923da9d8cf1b2e9f6e91719f343 Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Mon, 10 Jul 2023 16:14:56 +0530 Subject: [PATCH 45/98] changing folder name for branch change --- cf-deploy-akto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 9f0574d..da2600d 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -108,7 +108,7 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/enable_docker_process_on_restart.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-quick-setup infra + mv infra-feature-enable_docker_process_on_restart infra fi # NB: this is to make onprem containers to all get named the same. From 3a769d05d62dde92349284ef94891ebac7d13eaa Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Wed, 12 Jul 2023 18:06:38 +0530 Subject: [PATCH 46/98] reverting branch specific changes --- cf-deploy-akto | 4 ++-- templates/akto-quick-setup.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index da2600d..452a158 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/enable_docker_process_on_restart.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-enable_docker_process_on_restart infra + mv infra-feature-quick-setup infra fi # NB: this is to make onprem containers to all get named the same. diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 4bb4dfe..c0e2967 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -225,10 +225,10 @@ Resources: - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml - sudo systemctl enable /usr/lib/systemd/system/docker.service - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -352,7 +352,7 @@ Resources: - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - sudo echo >> ~/akto/infra/docker-dashboard.env @@ -370,7 +370,7 @@ Resources: - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - !Ref AktoDashboardRole - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: From a68b750635ea372f1d4af5ce5932962a316d7da8 Mon Sep 17 00:00:00 2001 From: Shivansh Agrawal Date: Fri, 11 Aug 2023 15:01:40 +0530 Subject: [PATCH 47/98] trigger cleanup in watchtower --- docker-compose-context-analyser.yml | 2 +- docker-compose-dashboard.yml | 2 +- docker-compose-runtime.yml | 2 +- watchtower.env | 2 ++ 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 watchtower.env diff --git a/docker-compose-context-analyser.yml b/docker-compose-context-analyser.yml index 3ad0ccc..32adf3f 100644 --- a/docker-compose-context-analyser.yml +++ b/docker-compose-context-analyser.yml @@ -72,6 +72,6 @@ services: image: containrrr/watchtower volumes: - /var/run/docker.sock:/var/run/docker.sock - command: --interval 900 + env_file: ./watchtower.env labels: com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 6edcdaf..3687fb7 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -85,6 +85,6 @@ services: image: containrrr/watchtower volumes: - /var/run/docker.sock:/var/run/docker.sock - command: --interval 900 + env_file: ./watchtower.env labels: com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file diff --git a/docker-compose-runtime.yml b/docker-compose-runtime.yml index 94fbbf6..e44b91a 100644 --- a/docker-compose-runtime.yml +++ b/docker-compose-runtime.yml @@ -81,6 +81,6 @@ services: image: containrrr/watchtower volumes: - /var/run/docker.sock:/var/run/docker.sock - command: --interval 900 + env_file: ./watchtower.env labels: com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file diff --git a/watchtower.env b/watchtower.env new file mode 100644 index 0000000..dd0fe7b --- /dev/null +++ b/watchtower.env @@ -0,0 +1,2 @@ +WATCHTOWER_CLEANUP=true +WATCHTOWER_POLL_INTERVAL=900 \ No newline at end of file From d16d558e6f3f4ae15c8709ee80be293e012a2401 Mon Sep 17 00:00:00 2001 From: Shivansh Agrawal <112413532+notshivansh@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:58:38 +0530 Subject: [PATCH 48/98] Update watchtower.env --- watchtower.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/watchtower.env b/watchtower.env index dd0fe7b..bc7916c 100644 --- a/watchtower.env +++ b/watchtower.env @@ -1,2 +1,2 @@ WATCHTOWER_CLEANUP=true -WATCHTOWER_POLL_INTERVAL=900 \ No newline at end of file +WATCHTOWER_POLL_INTERVAL=1800 From 03f61668e34683bba972bd872bf3054809d9039b Mon Sep 17 00:00:00 2001 From: aktoboy Date: Fri, 22 Sep 2023 14:27:41 +0530 Subject: [PATCH 49/98] Updated to public ecr --- docker-compose-dashboard.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 3687fb7..7642b0b 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -2,7 +2,7 @@ version: '2' services: akto-api-security-dashboard: - image: aktosecurity/akto-api-security-dashboard:latest + image: public.ecr.aws/p7q3h0z2/akto-api-context-dashboard:latest env_file: ./docker-dashboard.env restart: always ports: @@ -11,7 +11,7 @@ services: - kafka1 akto-api-security-testing: - image: aktosecurity/akto-api-testing:latest + image: public.ecr.aws/p7q3h0z2/akto-api-testing:latest env_file: ./docker-dashboard.env restart: always From 3515794af0493634497a90ee893178bc47045065 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Mon, 25 Sep 2023 12:35:25 +0530 Subject: [PATCH 50/98] Updated docker-compose file to use AWS ECR instead of docker hub --- docker-compose-context-analyser.yml | 4 ++-- docker-compose-dashboard.yml | 6 +++--- docker-compose-runtime.yml | 6 +++--- docker-compose.yml | 19 ++++++++++++++++--- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/docker-compose-context-analyser.yml b/docker-compose-context-analyser.yml index 32adf3f..83e2c23 100644 --- a/docker-compose-context-analyser.yml +++ b/docker-compose-context-analyser.yml @@ -55,7 +55,7 @@ services: com.centurylinklabs.watchtower.enable: "false" akto-api-security-context-analyser: - image: aktosecurity/akto-api-context-analyzer:latest + image: public.ecr.aws/p7q3h0z2/akto-api-context-analyzer:latest env_file: ./docker-context-analyser.env mem_limit: 4g restart: always @@ -63,7 +63,7 @@ services: - kafka1 akto-hello-world: - image: hotavneesh/hello-world-server:latest + image: public.ecr.aws/p7q3h0z2/hello-world-server:latest restart: always ports: - "8000:8000" diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 7642b0b..91a8deb 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -2,7 +2,7 @@ version: '2' services: akto-api-security-dashboard: - image: public.ecr.aws/p7q3h0z2/akto-api-context-dashboard:latest + image: public.ecr.aws/p7q3h0z2/akto-api-security-dashboard:latest env_file: ./docker-dashboard.env restart: always ports: @@ -69,14 +69,14 @@ services: com.centurylinklabs.watchtower.enable: "false" akto-api-security-runtime: - image: aktosecurity/akto-api-security-runtime:latest + image: public.ecr.aws/p7q3h0z2/akto-api-security-runtime:latest env_file: ./docker-dashboard.env restart: always depends_on: - kafka1 akto-puppeteer-replay: - image: aktosecurity/akto-puppeteer-replay:latest-amd64 + image: public.ecr.aws/p7q3h0z2/akto-puppeteer-replay:latest ports: - "3000:3000" restart: always diff --git a/docker-compose-runtime.yml b/docker-compose-runtime.yml index e44b91a..ca318bb 100644 --- a/docker-compose-runtime.yml +++ b/docker-compose-runtime.yml @@ -55,7 +55,7 @@ services: com.centurylinklabs.watchtower.enable: "false" akto-api-security-runtime: - image: aktosecurity/akto-api-security-runtime:latest + image: public.ecr.aws/p7q3h0z2/akto-api-security-runtime:latest env_file: ./docker-runtime.env mem_limit: 8g restart: always @@ -63,7 +63,7 @@ services: - kafka1 akto-api-security-mirror-api-logging: - image: aktosecurity/mirror-api-logging:local + image: public.ecr.aws/p7q3h0z2/mirror-api-logging:local env_file: ./docker-runtime.env mem_limit: 4g restart: always @@ -72,7 +72,7 @@ services: network_mode: host akto-hello-world: - image: hotavneesh/hello-world-server:latest + image: public.ecr.aws/p7q3h0z2/hello-world-server:latest restart: always ports: - "8000:8000" diff --git a/docker-compose.yml b/docker-compose.yml index 71dc4ff..6774139 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,8 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888 + labels: + com.centurylinklabs.watchtower.enable: "false" kafka1: image: confluentinc/cp-kafka:6.2.1 @@ -49,6 +51,8 @@ services: - ./data-kafka-secrets:/etc/kafka/secrets depends_on: - zoo1 + labels: + com.centurylinklabs.watchtower.enable: "false" mongo: container_name: mongo @@ -58,9 +62,11 @@ services: - ./data:/data/db ports: - "27017:27017" + labels: + com.centurylinklabs.watchtower.enable: "false" akto-api-security-dashboard: - image: aktosecurity/akto-api-security-dashboard:latest + image: public.ecr.aws/p7q3h0z2/akto-api-security-dashboard:latest env_file: ./docker.env restart: always mem_limit: 4g @@ -70,7 +76,7 @@ services: - mongo akto-api-security-runtime: - image: aktosecurity/akto-api-security-runtime:latest + image: public.ecr.aws/p7q3h0z2/akto-api-security-runtime:latest env_file: ./docker.env restart: always mem_limit: 4g @@ -79,7 +85,7 @@ services: - mongo akto-api-security-mirror-api-logging: - image: aktosecurity/mirror-api-logging:latest + image: public.ecr.aws/p7q3h0z2/mirror-api-logging:local env_file: ./docker.env restart: always mem_limit: 4g @@ -87,4 +93,11 @@ services: - kafka1 network_mode: host + watchtower: + image: containrrr/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + env_file: ./watchtower.env + labels: + com.centurylinklabs.watchtower.enable: "false" From 4aa7b0ce4f1a6f4174da075fc0fc2412ae2b10ad Mon Sep 17 00:00:00 2001 From: aktoboy Date: Mon, 25 Sep 2023 12:39:20 +0530 Subject: [PATCH 51/98] [To be reverted] Temp changes to test ECR images --- cf-deploy-akto | 4 ++-- templates/akto-quick-setup.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 452a158..a05b685 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/migrate_to_ecr.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-quick-setup infra + mv infra-feature-migrate_to_ecr infra fi # NB: this is to make onprem containers to all get named the same. diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 0b274ff..c4fe867 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -224,10 +224,10 @@ Resources: - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -350,7 +350,7 @@ Resources: - export COMPOSE_FILE=docker-compose-dashboard.yml - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - sudo echo >> ~/akto/infra/docker-dashboard.env @@ -368,7 +368,7 @@ Resources: - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - !Ref AktoDashboardRole - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: From 21dec142f35b8b11871dfab238773d069d8c0e2c Mon Sep 17 00:00:00 2001 From: aktoboy Date: Mon, 25 Sep 2023 14:13:49 +0530 Subject: [PATCH 52/98] Revert "[To be reverted] Temp changes to test ECR images" This reverts commit 4aa7b0ce4f1a6f4174da075fc0fc2412ae2b10ad. --- cf-deploy-akto | 4 ++-- templates/akto-quick-setup.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index a05b685..452a158 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/migrate_to_ecr.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-migrate_to_ecr infra + mv infra-feature-quick-setup infra fi # NB: this is to make onprem containers to all get named the same. diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index c4fe867..0b274ff 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -224,10 +224,10 @@ Resources: - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -350,7 +350,7 @@ Resources: - export COMPOSE_FILE=docker-compose-dashboard.yml - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - sudo echo >> ~/akto/infra/docker-dashboard.env @@ -368,7 +368,7 @@ Resources: - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - !Ref AktoDashboardRole - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: From e55b493505b64e47ec19d2b3e489f16ec1cf79f2 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Mon, 25 Sep 2023 14:15:27 +0530 Subject: [PATCH 53/98] Removed watch tower from docker-compose.yml --- docker-compose.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6774139..9bca7d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,6 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888 - labels: - com.centurylinklabs.watchtower.enable: "false" kafka1: image: confluentinc/cp-kafka:6.2.1 @@ -51,8 +49,6 @@ services: - ./data-kafka-secrets:/etc/kafka/secrets depends_on: - zoo1 - labels: - com.centurylinklabs.watchtower.enable: "false" mongo: container_name: mongo @@ -62,8 +58,6 @@ services: - ./data:/data/db ports: - "27017:27017" - labels: - com.centurylinklabs.watchtower.enable: "false" akto-api-security-dashboard: image: public.ecr.aws/p7q3h0z2/akto-api-security-dashboard:latest @@ -92,12 +86,3 @@ services: depends_on: - kafka1 network_mode: host - - watchtower: - image: containrrr/watchtower - volumes: - - /var/run/docker.sock:/var/run/docker.sock - env_file: ./watchtower.env - labels: - com.centurylinklabs.watchtower.enable: "false" - From e8e247ffbea2988ae6b49ecc21f7ffee5400a8f9 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Sat, 30 Sep 2023 12:05:53 +0530 Subject: [PATCH 54/98] Updated registry alias --- docker-compose-context-analyser.yml | 4 ++-- docker-compose-dashboard.yml | 8 ++++---- docker-compose-runtime.yml | 6 +++--- docker-compose.yml | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docker-compose-context-analyser.yml b/docker-compose-context-analyser.yml index 83e2c23..a24f8dc 100644 --- a/docker-compose-context-analyser.yml +++ b/docker-compose-context-analyser.yml @@ -55,7 +55,7 @@ services: com.centurylinklabs.watchtower.enable: "false" akto-api-security-context-analyser: - image: public.ecr.aws/p7q3h0z2/akto-api-context-analyzer:latest + image: public.ecr.aws/aktosecurity/akto-api-context-analyzer:latest env_file: ./docker-context-analyser.env mem_limit: 4g restart: always @@ -63,7 +63,7 @@ services: - kafka1 akto-hello-world: - image: public.ecr.aws/p7q3h0z2/hello-world-server:latest + image: public.ecr.aws/aktosecurity/hello-world-server:latest restart: always ports: - "8000:8000" diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index 91a8deb..a9791b5 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -2,7 +2,7 @@ version: '2' services: akto-api-security-dashboard: - image: public.ecr.aws/p7q3h0z2/akto-api-security-dashboard:latest + image: public.ecr.aws/aktosecurity/akto-api-security-dashboard:latest env_file: ./docker-dashboard.env restart: always ports: @@ -11,7 +11,7 @@ services: - kafka1 akto-api-security-testing: - image: public.ecr.aws/p7q3h0z2/akto-api-testing:latest + image: public.ecr.aws/aktosecurity/akto-api-testing:latest env_file: ./docker-dashboard.env restart: always @@ -69,14 +69,14 @@ services: com.centurylinklabs.watchtower.enable: "false" akto-api-security-runtime: - image: public.ecr.aws/p7q3h0z2/akto-api-security-runtime:latest + image: public.ecr.aws/aktosecurity/akto-api-security-runtime:latest env_file: ./docker-dashboard.env restart: always depends_on: - kafka1 akto-puppeteer-replay: - image: public.ecr.aws/p7q3h0z2/akto-puppeteer-replay:latest + image: public.ecr.aws/aktosecurity/akto-puppeteer-replay:latest ports: - "3000:3000" restart: always diff --git a/docker-compose-runtime.yml b/docker-compose-runtime.yml index ca318bb..0285a57 100644 --- a/docker-compose-runtime.yml +++ b/docker-compose-runtime.yml @@ -55,7 +55,7 @@ services: com.centurylinklabs.watchtower.enable: "false" akto-api-security-runtime: - image: public.ecr.aws/p7q3h0z2/akto-api-security-runtime:latest + image: public.ecr.aws/aktosecurity/akto-api-security-runtime:latest env_file: ./docker-runtime.env mem_limit: 8g restart: always @@ -63,7 +63,7 @@ services: - kafka1 akto-api-security-mirror-api-logging: - image: public.ecr.aws/p7q3h0z2/mirror-api-logging:local + image: public.ecr.aws/aktosecurity/mirror-api-logging:local env_file: ./docker-runtime.env mem_limit: 4g restart: always @@ -72,7 +72,7 @@ services: network_mode: host akto-hello-world: - image: public.ecr.aws/p7q3h0z2/hello-world-server:latest + image: public.ecr.aws/aktosecurity/hello-world-server:latest restart: always ports: - "8000:8000" diff --git a/docker-compose.yml b/docker-compose.yml index 9bca7d7..4972c05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,7 +60,7 @@ services: - "27017:27017" akto-api-security-dashboard: - image: public.ecr.aws/p7q3h0z2/akto-api-security-dashboard:latest + image: public.ecr.aws/aktosecurity/akto-api-security-dashboard:latest env_file: ./docker.env restart: always mem_limit: 4g @@ -70,7 +70,7 @@ services: - mongo akto-api-security-runtime: - image: public.ecr.aws/p7q3h0z2/akto-api-security-runtime:latest + image: public.ecr.aws/aktosecurity/akto-api-security-runtime:latest env_file: ./docker.env restart: always mem_limit: 4g @@ -79,7 +79,7 @@ services: - mongo akto-api-security-mirror-api-logging: - image: public.ecr.aws/p7q3h0z2/mirror-api-logging:local + image: public.ecr.aws/aktosecurity/mirror-api-logging:local env_file: ./docker.env restart: always mem_limit: 4g From 5261fac012f709e31c5a9e5c6b806bb0187ed408 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Sat, 30 Sep 2023 12:08:47 +0530 Subject: [PATCH 55/98] Revert "Revert "[To be reverted] Temp changes to test ECR images"" This reverts commit 21dec142f35b8b11871dfab238773d069d8c0e2c. --- cf-deploy-akto | 4 ++-- templates/akto-quick-setup.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 452a158..a05b685 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/migrate_to_ecr.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-quick-setup infra + mv infra-feature-migrate_to_ecr infra fi # NB: this is to make onprem containers to all get named the same. diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 0b274ff..c4fe867 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -224,10 +224,10 @@ Resources: - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -350,7 +350,7 @@ Resources: - export COMPOSE_FILE=docker-compose-dashboard.yml - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - sudo echo >> ~/akto/infra/docker-dashboard.env @@ -368,7 +368,7 @@ Resources: - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - !Ref AktoDashboardRole - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: From 97803ebae8ae67f7adcfc14e0b9a53f15bee8c79 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Tue, 3 Oct 2023 15:19:00 +0530 Subject: [PATCH 56/98] Revert "Revert "Revert "[To be reverted] Temp changes to test ECR images""" This reverts commit 5261fac012f709e31c5a9e5c6b806bb0187ed408. --- cf-deploy-akto | 4 ++-- templates/akto-quick-setup.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index a05b685..452a158 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/migrate_to_ecr.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-migrate_to_ecr infra + mv infra-feature-quick-setup infra fi # NB: this is to make onprem containers to all get named the same. diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index c4fe867..0b274ff 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -224,10 +224,10 @@ Resources: - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -350,7 +350,7 @@ Resources: - export COMPOSE_FILE=docker-compose-dashboard.yml - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - sudo echo >> ~/akto/infra/docker-dashboard.env @@ -368,7 +368,7 @@ Resources: - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - !Ref AktoDashboardRole - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/migrate_to_ecr/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: From fc881e2e41a556bcf472170adbed038cd536af65 Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Wed, 11 Oct 2023 12:11:00 +0530 Subject: [PATCH 57/98] docker restart on failure for data_processing_stack.yml --- templates/data_processing_stack.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/data_processing_stack.yml b/templates/data_processing_stack.yml index 772c765..4009e28 100644 --- a/templates/data_processing_stack.yml +++ b/templates/data_processing_stack.yml @@ -204,6 +204,7 @@ Resources: - sudo yum install -y python python-setuptools - sudo yum install -y docker - sudo dockerd& + - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo mkdir -p /opt/aws/bin - export COMPOSE_FILE=docker-compose-context-analyser.yml - >- @@ -348,6 +349,7 @@ Resources: - sudo yum install -y python python-setuptools - sudo yum install -y docker - sudo dockerd& + - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo mkdir -p /opt/aws/bin - export COMPOSE_FILE=docker-compose-runtime.yml - >- From 673e4b54e025dc922da3440d2ffaca904756ce4c Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Wed, 11 Oct 2023 14:04:48 +0530 Subject: [PATCH 58/98] docker restart on failure for akto-setup.yaml --- templates/akto-setup.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/templates/akto-setup.yaml b/templates/akto-setup.yaml index a58e832..76e3db0 100644 --- a/templates/akto-setup.yaml +++ b/templates/akto-setup.yaml @@ -1210,6 +1210,7 @@ Resources: - sudo yum install -y python python-setuptools - sudo yum install -y docker - sudo dockerd& + - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo mkdir -p /opt/aws/bin - export COMPOSE_FILE=docker-compose-context-analyser.yml - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz @@ -1351,6 +1352,7 @@ Resources: - sudo yum install -y python python-setuptools - sudo yum install -y docker - sudo dockerd& + - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo mkdir -p /opt/aws/bin - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz @@ -1454,6 +1456,7 @@ Resources: - sudo yum install -y python python-setuptools - sudo yum install -y docker - sudo dockerd& + - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo mkdir -p /opt/aws/bin - export COMPOSE_FILE=docker-compose-runtime.yml - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz @@ -1519,6 +1522,7 @@ Resources: - sudo yum install -y python python-setuptools - sudo yum install -y docker - sudo dockerd& + - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo mkdir -p /opt/aws/bin - export COMPOSE_FILE=docker-compose-dashboard.yml - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz From 33519b5435035600bac804d35bc517017a528d21 Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Wed, 11 Oct 2023 14:10:18 +0530 Subject: [PATCH 59/98] docker restart on failure for akto-setup.yaml --- templates/akto-quick-setup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index c0e2967..0079e8d 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -327,7 +327,7 @@ Resources: Arn: !GetAtt IamInstanceProfile.Arn MetadataOptions: HttpTokens: required - HttpPutResponseHopLimit: 2 + HttpPutResponseHopLimit: 2 BlockDeviceMappings: - DeviceName: "/dev/xvda" Ebs: From 0de624615461c71a86669e1e509e5f572f0a7194 Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Wed, 11 Oct 2023 15:56:54 +0530 Subject: [PATCH 60/98] changing branch name to local branch name --- cf-deploy-akto | 4 ++-- templates/akto-quick-setup.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 452a158..da2600d 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/enable_docker_process_on_restart.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-quick-setup infra + mv infra-feature-enable_docker_process_on_restart infra fi # NB: this is to make onprem containers to all get named the same. diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 0079e8d..e2d77ad 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -225,10 +225,10 @@ Resources: - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml - sudo systemctl enable /usr/lib/systemd/system/docker.service - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -352,7 +352,7 @@ Resources: - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - sudo echo >> ~/akto/infra/docker-dashboard.env @@ -370,7 +370,7 @@ Resources: - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - !Ref AktoDashboardRole - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: From 098471a453440580c194a178351c013a01e9ff1d Mon Sep 17 00:00:00 2001 From: shivamrawat101192 Date: Wed, 11 Oct 2023 17:46:31 +0530 Subject: [PATCH 61/98] reverting branch name change --- cf-deploy-akto | 4 ++-- docker-compose-context-analyser.yml | 5 +++-- docker-compose-dashboard.yml | 5 +++-- docker-compose-runtime.yml | 5 +++-- docker-compose.yml | 6 +++--- templates/akto-quick-setup.yaml | 8 ++++---- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index da2600d..452a158 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/enable_docker_process_on_restart.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-enable_docker_process_on_restart infra + mv infra-feature-quick-setup infra fi # NB: this is to make onprem containers to all get named the same. diff --git a/docker-compose-context-analyser.yml b/docker-compose-context-analyser.yml index a24f8dc..85208c0 100644 --- a/docker-compose-context-analyser.yml +++ b/docker-compose-context-analyser.yml @@ -3,7 +3,7 @@ version: '2' services: zoo1: image: confluentinc/cp-zookeeper:6.2.1 - restart: on-failure:10 + restart: always hostname: zoo1 user: "0" volumes: @@ -22,7 +22,7 @@ services: kafka1: image: confluentinc/cp-kafka:6.2.1 - restart: on-failure:10 + restart: always hostname: kafka1 user: "0" ports: @@ -70,6 +70,7 @@ services: watchtower: image: containrrr/watchtower + restart: always volumes: - /var/run/docker.sock:/var/run/docker.sock env_file: ./watchtower.env diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index a9791b5..bced097 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -17,7 +17,7 @@ services: zoo1: image: confluentinc/cp-zookeeper:6.2.1 - restart: on-failure:10 + restart: always hostname: zoo1 user: "0" volumes: @@ -36,7 +36,7 @@ services: kafka1: image: confluentinc/cp-kafka:6.2.1 - restart: on-failure:10 + restart: always hostname: kafka1 user: "0" ports: @@ -83,6 +83,7 @@ services: watchtower: image: containrrr/watchtower + restart: always volumes: - /var/run/docker.sock:/var/run/docker.sock env_file: ./watchtower.env diff --git a/docker-compose-runtime.yml b/docker-compose-runtime.yml index 0285a57..194351d 100644 --- a/docker-compose-runtime.yml +++ b/docker-compose-runtime.yml @@ -3,7 +3,7 @@ version: '2' services: zoo1: image: confluentinc/cp-zookeeper:6.2.1 - restart: on-failure:10 + restart: always hostname: zoo1 user: "0" volumes: @@ -22,7 +22,7 @@ services: kafka1: image: confluentinc/cp-kafka:6.2.1 - restart: on-failure:10 + restart: always hostname: kafka1 user: "0" ports: @@ -79,6 +79,7 @@ services: watchtower: image: containrrr/watchtower + restart: always volumes: - /var/run/docker.sock:/var/run/docker.sock env_file: ./watchtower.env diff --git a/docker-compose.yml b/docker-compose.yml index 4972c05..5f80b10 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '2' services: zoo1: image: confluentinc/cp-zookeeper:6.2.1 - restart: on-failure:10 + restart: always hostname: zoo1 user: "0" volumes: @@ -20,7 +20,7 @@ services: kafka1: image: confluentinc/cp-kafka:6.2.1 - restart: on-failure:10 + restart: always hostname: kafka1 user: "0" ports: @@ -53,7 +53,7 @@ services: mongo: container_name: mongo image: mongo - restart: on-failure:10 + restart: always volumes: - ./data:/data/db ports: diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index e2d77ad..0079e8d 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -225,10 +225,10 @@ Resources: - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml - sudo systemctl enable /usr/lib/systemd/system/docker.service - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -352,7 +352,7 @@ Resources: - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - sudo echo >> ~/akto/infra/docker-dashboard.env @@ -370,7 +370,7 @@ Resources: - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - !Ref AktoDashboardRole - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/enable_docker_process_on_restart/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: From 175881980db0fae5fa60f1d69c74b20ed81493a0 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Thu, 1 Feb 2024 16:12:44 +0530 Subject: [PATCH 62/98] Added mongo-cluster docker-compose file --- docker-compose-mongo-cluster.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docker-compose-mongo-cluster.yml diff --git a/docker-compose-mongo-cluster.yml b/docker-compose-mongo-cluster.yml new file mode 100644 index 0000000..df7719d --- /dev/null +++ b/docker-compose-mongo-cluster.yml @@ -0,0 +1,13 @@ +version: '3.3' + +services: + mongo: + container_name: mongo + image: mongo + restart: always + command: mongod --replSet "rs0" --bind_ip ${INSTANCE_IP} + volumes: + - ./data:/data/db + ports: + - "27017:27017" + network_mode: host \ No newline at end of file From ee43b0d4d3ab264f09f7a98bcfca47270ec2f9e6 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Thu, 1 Feb 2024 19:34:24 +0530 Subject: [PATCH 63/98] Changed branch name --- cf-deploy-akto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 452a158..b5f3756 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/mongo-cluster.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-quick-setup infra + mv infra-feature-mongo-cluster infra fi # NB: this is to make onprem containers to all get named the same. From 455ae4aa7b61bc0f701dc96dfa78a9acd76cb2b3 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Thu, 1 Feb 2024 20:25:35 +0530 Subject: [PATCH 64/98] Updated bind_ip --- docker-compose-mongo-cluster.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-mongo-cluster.yml b/docker-compose-mongo-cluster.yml index df7719d..78e99cc 100644 --- a/docker-compose-mongo-cluster.yml +++ b/docker-compose-mongo-cluster.yml @@ -5,7 +5,7 @@ services: container_name: mongo image: mongo restart: always - command: mongod --replSet "rs0" --bind_ip ${INSTANCE_IP} + command: mongod --replSet "rs0" --bind_ip ${INSTANCE_IP},localhost volumes: - ./data:/data/db ports: From afe2935be97d1818ab1cca249a8a0e9824cd2f16 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Fri, 2 Feb 2024 13:24:01 +0530 Subject: [PATCH 65/98] Working akto with mongo cluster setup --- templates/akto-quick-setup-enhanced.yaml | 619 +++++++++++++++++++++++ 1 file changed, 619 insertions(+) create mode 100644 templates/akto-quick-setup-enhanced.yaml diff --git a/templates/akto-quick-setup-enhanced.yaml b/templates/akto-quick-setup-enhanced.yaml new file mode 100644 index 0000000..0bcb6c3 --- /dev/null +++ b/templates/akto-quick-setup-enhanced.yaml @@ -0,0 +1,619 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: + This template does a quick setup of Akto modules. It sets up Akto dashboard and mongo on seperate ec2 instances. If you want a flexible setup, please + contact support@akto.io. +Parameters: + PrivateSubnetId: + Description: + Select a private subnetid where Akto EC2 instances will be deployed + Type: AWS::EC2::Subnet::Id + ConstraintDescription: must be an existing subnet + KeyPair: + Description: Select the key pair to connect to Akto EC2 instances + Type: AWS::EC2::KeyPair::KeyName + PublicSubnetIds: + Type: List + Description: + Select 2 public subnets. Atleast 1 should be in the same availability zone as the PrivateSubnetId you will provide below + ConstraintDescription: + must be a list of at least two existing subnets associated + with at least two different availability zones. They should be residing in the + selected Virtual Private Cloud. + UserEmail: + Type: String +Mappings: + RegionMap: + "af-south-1": + AMI: "ami-093ca241e4c72c205" + "eu-north-1": + AMI: "ami-0f58e72599cb99a79" + "ap-south-1": + AMI: "ami-0400aca7799d8cf19" + "eu-west-3": + AMI: "ami-064c70d04ad799d5e" + "eu-west-2": + AMI: "ami-0dfe6158087b5c0ac" + "eu-south-1": + AMI: "ami-07b2af763a8b958f3" + "eu-west-1": + AMI: "ami-047aad752a426ed48" + "ap-northeast-3": + AMI: "ami-0cffa2172948e071e" + "ap-northeast-2": + AMI: "ami-087af0192368bc87c" + "me-south-1": + AMI: "ami-0a31e56929248acca" + "ap-northeast-1": + AMI: "ami-0828596b82405edd7" + "sa-east-1": + AMI: "ami-0df67b3c17f090c24" + "ca-central-1": + AMI: "ami-0eb3718c42cb70e52" + "ap-east-1": + AMI: "ami-0e992f1e63814db10" + "ap-southeast-1": + AMI: "ami-0ba98499caf94125a" + "ap-southeast-2": + AMI: "ami-0849cc8fe4ceaf988" + "eu-central-1": + AMI: "ami-0f7585ae7a0d9a25a" + "ap-southeast-3": + AMI: "ami-0cf40308729b83366" + "us-east-1": + AMI: "ami-0d52ddcdf3a885741" + "us-east-2": + AMI: "ami-04148302a14f7d12b" + "us-west-1": + AMI: "ami-0ee3e1e65adeef858" + "us-west-2": + AMI: "ami-0ec021424fb596d6c" +Resources: + ## Setting up VpcDetailsLambda + GetVpcDetailsLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ec2:DescribeVpcs + - ec2:DescribeSubnets + Resource: "*" + GetVpcDetailsLambda: + Type: AWS::Lambda::Function + Properties: + Description: Look up info from a VPC and update deployment status + Handler: index.lambda_handler + Runtime: python3.9 + Timeout: 30 + Role: + Fn::GetAtt: + - GetVpcDetailsLambdaRole + - Arn + Environment: + Variables: + OWNER_EMAIL: + Ref: UserEmail + SUBNET_ID: + Ref: PrivateSubnetId + Code: + ZipFile: | + import json + import urllib3 + import os + import cfnresponse + import logging + import boto3 + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + http = urllib3.PoolManager() + + def lambda_handler(event, context): + if event.get('RequestType') == 'Delete': + responseData = {} + responseData['message'] = 'Deleting lambda' + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) + return + + try: + update_deployment_status() + except: + print('Failed to update deployment status') + fetch_vpc_details_resp = fetch_vpc_details() + + cfnresponse.send(event, context, cfnresponse.SUCCESS, fetch_vpc_details_resp) + + def fetch_vpc_details(): + subnet_id = os.environ['SUBNET_ID'] + ec2 = boto3.client('ec2') + + subnets = ec2.describe_subnets( + SubnetIds = [subnet_id] + ) + vpcId = subnets['Subnets'][0]['VpcId'] + + return { + 'VpcId': vpcId + } + + def update_deployment_status(): + url = "https://stairway.akto.io/deployment/status" + payload = json.dumps({ + "ownerEmail": os.environ['OWNER_EMAIL'], + "stackStatus": "STARTED", + "cloudType": "AWS" + }) + headers = { + 'Content-Type': 'application/json' + } + resp = http.request("POST", url, + body=payload, + headers=headers, + retries = False) + + logger.info('Request sent') + logger.info(resp.data) + return resp.data + CustomSourceGetVpcDetails: + Type: AWS::CloudFormation::CustomResource + Properties: + ServiceToken: + Fn::GetAtt: + - GetVpcDetailsLambda + - Arn + ## Setting up mongo + AktoMongoSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: Enable the ports Mongo requires (22, 27017) + SecurityGroupIngress: [] + SecurityGroupEgress: [] + AktoMongoLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateData: + ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] + InstanceType: m5a.xlarge + KeyName: + Ref: KeyPair + NetworkInterfaces: + - SubnetId: !Ref PrivateSubnetId + AssociatePublicIpAddress: "false" + DeviceIndex: 0 + Groups: [!Ref AktoMongoSecurityGroup] + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + VolumeType: gp3 + DeleteOnTermination: "false" + VolumeSize: "50" + Encrypted: true + MetadataOptions: + HttpTokens: required + HttpPutResponseHopLimit: 2 + AktoMongoPrimaryInstance: + Type: AWS::EC2::Instance + DependsOn: + - AktoMongoSecondary1Instance + - AktoMongoSecondary2Instance + Properties: + Tags: + - Key: Name + Value: Akto Mongo Primary instance + LaunchTemplate: + LaunchTemplateId: !Ref AktoMongoLaunchTemplate + Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-mongo-cluster.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' + - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Sub: export AKTO_SEC_1_IP='${AktoMongoSecondary1Instance.PrivateIp}' + - Fn::Sub: export AKTO_SEC_2_IP='${AktoMongoSecondary2Instance.PrivateIp}' + - sleep 20 + - docker exec -i mongo mongosh --eval "rs.initiate()" + - docker exec -i mongo mongosh --eval "rs.add('${AKTO_SEC_1_IP}:27017')" + - docker exec -i mongo mongosh --eval "rs.add('${AKTO_SEC_2_IP}:27017')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoMongoPrimaryInstance" + - " --region " + - Ref: AWS::Region + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 100 + ResourceSignal: + Timeout: PT10M + AktoMongoSecondary1Instance: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: Akto Mongo Secondary 1 instance + LaunchTemplate: + LaunchTemplateId: !Ref AktoMongoLaunchTemplate + Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-mongo-cluster.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' + - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoMongoSecondary1Instance" + - " --region " + - Ref: AWS::Region + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 100 + ResourceSignal: + Timeout: PT10M + AktoMongoSecondary2Instance: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: Akto Mongo Secondary 2 instance + LaunchTemplate: + LaunchTemplateId: !Ref AktoMongoLaunchTemplate + Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-mongo-cluster.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' + - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoMongoSecondary2Instance" + - " --region " + - Ref: AWS::Region + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 100 + ResourceSignal: + Timeout: PT10M + ## Setting up Dashboard + AktoDashboardTargetTrackingNetworkPolicy: + Type: AWS::AutoScaling::ScalingPolicy + Properties: + PolicyType: TargetTrackingScaling + AutoScalingGroupName: + Ref: AktoDashboardAutoScalingGroup + EstimatedInstanceWarmup: 30 + TargetTrackingConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ASGAverageCPUUtilization + TargetValue: 60 + AktoDashboardSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: Enable the ports Akto requires (22, 8080) + SecurityGroupIngress: [] + SecurityGroupEgress: [] + AktoDashboardAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + VPCZoneIdentifier: + - Ref: PrivateSubnetId + LaunchTemplate: + LaunchTemplateId: !Ref AktoDashboardLaunchTemplate + Version: !GetAtt AktoDashboardLaunchTemplate.LatestVersionNumber + TargetGroupARNs: + - Ref: AktoDashboardTargetGroup + MaxSize: "10" + MinSize: "1" + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 10 + ResourceSignal: + Timeout: PT10M + AktoDashboardRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + IamInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: "/" + Roles: + - Ref: AktoDashboardRole + AktoDashboardLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + DependsOn: + - AktoMongoPrimaryInstance + Properties: + LaunchTemplateData: + ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] + InstanceType: t2.medium + KeyName: + Ref: KeyPair + NetworkInterfaces: + - SubnetId: !Ref PrivateSubnetId + AssociatePublicIpAddress: "false" + DeviceIndex: 0 + Groups: [!Ref AktoDashboardSecurityGroup] + IamInstanceProfile: + Arn: !GetAtt IamInstanceProfile.Arn + MetadataOptions: + HttpTokens: required + HttpPutResponseHopLimit: 2 + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + VolumeType: gp2 + DeleteOnTermination: "true" + VolumeSize: "20" + Encrypted: true + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - Fn::Sub: export AKTO_MONGO_PRIMARY_IP='${AktoMongoPrimaryInstance.PrivateIp}' + - Fn::Sub: export AKTO_MONGO_SEC_1_IP='${AktoMongoSecondary1Instance.PrivateIp}' + - Fn::Sub: export AKTO_MONGO_SEC_2_IP='${AktoMongoSecondary2Instance.PrivateIp}' + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - export COMPOSE_FILE=docker-compose-dashboard.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - sudo echo >> ~/akto/infra/docker-dashboard.env + - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_PRIMARY_IP:27017,$AKTO_MONGO_SEC_1_IP:27017,$AKTO_MONGO_SEC_1_IP:27017/admini >> + ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo OWNER_EMAIL='${UserEmail}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo EC2_SUBNET_ID='${PrivateSubnetId}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AKTO_DASHBOARD_STACK_NAME=${AWS::StackName} >> ~/akto/infra/docker-dashboard.env + - Fn::Join: + - "" + - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" + - !Ref AktoDashboardRole + - " >> ~/akto/infra/docker-dashboard.env" + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoDashboardAutoScalingGroup" + - " --region " + - Ref: AWS::Region + AktoDashboardListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + LoadBalancerArn: + Ref: AktoLBDashboard + Port: "80" + Protocol: HTTP + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: AktoDashboardTargetGroup + AktoLBSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: Enable the port AktoLB requires + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIp: 0.0.0.0/0 + SecurityGroupEgress: [] + AktoLBDashboard: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Type: application + IpAddressType: ipv4 + Subnets: + Ref: PublicSubnetIds + SecurityGroups: [!Ref AktoLBSecurityGroup] + AktoDashboardTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: "8080" + Protocol: HTTP + HealthCheckEnabled: "true" + HealthCheckIntervalSeconds: 10 + HealthCheckPath: "/metrics" + HealthCheckPort: "8080" + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 6 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 2 + TargetType: instance + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId + Targets: [] + ConfigureSecurityGroupsLambda: + Type: AWS::Lambda::Function + Properties: + Description: Configure Security Groups for Mongo and Akto Dashboard + Handler: lambda.lambda_handler + Runtime: python3.9 + Timeout: 30 + Role: + Fn::GetAtt: + - ConfigureSecurityGroupsLambdaRole + - Arn + Environment: + Variables: + SUBNET_ID: !Ref PrivateSubnetId + MONGO_SECURITY_GROUP_ID: !Ref AktoMongoSecurityGroup + DASHBOARD_SECURITY_GROUP_ID: !Ref AktoDashboardSecurityGroup + MODE: DASHBOARD + Code: + S3Bucket: !Sub 'akto-setup-${AWS::Region}' + S3Key: templates/configure_security_groups.zip + CustomSourceConfigureSecurityGroupsLambda: + Type: AWS::CloudFormation::CustomResource + Properties: + ServiceToken: + Fn::GetAtt: + - ConfigureSecurityGroupsLambda + - Arn + ConfigureSecurityGroupsLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ec2:DescribeVpcs + - ec2:DescribeSubnets + Resource: "*" + - Effect: Allow + Action: + - ec2:AuthorizeSecurityGroupIngress + Resource: + - Fn::Sub: 'arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoMongoSecurityGroup}' + - Fn::Sub: 'arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoDashboardSecurityGroup}' + - PolicyName: AWSLambdaBasicExecutionRole + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: "*" +Outputs: + AktoLBDashboard: + Value: !Join + - '' + - - 'http://' + - !GetAtt + - AktoLBDashboard + - DNSName + - ':80' + Description: Url of Akto Dashboard LB From 60805b9010b65f872b015b0b71f88bf068a1b4f2 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Thu, 8 Feb 2024 11:02:02 +0530 Subject: [PATCH 66/98] Added mongo cluster yaml --- templates/prod-akto-setup.yaml | 619 +++++++++++++++++++++++++++++++++ 1 file changed, 619 insertions(+) create mode 100644 templates/prod-akto-setup.yaml diff --git a/templates/prod-akto-setup.yaml b/templates/prod-akto-setup.yaml new file mode 100644 index 0000000..173fbe3 --- /dev/null +++ b/templates/prod-akto-setup.yaml @@ -0,0 +1,619 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: + This template does a quick setup of Akto modules. It sets up Akto dashboard and mongo on seperate ec2 instances. If you want a flexible setup, please + contact support@akto.io. +Parameters: + PrivateSubnetId: + Description: + Select a private subnetid where Akto EC2 instances will be deployed + Type: AWS::EC2::Subnet::Id + ConstraintDescription: must be an existing subnet + KeyPair: + Description: Select the key pair to connect to Akto EC2 instances + Type: AWS::EC2::KeyPair::KeyName + PublicSubnetIds: + Type: List + Description: + Select 2 public subnets. Atleast 1 should be in the same availability zone as the PrivateSubnetId you will provide below + ConstraintDescription: + must be a list of at least two existing subnets associated + with at least two different availability zones. They should be residing in the + selected Virtual Private Cloud. + UserEmail: + Type: String +Mappings: + RegionMap: + "af-south-1": + AMI: "ami-093ca241e4c72c205" + "eu-north-1": + AMI: "ami-0f58e72599cb99a79" + "ap-south-1": + AMI: "ami-0400aca7799d8cf19" + "eu-west-3": + AMI: "ami-064c70d04ad799d5e" + "eu-west-2": + AMI: "ami-0dfe6158087b5c0ac" + "eu-south-1": + AMI: "ami-07b2af763a8b958f3" + "eu-west-1": + AMI: "ami-047aad752a426ed48" + "ap-northeast-3": + AMI: "ami-0cffa2172948e071e" + "ap-northeast-2": + AMI: "ami-087af0192368bc87c" + "me-south-1": + AMI: "ami-0a31e56929248acca" + "ap-northeast-1": + AMI: "ami-0828596b82405edd7" + "sa-east-1": + AMI: "ami-0df67b3c17f090c24" + "ca-central-1": + AMI: "ami-0eb3718c42cb70e52" + "ap-east-1": + AMI: "ami-0e992f1e63814db10" + "ap-southeast-1": + AMI: "ami-0ba98499caf94125a" + "ap-southeast-2": + AMI: "ami-0849cc8fe4ceaf988" + "eu-central-1": + AMI: "ami-0f7585ae7a0d9a25a" + "ap-southeast-3": + AMI: "ami-0cf40308729b83366" + "us-east-1": + AMI: "ami-0d52ddcdf3a885741" + "us-east-2": + AMI: "ami-04148302a14f7d12b" + "us-west-1": + AMI: "ami-0ee3e1e65adeef858" + "us-west-2": + AMI: "ami-0ec021424fb596d6c" +Resources: + ## Setting up VpcDetailsLambda + GetVpcDetailsLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ec2:DescribeVpcs + - ec2:DescribeSubnets + Resource: "*" + GetVpcDetailsLambda: + Type: AWS::Lambda::Function + Properties: + Description: Look up info from a VPC and update deployment status + Handler: index.lambda_handler + Runtime: python3.9 + Timeout: 30 + Role: + Fn::GetAtt: + - GetVpcDetailsLambdaRole + - Arn + Environment: + Variables: + OWNER_EMAIL: + Ref: UserEmail + SUBNET_ID: + Ref: PrivateSubnetId + Code: + ZipFile: | + import json + import urllib3 + import os + import cfnresponse + import logging + import boto3 + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + http = urllib3.PoolManager() + + def lambda_handler(event, context): + if event.get('RequestType') == 'Delete': + responseData = {} + responseData['message'] = 'Deleting lambda' + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) + return + + try: + update_deployment_status() + except: + print('Failed to update deployment status') + fetch_vpc_details_resp = fetch_vpc_details() + + cfnresponse.send(event, context, cfnresponse.SUCCESS, fetch_vpc_details_resp) + + def fetch_vpc_details(): + subnet_id = os.environ['SUBNET_ID'] + ec2 = boto3.client('ec2') + + subnets = ec2.describe_subnets( + SubnetIds = [subnet_id] + ) + vpcId = subnets['Subnets'][0]['VpcId'] + + return { + 'VpcId': vpcId + } + + def update_deployment_status(): + url = "https://stairway.akto.io/deployment/status" + payload = json.dumps({ + "ownerEmail": os.environ['OWNER_EMAIL'], + "stackStatus": "STARTED", + "cloudType": "AWS" + }) + headers = { + 'Content-Type': 'application/json' + } + resp = http.request("POST", url, + body=payload, + headers=headers, + retries = False) + + logger.info('Request sent') + logger.info(resp.data) + return resp.data + CustomSourceGetVpcDetails: + Type: AWS::CloudFormation::CustomResource + Properties: + ServiceToken: + Fn::GetAtt: + - GetVpcDetailsLambda + - Arn + ## Setting up mongo + AktoMongoSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: Enable the ports Mongo requires (22, 27017) + SecurityGroupIngress: [] + SecurityGroupEgress: [] + AktoMongoLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateData: + ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] + InstanceType: m5a.xlarge + KeyName: + Ref: KeyPair + NetworkInterfaces: + - SubnetId: !Ref PrivateSubnetId + AssociatePublicIpAddress: "false" + DeviceIndex: 0 + Groups: [!Ref AktoMongoSecurityGroup] + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + VolumeType: gp3 + DeleteOnTermination: "false" + VolumeSize: "50" + Encrypted: true + MetadataOptions: + HttpTokens: required + HttpPutResponseHopLimit: 2 + AktoMongoPrimaryInstance: + Type: AWS::EC2::Instance + DependsOn: + - AktoMongoSecondary1Instance + - AktoMongoSecondary2Instance + Properties: + Tags: + - Key: Name + Value: Akto Mongo Primary instance + LaunchTemplate: + LaunchTemplateId: !Ref AktoMongoLaunchTemplate + Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-mongo-cluster.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' + - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Sub: export AKTO_SEC_1_IP='${AktoMongoSecondary1Instance.PrivateIp}' + - Fn::Sub: export AKTO_SEC_2_IP='${AktoMongoSecondary2Instance.PrivateIp}' + - sleep 20 + - docker exec -i mongo mongosh --eval "rs.initiate()" + - docker exec -i mongo mongosh --eval "rs.add('${AKTO_SEC_1_IP}:27017')" + - docker exec -i mongo mongosh --eval "rs.add('${AKTO_SEC_2_IP}:27017')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoMongoPrimaryInstance" + - " --region " + - Ref: AWS::Region + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 100 + ResourceSignal: + Timeout: PT10M + AktoMongoSecondary1Instance: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: Akto Mongo Secondary 1 instance + LaunchTemplate: + LaunchTemplateId: !Ref AktoMongoLaunchTemplate + Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-mongo-cluster.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' + - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoMongoSecondary1Instance" + - " --region " + - Ref: AWS::Region + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 100 + ResourceSignal: + Timeout: PT10M + AktoMongoSecondary2Instance: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: Akto Mongo Secondary 2 instance + LaunchTemplate: + LaunchTemplateId: !Ref AktoMongoLaunchTemplate + Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-mongo-cluster.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' + - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoMongoSecondary2Instance" + - " --region " + - Ref: AWS::Region + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 100 + ResourceSignal: + Timeout: PT10M + ## Setting up Dashboard + AktoDashboardTargetTrackingNetworkPolicy: + Type: AWS::AutoScaling::ScalingPolicy + Properties: + PolicyType: TargetTrackingScaling + AutoScalingGroupName: + Ref: AktoDashboardAutoScalingGroup + EstimatedInstanceWarmup: 30 + TargetTrackingConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ASGAverageCPUUtilization + TargetValue: 60 + AktoDashboardSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: Enable the ports Akto requires (22, 8080) + SecurityGroupIngress: [] + SecurityGroupEgress: [] + AktoDashboardAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + VPCZoneIdentifier: + - Ref: PrivateSubnetId + LaunchTemplate: + LaunchTemplateId: !Ref AktoDashboardLaunchTemplate + Version: !GetAtt AktoDashboardLaunchTemplate.LatestVersionNumber + TargetGroupARNs: + - Ref: AktoDashboardTargetGroup + MaxSize: "10" + MinSize: "1" + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 10 + ResourceSignal: + Timeout: PT10M + AktoDashboardRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + IamInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: "/" + Roles: + - Ref: AktoDashboardRole + AktoDashboardLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + DependsOn: + - AktoMongoPrimaryInstance + Properties: + LaunchTemplateData: + ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] + InstanceType: m5a.xlarge + KeyName: + Ref: KeyPair + NetworkInterfaces: + - SubnetId: !Ref PrivateSubnetId + AssociatePublicIpAddress: "false" + DeviceIndex: 0 + Groups: [!Ref AktoDashboardSecurityGroup] + IamInstanceProfile: + Arn: !GetAtt IamInstanceProfile.Arn + MetadataOptions: + HttpTokens: required + HttpPutResponseHopLimit: 2 + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + VolumeType: gp2 + DeleteOnTermination: "true" + VolumeSize: "20" + Encrypted: true + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - Fn::Sub: export AKTO_MONGO_PRIMARY_IP='${AktoMongoPrimaryInstance.PrivateIp}' + - Fn::Sub: export AKTO_MONGO_SEC_1_IP='${AktoMongoSecondary1Instance.PrivateIp}' + - Fn::Sub: export AKTO_MONGO_SEC_2_IP='${AktoMongoSecondary2Instance.PrivateIp}' + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - export COMPOSE_FILE=docker-compose-dashboard.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - sudo echo >> ~/akto/infra/docker-dashboard.env + - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_PRIMARY_IP:27017,$AKTO_MONGO_SEC_1_IP:27017,$AKTO_MONGO_SEC_1_IP:27017/admini >> + ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo OWNER_EMAIL='${UserEmail}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo EC2_SUBNET_ID='${PrivateSubnetId}' >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env + - Fn::Sub: sudo echo AKTO_DASHBOARD_STACK_NAME=${AWS::StackName} >> ~/akto/infra/docker-dashboard.env + - Fn::Join: + - "" + - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" + - !Ref AktoDashboardRole + - " >> ~/akto/infra/docker-dashboard.env" + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoDashboardAutoScalingGroup" + - " --region " + - Ref: AWS::Region + AktoDashboardListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + LoadBalancerArn: + Ref: AktoLBDashboard + Port: "80" + Protocol: HTTP + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: AktoDashboardTargetGroup + AktoLBSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: Enable the port AktoLB requires + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIp: 0.0.0.0/0 + SecurityGroupEgress: [] + AktoLBDashboard: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Type: application + IpAddressType: ipv4 + Subnets: + Ref: PublicSubnetIds + SecurityGroups: [!Ref AktoLBSecurityGroup] + AktoDashboardTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: "8080" + Protocol: HTTP + HealthCheckEnabled: "true" + HealthCheckIntervalSeconds: 10 + HealthCheckPath: "/metrics" + HealthCheckPort: "8080" + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 6 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 2 + TargetType: instance + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId + Targets: [] + ConfigureSecurityGroupsLambda: + Type: AWS::Lambda::Function + Properties: + Description: Configure Security Groups for Mongo and Akto Dashboard + Handler: lambda.lambda_handler + Runtime: python3.9 + Timeout: 30 + Role: + Fn::GetAtt: + - ConfigureSecurityGroupsLambdaRole + - Arn + Environment: + Variables: + SUBNET_ID: !Ref PrivateSubnetId + MONGO_SECURITY_GROUP_ID: !Ref AktoMongoSecurityGroup + DASHBOARD_SECURITY_GROUP_ID: !Ref AktoDashboardSecurityGroup + MODE: DASHBOARD + Code: + S3Bucket: !Sub 'akto-setup-${AWS::Region}' + S3Key: templates/configure_security_groups.zip + CustomSourceConfigureSecurityGroupsLambda: + Type: AWS::CloudFormation::CustomResource + Properties: + ServiceToken: + Fn::GetAtt: + - ConfigureSecurityGroupsLambda + - Arn + ConfigureSecurityGroupsLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ec2:DescribeVpcs + - ec2:DescribeSubnets + Resource: "*" + - Effect: Allow + Action: + - ec2:AuthorizeSecurityGroupIngress + Resource: + - Fn::Sub: 'arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoMongoSecurityGroup}' + - Fn::Sub: 'arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoDashboardSecurityGroup}' + - PolicyName: AWSLambdaBasicExecutionRole + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: "*" +Outputs: + AktoLBDashboard: + Value: !Join + - '' + - - 'http://' + - !GetAtt + - AktoLBDashboard + - DNSName + - ':80' + Description: Url of Akto Dashboard LB From 87baa2d4cdf5abcf0cb5a859b89c766669b23bb5 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Thu, 8 Feb 2024 11:04:35 +0530 Subject: [PATCH 67/98] Moved all dependnecies to quick-setup branch --- cf-deploy-akto | 4 ++-- templates/prod-akto-setup.yaml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index b5f3756..452a158 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/mongo-cluster.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-mongo-cluster infra + mv infra-feature-quick-setup infra fi # NB: this is to make onprem containers to all get named the same. diff --git a/templates/prod-akto-setup.yaml b/templates/prod-akto-setup.yaml index 173fbe3..cda2807 100644 --- a/templates/prod-akto-setup.yaml +++ b/templates/prod-akto-setup.yaml @@ -237,13 +237,13 @@ Resources: - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo-cluster.yml - sudo systemctl enable /usr/lib/systemd/system/docker.service - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Sub: export AKTO_SEC_1_IP='${AktoMongoSecondary1Instance.PrivateIp}' @@ -290,13 +290,13 @@ Resources: - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo-cluster.yml - sudo systemctl enable /usr/lib/systemd/system/docker.service - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -337,13 +337,13 @@ Resources: - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo-cluster.yml - sudo systemctl enable /usr/lib/systemd/system/docker.service - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: From 5d809e93cd8a8fe7e691cb1fc7e9cedeeec1f6ee Mon Sep 17 00:00:00 2001 From: aktoboy Date: Thu, 8 Feb 2024 11:06:43 +0530 Subject: [PATCH 68/98] Removed file --- templates/akto-quick-setup-enhanced.yaml | 619 ----------------------- 1 file changed, 619 deletions(-) delete mode 100644 templates/akto-quick-setup-enhanced.yaml diff --git a/templates/akto-quick-setup-enhanced.yaml b/templates/akto-quick-setup-enhanced.yaml deleted file mode 100644 index 0bcb6c3..0000000 --- a/templates/akto-quick-setup-enhanced.yaml +++ /dev/null @@ -1,619 +0,0 @@ -AWSTemplateFormatVersion: "2010-09-09" -Description: - This template does a quick setup of Akto modules. It sets up Akto dashboard and mongo on seperate ec2 instances. If you want a flexible setup, please - contact support@akto.io. -Parameters: - PrivateSubnetId: - Description: - Select a private subnetid where Akto EC2 instances will be deployed - Type: AWS::EC2::Subnet::Id - ConstraintDescription: must be an existing subnet - KeyPair: - Description: Select the key pair to connect to Akto EC2 instances - Type: AWS::EC2::KeyPair::KeyName - PublicSubnetIds: - Type: List - Description: - Select 2 public subnets. Atleast 1 should be in the same availability zone as the PrivateSubnetId you will provide below - ConstraintDescription: - must be a list of at least two existing subnets associated - with at least two different availability zones. They should be residing in the - selected Virtual Private Cloud. - UserEmail: - Type: String -Mappings: - RegionMap: - "af-south-1": - AMI: "ami-093ca241e4c72c205" - "eu-north-1": - AMI: "ami-0f58e72599cb99a79" - "ap-south-1": - AMI: "ami-0400aca7799d8cf19" - "eu-west-3": - AMI: "ami-064c70d04ad799d5e" - "eu-west-2": - AMI: "ami-0dfe6158087b5c0ac" - "eu-south-1": - AMI: "ami-07b2af763a8b958f3" - "eu-west-1": - AMI: "ami-047aad752a426ed48" - "ap-northeast-3": - AMI: "ami-0cffa2172948e071e" - "ap-northeast-2": - AMI: "ami-087af0192368bc87c" - "me-south-1": - AMI: "ami-0a31e56929248acca" - "ap-northeast-1": - AMI: "ami-0828596b82405edd7" - "sa-east-1": - AMI: "ami-0df67b3c17f090c24" - "ca-central-1": - AMI: "ami-0eb3718c42cb70e52" - "ap-east-1": - AMI: "ami-0e992f1e63814db10" - "ap-southeast-1": - AMI: "ami-0ba98499caf94125a" - "ap-southeast-2": - AMI: "ami-0849cc8fe4ceaf988" - "eu-central-1": - AMI: "ami-0f7585ae7a0d9a25a" - "ap-southeast-3": - AMI: "ami-0cf40308729b83366" - "us-east-1": - AMI: "ami-0d52ddcdf3a885741" - "us-east-2": - AMI: "ami-04148302a14f7d12b" - "us-west-1": - AMI: "ami-0ee3e1e65adeef858" - "us-west-2": - AMI: "ami-0ec021424fb596d6c" -Resources: - ## Setting up VpcDetailsLambda - GetVpcDetailsLambdaRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: "/" - Policies: - - PolicyName: DescribeAssetsPolicy - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - ec2:DescribeVpcs - - ec2:DescribeSubnets - Resource: "*" - GetVpcDetailsLambda: - Type: AWS::Lambda::Function - Properties: - Description: Look up info from a VPC and update deployment status - Handler: index.lambda_handler - Runtime: python3.9 - Timeout: 30 - Role: - Fn::GetAtt: - - GetVpcDetailsLambdaRole - - Arn - Environment: - Variables: - OWNER_EMAIL: - Ref: UserEmail - SUBNET_ID: - Ref: PrivateSubnetId - Code: - ZipFile: | - import json - import urllib3 - import os - import cfnresponse - import logging - import boto3 - - logger = logging.getLogger() - logger.setLevel(logging.INFO) - http = urllib3.PoolManager() - - def lambda_handler(event, context): - if event.get('RequestType') == 'Delete': - responseData = {} - responseData['message'] = 'Deleting lambda' - cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) - return - - try: - update_deployment_status() - except: - print('Failed to update deployment status') - fetch_vpc_details_resp = fetch_vpc_details() - - cfnresponse.send(event, context, cfnresponse.SUCCESS, fetch_vpc_details_resp) - - def fetch_vpc_details(): - subnet_id = os.environ['SUBNET_ID'] - ec2 = boto3.client('ec2') - - subnets = ec2.describe_subnets( - SubnetIds = [subnet_id] - ) - vpcId = subnets['Subnets'][0]['VpcId'] - - return { - 'VpcId': vpcId - } - - def update_deployment_status(): - url = "https://stairway.akto.io/deployment/status" - payload = json.dumps({ - "ownerEmail": os.environ['OWNER_EMAIL'], - "stackStatus": "STARTED", - "cloudType": "AWS" - }) - headers = { - 'Content-Type': 'application/json' - } - resp = http.request("POST", url, - body=payload, - headers=headers, - retries = False) - - logger.info('Request sent') - logger.info(resp.data) - return resp.data - CustomSourceGetVpcDetails: - Type: AWS::CloudFormation::CustomResource - Properties: - ServiceToken: - Fn::GetAtt: - - GetVpcDetailsLambda - - Arn - ## Setting up mongo - AktoMongoSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - VpcId: - Fn::GetAtt: - - CustomSourceGetVpcDetails - - VpcId - GroupDescription: Enable the ports Mongo requires (22, 27017) - SecurityGroupIngress: [] - SecurityGroupEgress: [] - AktoMongoLaunchTemplate: - Type: AWS::EC2::LaunchTemplate - Properties: - LaunchTemplateData: - ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] - InstanceType: m5a.xlarge - KeyName: - Ref: KeyPair - NetworkInterfaces: - - SubnetId: !Ref PrivateSubnetId - AssociatePublicIpAddress: "false" - DeviceIndex: 0 - Groups: [!Ref AktoMongoSecurityGroup] - BlockDeviceMappings: - - DeviceName: "/dev/xvda" - Ebs: - VolumeType: gp3 - DeleteOnTermination: "false" - VolumeSize: "50" - Encrypted: true - MetadataOptions: - HttpTokens: required - HttpPutResponseHopLimit: 2 - AktoMongoPrimaryInstance: - Type: AWS::EC2::Instance - DependsOn: - - AktoMongoSecondary1Instance - - AktoMongoSecondary2Instance - Properties: - Tags: - - Key: Name - Value: Akto Mongo Primary instance - LaunchTemplate: - LaunchTemplateId: !Ref AktoMongoLaunchTemplate - Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber - UserData: - Fn::Base64: - Fn::Join: - - "\n" - - - "#!/bin/bash -xe" - - touch /tmp/hello.txt - - touch ~/hello.txt - - sudo yum update -y - - sudo yum install -y python python-setuptools - - sudo yum install -y docker - - sudo dockerd& - - sudo mkdir -p /opt/aws/bin - - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - export COMPOSE_FILE=docker-compose-mongo-cluster.yml - - sudo systemctl enable /usr/lib/systemd/system/docker.service - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto - - sudo chmod 700 cf-deploy-akto - - "./cf-deploy-akto < <(echo 'test')" - - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' - - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' - - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start - - sudo chmod 700 cf-deploy-akto-start - - "./cf-deploy-akto-start < <(echo 'test')" - - Fn::Sub: export AKTO_SEC_1_IP='${AktoMongoSecondary1Instance.PrivateIp}' - - Fn::Sub: export AKTO_SEC_2_IP='${AktoMongoSecondary2Instance.PrivateIp}' - - sleep 20 - - docker exec -i mongo mongosh --eval "rs.initiate()" - - docker exec -i mongo mongosh --eval "rs.add('${AKTO_SEC_1_IP}:27017')" - - docker exec -i mongo mongosh --eval "rs.add('${AKTO_SEC_2_IP}:27017')" - - Fn::Join: - - '' - - - sudo /opt/aws/bin/cfn-signal -s true - - " --stack " - - Ref: AWS::StackName - - " --resource AktoMongoPrimaryInstance" - - " --region " - - Ref: AWS::Region - CreationPolicy: - AutoScalingCreationPolicy: - MinSuccessfulInstancesPercent: 100 - ResourceSignal: - Timeout: PT10M - AktoMongoSecondary1Instance: - Type: AWS::EC2::Instance - Properties: - Tags: - - Key: Name - Value: Akto Mongo Secondary 1 instance - LaunchTemplate: - LaunchTemplateId: !Ref AktoMongoLaunchTemplate - Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber - UserData: - Fn::Base64: - Fn::Join: - - "\n" - - - "#!/bin/bash -xe" - - touch /tmp/hello.txt - - touch ~/hello.txt - - sudo yum update -y - - sudo yum install -y python python-setuptools - - sudo yum install -y docker - - sudo dockerd& - - sudo mkdir -p /opt/aws/bin - - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - export COMPOSE_FILE=docker-compose-mongo-cluster.yml - - sudo systemctl enable /usr/lib/systemd/system/docker.service - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto - - sudo chmod 700 cf-deploy-akto - - "./cf-deploy-akto < <(echo 'test')" - - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' - - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' - - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start - - sudo chmod 700 cf-deploy-akto-start - - "./cf-deploy-akto-start < <(echo 'test')" - - Fn::Join: - - '' - - - sudo /opt/aws/bin/cfn-signal -s true - - " --stack " - - Ref: AWS::StackName - - " --resource AktoMongoSecondary1Instance" - - " --region " - - Ref: AWS::Region - CreationPolicy: - AutoScalingCreationPolicy: - MinSuccessfulInstancesPercent: 100 - ResourceSignal: - Timeout: PT10M - AktoMongoSecondary2Instance: - Type: AWS::EC2::Instance - Properties: - Tags: - - Key: Name - Value: Akto Mongo Secondary 2 instance - LaunchTemplate: - LaunchTemplateId: !Ref AktoMongoLaunchTemplate - Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber - UserData: - Fn::Base64: - Fn::Join: - - "\n" - - - "#!/bin/bash -xe" - - touch /tmp/hello.txt - - touch ~/hello.txt - - sudo yum update -y - - sudo yum install -y python python-setuptools - - sudo yum install -y docker - - sudo dockerd& - - sudo mkdir -p /opt/aws/bin - - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - export COMPOSE_FILE=docker-compose-mongo-cluster.yml - - sudo systemctl enable /usr/lib/systemd/system/docker.service - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto' > cf-deploy-akto - - sudo chmod 700 cf-deploy-akto - - "./cf-deploy-akto < <(echo 'test')" - - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' - - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' - - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mongo-cluster/cf-deploy-akto-start' > cf-deploy-akto-start - - sudo chmod 700 cf-deploy-akto-start - - "./cf-deploy-akto-start < <(echo 'test')" - - Fn::Join: - - '' - - - sudo /opt/aws/bin/cfn-signal -s true - - " --stack " - - Ref: AWS::StackName - - " --resource AktoMongoSecondary2Instance" - - " --region " - - Ref: AWS::Region - CreationPolicy: - AutoScalingCreationPolicy: - MinSuccessfulInstancesPercent: 100 - ResourceSignal: - Timeout: PT10M - ## Setting up Dashboard - AktoDashboardTargetTrackingNetworkPolicy: - Type: AWS::AutoScaling::ScalingPolicy - Properties: - PolicyType: TargetTrackingScaling - AutoScalingGroupName: - Ref: AktoDashboardAutoScalingGroup - EstimatedInstanceWarmup: 30 - TargetTrackingConfiguration: - PredefinedMetricSpecification: - PredefinedMetricType: ASGAverageCPUUtilization - TargetValue: 60 - AktoDashboardSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - VpcId: - Fn::GetAtt: - - CustomSourceGetVpcDetails - - VpcId - GroupDescription: Enable the ports Akto requires (22, 8080) - SecurityGroupIngress: [] - SecurityGroupEgress: [] - AktoDashboardAutoScalingGroup: - Type: AWS::AutoScaling::AutoScalingGroup - Properties: - VPCZoneIdentifier: - - Ref: PrivateSubnetId - LaunchTemplate: - LaunchTemplateId: !Ref AktoDashboardLaunchTemplate - Version: !GetAtt AktoDashboardLaunchTemplate.LatestVersionNumber - TargetGroupARNs: - - Ref: AktoDashboardTargetGroup - MaxSize: "10" - MinSize: "1" - CreationPolicy: - AutoScalingCreationPolicy: - MinSuccessfulInstancesPercent: 10 - ResourceSignal: - Timeout: PT10M - AktoDashboardRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: ec2.amazonaws.com - Action: sts:AssumeRole - IamInstanceProfile: - Type: AWS::IAM::InstanceProfile - Properties: - Path: "/" - Roles: - - Ref: AktoDashboardRole - AktoDashboardLaunchTemplate: - Type: AWS::EC2::LaunchTemplate - DependsOn: - - AktoMongoPrimaryInstance - Properties: - LaunchTemplateData: - ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] - InstanceType: t2.medium - KeyName: - Ref: KeyPair - NetworkInterfaces: - - SubnetId: !Ref PrivateSubnetId - AssociatePublicIpAddress: "false" - DeviceIndex: 0 - Groups: [!Ref AktoDashboardSecurityGroup] - IamInstanceProfile: - Arn: !GetAtt IamInstanceProfile.Arn - MetadataOptions: - HttpTokens: required - HttpPutResponseHopLimit: 2 - BlockDeviceMappings: - - DeviceName: "/dev/xvda" - Ebs: - VolumeType: gp2 - DeleteOnTermination: "true" - VolumeSize: "20" - Encrypted: true - UserData: - Fn::Base64: - Fn::Join: - - "\n" - - - "#!/bin/bash -xe" - - touch /tmp/hello.txt - - touch ~/hello.txt - - Fn::Sub: export AKTO_MONGO_PRIMARY_IP='${AktoMongoPrimaryInstance.PrivateIp}' - - Fn::Sub: export AKTO_MONGO_SEC_1_IP='${AktoMongoSecondary1Instance.PrivateIp}' - - Fn::Sub: export AKTO_MONGO_SEC_2_IP='${AktoMongoSecondary2Instance.PrivateIp}' - - sudo yum update -y - - sudo yum install -y python python-setuptools - - sudo yum install -y docker - - sudo dockerd& - - sudo mkdir -p /opt/aws/bin - - export COMPOSE_FILE=docker-compose-dashboard.yml - - sudo systemctl enable /usr/lib/systemd/system/docker.service - - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto - - sudo chmod 700 cf-deploy-akto - - "./cf-deploy-akto < <(echo 'test')" - - sudo echo >> ~/akto/infra/docker-dashboard.env - - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_PRIMARY_IP:27017,$AKTO_MONGO_SEC_1_IP:27017,$AKTO_MONGO_SEC_1_IP:27017/admini >> - ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo OWNER_EMAIL='${UserEmail}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo EC2_SUBNET_ID='${PrivateSubnetId}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AKTO_DASHBOARD_STACK_NAME=${AWS::StackName} >> ~/akto/infra/docker-dashboard.env - - Fn::Join: - - "" - - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - - !Ref AktoDashboardRole - - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start - - sudo chmod 700 cf-deploy-akto-start - - "./cf-deploy-akto-start < <(echo 'test')" - - Fn::Join: - - '' - - - sudo /opt/aws/bin/cfn-signal -s true - - " --stack " - - Ref: AWS::StackName - - " --resource AktoDashboardAutoScalingGroup" - - " --region " - - Ref: AWS::Region - AktoDashboardListener: - Type: AWS::ElasticLoadBalancingV2::Listener - Properties: - LoadBalancerArn: - Ref: AktoLBDashboard - Port: "80" - Protocol: HTTP - DefaultActions: - - Type: forward - TargetGroupArn: - Ref: AktoDashboardTargetGroup - AktoLBSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - VpcId: - Fn::GetAtt: - - CustomSourceGetVpcDetails - - VpcId - GroupDescription: Enable the port AktoLB requires - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 80 - ToPort: 80 - CidrIp: 0.0.0.0/0 - SecurityGroupEgress: [] - AktoLBDashboard: - Type: AWS::ElasticLoadBalancingV2::LoadBalancer - Properties: - Type: application - IpAddressType: ipv4 - Subnets: - Ref: PublicSubnetIds - SecurityGroups: [!Ref AktoLBSecurityGroup] - AktoDashboardTargetGroup: - Type: AWS::ElasticLoadBalancingV2::TargetGroup - Properties: - Port: "8080" - Protocol: HTTP - HealthCheckEnabled: "true" - HealthCheckIntervalSeconds: 10 - HealthCheckPath: "/metrics" - HealthCheckPort: "8080" - HealthCheckProtocol: HTTP - HealthCheckTimeoutSeconds: 6 - HealthyThresholdCount: 2 - UnhealthyThresholdCount: 2 - TargetType: instance - VpcId: - Fn::GetAtt: - - CustomSourceGetVpcDetails - - VpcId - Targets: [] - ConfigureSecurityGroupsLambda: - Type: AWS::Lambda::Function - Properties: - Description: Configure Security Groups for Mongo and Akto Dashboard - Handler: lambda.lambda_handler - Runtime: python3.9 - Timeout: 30 - Role: - Fn::GetAtt: - - ConfigureSecurityGroupsLambdaRole - - Arn - Environment: - Variables: - SUBNET_ID: !Ref PrivateSubnetId - MONGO_SECURITY_GROUP_ID: !Ref AktoMongoSecurityGroup - DASHBOARD_SECURITY_GROUP_ID: !Ref AktoDashboardSecurityGroup - MODE: DASHBOARD - Code: - S3Bucket: !Sub 'akto-setup-${AWS::Region}' - S3Key: templates/configure_security_groups.zip - CustomSourceConfigureSecurityGroupsLambda: - Type: AWS::CloudFormation::CustomResource - Properties: - ServiceToken: - Fn::GetAtt: - - ConfigureSecurityGroupsLambda - - Arn - ConfigureSecurityGroupsLambdaRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: "/" - Policies: - - PolicyName: DescribeAssetsPolicy - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - ec2:DescribeVpcs - - ec2:DescribeSubnets - Resource: "*" - - Effect: Allow - Action: - - ec2:AuthorizeSecurityGroupIngress - Resource: - - Fn::Sub: 'arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoMongoSecurityGroup}' - - Fn::Sub: 'arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoDashboardSecurityGroup}' - - PolicyName: AWSLambdaBasicExecutionRole - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: "*" -Outputs: - AktoLBDashboard: - Value: !Join - - '' - - - 'http://' - - !GetAtt - - AktoLBDashboard - - DNSName - - ':80' - Description: Url of Akto Dashboard LB From 4f4aac2577cce314837114f2aefedeea406bcfda Mon Sep 17 00:00:00 2001 From: aktoboy Date: Thu, 8 Feb 2024 13:23:39 +0530 Subject: [PATCH 69/98] Upgraded instance types to 6th gen --- templates/akto-quick-setup.yaml | 4 ++-- templates/prod-akto-setup.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 0079e8d..188f2ff 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -191,7 +191,7 @@ Resources: Properties: LaunchTemplateData: ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] - InstanceType: m5a.xlarge + InstanceType: m6a.large KeyName: Ref: KeyPair NetworkInterfaces: @@ -315,7 +315,7 @@ Resources: Properties: LaunchTemplateData: ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] - InstanceType: m5a.xlarge + InstanceType: m6a.large KeyName: Ref: KeyPair NetworkInterfaces: diff --git a/templates/prod-akto-setup.yaml b/templates/prod-akto-setup.yaml index cda2807..08c58ff 100644 --- a/templates/prod-akto-setup.yaml +++ b/templates/prod-akto-setup.yaml @@ -191,7 +191,7 @@ Resources: Properties: LaunchTemplateData: ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] - InstanceType: m5a.xlarge + InstanceType: m6a.xlarge KeyName: Ref: KeyPair NetworkInterfaces: @@ -421,7 +421,7 @@ Resources: Properties: LaunchTemplateData: ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] - InstanceType: m5a.xlarge + InstanceType: m6a.xlarge KeyName: Ref: KeyPair NetworkInterfaces: From c1a0b169f08cba4214298b869163d03326be0167 Mon Sep 17 00:00:00 2001 From: Bhavik Date: Fri, 17 May 2024 13:33:32 +0530 Subject: [PATCH 70/98] Added template for mongo cluster --- templates/mongo-cluster-template.yml | 348 +++++++++++++++++++++++++++ 1 file changed, 348 insertions(+) create mode 100644 templates/mongo-cluster-template.yml diff --git a/templates/mongo-cluster-template.yml b/templates/mongo-cluster-template.yml new file mode 100644 index 0000000..8d4ecfb --- /dev/null +++ b/templates/mongo-cluster-template.yml @@ -0,0 +1,348 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: + This template does a quick setup of Akto modules. It sets up Akto dashboard and mongo on seperate ec2 instances. If you want a flexible setup, please + contact support@akto.io. +Parameters: + PrivateSubnetId: + Description: + Select a private subnetid where Akto EC2 instances will be deployed + Type: AWS::EC2::Subnet::Id + ConstraintDescription: must be an existing subnet + KeyPair: + Description: Select the key pair to connect to Mongo cluster + Type: AWS::EC2::KeyPair::KeyName +Mappings: + RegionMap: + "af-south-1": + AMI: "ami-093ca241e4c72c205" + "eu-north-1": + AMI: "ami-0f58e72599cb99a79" + "ap-south-1": + AMI: "ami-0400aca7799d8cf19" + "eu-west-3": + AMI: "ami-064c70d04ad799d5e" + "eu-west-2": + AMI: "ami-0dfe6158087b5c0ac" + "eu-south-1": + AMI: "ami-07b2af763a8b958f3" + "eu-west-1": + AMI: "ami-047aad752a426ed48" + "ap-northeast-3": + AMI: "ami-0cffa2172948e071e" + "ap-northeast-2": + AMI: "ami-087af0192368bc87c" + "me-south-1": + AMI: "ami-0a31e56929248acca" + "ap-northeast-1": + AMI: "ami-0828596b82405edd7" + "sa-east-1": + AMI: "ami-0df67b3c17f090c24" + "ca-central-1": + AMI: "ami-0eb3718c42cb70e52" + "ap-east-1": + AMI: "ami-0e992f1e63814db10" + "ap-southeast-1": + AMI: "ami-0ba98499caf94125a" + "ap-southeast-2": + AMI: "ami-0849cc8fe4ceaf988" + "eu-central-1": + AMI: "ami-0f7585ae7a0d9a25a" + "ap-southeast-3": + AMI: "ami-0cf40308729b83366" + "us-east-1": + AMI: "ami-0d52ddcdf3a885741" + "us-east-2": + AMI: "ami-04148302a14f7d12b" + "us-west-1": + AMI: "ami-0ee3e1e65adeef858" + "us-west-2": + AMI: "ami-0ec021424fb596d6c" +Resources: + ## Setting up VpcDetailsLambda + GetVpcDetailsLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ec2:DescribeVpcs + - ec2:DescribeSubnets + Resource: "*" + GetVpcDetailsLambda: + Type: AWS::Lambda::Function + Properties: + Description: Look up info from a VPC and update deployment status + Handler: index.lambda_handler + Runtime: python3.9 + Timeout: 30 + Role: + Fn::GetAtt: + - GetVpcDetailsLambdaRole + - Arn + Environment: + Variables: + SUBNET_ID: + Ref: PrivateSubnetId + Code: + ZipFile: | + import json + import urllib3 + import os + import cfnresponse + import logging + import boto3 + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + http = urllib3.PoolManager() + + def lambda_handler(event, context): + if event.get('RequestType') == 'Delete': + responseData = {} + responseData['message'] = 'Deleting lambda' + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData) + return + + fetch_vpc_details_resp = fetch_vpc_details() + + cfnresponse.send(event, context, cfnresponse.SUCCESS, fetch_vpc_details_resp) + + def fetch_vpc_details(): + subnet_id = os.environ['SUBNET_ID'] + ec2 = boto3.client('ec2') + + subnets = ec2.describe_subnets( + SubnetIds = [subnet_id] + ) + vpcId = subnets['Subnets'][0]['VpcId'] + vpc_response = ec2.describe_vpcs(VpcIds=[vpcId]) + cidr_block = vpc_response['Vpcs'][0]['CidrBlock'] + + return { + 'VpcId': vpcId, + 'CidrBlock': cidr_block + } + + CustomSourceGetVpcDetails: + Type: AWS::CloudFormation::CustomResource + Properties: + ServiceToken: + Fn::GetAtt: + - GetVpcDetailsLambda + - Arn + ## Setting up mongo + AktoMongoSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: Enable the ports Mongo requires (22, 27017) + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - CidrBlock + - IpProtocol: tcp + FromPort: 27017 + ToPort: 27017 + CidrIp: + Fn::GetAtt: + - CustomSourceGetVpcDetails + - CidrBlock + SecurityGroupEgress: [] + AktoMongoLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateData: + ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] + InstanceType: m6a.large + KeyName: + Ref: KeyPair + NetworkInterfaces: + - SubnetId: !Ref PrivateSubnetId + AssociatePublicIpAddress: "false" + DeviceIndex: 0 + Groups: [!Ref AktoMongoSecurityGroup] + BlockDeviceMappings: + - DeviceName: "/dev/xvda" + Ebs: + VolumeType: gp3 + DeleteOnTermination: "false" + VolumeSize: "50" + Encrypted: true + MetadataOptions: + HttpTokens: required + HttpPutResponseHopLimit: 2 + AktoMongoPrimaryInstance: + Type: AWS::EC2::Instance + DependsOn: + - AktoMongoSecondary1Instance + - AktoMongoSecondary2Instance + Properties: + Tags: + - Key: Name + Value: Akto Mongo Primary instance + LaunchTemplate: + LaunchTemplateId: !Ref AktoMongoLaunchTemplate + Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-mongo-cluster.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' + - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Sub: export AKTO_SEC_1_IP='${AktoMongoSecondary1Instance.PrivateIp}' + - Fn::Sub: export AKTO_SEC_2_IP='${AktoMongoSecondary2Instance.PrivateIp}' + - sleep 20 + - docker exec -i mongo mongosh --eval "rs.initiate()" + - docker exec -i mongo mongosh --eval "rs.add('${AKTO_SEC_1_IP}:27017')" + - docker exec -i mongo mongosh --eval "rs.add('${AKTO_SEC_2_IP}:27017')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoMongoPrimaryInstance" + - " --region " + - Ref: AWS::Region + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 100 + ResourceSignal: + Timeout: PT10M + AktoMongoSecondary1Instance: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: Akto Mongo Secondary 1 instance + LaunchTemplate: + LaunchTemplateId: !Ref AktoMongoLaunchTemplate + Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-mongo-cluster.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' + - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoMongoSecondary1Instance" + - " --region " + - Ref: AWS::Region + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 100 + ResourceSignal: + Timeout: PT10M + AktoMongoSecondary2Instance: + Type: AWS::EC2::Instance + Properties: + Tags: + - Key: Name + Value: Akto Mongo Secondary 2 instance + LaunchTemplate: + LaunchTemplateId: !Ref AktoMongoLaunchTemplate + Version: !GetAtt AktoMongoLaunchTemplate.LatestVersionNumber + UserData: + Fn::Base64: + Fn::Join: + - "\n" + - - "#!/bin/bash -xe" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo mkdir -p /opt/aws/bin + - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + - export COMPOSE_FILE=docker-compose-mongo-cluster.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - "./cf-deploy-akto < <(echo 'test')" + - 'TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`' + - 'PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - sed -i "s/\${INSTANCE_IP}/$PRIVATE_IP/g" ~/akto/infra/docker-compose-mongo-cluster.yml + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - "./cf-deploy-akto-start < <(echo 'test')" + - Fn::Join: + - '' + - - sudo /opt/aws/bin/cfn-signal -s true + - " --stack " + - Ref: AWS::StackName + - " --resource AktoMongoSecondary2Instance" + - " --region " + - Ref: AWS::Region + CreationPolicy: + AutoScalingCreationPolicy: + MinSuccessfulInstancesPercent: 100 + ResourceSignal: + Timeout: PT10M +Outputs: + MongoConnURL: + Value: !Sub mongodb://${AktoMongoPrimaryInstance.PrivateIp}:27017,${AktoMongoSecondary1Instance.PrivateIp}:27017,${AktoMongoSecondary2Instance.PrivateIp}:27017/admini + Description: Mongo connection url \ No newline at end of file From 28c45ecb80e553c0896481f02444301dfe9a5989 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Thu, 23 May 2024 01:15:41 +0530 Subject: [PATCH 71/98] mini runtime cft --- docker-compose-mini-runtime.yaml | 87 ++++++++++ docker-mini-runtime.env | 13 ++ templates/data_processing_stack.yml | 247 ++++++++++++++-------------- 3 files changed, 223 insertions(+), 124 deletions(-) create mode 100644 docker-compose-mini-runtime.yaml create mode 100644 docker-mini-runtime.env diff --git a/docker-compose-mini-runtime.yaml b/docker-compose-mini-runtime.yaml new file mode 100644 index 0000000..5a63041 --- /dev/null +++ b/docker-compose-mini-runtime.yaml @@ -0,0 +1,87 @@ +version: '2' + +services: + zoo1: + image: confluentinc/cp-zookeeper:6.2.1 + restart: always + hostname: zoo1 + user: "0" + volumes: + - ./data-zoo-data:/var/lib/zookeeper/data + - ./data-zoo-logs:/var/lib/zookeeper/log + - ./data-zoo-secrets:/etc/zookeeper/secrets + container_name: zoo1 + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_SERVER_ID: 1 + ZOOKEEPER_SERVERS: zoo1:2888:3888 + labels: + com.centurylinklabs.watchtower.enable: "false" + + kafka1: + image: confluentinc/cp-kafka:6.2.1 + restart: always + hostname: kafka1 + user: "0" + ports: + - "9092:9092" + - "19092:19092" + - "29092:29092" + - "9999:9999" + environment: + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://${AKTO_KAFKA_IP}:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" + KAFKA_BROKER_ID: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_CREATE_TOPICS: "akto.api.logs:3:3" + KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 60000 + KAFKA_LOG_RETENTION_HOURS: 5 + KAFKA_LOG_SEGMENT_BYTES: 104857600 + KAFKA_LOG_CLEANER_ENABLE: "true" + KAFKA_CLEANUP_POLICY: "delete" + KAFKA_LOG_RETENTION_BYTES: 10737418240 + volumes: + - ./data-kafka-data:/var/lib/kafka/data + - ./data-kafka-secrets:/etc/kafka/secrets + depends_on: + - zoo1 + labels: + com.centurylinklabs.watchtower.enable: "false" + + akto-api-security-runtime: + image: public.ecr.aws/aktosecurity/akto-api-security-mini-runtime:testruntime + env_file: ./docker-mini-runtime.env + mem_limit: 8g + restart: always + depends_on: + - kafka1 + + akto-api-security-mirror-api-logging: + image: public.ecr.aws/aktosecurity/mirror-api-logging:local + env_file: ./docker-mini-runtime.env + mem_limit: 4g + restart: always + depends_on: + - kafka1 + network_mode: host + + akto-hello-world: + image: public.ecr.aws/aktosecurity/hello-world-server:latest + restart: always + ports: + - "8000:8000" + + watchtower: + image: containrrr/watchtower + restart: always + volumes: + - /var/run/docker.sock:/var/run/docker.sock + env_file: ./watchtower.env + labels: + com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file diff --git a/docker-mini-runtime.env b/docker-mini-runtime.env new file mode 100644 index 0000000..054c7ac --- /dev/null +++ b/docker-mini-runtime.env @@ -0,0 +1,13 @@ +AKTO_CONFIG_NAME=staging +AKTO_KAFKA_TOPIC_NAME=akto.api.logs +AKTO_KAFKA_BROKER_URL=kafka1:19092 +AKTO_KAFKA_BROKER_MAL=localhost:29092 +AKTO_KAFKA_GROUP_ID_CONFIG=asdf +AKTO_KAFKA_MAX_POLL_RECORDS_CONFIG=100 +AKTO_ACCOUNT_NAME=Helios +AKTO_TRAFFIC_BATCH_SIZE=100 +AKTO_TRAFFIC_BATCH_TIME_SECS=10 +USE_HOSTNAME=true +AKTO_INSTANCE_TYPE=RUNTIME +DATABASE_ABSTRACTOR_SERVICE_TOKEN="eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJBa3RvIiwic3ViIjoiaW52aXRlX3VzZXIiLCJhY2NvdW50SWQiOjEwMDAwMDAsImlhdCI6MTcxNjQwNjY2NiwiZXhwIjoxNzMyMzA0MjY2fQ.Up90fQoemV0SND8d3O78wcheLXnyAkiB0kJJYYNJ_u1InwwfCTo0aq5vl9QsDZj-qC75hg01iVKhuZmVuvh9l_65j9lTx_oDF2vkB2P5TLMkdyjriL7lTSp3RE4ceJSGVWCdM8rG5p8fG0Sf3FxepMJAFmX3JbpXi7bb31-K6lwddsc-EGNEMiL_KE4lWzmpNVYgqjBxAFk8U9TSN703AocQX7X4u624gTD6cFasbiBBgZTGMaFdOBLY_rwjLCu71Jkoogvg6Klp-zGrsrv7vFURUNreaUOudLC6Ppu1iqCnUfD5IAnZ_7Lnpdud9UbwRIpoMbivQJYBVYIJJ7Ab_Q" +DATABASE_ABSTRACTOR_SERVICE_URL="http://13.229.55.224:9000" diff --git a/templates/data_processing_stack.yml b/templates/data_processing_stack.yml index 4009e28..8aa10c9 100644 --- a/templates/data_processing_stack.yml +++ b/templates/data_processing_stack.yml @@ -8,8 +8,8 @@ Parameters: Type: 'AWS::EC2::Subnet::Id' KeyPair: Type: 'AWS::EC2::KeyPair::KeyName' - MongoIp: - Type: String + # MongoIp: + # Type: String Mappings: RegionMap: af-south-1: @@ -162,120 +162,120 @@ Resources: Path: / Roles: - !Ref RefreshHandlerLambdaBasicExecutionRole - AktoContextAnalyzerSecurityGroup: - Type: 'AWS::EC2::SecurityGroup' - Properties: - VpcId: !GetAtt - - CustomSourceGetVpcDetails - - VpcId - GroupDescription: 'Enable the ports Akto requires (22, 9092)' - SecurityGroupIngress: [] - SecurityGroupEgress: [] - AktoContextAnalyzerASGLaunchConfiguration: - Type: 'AWS::AutoScaling::LaunchConfiguration' - Properties: - ImageId: !FindInMap - - RegionMap - - !Ref 'AWS::Region' - - AMI - InstanceType: m5a.xlarge - KeyName: !Ref KeyPair - AssociatePublicIpAddress: 'false' - SecurityGroups: - - !Ref AktoContextAnalyzerSecurityGroup - BlockDeviceMappings: - - DeviceName: /dev/xvda - Ebs: - VolumeType: gp2 - DeleteOnTermination: 'true' - VolumeSize: '50' - Encrypted: true - MetadataOptions: - HttpTokens: required - UserData: !Base64 - 'Fn::Join': - - |+ + # AktoContextAnalyzerSecurityGroup: + # Type: 'AWS::EC2::SecurityGroup' + # Properties: + # VpcId: !GetAtt + # - CustomSourceGetVpcDetails + # - VpcId + # GroupDescription: 'Enable the ports Akto requires (22, 9092)' + # SecurityGroupIngress: [] + # SecurityGroupEgress: [] + # AktoContextAnalyzerASGLaunchConfiguration: + # Type: 'AWS::AutoScaling::LaunchConfiguration' + # Properties: + # ImageId: !FindInMap + # - RegionMap + # - !Ref 'AWS::Region' + # - AMI + # InstanceType: m5a.xlarge + # KeyName: !Ref KeyPair + # AssociatePublicIpAddress: 'false' + # SecurityGroups: + # - !Ref AktoContextAnalyzerSecurityGroup + # BlockDeviceMappings: + # - DeviceName: /dev/xvda + # Ebs: + # VolumeType: gp2 + # DeleteOnTermination: 'true' + # VolumeSize: '50' + # Encrypted: true + # MetadataOptions: + # HttpTokens: required + # UserData: !Base64 + # 'Fn::Join': + # - |+ - - - '#!/bin/bash -xe' - - !Sub 'export AKTO_MONGO_CONN=''${MongoIp}''' - - touch /tmp/hello.txt - - touch ~/hello.txt - - sudo yum update -y - - sudo yum install -y python python-setuptools - - sudo yum install -y docker - - sudo dockerd& - - sudo systemctl enable /usr/lib/systemd/system/docker.service - - sudo mkdir -p /opt/aws/bin - - export COMPOSE_FILE=docker-compose-context-analyser.yml - - >- - sudo wget - https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - - >- - sudo python -m easy_install --script-dir /opt/aws/bin - aws-cfn-bootstrap-latest.tar.gz - - >- - curl -fsSL - 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' - > cf-deploy-akto - - sudo chmod 700 cf-deploy-akto - - ./cf-deploy-akto < <(echo 'test') - - sudo echo >> ~/akto/infra/docker-context-analyser.env - - >- - sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> - ~/akto/infra/docker-context-analyser.env - - >- - export TOKEN=$(curl -X PUT - 'http://169.254.169.254/latest/api/token' -H - 'X-aws-ec2-metadata-token-ttl-seconds: 600') - - !Join - - ':' - - - >- - export AKTO_CURRENT_INSTANCE_IP=$(curl -H - "X-aws-ec2-metadata-token - - '$TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' - - >- - echo AKTO_CURRENT_INSTANCE_IP=$AKTO_CURRENT_INSTANCE_IP >> - ~/akto/infra/docker-context-analyser.env - - >- - curl -fsSL - 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' - > cf-deploy-akto-start - - sudo chmod 700 cf-deploy-akto-start - - ./cf-deploy-akto-start < <(echo 'test') - AktoContextAnalyzerAutoScalingGroup: - Type: 'AWS::AutoScaling::AutoScalingGroup' - Properties: - LaunchConfigurationName: !Ref AktoContextAnalyzerASGLaunchConfiguration - VPCZoneIdentifier: - - !Ref SubnetId - MaxSize: '1' - MinSize: '1' - AktoContextAnalyzerInstanceRefreshHandler: - Type: 'AWS::Lambda::Function' - Properties: - Handler: index.handler - Runtime: nodejs16.x - Timeout: 30 - Role: !GetAtt - - InstanceRefreshHandlerLambdaRole - - Arn - Code: - ZipFile: > - var aws = require('aws-sdk'); var autoscaling = new aws.AutoScaling(); - exports.handler = function(event, context) { - var params = { - AutoScalingGroupName: 'AktoContextAnalyzerAutoScalingGroup', - Preferences: { - InstanceWarmup: 200, - MinHealthyPercentage: 0 - } - }; + # - - '#!/bin/bash -xe' + # - !Sub 'export AKTO_MONGO_CONN=''${MongoIp}''' + # - touch /tmp/hello.txt + # - touch ~/hello.txt + # - sudo yum update -y + # - sudo yum install -y python python-setuptools + # - sudo yum install -y docker + # - sudo dockerd& + # - sudo systemctl enable /usr/lib/systemd/system/docker.service + # - sudo mkdir -p /opt/aws/bin + # - export COMPOSE_FILE=docker-compose-context-analyser.yml + # - >- + # sudo wget + # https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + # - >- + # sudo python -m easy_install --script-dir /opt/aws/bin + # aws-cfn-bootstrap-latest.tar.gz + # - >- + # curl -fsSL + # 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' + # > cf-deploy-akto + # - sudo chmod 700 cf-deploy-akto + # - ./cf-deploy-akto < <(echo 'test') + # - sudo echo >> ~/akto/infra/docker-context-analyser.env + # - >- + # sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> + # ~/akto/infra/docker-context-analyser.env + # - >- + # export TOKEN=$(curl -X PUT + # 'http://169.254.169.254/latest/api/token' -H + # 'X-aws-ec2-metadata-token-ttl-seconds: 600') + # - !Join + # - ':' + # - - >- + # export AKTO_CURRENT_INSTANCE_IP=$(curl -H + # "X-aws-ec2-metadata-token + # - '$TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + # - >- + # echo AKTO_CURRENT_INSTANCE_IP=$AKTO_CURRENT_INSTANCE_IP >> + # ~/akto/infra/docker-context-analyser.env + # - >- + # curl -fsSL + # 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' + # > cf-deploy-akto-start + # - sudo chmod 700 cf-deploy-akto-start + # - ./cf-deploy-akto-start < <(echo 'test') + # AktoContextAnalyzerAutoScalingGroup: + # Type: 'AWS::AutoScaling::AutoScalingGroup' + # Properties: + # LaunchConfigurationName: !Ref AktoContextAnalyzerASGLaunchConfiguration + # VPCZoneIdentifier: + # - !Ref SubnetId + # MaxSize: '1' + # MinSize: '1' + # AktoContextAnalyzerInstanceRefreshHandler: + # Type: 'AWS::Lambda::Function' + # Properties: + # Handler: index.handler + # Runtime: nodejs16.x + # Timeout: 30 + # Role: !GetAtt + # - InstanceRefreshHandlerLambdaRole + # - Arn + # Code: + # ZipFile: > + # var aws = require('aws-sdk'); var autoscaling = new aws.AutoScaling(); + # exports.handler = function(event, context) { + # var params = { + # AutoScalingGroupName: 'AktoContextAnalyzerAutoScalingGroup', + # Preferences: { + # InstanceWarmup: 200, + # MinHealthyPercentage: 0 + # } + # }; - autoscaling.startInstanceRefresh(params, function(err, data) { - if(err) { console.log(err) } - else { console.log(data) } - }) - }; + # autoscaling.startInstanceRefresh(params, function(err, data) { + # if(err) { console.log(err) } + # else { console.log(data) } + # }) + # }; RefreshHandlerLambdaBasicExecutionRole: Type: 'AWS::IAM::Role' Properties: @@ -299,9 +299,9 @@ Resources: - !GetAtt - TrafficMirroringInstanceRefreshHandler - Arn - - !GetAtt - - AktoContextAnalyzerInstanceRefreshHandler - - Arn + # - !GetAtt + # - AktoContextAnalyzerInstanceRefreshHandler + # - Arn Action: 'lambda:InvokeFunction' AktoSecurityGroup: Type: 'AWS::EC2::SecurityGroup' @@ -341,7 +341,6 @@ Resources: - |+ - - '#!/bin/bash -xe' - - !Sub 'export AKTO_MONGO_CONN=''${MongoIp}''' - !Sub 'export AKTO_KAFKA_IP=''${AktoNLB.DNSName}''' - touch /tmp/hello.txt - touch ~/hello.txt @@ -351,7 +350,7 @@ Resources: - sudo dockerd& - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo mkdir -p /opt/aws/bin - - export COMPOSE_FILE=docker-compose-runtime.yml + - export COMPOSE_FILE=docker-compose-mini-runtime.yml - >- sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz @@ -360,7 +359,7 @@ Resources: aws-cfn-bootstrap-latest.tar.gz - >- curl -fsSL - 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - ./cf-deploy-akto < <(echo 'test') @@ -371,7 +370,7 @@ Resources: - sudo echo AKTO_KAFKA_IP=$AKTO_KAFKA_IP >> ~/akto/infra/.env - >- curl -fsSL - 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - ./cf-deploy-akto-start < <(echo 'test') @@ -555,7 +554,7 @@ Resources: Environment: Variables: SUBNET_ID: !Ref SubnetId - CONTEXT_ANALYZER_SECURITY_GROUP_ID: !Ref AktoContextAnalyzerSecurityGroup + # CONTEXT_ANALYZER_SECURITY_GROUP_ID: !Ref AktoContextAnalyzerSecurityGroup RUNTIME_PROCESSOR_SECURITY_GROUP_ID: !Ref AktoSecurityGroup MODE: RUNTIME Code: @@ -596,8 +595,8 @@ Resources: Resource: - !Sub >- arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoSecurityGroup} - - !Sub >- - arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoContextAnalyzerSecurityGroup} + # - !Sub >- + # arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoContextAnalyzerSecurityGroup} - PolicyName: AWSLambdaBasicExecutionRole PolicyDocument: Version: 2012-10-17 From 033a3db4e669d578d298029ee1be198d05da0147 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Sat, 25 May 2024 15:17:38 +0530 Subject: [PATCH 72/98] modify docker file name --- ...r-compose-mini-runtime.yaml => docker-compose-mini-runtime.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker-compose-mini-runtime.yaml => docker-compose-mini-runtime.yml (100%) diff --git a/docker-compose-mini-runtime.yaml b/docker-compose-mini-runtime.yml similarity index 100% rename from docker-compose-mini-runtime.yaml rename to docker-compose-mini-runtime.yml From db740d1889cd0033acb22de4e377a8bc74dffbf8 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Sat, 25 May 2024 15:30:28 +0530 Subject: [PATCH 73/98] modify cf deploy --- cf-deploy-akto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf-deploy-akto b/cf-deploy-akto index 452a158..ed77312 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/mini-runtime-cft.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-quick-setup infra + mv infra-feature-mini-runtime-cft infra fi # NB: this is to make onprem containers to all get named the same. From 58fba759a86f1e864037a2e32fca18c3c6747216 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Sat, 25 May 2024 15:51:22 +0530 Subject: [PATCH 74/98] add var in env file --- docker-mini-runtime.env | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-mini-runtime.env b/docker-mini-runtime.env index 054c7ac..28330de 100644 --- a/docker-mini-runtime.env +++ b/docker-mini-runtime.env @@ -11,3 +11,4 @@ USE_HOSTNAME=true AKTO_INSTANCE_TYPE=RUNTIME DATABASE_ABSTRACTOR_SERVICE_TOKEN="eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJBa3RvIiwic3ViIjoiaW52aXRlX3VzZXIiLCJhY2NvdW50SWQiOjEwMDAwMDAsImlhdCI6MTcxNjQwNjY2NiwiZXhwIjoxNzMyMzA0MjY2fQ.Up90fQoemV0SND8d3O78wcheLXnyAkiB0kJJYYNJ_u1InwwfCTo0aq5vl9QsDZj-qC75hg01iVKhuZmVuvh9l_65j9lTx_oDF2vkB2P5TLMkdyjriL7lTSp3RE4ceJSGVWCdM8rG5p8fG0Sf3FxepMJAFmX3JbpXi7bb31-K6lwddsc-EGNEMiL_KE4lWzmpNVYgqjBxAFk8U9TSN703AocQX7X4u624gTD6cFasbiBBgZTGMaFdOBLY_rwjLCu71Jkoogvg6Klp-zGrsrv7vFURUNreaUOudLC6Ppu1iqCnUfD5IAnZ_7Lnpdud9UbwRIpoMbivQJYBVYIJJ7Ab_Q" DATABASE_ABSTRACTOR_SERVICE_URL="http://13.229.55.224:9000" +RUNTIME_MODE="hybrid" From cedf606661dcf07330badac3e87ec4564d381fe4 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Sat, 25 May 2024 16:38:21 +0530 Subject: [PATCH 75/98] add var in env file --- docker-mini-runtime.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-mini-runtime.env b/docker-mini-runtime.env index 28330de..5dcfd55 100644 --- a/docker-mini-runtime.env +++ b/docker-mini-runtime.env @@ -11,4 +11,4 @@ USE_HOSTNAME=true AKTO_INSTANCE_TYPE=RUNTIME DATABASE_ABSTRACTOR_SERVICE_TOKEN="eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJBa3RvIiwic3ViIjoiaW52aXRlX3VzZXIiLCJhY2NvdW50SWQiOjEwMDAwMDAsImlhdCI6MTcxNjQwNjY2NiwiZXhwIjoxNzMyMzA0MjY2fQ.Up90fQoemV0SND8d3O78wcheLXnyAkiB0kJJYYNJ_u1InwwfCTo0aq5vl9QsDZj-qC75hg01iVKhuZmVuvh9l_65j9lTx_oDF2vkB2P5TLMkdyjriL7lTSp3RE4ceJSGVWCdM8rG5p8fG0Sf3FxepMJAFmX3JbpXi7bb31-K6lwddsc-EGNEMiL_KE4lWzmpNVYgqjBxAFk8U9TSN703AocQX7X4u624gTD6cFasbiBBgZTGMaFdOBLY_rwjLCu71Jkoogvg6Klp-zGrsrv7vFURUNreaUOudLC6Ppu1iqCnUfD5IAnZ_7Lnpdud9UbwRIpoMbivQJYBVYIJJ7Ab_Q" DATABASE_ABSTRACTOR_SERVICE_URL="http://13.229.55.224:9000" -RUNTIME_MODE="hybrid" +RUNTIME_MODE=hybrid From 376f4134588a679d2bbd188d5fd3cc15b245514f Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Mon, 3 Jun 2024 20:34:38 +0530 Subject: [PATCH 76/98] modify env file --- docker-mini-runtime.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-mini-runtime.env b/docker-mini-runtime.env index 5dcfd55..5166657 100644 --- a/docker-mini-runtime.env +++ b/docker-mini-runtime.env @@ -9,6 +9,6 @@ AKTO_TRAFFIC_BATCH_SIZE=100 AKTO_TRAFFIC_BATCH_TIME_SECS=10 USE_HOSTNAME=true AKTO_INSTANCE_TYPE=RUNTIME -DATABASE_ABSTRACTOR_SERVICE_TOKEN="eyJhbGciOiJSUzI1NiJ9.eyJpc3MiOiJBa3RvIiwic3ViIjoiaW52aXRlX3VzZXIiLCJhY2NvdW50SWQiOjEwMDAwMDAsImlhdCI6MTcxNjQwNjY2NiwiZXhwIjoxNzMyMzA0MjY2fQ.Up90fQoemV0SND8d3O78wcheLXnyAkiB0kJJYYNJ_u1InwwfCTo0aq5vl9QsDZj-qC75hg01iVKhuZmVuvh9l_65j9lTx_oDF2vkB2P5TLMkdyjriL7lTSp3RE4ceJSGVWCdM8rG5p8fG0Sf3FxepMJAFmX3JbpXi7bb31-K6lwddsc-EGNEMiL_KE4lWzmpNVYgqjBxAFk8U9TSN703AocQX7X4u624gTD6cFasbiBBgZTGMaFdOBLY_rwjLCu71Jkoogvg6Klp-zGrsrv7vFURUNreaUOudLC6Ppu1iqCnUfD5IAnZ_7Lnpdud9UbwRIpoMbivQJYBVYIJJ7Ab_Q" -DATABASE_ABSTRACTOR_SERVICE_URL="http://13.229.55.224:9000" +DATABASE_ABSTRACTOR_SERVICE_TOKEN=token +DATABASE_ABSTRACTOR_SERVICE_URL=http://13.229.55.224:9000 RUNTIME_MODE=hybrid From 53f6bd64ed57b2615b87557f165f655d0aa0eade Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Mon, 3 Jun 2024 20:52:13 +0530 Subject: [PATCH 77/98] modify env file --- docker-mini-runtime.env | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-mini-runtime.env b/docker-mini-runtime.env index 5166657..5070d9c 100644 --- a/docker-mini-runtime.env +++ b/docker-mini-runtime.env @@ -9,6 +9,5 @@ AKTO_TRAFFIC_BATCH_SIZE=100 AKTO_TRAFFIC_BATCH_TIME_SECS=10 USE_HOSTNAME=true AKTO_INSTANCE_TYPE=RUNTIME -DATABASE_ABSTRACTOR_SERVICE_TOKEN=token DATABASE_ABSTRACTOR_SERVICE_URL=http://13.229.55.224:9000 RUNTIME_MODE=hybrid From a1a4bb1635e09a62cdb0f9cb315686ecba141438 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Mon, 3 Jun 2024 21:43:30 +0530 Subject: [PATCH 78/98] add mini runtime cft --- cf-deploy-akto | 4 +- templates/mini-runtime.yml | 502 +++++++++++++++++++++++++++++++++++++ 2 files changed, 504 insertions(+), 2 deletions(-) create mode 100644 templates/mini-runtime.yml diff --git a/cf-deploy-akto b/cf-deploy-akto index ed77312..452a158 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/mini-runtime-cft.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-mini-runtime-cft infra + mv infra-feature-quick-setup infra fi # NB: this is to make onprem containers to all get named the same. diff --git a/templates/mini-runtime.yml b/templates/mini-runtime.yml new file mode 100644 index 0000000..22c1e12 --- /dev/null +++ b/templates/mini-runtime.yml @@ -0,0 +1,502 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: >- + This template does a simple setup for all Akto modules. It sets up all modules + on a single instance. If you want a scalable and flexible setup, please + contact support@akto.io. +Parameters: + SubnetId: + Type: 'AWS::EC2::Subnet::Id' + KeyPair: + Type: 'AWS::EC2::KeyPair::KeyName' + DatabaseAbstractorToken: + Type: String +Mappings: + RegionMap: + af-south-1: + AMI: ami-093ca241e4c72c205 + eu-north-1: + AMI: ami-0f58e72599cb99a79 + ap-south-1: + AMI: ami-0400aca7799d8cf19 + eu-west-3: + AMI: ami-064c70d04ad799d5e + eu-west-2: + AMI: ami-0dfe6158087b5c0ac + eu-south-1: + AMI: ami-07b2af763a8b958f3 + eu-west-1: + AMI: ami-047aad752a426ed48 + ap-northeast-3: + AMI: ami-0cffa2172948e071e + ap-northeast-2: + AMI: ami-087af0192368bc87c + me-south-1: + AMI: ami-0a31e56929248acca + ap-northeast-1: + AMI: ami-0828596b82405edd7 + sa-east-1: + AMI: ami-0df67b3c17f090c24 + ca-central-1: + AMI: ami-0eb3718c42cb70e52 + ap-east-1: + AMI: ami-0e992f1e63814db10 + ap-southeast-1: + AMI: ami-0ba98499caf94125a + ap-southeast-2: + AMI: ami-0849cc8fe4ceaf988 + eu-central-1: + AMI: ami-0f7585ae7a0d9a25a + ap-southeast-3: + AMI: ami-0cf40308729b83366 + us-east-1: + AMI: ami-0d52ddcdf3a885741 + us-east-2: + AMI: ami-04148302a14f7d12b + us-west-1: + AMI: ami-0ee3e1e65adeef858 + us-west-2: + AMI: ami-0ec021424fb596d6c +Resources: + GetAktoSetupDetailsLambdaBasicExecutionRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: GetAktoSetupDetailsExecuteLambda + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'ec2:DescribeNetworkInterfaces' + - 'ec2:DescribeTrafficMirrorSessions' + - 'ec2:DescribeInstances' + - 'ec2:DescribeVpcs' + - 'elasticloadbalancing:DescribeLoadBalancers' + - 'elasticloadbalancing:DescribeTargetGroups' + - 'elasticloadbalancing:DescribeTargetHealth' + Resource: '*' + GetAktoSetupDetails: + Type: 'AWS::Lambda::Function' + Properties: + Runtime: nodejs16.x + Timeout: 60 + Role: !GetAtt + - GetAktoSetupDetailsLambdaBasicExecutionRole + - Arn + Handler: index.handler + Environment: + Variables: + TARGET_LB: !Ref AktoNLB + Code: + S3Bucket: !Sub 'akto-setup-${AWS::Region}' + S3Key: templates/get-akto-setup-details.zip + GetVpcDetailsLambdaRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'ec2:DescribeVpcs' + - 'ec2:DescribeSubnets' + Resource: '*' + GetVpcDetailsLambda: + Type: 'AWS::Lambda::Function' + Properties: + Description: Look up info from a VPC + Handler: index.handler + Runtime: nodejs16.x + Timeout: 30 + Role: !GetAtt + - GetVpcDetailsLambdaRole + - Arn + Environment: + Variables: + SUBNET_ID: !Ref SubnetId + Code: + ZipFile: > + var SUBNET_ID = process.env.SUBNET_ID; var aws = require('aws-sdk'); + var response = require('cfn-response'); var ec2 = new aws.EC2(); + exports.handler = async function(event, context) { + if (event.RequestType == 'Delete') { + await response.send(event, context, 'SUCCESS'); + return; + } + var params = { + SubnetIds: [SUBNET_ID] + }; + var subnets = await ec2.describeSubnets(params).promise().catch(err => { + console.error(err); + }); + await response.send(event, context, 'SUCCESS', {VpcId: subnets['Subnets'][0]['VpcId']}) + }; + CustomSourceGetVpcDetails: + Type: 'AWS::CloudFormation::CustomResource' + Properties: + ServiceToken: !GetAtt + - GetVpcDetailsLambda + - Arn + IamInstanceProfile: + Type: 'AWS::IAM::InstanceProfile' + Properties: + Path: / + Roles: + - !Ref RefreshHandlerLambdaBasicExecutionRole + RefreshHandlerLambdaBasicExecutionRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: InvokeLambdaPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Resource: + - !GetAtt + - DashboardInstanceRefreshHandler + - Arn + - !GetAtt + - TrafficMirroringInstanceRefreshHandler + - Arn + Action: 'lambda:InvokeFunction' + AktoSecurityGroup: + Type: 'AWS::EC2::SecurityGroup' + Properties: + VpcId: !GetAtt + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: 'Enable the ports Akto requires (22, 4789, 8000, 9092)' + SecurityGroupIngress: [] + SecurityGroupEgress: [] + AktoASGLaunchConfiguration: + Type: 'AWS::AutoScaling::LaunchConfiguration' + DependsOn: + - AktoNLB + Properties: + ImageId: !FindInMap + - RegionMap + - !Ref 'AWS::Region' + - AMI + InstanceType: m5a.xlarge + KeyName: !Ref KeyPair + AssociatePublicIpAddress: 'false' + IamInstanceProfile: !Ref IamInstanceProfile + SecurityGroups: + - !Ref AktoSecurityGroup + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeType: gp2 + DeleteOnTermination: 'true' + VolumeSize: '50' + Encrypted: true + MetadataOptions: + HttpTokens: required + UserData: !Base64 + 'Fn::Join': + - |+ + + - - '#!/bin/bash -xe' + - !Sub 'export DATABASE_ABSTRACTOR_SERVICE_TOKEN=''${DatabaseAbstractorToken}''' + - !Sub 'export AKTO_KAFKA_IP=''${AktoNLB.DNSName}''' + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - sudo mkdir -p /opt/aws/bin + - export COMPOSE_FILE=docker-compose-mini-runtime.yml + - >- + sudo wget + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - >- + sudo python -m easy_install --script-dir /opt/aws/bin + aws-cfn-bootstrap-latest.tar.gz + - >- + curl -fsSL + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto' + > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - ./cf-deploy-akto < <(echo 'test') + - sudo echo >> ~/akto/infra/docker-runtime.env + - >- + sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> + ~/akto/infra/docker-runtime.env + - >- + sudo echo DATABASE_ABSTRACTOR_SERVICE_TOKEN=$DATABASE_ABSTRACTOR_SERVICE_TOKEN >> + ~/akto/infra/docker-mini-runtime.env + - sudo echo AKTO_KAFKA_IP=$AKTO_KAFKA_IP >> ~/akto/infra/.env + - >- + curl -fsSL + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto-start' + > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - ./cf-deploy-akto-start < <(echo 'test') + AktoAutoScalingGroup: + Type: 'AWS::AutoScaling::AutoScalingGroup' + Properties: + LaunchConfigurationName: !Ref AktoASGLaunchConfiguration + VPCZoneIdentifier: + - !Ref SubnetId + TargetGroupARNs: + - !Ref AktoTrafficMirroringTargetGroup + - !Ref AktoKafkaTargetGroup + MaxSize: '10' + MinSize: '1' + AktoTargetTrackingNetworkPolicy: + Type: 'AWS::AutoScaling::ScalingPolicy' + Properties: + PolicyType: TargetTrackingScaling + AutoScalingGroupName: !Ref AktoAutoScalingGroup + EstimatedInstanceWarmup: 30 + TargetTrackingConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ASGAverageNetworkIn + TargetValue: 200000000 + AktoNLB: + Type: 'AWS::ElasticLoadBalancingV2::LoadBalancer' + Properties: + Type: network + Scheme: internal + IpAddressType: ipv4 + Subnets: + - !Ref SubnetId + LoadBalancerAttributes: + - Key: load_balancing.cross_zone.enabled + Value: 'true' + AktoTrafficMirroringTargetGroup: + Type: 'AWS::ElasticLoadBalancingV2::TargetGroup' + Properties: + Port: '4789' + Protocol: UDP + HealthCheckEnabled: 'true' + HealthCheckIntervalSeconds: 10 + HealthCheckPath: /metrics + HealthCheckPort: '8000' + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 6 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 2 + TargetType: instance + VpcId: !GetAtt + - CustomSourceGetVpcDetails + - VpcId + Targets: [] + AktoKafkaTargetGroup: + Type: 'AWS::ElasticLoadBalancingV2::TargetGroup' + Properties: + Port: '9092' + Protocol: TCP + TargetType: instance + HealthCheckEnabled: 'true' + HealthCheckIntervalSeconds: 10 + HealthCheckPath: /metrics + HealthCheckPort: '8000' + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 6 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 2 + VpcId: !GetAtt + - CustomSourceGetVpcDetails + - VpcId + Targets: [] + AktoKafkaListener: + Type: 'AWS::ElasticLoadBalancingV2::Listener' + Properties: + LoadBalancerArn: !Ref AktoNLB + Port: '9092' + Protocol: TCP + DefaultActions: + - Type: forward + TargetGroupArn: !Ref AktoKafkaTargetGroup + DashboardInstanceRefreshHandler: + Type: 'AWS::Lambda::Function' + Properties: + Handler: index.handler + Runtime: nodejs16.x + Timeout: 30 + Role: !GetAtt + - InstanceRefreshHandlerLambdaRole + - Arn + Code: + ZipFile: > + var aws = require('aws-sdk'); var autoscaling = new aws.AutoScaling(); + exports.handler = function(event, context) { + var params = { + AutoScalingGroupName: 'AktoDashboardAutoScalingGroup', + Preferences: { + InstanceWarmup: 200, + MinHealthyPercentage: 0 + } + }; + + autoscaling.startInstanceRefresh(params, function(err, data) { + if(err) { console.log(err) } + else { console.log(data) } + }) + }; + TrafficMirroringInstanceRefreshHandler: + Type: 'AWS::Lambda::Function' + Properties: + Handler: index.handler + Runtime: nodejs16.x + Timeout: 30 + Role: !GetAtt + - InstanceRefreshHandlerLambdaRole + - Arn + Code: + ZipFile: > + var aws = require('aws-sdk'); var autoscaling = new aws.AutoScaling(); + exports.handler = function(event, context) { + var params = { + AutoScalingGroupName: 'AktoAutoScalingGroup', + Preferences: { + InstanceWarmup: 200, + MinHealthyPercentage: 0 + } + }; + + autoscaling.startInstanceRefresh(params, function(err, data) { + if(err) { console.log(err) } + else { console.log(data) } + }) + }; + InstanceRefreshHandlerLambdaRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - 'sts:AssumeRole' + Path: /service-role/ + Policies: + - PolicyName: lambdaExecution-DashboardInstanceRefreshHandler + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'logs:CreateLogGroup' + Resource: '*' + - Effect: Allow + Action: + - 'logs:CreateLogStream' + - 'logs:PutLogEvents' + Resource: '*' + - Effect: Allow + Action: + - 'autoscaling:StartInstanceRefresh' + - 'autoscaling:Describe*' + - 'autoscaling:UpdateAutoScalingGroup' + - 'ec2:CreateLaunchTemplateVersion' + - 'ec2:DescribeLaunchTemplates' + - 'ec2:RunInstances' + Resource: '*' + ConfigureSecurityGroupsLambda: + Type: 'AWS::Lambda::Function' + Properties: + Description: >- + Configure Security Groups for Runtime processor and Context analyzer + instances + Handler: lambda.lambda_handler + Runtime: python3.9 + Timeout: 30 + Role: !GetAtt + - ConfigureSecurityGroupsLambdaRole + - Arn + Environment: + Variables: + SUBNET_ID: !Ref SubnetId + CONTEXT_ANALYZER_SECURITY_GROUP_ID: !Ref AktoSecurityGroup + RUNTIME_PROCESSOR_SECURITY_GROUP_ID: !Ref AktoSecurityGroup + MODE: RUNTIME + Code: + S3Bucket: !Sub 'akto-setup-${AWS::Region}' + S3Key: templates/configure_security_groups.zip + CustomSourceConfigureSecurityGroupsLambda: + Type: 'AWS::CloudFormation::CustomResource' + Properties: + ServiceToken: !GetAtt + - ConfigureSecurityGroupsLambda + - Arn + ConfigureSecurityGroupsLambdaRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: DescribeAssetsPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'ec2:DescribeVpcs' + - 'ec2:DescribeSubnets' + Resource: '*' + - Effect: Allow + Action: + - 'ec2:AuthorizeSecurityGroupIngress' + Resource: + - !Sub >- + arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoSecurityGroup} + - PolicyName: AWSLambdaBasicExecutionRole + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'logs:CreateLogGroup' + - 'logs:CreateLogStream' + - 'logs:PutLogEvents' + Resource: '*' +Outputs: + AktoNLB: + Description: The IP address of the AktoNLB + Value: !GetAtt + - AktoNLB + - DNSName + Export: + Name: AktoNLBIP \ No newline at end of file From cd6a2394ba1e57b8096f7fd79327c10bd554029e Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Mon, 3 Jun 2024 21:45:01 +0530 Subject: [PATCH 79/98] revert runtime yml changes --- templates/data_processing_stack.yml | 229 ++++++++++++++-------------- 1 file changed, 115 insertions(+), 114 deletions(-) diff --git a/templates/data_processing_stack.yml b/templates/data_processing_stack.yml index 8aa10c9..0c05996 100644 --- a/templates/data_processing_stack.yml +++ b/templates/data_processing_stack.yml @@ -8,8 +8,8 @@ Parameters: Type: 'AWS::EC2::Subnet::Id' KeyPair: Type: 'AWS::EC2::KeyPair::KeyName' - # MongoIp: - # Type: String + MongoIp: + Type: String Mappings: RegionMap: af-south-1: @@ -171,111 +171,111 @@ Resources: # GroupDescription: 'Enable the ports Akto requires (22, 9092)' # SecurityGroupIngress: [] # SecurityGroupEgress: [] - # AktoContextAnalyzerASGLaunchConfiguration: - # Type: 'AWS::AutoScaling::LaunchConfiguration' - # Properties: - # ImageId: !FindInMap - # - RegionMap - # - !Ref 'AWS::Region' - # - AMI - # InstanceType: m5a.xlarge - # KeyName: !Ref KeyPair - # AssociatePublicIpAddress: 'false' - # SecurityGroups: - # - !Ref AktoContextAnalyzerSecurityGroup - # BlockDeviceMappings: - # - DeviceName: /dev/xvda - # Ebs: - # VolumeType: gp2 - # DeleteOnTermination: 'true' - # VolumeSize: '50' - # Encrypted: true - # MetadataOptions: - # HttpTokens: required - # UserData: !Base64 - # 'Fn::Join': - # - |+ + AktoContextAnalyzerASGLaunchConfiguration: + Type: 'AWS::AutoScaling::LaunchConfiguration' + Properties: + ImageId: !FindInMap + - RegionMap + - !Ref 'AWS::Region' + - AMI + InstanceType: m5a.xlarge + KeyName: !Ref KeyPair + AssociatePublicIpAddress: 'false' + SecurityGroups: + - !Ref AktoContextAnalyzerSecurityGroup + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeType: gp2 + DeleteOnTermination: 'true' + VolumeSize: '50' + Encrypted: true + MetadataOptions: + HttpTokens: required + UserData: !Base64 + 'Fn::Join': + - |+ - # - - '#!/bin/bash -xe' - # - !Sub 'export AKTO_MONGO_CONN=''${MongoIp}''' - # - touch /tmp/hello.txt - # - touch ~/hello.txt - # - sudo yum update -y - # - sudo yum install -y python python-setuptools - # - sudo yum install -y docker - # - sudo dockerd& - # - sudo systemctl enable /usr/lib/systemd/system/docker.service - # - sudo mkdir -p /opt/aws/bin - # - export COMPOSE_FILE=docker-compose-context-analyser.yml - # - >- - # sudo wget - # https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - # - >- - # sudo python -m easy_install --script-dir /opt/aws/bin - # aws-cfn-bootstrap-latest.tar.gz - # - >- - # curl -fsSL - # 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' - # > cf-deploy-akto - # - sudo chmod 700 cf-deploy-akto - # - ./cf-deploy-akto < <(echo 'test') - # - sudo echo >> ~/akto/infra/docker-context-analyser.env - # - >- - # sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> - # ~/akto/infra/docker-context-analyser.env - # - >- - # export TOKEN=$(curl -X PUT - # 'http://169.254.169.254/latest/api/token' -H - # 'X-aws-ec2-metadata-token-ttl-seconds: 600') - # - !Join - # - ':' - # - - >- - # export AKTO_CURRENT_INSTANCE_IP=$(curl -H - # "X-aws-ec2-metadata-token - # - '$TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' - # - >- - # echo AKTO_CURRENT_INSTANCE_IP=$AKTO_CURRENT_INSTANCE_IP >> - # ~/akto/infra/docker-context-analyser.env - # - >- - # curl -fsSL - # 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' - # > cf-deploy-akto-start - # - sudo chmod 700 cf-deploy-akto-start - # - ./cf-deploy-akto-start < <(echo 'test') - # AktoContextAnalyzerAutoScalingGroup: - # Type: 'AWS::AutoScaling::AutoScalingGroup' - # Properties: - # LaunchConfigurationName: !Ref AktoContextAnalyzerASGLaunchConfiguration - # VPCZoneIdentifier: - # - !Ref SubnetId - # MaxSize: '1' - # MinSize: '1' - # AktoContextAnalyzerInstanceRefreshHandler: - # Type: 'AWS::Lambda::Function' - # Properties: - # Handler: index.handler - # Runtime: nodejs16.x - # Timeout: 30 - # Role: !GetAtt - # - InstanceRefreshHandlerLambdaRole - # - Arn - # Code: - # ZipFile: > - # var aws = require('aws-sdk'); var autoscaling = new aws.AutoScaling(); - # exports.handler = function(event, context) { - # var params = { - # AutoScalingGroupName: 'AktoContextAnalyzerAutoScalingGroup', - # Preferences: { - # InstanceWarmup: 200, - # MinHealthyPercentage: 0 - # } - # }; + - - '#!/bin/bash -xe' + - !Sub 'export AKTO_MONGO_CONN=''${MongoIp}''' + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - sudo mkdir -p /opt/aws/bin + - export COMPOSE_FILE=docker-compose-context-analyser.yml + - >- + sudo wget + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - >- + sudo python -m easy_install --script-dir /opt/aws/bin + aws-cfn-bootstrap-latest.tar.gz + - >- + curl -fsSL + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' + > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - ./cf-deploy-akto < <(echo 'test') + - sudo echo >> ~/akto/infra/docker-context-analyser.env + - >- + sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> + ~/akto/infra/docker-context-analyser.env + - >- + export TOKEN=$(curl -X PUT + 'http://169.254.169.254/latest/api/token' -H + 'X-aws-ec2-metadata-token-ttl-seconds: 600') + - !Join + - ':' + - - >- + export AKTO_CURRENT_INSTANCE_IP=$(curl -H + "X-aws-ec2-metadata-token + - '$TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)' + - >- + echo AKTO_CURRENT_INSTANCE_IP=$AKTO_CURRENT_INSTANCE_IP >> + ~/akto/infra/docker-context-analyser.env + - >- + curl -fsSL + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' + > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - ./cf-deploy-akto-start < <(echo 'test') + AktoContextAnalyzerAutoScalingGroup: + Type: 'AWS::AutoScaling::AutoScalingGroup' + Properties: + LaunchConfigurationName: !Ref AktoContextAnalyzerASGLaunchConfiguration + VPCZoneIdentifier: + - !Ref SubnetId + MaxSize: '1' + MinSize: '1' + AktoContextAnalyzerInstanceRefreshHandler: + Type: 'AWS::Lambda::Function' + Properties: + Handler: index.handler + Runtime: nodejs16.x + Timeout: 30 + Role: !GetAtt + - InstanceRefreshHandlerLambdaRole + - Arn + Code: + ZipFile: > + var aws = require('aws-sdk'); var autoscaling = new aws.AutoScaling(); + exports.handler = function(event, context) { + var params = { + AutoScalingGroupName: 'AktoContextAnalyzerAutoScalingGroup', + Preferences: { + InstanceWarmup: 200, + MinHealthyPercentage: 0 + } + }; - # autoscaling.startInstanceRefresh(params, function(err, data) { - # if(err) { console.log(err) } - # else { console.log(data) } - # }) - # }; + autoscaling.startInstanceRefresh(params, function(err, data) { + if(err) { console.log(err) } + else { console.log(data) } + }) + }; RefreshHandlerLambdaBasicExecutionRole: Type: 'AWS::IAM::Role' Properties: @@ -299,9 +299,9 @@ Resources: - !GetAtt - TrafficMirroringInstanceRefreshHandler - Arn - # - !GetAtt - # - AktoContextAnalyzerInstanceRefreshHandler - # - Arn + - !GetAtt + - AktoContextAnalyzerInstanceRefreshHandler + - Arn Action: 'lambda:InvokeFunction' AktoSecurityGroup: Type: 'AWS::EC2::SecurityGroup' @@ -341,6 +341,7 @@ Resources: - |+ - - '#!/bin/bash -xe' + - !Sub 'export AKTO_MONGO_CONN=''${MongoIp}''' - !Sub 'export AKTO_KAFKA_IP=''${AktoNLB.DNSName}''' - touch /tmp/hello.txt - touch ~/hello.txt @@ -350,7 +351,7 @@ Resources: - sudo dockerd& - sudo systemctl enable /usr/lib/systemd/system/docker.service - sudo mkdir -p /opt/aws/bin - - export COMPOSE_FILE=docker-compose-mini-runtime.yml + - export COMPOSE_FILE=docker-compose-runtime.yml - >- sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz @@ -359,7 +360,7 @@ Resources: aws-cfn-bootstrap-latest.tar.gz - >- curl -fsSL - 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto' + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - ./cf-deploy-akto < <(echo 'test') @@ -370,7 +371,7 @@ Resources: - sudo echo AKTO_KAFKA_IP=$AKTO_KAFKA_IP >> ~/akto/infra/.env - >- curl -fsSL - 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto-start' + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/segregation_2/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - ./cf-deploy-akto-start < <(echo 'test') @@ -554,7 +555,7 @@ Resources: Environment: Variables: SUBNET_ID: !Ref SubnetId - # CONTEXT_ANALYZER_SECURITY_GROUP_ID: !Ref AktoContextAnalyzerSecurityGroup + CONTEXT_ANALYZER_SECURITY_GROUP_ID: !Ref AktoContextAnalyzerSecurityGroup RUNTIME_PROCESSOR_SECURITY_GROUP_ID: !Ref AktoSecurityGroup MODE: RUNTIME Code: @@ -595,8 +596,8 @@ Resources: Resource: - !Sub >- arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoSecurityGroup} - # - !Sub >- - # arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoContextAnalyzerSecurityGroup} + - !Sub >- + arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:security-group/${AktoContextAnalyzerSecurityGroup} - PolicyName: AWSLambdaBasicExecutionRole PolicyDocument: Version: 2012-10-17 From 677d4eaaeb2ed201a6df81d14b62da711363dee0 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Mon, 3 Jun 2024 21:45:48 +0530 Subject: [PATCH 80/98] uncomment changes --- templates/data_processing_stack.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/templates/data_processing_stack.yml b/templates/data_processing_stack.yml index 0c05996..4009e28 100644 --- a/templates/data_processing_stack.yml +++ b/templates/data_processing_stack.yml @@ -162,15 +162,15 @@ Resources: Path: / Roles: - !Ref RefreshHandlerLambdaBasicExecutionRole - # AktoContextAnalyzerSecurityGroup: - # Type: 'AWS::EC2::SecurityGroup' - # Properties: - # VpcId: !GetAtt - # - CustomSourceGetVpcDetails - # - VpcId - # GroupDescription: 'Enable the ports Akto requires (22, 9092)' - # SecurityGroupIngress: [] - # SecurityGroupEgress: [] + AktoContextAnalyzerSecurityGroup: + Type: 'AWS::EC2::SecurityGroup' + Properties: + VpcId: !GetAtt + - CustomSourceGetVpcDetails + - VpcId + GroupDescription: 'Enable the ports Akto requires (22, 9092)' + SecurityGroupIngress: [] + SecurityGroupEgress: [] AktoContextAnalyzerASGLaunchConfiguration: Type: 'AWS::AutoScaling::LaunchConfiguration' Properties: From 6114fefb9ccf07b68a8b8b2a4f1d17df300a87a0 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Mon, 3 Jun 2024 22:04:10 +0530 Subject: [PATCH 81/98] add url var --- docker-mini-runtime.env | 3 ++- templates/mini-runtime.yml | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docker-mini-runtime.env b/docker-mini-runtime.env index 5070d9c..93052c5 100644 --- a/docker-mini-runtime.env +++ b/docker-mini-runtime.env @@ -9,5 +9,6 @@ AKTO_TRAFFIC_BATCH_SIZE=100 AKTO_TRAFFIC_BATCH_TIME_SECS=10 USE_HOSTNAME=true AKTO_INSTANCE_TYPE=RUNTIME -DATABASE_ABSTRACTOR_SERVICE_URL=http://13.229.55.224:9000 +DATABASE_ABSTRACTOR_SERVICE_URL=url +DATABASE_ABSTRACTOR_SERVICE_TOKEN=token RUNTIME_MODE=hybrid diff --git a/templates/mini-runtime.yml b/templates/mini-runtime.yml index 22c1e12..b07d620 100644 --- a/templates/mini-runtime.yml +++ b/templates/mini-runtime.yml @@ -8,6 +8,8 @@ Parameters: Type: 'AWS::EC2::Subnet::Id' KeyPair: Type: 'AWS::EC2::KeyPair::KeyName' + DatabaseAbstractorUrl: + Type: String DatabaseAbstractorToken: Type: String Mappings: @@ -224,6 +226,7 @@ Resources: - |+ - - '#!/bin/bash -xe' + - !Sub 'export DATABASE_ABSTRACTOR_SERVICE_URL=''${DatabaseAbstractorUrl}''' - !Sub 'export DATABASE_ABSTRACTOR_SERVICE_TOKEN=''${DatabaseAbstractorToken}''' - !Sub 'export AKTO_KAFKA_IP=''${AktoNLB.DNSName}''' - touch /tmp/hello.txt @@ -251,6 +254,9 @@ Resources: - >- sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> ~/akto/infra/docker-runtime.env + - >- + sudo echo DATABASE_ABSTRACTOR_SERVICE_URL=$DATABASE_ABSTRACTOR_SERVICE_URL >> + ~/akto/infra/docker-mini-runtime.env - >- sudo echo DATABASE_ABSTRACTOR_SERVICE_TOKEN=$DATABASE_ABSTRACTOR_SERVICE_TOKEN >> ~/akto/infra/docker-mini-runtime.env From 322f9d00305bff0f7eb5d15f4b55625b94c8126b Mon Sep 17 00:00:00 2001 From: Avneesh Hota <91518451+avneesh-akto@users.noreply.github.com> Date: Sat, 22 Jun 2024 13:18:27 +0530 Subject: [PATCH 82/98] Update docker-compose-context-analyser.yml --- docker-compose-context-analyser.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-context-analyser.yml b/docker-compose-context-analyser.yml index 85208c0..279e6f4 100644 --- a/docker-compose-context-analyser.yml +++ b/docker-compose-context-analyser.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.1' services: zoo1: @@ -75,4 +75,4 @@ services: - /var/run/docker.sock:/var/run/docker.sock env_file: ./watchtower.env labels: - com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file + com.centurylinklabs.watchtower.enable: "false" From b93a26df4aac5e9747aa80459c5acfdaab677544 Mon Sep 17 00:00:00 2001 From: Avneesh Hota <91518451+avneesh-akto@users.noreply.github.com> Date: Sat, 22 Jun 2024 13:18:44 +0530 Subject: [PATCH 83/98] Update docker-compose-dashboard.yml --- docker-compose-dashboard.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-dashboard.yml b/docker-compose-dashboard.yml index bced097..78335f6 100644 --- a/docker-compose-dashboard.yml +++ b/docker-compose-dashboard.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.1' services: akto-api-security-dashboard: @@ -88,4 +88,4 @@ services: - /var/run/docker.sock:/var/run/docker.sock env_file: ./watchtower.env labels: - com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file + com.centurylinklabs.watchtower.enable: "false" From 957cf5ae95c35a0481cd745df40da3492c152103 Mon Sep 17 00:00:00 2001 From: Avneesh Hota <91518451+avneesh-akto@users.noreply.github.com> Date: Sat, 22 Jun 2024 13:18:56 +0530 Subject: [PATCH 84/98] Update docker-compose-mini-runtime.yml --- docker-compose-mini-runtime.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-mini-runtime.yml b/docker-compose-mini-runtime.yml index 5a63041..1d909c1 100644 --- a/docker-compose-mini-runtime.yml +++ b/docker-compose-mini-runtime.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.1' services: zoo1: @@ -84,4 +84,4 @@ services: - /var/run/docker.sock:/var/run/docker.sock env_file: ./watchtower.env labels: - com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file + com.centurylinklabs.watchtower.enable: "false" From c6aa1c5f3f5d648e65e8527aed85515f0fc758b6 Mon Sep 17 00:00:00 2001 From: Avneesh Hota <91518451+avneesh-akto@users.noreply.github.com> Date: Sat, 22 Jun 2024 13:19:08 +0530 Subject: [PATCH 85/98] Update docker-compose-runtime.yml --- docker-compose-runtime.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-runtime.yml b/docker-compose-runtime.yml index 194351d..fd5d5e1 100644 --- a/docker-compose-runtime.yml +++ b/docker-compose-runtime.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.1' services: zoo1: @@ -84,4 +84,4 @@ services: - /var/run/docker.sock:/var/run/docker.sock env_file: ./watchtower.env labels: - com.centurylinklabs.watchtower.enable: "false" \ No newline at end of file + com.centurylinklabs.watchtower.enable: "false" From de0f5b1374f6df983bfcb6fad43b079c590dcea2 Mon Sep 17 00:00:00 2001 From: Avneesh Hota <91518451+avneesh-akto@users.noreply.github.com> Date: Sat, 22 Jun 2024 13:19:38 +0530 Subject: [PATCH 86/98] Update docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5f80b10..77c21af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.1' services: zoo1: From d163df4ebc0e13364dec37c005616ad77a27522d Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Wed, 11 Sep 2024 15:29:39 +0530 Subject: [PATCH 87/98] add terraform script for mini runtime setup --- templates/mini-runtime.tf | 333 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 templates/mini-runtime.tf diff --git a/templates/mini-runtime.tf b/templates/mini-runtime.tf new file mode 100644 index 0000000..625686a --- /dev/null +++ b/templates/mini-runtime.tf @@ -0,0 +1,333 @@ +# Provider configuration +provider "aws" { + region = var.region +} + +# Input Variables +variable "region" { + description = "AWS region where resources will be deployed" + type = string +} + +variable "key_pair" { + description = "Key pair for SSH access to the EC2 instances" + type = string +} + +variable "subnet_id" { + description = "Subnet ID where the EC2 instances will be launched" + type = string +} + +variable "database_abstractor_url" { + description = "URL for the Database Abstractor service" + type = string +} + +variable "database_abstractor_token" { + description = "Token for the Database Abstractor service" + type = string +} + +locals { + mappings = { + RegionMap = { + af-south-1 = { + AMI = "ami-093ca241e4c72c205" + } + eu-north-1 = { + AMI = "ami-0f58e72599cb99a79" + } + ap-south-1 = { + AMI = "ami-0400aca7799d8cf19" + } + eu-west-3 = { + AMI = "ami-064c70d04ad799d5e" + } + eu-west-2 = { + AMI = "ami-0dfe6158087b5c0ac" + } + eu-south-1 = { + AMI = "ami-07b2af763a8b958f3" + } + eu-west-1 = { + AMI = "ami-047aad752a426ed48" + } + ap-northeast-3 = { + AMI = "ami-0cffa2172948e071e" + } + ap-northeast-2 = { + AMI = "ami-087af0192368bc87c" + } + me-south-1 = { + AMI = "ami-0a31e56929248acca" + } + ap-northeast-1 = { + AMI = "ami-0828596b82405edd7" + } + sa-east-1 = { + AMI = "ami-0df67b3c17f090c24" + } + ca-central-1 = { + AMI = "ami-0eb3718c42cb70e52" + } + ap-east-1 = { + AMI = "ami-0e992f1e63814db10" + } + ap-southeast-1 = { + AMI = "ami-0ba98499caf94125a" + } + ap-southeast-2 = { + AMI = "ami-0849cc8fe4ceaf988" + } + eu-central-1 = { + AMI = "ami-0f7585ae7a0d9a25a" + } + ap-southeast-3 = { + AMI = "ami-0cf40308729b83366" + } + us-east-1 = { + AMI = "ami-0d52ddcdf3a885741" + } + us-east-2 = { + AMI = "ami-04148302a14f7d12b" + } + us-west-1 = { + AMI = "ami-0ee3e1e65adeef858" + } + us-west-2 = { + AMI = "ami-0ec021424fb596d6c" + } + } + } + # Lookup the AMI ID based on the region + ami_id = local.mappings.RegionMap[var.region].AMI +} + +# Data source for Subnet +data "aws_subnet" "selected" { + id = var.subnet_id +} + +# Data source for VPC +data "aws_vpc" "selected" { + id = data.aws_subnet.selected.vpc_id +} + + +# IAM Role for Lambda functions +resource "aws_iam_role" "lambda_execution_role" { + name = "GetAktoSetupDetailsLambdaBasicExecutionRole" + assume_role_policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [{ + "Effect" : "Allow", + "Principal" : { + "Service" : "lambda.amazonaws.com" + }, + "Action" : "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_role_policy" "get_akto_lambda_policy" { + name = "GetAktoSetupDetailsExecuteLambda" + role = aws_iam_role.lambda_execution_role.id + policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [{ + "Effect" : "Allow", + "Action" : [ + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTrafficMirrorSessions", + "ec2:DescribeInstances", + "ec2:DescribeVpcs", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth" + ], + "Resource" : "*" + }] + }) +} + +# Lambda Function +resource "aws_lambda_function" "get_akto_setup_details" { + function_name = "GetAktoSetupDetails" + runtime = "nodejs16.x" + role = aws_iam_role.lambda_execution_role.arn + handler = "index.handler" + timeout = 60 + environment { + variables = { + TARGET_LB = aws_lb.akto_nlb.dns_name + } + } + s3_bucket = "akto-setup-${var.region}" + s3_key = "templates/get-akto-setup-details.zip" +} + +# IAM Role for EC2 Instances +resource "aws_iam_instance_profile" "iam_instance_profile" { + name = "AktoInstanceProfile" + role = aws_iam_role.lambda_execution_role.name +} + +# Security Group +resource "aws_security_group" "akto_security_group" { + name_prefix = "akto-security-group" + vpc_id = data.aws_vpc.selected.id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + ingress { + from_port = 4789 + to_port = 4789 + protocol = "udp" + cidr_blocks = ["0.0.0.0/0"] + } + ingress { + from_port = 8000 + to_port = 8000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + ingress { + from_port = 9092 + to_port = 9092 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +# Launch Configuration +resource "aws_launch_configuration" "akto_launch_configuration" { + name_prefix = "AktoASGLaunchConfiguration" + image_id = local.ami_id + instance_type = "m5a.xlarge" + key_name = var.key_pair + iam_instance_profile = aws_iam_instance_profile.iam_instance_profile.id + security_groups = [aws_security_group.akto_security_group.id] + associate_public_ip_address = false + + user_data = <<-EOF + #!/bin/bash -xe + export DATABASE_ABSTRACTOR_SERVICE_URL='${var.database_abstractor_url}' + export DATABASE_ABSTRACTOR_SERVICE_TOKEN='${var.database_abstractor_token}' + export DATABASE_ABSTRACTOR_SERVICE_URL='${var.database_abstractor_url}' + export DATABASE_ABSTRACTOR_SERVICE_TOKEN='${var.database_abstractor_token}' + export AKTO_KAFKA_IP='${aws_lb.akto_nlb.dns_name}' + touch /tmp/hello.txt + touch ~/hello.txt + sudo yum update -y + sudo yum install -y python python-setuptools + sudo yum install -y docker + sudo dockerd& + sudo systemctl enable /usr/lib/systemd/system/docker.service + sudo mkdir -p /opt/aws/bin + export COMPOSE_FILE=docker-compose-mini-runtime.yml + sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz + curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto' > cf-deploy-akto + sudo chmod 700 cf-deploy-akto + ./cf-deploy-akto < <(echo 'test') + sudo echo >> ~/akto/infra/docker-runtime.env + sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> ~/akto/infra/docker-runtime.env + sudo echo DATABASE_ABSTRACTOR_SERVICE_URL=$DATABASE_ABSTRACTOR_SERVICE_URL >> ~/akto/infra/docker-mini-runtime.env + sudo echo DATABASE_ABSTRACTOR_SERVICE_TOKEN=$DATABASE_ABSTRACTOR_SERVICE_TOKEN >> ~/akto/infra/docker-mini-runtime.env + sudo echo AKTO_KAFKA_IP=$AKTO_KAFKA_IP >> ~/akto/infra/.env + curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto-start' > cf-deploy-akto-start + sudo chmod 700 cf-deploy-akto-start + ./cf-deploy-akto-start < <(echo 'test') + EOF +} + +# Auto Scaling Group +resource "aws_autoscaling_group" "akto_autoscaling_group" { + desired_capacity = 1 + max_size = 10 + min_size = 1 + vpc_zone_identifier = [var.subnet_id] + target_group_arns = [ + aws_lb_target_group.akto_traffic_mirroring_target_group.arn, + aws_lb_target_group.akto_kafka_target_group.arn, + ] + launch_configuration = aws_launch_configuration.akto_launch_configuration.name +} + +# Network Load Balancer +resource "aws_lb" "akto_nlb" { + name = "AktoNLB" + internal = true + load_balancer_type = "network" + ip_address_type = "ipv4" + subnets = [var.subnet_id] + + enable_cross_zone_load_balancing = true +} + +# Target Groups +resource "aws_lb_target_group" "akto_traffic_mirroring_target_group" { + name = "AktoTrafficMirroringTG" + port = 4789 + protocol = "UDP" + vpc_id = data.aws_vpc.selected.id + + health_check { + enabled = true + interval = 10 + path = "/metrics" + port = "8000" + protocol = "HTTP" + timeout = 6 + healthy_threshold = 2 + unhealthy_threshold = 2 + } +} + +resource "aws_lb_target_group" "akto_kafka_target_group" { + name = "AktoKafkaTG" + port = 9092 + protocol = "TCP" + vpc_id = data.aws_vpc.selected.id + + health_check { + enabled = true + interval = 10 + path = "/metrics" + port = "8000" + protocol = "HTTP" + timeout = 6 + healthy_threshold = 2 + unhealthy_threshold = 2 + } +} + +# Load Balancer Listeners +resource "aws_lb_listener" "akto_kafka_listener" { + load_balancer_arn = aws_lb.akto_nlb.arn + port = 9092 + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.akto_kafka_target_group.arn + } +} + +# Outputs +output "akto_nlb_dns" { + description = "The DNS name of the Akto NLB" + value = aws_lb.akto_nlb.dns_name +} From 449c1cf34f0826f6154e103c7e49b9d25bd70c85 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Wed, 11 Sep 2024 16:53:03 +0530 Subject: [PATCH 88/98] add storage block --- templates/mini-runtime.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/templates/mini-runtime.tf b/templates/mini-runtime.tf index 625686a..101ce46 100644 --- a/templates/mini-runtime.tf +++ b/templates/mini-runtime.tf @@ -221,6 +221,13 @@ resource "aws_launch_configuration" "akto_launch_configuration" { security_groups = [aws_security_group.akto_security_group.id] associate_public_ip_address = false + # Specify root block device for the instance + root_block_device { + volume_size = 20 # Specify 20 GB volume size + volume_type = "gp2" + delete_on_termination = true + } + user_data = <<-EOF #!/bin/bash -xe export DATABASE_ABSTRACTOR_SERVICE_URL='${var.database_abstractor_url}' From f87b35b4d7c2488784a4ed1484a0c414ba60c8b5 Mon Sep 17 00:00:00 2001 From: notshivansh Date: Fri, 18 Oct 2024 15:13:03 +0530 Subject: [PATCH 89/98] add mini-testing --- docker-compose-mini-testing.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docker-compose-mini-testing.yml diff --git a/docker-compose-mini-testing.yml b/docker-compose-mini-testing.yml new file mode 100644 index 0000000..28167cc --- /dev/null +++ b/docker-compose-mini-testing.yml @@ -0,0 +1,28 @@ +version: '3.8' +services: + akto-api-security-testing: + image: public.ecr.aws/aktosecurity/akto-api-security-mini-testing:latest + environment: + RUNTIME_MODE: hybrid + DATABASE_ABSTRACTOR_SERVICE_TOKEN: + PUPPETEER_REPLAY_SERVICE_URL: "http://akto-puppeteer-replay:3000" + restart: always + + akto-api-security-puppeteer-replay: + image: public.ecr.aws/aktosecurity/akto-puppeteer-replay:latest + ports: + - "3000:3000" + environment: + NODE_ENV: production + restart: always + + watchtower: + image: containrrr/watchtower + restart: always + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + WATCHTOWER_CLEANUP: true + WATCHTOWER_POLL_INTERVAL: 1800 + labels: + com.centurylinklabs.watchtower.enable: "false" From 70e20d8754fb35b9ffbbef8be8229476fcba062d Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Mon, 4 Nov 2024 00:18:57 +0530 Subject: [PATCH 90/98] add data ingestion docker compose --- data-ingestion-docker.env | 6 ++ docker-compose-data-ingestion-runtime.yml | 81 +++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 data-ingestion-docker.env create mode 100644 docker-compose-data-ingestion-runtime.yml diff --git a/data-ingestion-docker.env b/data-ingestion-docker.env new file mode 100644 index 0000000..c4033a0 --- /dev/null +++ b/data-ingestion-docker.env @@ -0,0 +1,6 @@ +AKTO_TRAFFIC_BATCH_SIZE=100 +AKTO_TRAFFIC_BATCH_TIME_SECS=10 +AKTO_KAFKA_BROKER_URL=kafka1:19092 +AKTO_KAFKA_PRODUCER_BATCH_SIZE=10 +AKTO_KAFKA_PRODUCER_LINGER_MS=10 +AKTO_KAFKA_TOPIC_NAME="akto.api.logs" \ No newline at end of file diff --git a/docker-compose-data-ingestion-runtime.yml b/docker-compose-data-ingestion-runtime.yml new file mode 100644 index 0000000..d8575ec --- /dev/null +++ b/docker-compose-data-ingestion-runtime.yml @@ -0,0 +1,81 @@ +version: '2.1' + +services: + zoo1: + image: confluentinc/cp-zookeeper:6.2.1 + restart: always + hostname: zoo1 + user: "0" + volumes: + - ./data-zoo-data:/var/lib/zookeeper/data + - ./data-zoo-logs:/var/lib/zookeeper/log + - ./data-zoo-secrets:/etc/zookeeper/secrets + container_name: zoo1 + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_SERVER_ID: 1 + ZOOKEEPER_SERVERS: zoo1:2888:3888 + labels: + com.centurylinklabs.watchtower.enable: "false" + + kafka1: + image: confluentinc/cp-kafka:6.2.1 + restart: always + hostname: kafka1 + user: "0" + ports: + - "9092:9092" + - "19092:19092" + - "29092:29092" + - "9999:9999" + environment: + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://${AKTO_KAFKA_IP}:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" + KAFKA_BROKER_ID: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_CREATE_TOPICS: "akto.api.logs:3:3" + KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 60000 + KAFKA_LOG_RETENTION_HOURS: 5 + KAFKA_LOG_SEGMENT_BYTES: 104857600 + KAFKA_LOG_CLEANER_ENABLE: "true" + KAFKA_CLEANUP_POLICY: "delete" + KAFKA_LOG_RETENTION_BYTES: 10737418240 + volumes: + - ./data-kafka-data:/var/lib/kafka/data + - ./data-kafka-secrets:/etc/kafka/secrets + depends_on: + - zoo1 + labels: + com.centurylinklabs.watchtower.enable: "false" + + akto-api-security-runtime: + image: public.ecr.aws/aktosecurity/akto-api-security-mini-runtime:latest + env_file: ./docker-mini-runtime.env + mem_limit: 8g + restart: always + depends_on: + - kafka1 + + data-ingestion-service: + image: aktosecurity/data-ingestion-service:latest + env_file: ./data-ingestion-docker.env + restart: always + ports: + - "9091:8080" + depends_on: + - kafka1 + + watchtower: + image: containrrr/watchtower + restart: always + volumes: + - /var/run/docker.sock:/var/run/docker.sock + env_file: ./watchtower.env + labels: + com.centurylinklabs.watchtower.enable: "false" From 18d6758426b2412e64beb26e65d8749f3e85cc9e Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Mon, 4 Nov 2024 00:53:03 +0530 Subject: [PATCH 91/98] modify env file --- docker-mini-runtime.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-mini-runtime.env b/docker-mini-runtime.env index 93052c5..ed59e59 100644 --- a/docker-mini-runtime.env +++ b/docker-mini-runtime.env @@ -9,6 +9,6 @@ AKTO_TRAFFIC_BATCH_SIZE=100 AKTO_TRAFFIC_BATCH_TIME_SECS=10 USE_HOSTNAME=true AKTO_INSTANCE_TYPE=RUNTIME -DATABASE_ABSTRACTOR_SERVICE_URL=url -DATABASE_ABSTRACTOR_SERVICE_TOKEN=token +DATABASE_ABSTRACTOR_SERVICE_URL=https://cyborg.akto.io +DATABASE_ABSTRACTOR_SERVICE_TOKEN= RUNTIME_MODE=hybrid From 7eb5a83234a39371f6e8fd013741e22c3dd0efd5 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Thu, 7 Nov 2024 13:51:49 +0530 Subject: [PATCH 92/98] add rolling logs --- docker-compose-mini-runtime.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker-compose-mini-runtime.yml b/docker-compose-mini-runtime.yml index 1d909c1..afe199b 100644 --- a/docker-compose-mini-runtime.yml +++ b/docker-compose-mini-runtime.yml @@ -61,6 +61,11 @@ services: restart: always depends_on: - kafka1 + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "2" akto-api-security-mirror-api-logging: image: public.ecr.aws/aktosecurity/mirror-api-logging:local @@ -69,6 +74,11 @@ services: restart: always depends_on: - kafka1 + logging: + driver: "json-file" + options: + max-size: "100m" + max-file: "2" network_mode: host akto-hello-world: From 4c138fd2a651271f447caf4a99679d69463c88cf Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Tue, 19 Nov 2024 23:40:08 +0530 Subject: [PATCH 93/98] mini runtime arm docker compose setup file --- templates/docker-compose-mini-runtime-arm.yml | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 templates/docker-compose-mini-runtime-arm.yml diff --git a/templates/docker-compose-mini-runtime-arm.yml b/templates/docker-compose-mini-runtime-arm.yml new file mode 100644 index 0000000..d1620e4 --- /dev/null +++ b/templates/docker-compose-mini-runtime-arm.yml @@ -0,0 +1,87 @@ +version: '2.1' + +services: + zoo1: + image: confluentinc/cp-zookeeper:7.0.6 + restart: always + hostname: zoo1 + user: "0" + volumes: + - ./data-zoo-data:/var/lib/zookeeper/data + - ./data-zoo-logs:/var/lib/zookeeper/log + - ./data-zoo-secrets:/etc/zookeeper/secrets + container_name: zoo1 + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_SERVER_ID: 1 + ZOOKEEPER_SERVERS: zoo1:2888:3888 + labels: + com.centurylinklabs.watchtower.enable: "false" + + kafka1: + image: confluentinc/cp-kafka:7.0.5 + restart: always + hostname: kafka1 + user: "0" + ports: + - "9092:9092" + - "19092:19092" + - "29092:29092" + - "9999:9999" + environment: + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://${AKTO_KAFKA_IP}:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" + KAFKA_BROKER_ID: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_CREATE_TOPICS: "akto.api.logs:3:3" + KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 60000 + KAFKA_LOG_RETENTION_HOURS: 5 + KAFKA_LOG_SEGMENT_BYTES: 104857600 + KAFKA_LOG_CLEANER_ENABLE: "true" + KAFKA_CLEANUP_POLICY: "delete" + KAFKA_LOG_RETENTION_BYTES: 10737418240 + volumes: + - ./data-kafka-data:/var/lib/kafka/data + - ./data-kafka-secrets:/etc/kafka/secrets + depends_on: + - zoo1 + labels: + com.centurylinklabs.watchtower.enable: "false" + + akto-api-security-runtime: + image: public.ecr.aws/aktosecurity/akto-api-security-mini-runtime:testruntime + env_file: ./docker-mini-runtime.env + mem_limit: 8g + restart: always + depends_on: + - kafka1 + + akto-api-security-mirror-api-logging: + image: public.ecr.aws/aktosecurity/mirror-api-logging:local + env_file: ./docker-mini-runtime.env + mem_limit: 4g + restart: always + depends_on: + - kafka1 + network_mode: host + + akto-hello-world: + image: public.ecr.aws/aktosecurity/hello-world-server:latest + restart: always + ports: + - "8000:8000" + + watchtower: + image: containrrr/watchtower + restart: always + volumes: + - /var/run/docker.sock:/var/run/docker.sock + env_file: ./watchtower.env + labels: + com.centurylinklabs.watchtower.enable: "false" From cc25f89df583738323b83eb813cab379bc24ddaf Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Wed, 20 Nov 2024 15:10:45 +0530 Subject: [PATCH 94/98] Add terraform script for mini testing setup --- templates/mini-testing.tf | 204 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 templates/mini-testing.tf diff --git a/templates/mini-testing.tf b/templates/mini-testing.tf new file mode 100644 index 0000000..5b62b93 --- /dev/null +++ b/templates/mini-testing.tf @@ -0,0 +1,204 @@ +# Provider configuration +provider "aws" { + region = var.region +} + +# Input Variables +variable "region" { + description = "AWS region where resources will be deployed" + type = string +} + +variable "key_pair" { + description = "Key pair for SSH access to the EC2 instances" + type = string +} + +variable "subnet_id" { + description = "Subnet ID where the EC2 instances will be launched" + type = string +} + +# Input Variable for Token +variable "database_abstractor_service_token" { + description = "Token for the Database Abstractor service" + type = string +} + +locals { + mappings = { + RegionMap = { + af-south-1 = { + AMI = "ami-093ca241e4c72c205" + } + eu-north-1 = { + AMI = "ami-0f58e72599cb99a79" + } + ap-south-1 = { + AMI = "ami-0400aca7799d8cf19" + } + eu-west-3 = { + AMI = "ami-064c70d04ad799d5e" + } + eu-west-2 = { + AMI = "ami-0dfe6158087b5c0ac" + } + eu-south-1 = { + AMI = "ami-07b2af763a8b958f3" + } + eu-west-1 = { + AMI = "ami-047aad752a426ed48" + } + ap-northeast-3 = { + AMI = "ami-0cffa2172948e071e" + } + ap-northeast-2 = { + AMI = "ami-087af0192368bc87c" + } + me-south-1 = { + AMI = "ami-0a31e56929248acca" + } + ap-northeast-1 = { + AMI = "ami-0828596b82405edd7" + } + sa-east-1 = { + AMI = "ami-0df67b3c17f090c24" + } + ca-central-1 = { + AMI = "ami-0eb3718c42cb70e52" + } + ap-east-1 = { + AMI = "ami-0e992f1e63814db10" + } + ap-southeast-1 = { + AMI = "ami-0ba98499caf94125a" + } + ap-southeast-2 = { + AMI = "ami-0849cc8fe4ceaf988" + } + eu-central-1 = { + AMI = "ami-0f7585ae7a0d9a25a" + } + ap-southeast-3 = { + AMI = "ami-0cf40308729b83366" + } + us-east-1 = { + AMI = "ami-0d52ddcdf3a885741" + } + us-east-2 = { + AMI = "ami-04148302a14f7d12b" + } + us-west-1 = { + AMI = "ami-0ee3e1e65adeef858" + } + us-west-2 = { + AMI = "ami-0ec021424fb596d6c" + } + } + } + # Lookup the AMI ID based on the region + ami_id = local.mappings.RegionMap[var.region].AMI +} + +# IAM Role for EC2 Instances +resource "aws_iam_role" "ec2_execution_role" { + name = "TestServiceExecutionRole" + assume_role_policy = jsonencode({ + Version : "2012-10-17", + Statement : [ + { + Effect : "Allow", + Principal : { + Service : "ec2.amazonaws.com" + }, + Action : "sts:AssumeRole" + } + ] + }) +} + +resource "aws_iam_instance_profile" "instance_profile" { + name = "TestServiceInstanceProfile" + role = aws_iam_role.ec2_execution_role.name +} + +# Security Group +resource "aws_security_group" "service_security_group" { + name_prefix = "service-security-group" + vpc_id = data.aws_subnet.selected.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +# Data source for Subnet and VPC +data "aws_subnet" "selected" { + id = var.subnet_id +} + +data "aws_vpc" "selected" { + id = data.aws_subnet.selected.vpc_id +} + +# Launch Configuration +resource "aws_launch_configuration" "test_service_launch_configuration" { + name_prefix = "TestServiceLaunchConfiguration" + image_id = local.ami_id + instance_type = "t3.medium" + key_name = var.key_pair + iam_instance_profile = aws_iam_instance_profile.instance_profile.id + security_groups = [aws_security_group.service_security_group.id] + + user_data = <<-EOF + #!/bin/bash -xe + sudo yum update -y + sudo yum install -y docker + sudo systemctl start docker + sudo systemctl enable docker + sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose + curl -L https://raw.githubusercontent.com/akto-api-security/infra/refs/heads/feature/quick-setup/docker-compose-mini-testing.yml -o docker-compose.yml + sudo mv docker-compose.yml /root + sed -i 's||${var.database_abstractor_service_token}|g' /root/docker-compose.yml + sudo docker-compose -f /root/docker-compose.yml up -d + EOF +} + +# Auto Scaling Group +resource "aws_autoscaling_group" "test_service_autoscaling_group" { + desired_capacity = 1 + max_size = 3 + min_size = 1 + vpc_zone_identifier = [var.subnet_id] + launch_configuration = aws_launch_configuration.test_service_launch_configuration.name + + tag { + key = "Name" + value = "TestServiceInstance" + propagate_at_launch = true + } +} + +# Outputs +output "autoscaling_group_name" { + description = "Name of the Auto Scaling Group" + value = aws_autoscaling_group.test_service_autoscaling_group.name +} + +output "security_group_id" { + description = "Security Group ID for the service" + value = aws_security_group.service_security_group.id +} + From ee392d4ac241db5aa8061d9e53b4c9b92b6c8784 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Wed, 20 Nov 2024 15:36:38 +0530 Subject: [PATCH 95/98] modify instance type --- templates/mini-testing.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mini-testing.tf b/templates/mini-testing.tf index 5b62b93..22eb3e1 100644 --- a/templates/mini-testing.tf +++ b/templates/mini-testing.tf @@ -155,7 +155,7 @@ data "aws_vpc" "selected" { resource "aws_launch_configuration" "test_service_launch_configuration" { name_prefix = "TestServiceLaunchConfiguration" image_id = local.ami_id - instance_type = "t3.medium" + instance_type = "m5a.large" key_name = var.key_pair iam_instance_profile = aws_iam_instance_profile.instance_profile.id security_groups = [aws_security_group.service_security_group.id] From 4ca984a7dbfdc4ad2c8b96b2f29da55da7e8dc95 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Tue, 3 Dec 2024 16:07:46 +0530 Subject: [PATCH 96/98] fix tf script --- templates/mini-runtime.tf | 65 ++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/templates/mini-runtime.tf b/templates/mini-runtime.tf index 101ce46..5d32da6 100644 --- a/templates/mini-runtime.tf +++ b/templates/mini-runtime.tf @@ -151,22 +151,6 @@ resource "aws_iam_role_policy" "get_akto_lambda_policy" { }) } -# Lambda Function -resource "aws_lambda_function" "get_akto_setup_details" { - function_name = "GetAktoSetupDetails" - runtime = "nodejs16.x" - role = aws_iam_role.lambda_execution_role.arn - handler = "index.handler" - timeout = 60 - environment { - variables = { - TARGET_LB = aws_lb.akto_nlb.dns_name - } - } - s3_bucket = "akto-setup-${var.region}" - s3_key = "templates/get-akto-setup-details.zip" -} - # IAM Role for EC2 Instances resource "aws_iam_instance_profile" "iam_instance_profile" { name = "AktoInstanceProfile" @@ -211,6 +195,26 @@ resource "aws_security_group" "akto_security_group" { } } +# Security Group +resource "aws_security_group" "lb_security_group" { + name_prefix = "lb-security-group" + vpc_id = data.aws_vpc.selected.id + + ingress { + from_port = 9092 + to_port = 9092 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + # Launch Configuration resource "aws_launch_configuration" "akto_launch_configuration" { name_prefix = "AktoASGLaunchConfiguration" @@ -276,10 +280,11 @@ resource "aws_autoscaling_group" "akto_autoscaling_group" { # Network Load Balancer resource "aws_lb" "akto_nlb" { name = "AktoNLB" - internal = true + internal = false load_balancer_type = "network" ip_address_type = "ipv4" subnets = [var.subnet_id] + security_groups = [aws_security_group.lb_security_group.id] enable_cross_zone_load_balancing = true } @@ -293,13 +298,12 @@ resource "aws_lb_target_group" "akto_traffic_mirroring_target_group" { health_check { enabled = true - interval = 10 - path = "/metrics" - port = "8000" - protocol = "HTTP" - timeout = 6 - healthy_threshold = 2 - unhealthy_threshold = 2 + interval = 10 # Interval between health checks in seconds + port = "9092" # Change the health check port to 9092 + protocol = "TCP" # Change the protocol to TCP + timeout = 6 # Timeout for health check response in seconds + healthy_threshold = 2 # Number of successful checks before marking healthy + unhealthy_threshold = 2 # Number of failed checks before marking unhealthy } } @@ -311,13 +315,12 @@ resource "aws_lb_target_group" "akto_kafka_target_group" { health_check { enabled = true - interval = 10 - path = "/metrics" - port = "8000" - protocol = "HTTP" - timeout = 6 - healthy_threshold = 2 - unhealthy_threshold = 2 + interval = 10 # Interval between health checks in seconds + port = "9092" # Change the health check port to 9092 + protocol = "TCP" # Change the protocol to TCP + timeout = 6 # Timeout for health check response in seconds + healthy_threshold = 2 # Number of successful checks before marking healthy + unhealthy_threshold = 2 # Number of failed checks before marking unhealthy } } From 6999f8103abfc5f38712c0f22d2aa75101e4ce7d Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Tue, 3 Dec 2024 16:12:58 +0530 Subject: [PATCH 97/98] mark lb internal --- templates/mini-runtime.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/mini-runtime.tf b/templates/mini-runtime.tf index 5d32da6..2908032 100644 --- a/templates/mini-runtime.tf +++ b/templates/mini-runtime.tf @@ -280,7 +280,7 @@ resource "aws_autoscaling_group" "akto_autoscaling_group" { # Network Load Balancer resource "aws_lb" "akto_nlb" { name = "AktoNLB" - internal = false + internal = true load_balancer_type = "network" ip_address_type = "ipv4" subnets = [var.subnet_id] From c63450062d5eb4e03c2834c0b42377522ac83c40 Mon Sep 17 00:00:00 2001 From: ayushaga14 Date: Fri, 24 Jan 2025 14:53:47 +0530 Subject: [PATCH 98/98] modify launch config to launch template --- templates/mini-runtime.yml | 145 +++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 69 deletions(-) diff --git a/templates/mini-runtime.yml b/templates/mini-runtime.yml index b07d620..a01900e 100644 --- a/templates/mini-runtime.yml +++ b/templates/mini-runtime.yml @@ -197,80 +197,87 @@ Resources: GroupDescription: 'Enable the ports Akto requires (22, 4789, 8000, 9092)' SecurityGroupIngress: [] SecurityGroupEgress: [] - AktoASGLaunchConfiguration: - Type: 'AWS::AutoScaling::LaunchConfiguration' - DependsOn: - - AktoNLB + + AktoLaunchTemplate: + Type: 'AWS::EC2::LaunchTemplate' Properties: - ImageId: !FindInMap - - RegionMap - - !Ref 'AWS::Region' - - AMI - InstanceType: m5a.xlarge - KeyName: !Ref KeyPair - AssociatePublicIpAddress: 'false' - IamInstanceProfile: !Ref IamInstanceProfile - SecurityGroups: - - !Ref AktoSecurityGroup - BlockDeviceMappings: - - DeviceName: /dev/xvda - Ebs: - VolumeType: gp2 - DeleteOnTermination: 'true' - VolumeSize: '50' - Encrypted: true - MetadataOptions: - HttpTokens: required - UserData: !Base64 - 'Fn::Join': - - |+ + LaunchTemplateName: AktoLaunchTemplate + LaunchTemplateData: + ImageId: !FindInMap + - RegionMap + - !Ref 'AWS::Region' + - AMI + InstanceType: m5a.xlarge + KeyName: !Ref KeyPair + NetworkInterfaces: + - AssociatePublicIpAddress: false + SubnetId: !Ref SubnetId + DeviceIndex: 0 + Groups: + - !Ref AktoSecurityGroup + IamInstanceProfile: + Arn: !GetAtt IamInstanceProfile.Arn + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeType: gp2 + DeleteOnTermination: true + VolumeSize: 50 + Encrypted: true + MetadataOptions: + HttpTokens: required + UserData: !Base64 + Fn::Join: + - "\n" + - - '#!/bin/bash -xe' + - !Sub "export DATABASE_ABSTRACTOR_SERVICE_URL=${DatabaseAbstractorUrl}" + - !Sub "export DATABASE_ABSTRACTOR_SERVICE_TOKEN=${DatabaseAbstractorToken}" + - !Sub "export AKTO_KAFKA_IP=${AktoNLB.DNSName}" + - touch /tmp/hello.txt + - touch ~/hello.txt + - sudo yum update -y + - sudo yum install -y python python-setuptools + - sudo yum install -y docker + - sudo dockerd& + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - sudo mkdir -p /opt/aws/bin + - export COMPOSE_FILE=docker-compose-mini-runtime.yml + - >- + sudo wget + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz + - >- + sudo python -m easy_install --script-dir /opt/aws/bin + aws-cfn-bootstrap-latest.tar.gz + - >- + curl -fsSL + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto' + > cf-deploy-akto + - sudo chmod 700 cf-deploy-akto + - ./cf-deploy-akto < <(echo 'test') + - sudo echo >> ~/akto/infra/docker-runtime.env + - >- + sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> + ~/akto/infra/docker-runtime.env + - >- + sudo echo DATABASE_ABSTRACTOR_SERVICE_URL=$DATABASE_ABSTRACTOR_SERVICE_URL >> + ~/akto/infra/docker-mini-runtime.env + - >- + sudo echo DATABASE_ABSTRACTOR_SERVICE_TOKEN=$DATABASE_ABSTRACTOR_SERVICE_TOKEN >> + ~/akto/infra/docker-mini-runtime.env + - sudo echo AKTO_KAFKA_IP=$AKTO_KAFKA_IP >> ~/akto/infra/.env + - >- + curl -fsSL + 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto-start' + > cf-deploy-akto-start + - sudo chmod 700 cf-deploy-akto-start + - ./cf-deploy-akto-start < <(echo 'test') - - - '#!/bin/bash -xe' - - !Sub 'export DATABASE_ABSTRACTOR_SERVICE_URL=''${DatabaseAbstractorUrl}''' - - !Sub 'export DATABASE_ABSTRACTOR_SERVICE_TOKEN=''${DatabaseAbstractorToken}''' - - !Sub 'export AKTO_KAFKA_IP=''${AktoNLB.DNSName}''' - - touch /tmp/hello.txt - - touch ~/hello.txt - - sudo yum update -y - - sudo yum install -y python python-setuptools - - sudo yum install -y docker - - sudo dockerd& - - sudo systemctl enable /usr/lib/systemd/system/docker.service - - sudo mkdir -p /opt/aws/bin - - export COMPOSE_FILE=docker-compose-mini-runtime.yml - - >- - sudo wget - https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - - >- - sudo python -m easy_install --script-dir /opt/aws/bin - aws-cfn-bootstrap-latest.tar.gz - - >- - curl -fsSL - 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto' - > cf-deploy-akto - - sudo chmod 700 cf-deploy-akto - - ./cf-deploy-akto < <(echo 'test') - - sudo echo >> ~/akto/infra/docker-runtime.env - - >- - sudo echo AKTO_MONGO_CONN=$AKTO_MONGO_CONN >> - ~/akto/infra/docker-runtime.env - - >- - sudo echo DATABASE_ABSTRACTOR_SERVICE_URL=$DATABASE_ABSTRACTOR_SERVICE_URL >> - ~/akto/infra/docker-mini-runtime.env - - >- - sudo echo DATABASE_ABSTRACTOR_SERVICE_TOKEN=$DATABASE_ABSTRACTOR_SERVICE_TOKEN >> - ~/akto/infra/docker-mini-runtime.env - - sudo echo AKTO_KAFKA_IP=$AKTO_KAFKA_IP >> ~/akto/infra/.env - - >- - curl -fsSL - 'https://raw.githubusercontent.com/akto-api-security/infra/feature/mini-runtime-cft/cf-deploy-akto-start' - > cf-deploy-akto-start - - sudo chmod 700 cf-deploy-akto-start - - ./cf-deploy-akto-start < <(echo 'test') AktoAutoScalingGroup: Type: 'AWS::AutoScaling::AutoScalingGroup' Properties: - LaunchConfigurationName: !Ref AktoASGLaunchConfiguration + LaunchTemplate: + LaunchTemplateId: !Ref AktoLaunchTemplate + Version: !GetAtt AktoLaunchTemplate.LatestVersionNumber VPCZoneIdentifier: - !Ref SubnetId TargetGroupARNs: