From ef157db2fa29b315069517b025d2a2755c27c70a Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Fri, 29 Nov 2024 09:16:03 +0000 Subject: [PATCH] WIP --- .../service-catalogue.test.ts.snap | 9128 +++++++++-------- 1 file changed, 4601 insertions(+), 4527 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index dad3d702..ab1f62f7 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -64,9 +64,9 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` }, }, "Resources": { - "CloudquerySourceAwsCostExplorerScheduledEventRule85BE97F8": { + "CloudquerySourceAmigoBakePackagesScheduledEventRule3FDBCEB5": { "Properties": { - "ScheduleExpression": "rate(7 days)", + "ScheduleExpression": "rate(1 day)", "State": "ENABLED", "Targets": [ { @@ -97,14 +97,14 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsCostExplorerTaskDefinition62DC1A04", + "Ref": "CloudquerySourceAmigoBakePackagesTaskDefinitionF04CFC72", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsCostExplorerTaskDefinitionEventsRole71FB003A", + "CloudquerySourceAmigoBakePackagesTaskDefinitionEventsRoleB18B35DF", "Arn", ], }, @@ -113,7 +113,251 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsCostExplorerTaskDefinition62DC1A04": { + "CloudquerySourceAmigoBakePackagesTaskDefinitionCloudquerySourceAmigoBakePackagesFirelensLogGroupA4EF0BA2": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AmigoBakePackages", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionEventsRoleB18B35DF": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "events.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AmigoBakePackages", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionEventsRoleDefaultPolicy145F68FA": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ecs:RunTask", + "Condition": { + "ArnEquals": { + "ecs:cluster": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + }, + }, + "Effect": "Allow", + "Resource": { + "Ref": "CloudquerySourceAmigoBakePackagesTaskDefinitionF04CFC72", + }, + }, + { + "Action": "ecs:TagResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":ecs:", + { + "Ref": "AWS::Region", + }, + ":*:task/", + { + "Ref": "servicecatalogueCluster5FC34DC5", + }, + "/*", + ], + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleD495DC33", + "Arn", + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAmigoBakePackagesE3F44845", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceAmigoBakePackagesTaskDefinitionEventsRoleDefaultPolicy145F68FA", + "Roles": [ + { + "Ref": "CloudquerySourceAmigoBakePackagesTaskDefinitionEventsRoleB18B35DF", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleD495DC33": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AmigoBakePackages", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleDefaultPolicyBD5A8255": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "amigobakepackagesE494D60D", + }, + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "cloudqueryapikeyCCF82F53", + }, + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceAmigoBakePackagesTaskDefinitionCloudquerySourceAmigoBakePackagesFirelensLogGroupA4EF0BA2", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleDefaultPolicyBD5A8255", + "Roles": [ + { + "Ref": "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleD495DC33", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceAmigoBakePackagesTaskDefinitionF04CFC72": { "Properties": { "ContainerDefinitions": [ { @@ -122,20 +366,19 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` "-c", "printf 'kind: source spec: - name: aws - path: cloudquery/aws - version: v27.5.0 - tables: - - aws_costexplorer_* + name: image-packages + registry: github + path: guardian/image-packages + version: v0.0.13 destinations: - postgresql - otel_endpoint: 0.0.0.0:4318 - otel_endpoint_insecure: true + tables: + - amigo_bake_packages spec: - org: - member_role_name: cloudquery-access - organization_units: - - ou-123 + base_images_table: \${BASE_IMAGES_TABLE_NAME} + recipes_table: \${RECIPES_TABLE_NAME} + bakes_table: \${BAKES_TABLE_NAME} + bucket: \${PACKAGES_BUCKET_NAME} ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -152,12 +395,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsCostExplorerAWSOTELCollector", + "ContainerName": "CloudquerySource-AmigoBakePackagesAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsCostExplorer", + "Name": "AmigoBakePackages", "Stack": "deploy", "Stage": "TEST", }, @@ -202,25 +445,81 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsCostExplorerContainer", + "Name": "CloudquerySource-AmigoBakePackagesContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { - "Name": "DB_USERNAME", + "Name": "BASE_IMAGES_TABLE_NAME", "ValueFrom": { "Fn::Join": [ "", [ { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + "Ref": "amigobakepackagesE494D60D", }, - ":username::", + ":base-images-table-name::", ], ], }, }, { - "Name": "DB_HOST", + "Name": "RECIPES_TABLE_NAME", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "amigobakepackagesE494D60D", + }, + ":recipes-table-name::", + ], + ], + }, + }, + { + "Name": "BAKES_TABLE_NAME", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "amigobakepackagesE494D60D", + }, + ":bakes-table-name::", + ], + ], + }, + }, + { + "Name": "PACKAGES_BUCKET_NAME", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "amigobakepackagesE494D60D", + }, + ":packages-bucket-name::", + ], + ], + }, + }, + { + "Name": "DB_USERNAME", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":username::", + ], + ], + }, + }, + { + "Name": "DB_HOST", "ValueFrom": { "Fn::Join": [ "", @@ -291,7 +590,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsCostExplorerAWSOTELCollector", + "Name": "CloudquerySource-AmigoBakePackagesAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -304,11 +603,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_costexplorer_%', 'WEEKLY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'WEEKLY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('amigo_bake_packages', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsCostExplorer", + "Name": "AmigoBakePackages", "Stack": "deploy", "Stage": "TEST", }, @@ -330,7 +629,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsCostExplorerPostgresContainer", + "Name": "CloudquerySource-AmigoBakePackagesPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -405,7 +704,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsCostExplorerTaskDefinitionCloudquerySourceAwsCostExplorerFirelensLogGroup0D0D76FF", + "Ref": "CloudquerySourceAmigoBakePackagesTaskDefinitionCloudquerySourceAmigoBakePackagesFirelensLogGroupA4EF0BA2", }, "awslogs-region": { "Ref": "AWS::Region", @@ -420,18 +719,18 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsCostExplorerFirelens", + "Name": "CloudquerySource-AmigoBakePackagesFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleD508639D", + "CloudquerySourceAmigoBakePackagesTaskDefinitionExecutionRoleD495DC33", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsCostExplorerTaskDefinitionD8C37FA5", + "Family": "ServiceCatalogueCloudquerySourceAmigoBakePackagesTaskDefinition07388B36", "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ @@ -448,7 +747,7 @@ spec: }, { "Key": "Name", - "Value": "AwsCostExplorer", + "Value": "AmigoBakePackages", }, { "Key": "Stack", @@ -461,7 +760,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsCostExplorer78777A06", + "servicecatalogueTESTtaskAmigoBakePackagesE3F44845", "Arn", ], }, @@ -482,292 +781,58 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsCostExplorerTaskDefinitionCloudquerySourceAwsCostExplorerFirelensLogGroup0D0D76FF": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsCostExplorer", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", - }, - "CloudquerySourceAwsCostExplorerTaskDefinitionEventsRole71FB003A": { + "CloudquerySourceAwsCostExplorerScheduledEventRule85BE97F8": { "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "events.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsCostExplorer", - }, - { - "Key": "Stack", - "Value": "deploy", - }, + "ScheduleExpression": "rate(7 days)", + "State": "ENABLED", + "Targets": [ { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceAwsCostExplorerTaskDefinitionEventsRoleDefaultPolicy6D27ED91": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": "ecs:RunTask", - "Condition": { - "ArnEquals": { - "ecs:cluster": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], - }, - }, - }, - "Effect": "Allow", - "Resource": { - "Ref": "CloudquerySourceAwsCostExplorerTaskDefinition62DC1A04", - }, + "Arn": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], }, - { - "Action": "ecs:TagResource", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":ecs:", - { - "Ref": "AWS::Region", - }, - ":*:task/", + "EcsParameters": { + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsVpcConfiguration": { + "AssignPublicIp": "DISABLED", + "SecurityGroups": [ { - "Ref": "servicecatalogueCluster5FC34DC5", + "Fn::GetAtt": [ + "PostgresAccessSecurityGroupServicecatalogue03C78F14", + "GroupId", + ], }, - "/*", ], - ], + "Subnets": { + "Ref": "PrivateSubnets", + }, + }, }, - }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleD508639D", - "Arn", - ], + "PropagateTags": "TASK_DEFINITION", + "TaskCount": 1, + "TaskDefinitionArn": { + "Ref": "CloudquerySourceAwsCostExplorerTaskDefinition62DC1A04", }, }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsCostExplorer78777A06", - "Arn", - ], - }, + "Id": "Target0", + "Input": "{}", + "RoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAwsCostExplorerTaskDefinitionEventsRole71FB003A", + "Arn", + ], }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceAwsCostExplorerTaskDefinitionEventsRoleDefaultPolicy6D27ED91", - "Roles": [ - { - "Ref": "CloudquerySourceAwsCostExplorerTaskDefinitionEventsRole71FB003A", }, ], }, - "Type": "AWS::IAM::Policy", + "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleD508639D": { + "CloudquerySourceAwsCostExplorerTaskDefinition62DC1A04": { "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ecs-tasks.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsCostExplorer", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleDefaultPolicyEBFC45ED": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - }, - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "cloudqueryapikeyCCF82F53", - }, - }, - { - "Action": [ - "logs:CreateLogStream", - "logs:PutLogEvents", - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceAwsCostExplorerTaskDefinitionCloudquerySourceAwsCostExplorerFirelensLogGroup0D0D76FF", - "Arn", - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleDefaultPolicyEBFC45ED", - "Roles": [ - { - "Ref": "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleD508639D", - }, - ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceAwsDelegatedToSecurityAccountScheduledEventRuleD23A8E56": { - "Properties": { - "ScheduleExpression": "cron(0 22 * * ? *)", - "State": "ENABLED", - "Targets": [ - { - "Arn": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], - }, - "EcsParameters": { - "LaunchType": "FARGATE", - "NetworkConfiguration": { - "AwsVpcConfiguration": { - "AssignPublicIp": "DISABLED", - "SecurityGroups": [ - { - "Fn::GetAtt": [ - "PostgresAccessSecurityGroupServicecatalogue03C78F14", - "GroupId", - ], - }, - ], - "Subnets": { - "Ref": "PrivateSubnets", - }, - }, - }, - "PropagateTags": "TASK_DEFINITION", - "TaskCount": 1, - "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinition8FFEB633", - }, - }, - "Id": "Target0", - "Input": "{}", - "RoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionEventsRole84A3EC34", - "Arn", - ], - }, - }, - ], - }, - "Type": "AWS::Events::Rule", - }, - "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinition8FFEB633": { - "Properties": { - "ContainerDefinitions": [ + "ContainerDefinitions": [ { "Command": [ "/bin/sh", @@ -778,30 +843,16 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_accessanalyzer_* - - aws_securityhub_* + - aws_costexplorer_* destinations: - postgresql otel_endpoint: 0.0.0.0:4318 otel_endpoint_insecure: true spec: - concurrency: 2000 - accounts: - - id: cq-for-000000000015 - role_arn: arn:aws:iam::000000000015:role/cloudquery-access - table_options: - aws_securityhub_findings: - get_findings: - - filters: - record_state: - - comparison: EQUALS - value: ACTIVE - compliance_status: - - comparison: NOT_EQUALS - value: PASSED - workflow_status: - - comparison: NOT_EQUALS - value: RESOLVED + org: + member_role_name: cloudquery-access + organization_units: + - ou-123 ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -818,12 +869,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsDelegatedToSecurityAccountAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsCostExplorerAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsDelegatedToSecurityAccount", + "Name": "AwsCostExplorer", "Stack": "deploy", "Stage": "TEST", }, @@ -833,7 +884,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "1638MiB", + "Value": "409MiB", }, ], "Essential": true, @@ -868,7 +919,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsDelegatedToSecurityAccountContainer", + "Name": "CloudquerySource-AwsCostExplorerContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -957,7 +1008,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsDelegatedToSecurityAccountAWSOTELCollector", + "Name": "CloudquerySource-AwsCostExplorerAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -970,11 +1021,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_accessanalyzer_%', 'DAILY'),('aws_securityhub_%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_costexplorer_%', 'WEEKLY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'WEEKLY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsDelegatedToSecurityAccount", + "Name": "AwsCostExplorer", "Stack": "deploy", "Stage": "TEST", }, @@ -996,7 +1047,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsDelegatedToSecurityAccountPostgresContainer", + "Name": "CloudquerySource-AwsCostExplorerPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -1071,7 +1122,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionCloudquerySourceAwsDelegatedToSecurityAccountFirelensLogGroupE468B6F3", + "Ref": "CloudquerySourceAwsCostExplorerTaskDefinitionCloudquerySourceAwsCostExplorerFirelensLogGroup0D0D76FF", }, "awslogs-region": { "Ref": "AWS::Region", @@ -1086,19 +1137,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsDelegatedToSecurityAccountFirelens", + "Name": "CloudquerySource-AwsCostExplorerFirelens", "ReadonlyRootFilesystem": true, }, ], - "Cpu": "1024", + "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRole390812B0", + "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleD508639D", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionD9E19358", - "Memory": "2048", + "Family": "ServiceCatalogueCloudquerySourceAwsCostExplorerTaskDefinitionD8C37FA5", + "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -1114,7 +1165,7 @@ spec: }, { "Key": "Name", - "Value": "AwsDelegatedToSecurityAccount", + "Value": "AwsCostExplorer", }, { "Key": "Stack", @@ -1127,7 +1178,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsDelegatedToSecurityAccountEDD7C370", + "servicecatalogueTESTtaskAwsCostExplorer78777A06", "Arn", ], }, @@ -1148,7 +1199,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionCloudquerySourceAwsDelegatedToSecurityAccountFirelensLogGroupE468B6F3": { + "CloudquerySourceAwsCostExplorerTaskDefinitionCloudquerySourceAwsCostExplorerFirelensLogGroup0D0D76FF": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -1163,7 +1214,7 @@ spec: }, { "Key": "Name", - "Value": "AwsDelegatedToSecurityAccount", + "Value": "AwsCostExplorer", }, { "Key": "Stack", @@ -1178,7 +1229,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionEventsRole84A3EC34": { + "CloudquerySourceAwsCostExplorerTaskDefinitionEventsRole71FB003A": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -1203,7 +1254,7 @@ spec: }, { "Key": "Name", - "Value": "AwsDelegatedToSecurityAccount", + "Value": "AwsCostExplorer", }, { "Key": "Stack", @@ -1217,7 +1268,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionEventsRoleDefaultPolicy33EB3CCB": { + "CloudquerySourceAwsCostExplorerTaskDefinitionEventsRoleDefaultPolicy6D27ED91": { "Properties": { "PolicyDocument": { "Statement": [ @@ -1235,7 +1286,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinition8FFEB633", + "Ref": "CloudquerySourceAwsCostExplorerTaskDefinition62DC1A04", }, }, { @@ -1267,7 +1318,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRole390812B0", + "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleD508639D", "Arn", ], }, @@ -1277,7 +1328,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsDelegatedToSecurityAccountEDD7C370", + "servicecatalogueTESTtaskAwsCostExplorer78777A06", "Arn", ], }, @@ -1285,16 +1336,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionEventsRoleDefaultPolicy33EB3CCB", + "PolicyName": "CloudquerySourceAwsCostExplorerTaskDefinitionEventsRoleDefaultPolicy6D27ED91", "Roles": [ { - "Ref": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionEventsRole84A3EC34", + "Ref": "CloudquerySourceAwsCostExplorerTaskDefinitionEventsRole71FB003A", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRole390812B0": { + "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleD508639D": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -1319,7 +1370,7 @@ spec: }, { "Key": "Name", - "Value": "AwsDelegatedToSecurityAccount", + "Value": "AwsCostExplorer", }, { "Key": "Stack", @@ -1333,7 +1384,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRoleDefaultPolicy36910251": { + "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleDefaultPolicyEBFC45ED": { "Properties": { "PolicyDocument": { "Statement": [ @@ -1365,7 +1416,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionCloudquerySourceAwsDelegatedToSecurityAccountFirelensLogGroupE468B6F3", + "CloudquerySourceAwsCostExplorerTaskDefinitionCloudquerySourceAwsCostExplorerFirelensLogGroup0D0D76FF", "Arn", ], }, @@ -1373,18 +1424,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRoleDefaultPolicy36910251", + "PolicyName": "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleDefaultPolicyEBFC45ED", "Roles": [ { - "Ref": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRole390812B0", + "Ref": "CloudquerySourceAwsCostExplorerTaskDefinitionExecutionRoleD508639D", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsLambdaScheduledEventRuleC1529FB1": { + "CloudquerySourceAwsDelegatedToSecurityAccountScheduledEventRuleD23A8E56": { "Properties": { - "ScheduleExpression": "cron(10 1 * * ? *)", + "ScheduleExpression": "cron(0 22 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -1415,14 +1466,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsLambdaTaskDefinitionD9609861", + "Ref": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinition8FFEB633", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsLambdaTaskDefinitionEventsRole4D97B167", + "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionEventsRole84A3EC34", "Arn", ], }, @@ -1431,37 +1482,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsLambdaTaskDefinitionCloudquerySourceAwsLambdaFirelensLogGroupBB74129A": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsLambda", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", - }, - "CloudquerySourceAwsLambdaTaskDefinitionD9609861": { + "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinition8FFEB633": { "Properties": { "ContainerDefinitions": [ { @@ -1474,38 +1495,52 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_lambda_* + - aws_accessanalyzer_* + - aws_securityhub_* destinations: - postgresql otel_endpoint: 0.0.0.0:4318 otel_endpoint_insecure: true spec: - org: - member_role_name: cloudquery-access - organization_units: - - ou-123 -' > /usr/share/cloudquery/source.yaml;printf 'kind: destination -spec: - name: postgresql - registry: github - path: cloudquery/postgresql - version: v7.2.0 - migrate_mode: forced - spec: - connection_string: >- - user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 + concurrency: 2000 + accounts: + - id: cq-for-000000000015 + role_arn: arn:aws:iam::000000000015:role/cloudquery-access + table_options: + aws_securityhub_findings: + get_findings: + - filters: + record_state: + - comparison: EQUALS + value: ACTIVE + compliance_status: + - comparison: NOT_EQUALS + value: PASSED + workflow_status: + - comparison: NOT_EQUALS + value: RESOLVED +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination +spec: + name: postgresql + registry: github + path: cloudquery/postgresql + version: v7.2.0 + migrate_mode: forced + spec: + connection_string: >- + user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full ' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsLambdaAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsDelegatedToSecurityAccountAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsLambda", + "Name": "AwsDelegatedToSecurityAccount", "Stack": "deploy", "Stage": "TEST", }, @@ -1515,7 +1550,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "409MiB", + "Value": "1638MiB", }, ], "Essential": true, @@ -1550,7 +1585,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsLambdaContainer", + "Name": "CloudquerySource-AwsDelegatedToSecurityAccountContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -1639,7 +1674,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsLambdaAWSOTELCollector", + "Name": "CloudquerySource-AwsDelegatedToSecurityAccountAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -1652,11 +1687,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_lambda_%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_accessanalyzer_%', 'DAILY'),('aws_securityhub_%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsLambda", + "Name": "AwsDelegatedToSecurityAccount", "Stack": "deploy", "Stage": "TEST", }, @@ -1678,7 +1713,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsLambdaPostgresContainer", + "Name": "CloudquerySource-AwsDelegatedToSecurityAccountPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -1753,7 +1788,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsLambdaTaskDefinitionCloudquerySourceAwsLambdaFirelensLogGroupBB74129A", + "Ref": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionCloudquerySourceAwsDelegatedToSecurityAccountFirelensLogGroupE468B6F3", }, "awslogs-region": { "Ref": "AWS::Region", @@ -1768,19 +1803,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsLambdaFirelens", + "Name": "CloudquerySource-AwsDelegatedToSecurityAccountFirelens", "ReadonlyRootFilesystem": true, }, ], - "Cpu": "256", + "Cpu": "1024", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsLambdaTaskDefinitionExecutionRole9469B1E9", + "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRole390812B0", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsLambdaTaskDefinitionDAA42514", - "Memory": "512", + "Family": "ServiceCatalogueCloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionD9E19358", + "Memory": "2048", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -1796,7 +1831,7 @@ spec: }, { "Key": "Name", - "Value": "AwsLambda", + "Value": "AwsDelegatedToSecurityAccount", }, { "Key": "Stack", @@ -1809,7 +1844,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsLambda20C12A7D", + "servicecatalogueTESTtaskAwsDelegatedToSecurityAccountEDD7C370", "Arn", ], }, @@ -1830,7 +1865,37 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsLambdaTaskDefinitionEventsRole4D97B167": { + "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionCloudquerySourceAwsDelegatedToSecurityAccountFirelensLogGroupE468B6F3": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsDelegatedToSecurityAccount", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionEventsRole84A3EC34": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -1855,7 +1920,7 @@ spec: }, { "Key": "Name", - "Value": "AwsLambda", + "Value": "AwsDelegatedToSecurityAccount", }, { "Key": "Stack", @@ -1869,7 +1934,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsLambdaTaskDefinitionEventsRoleDefaultPolicyCA44BFE7": { + "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionEventsRoleDefaultPolicy33EB3CCB": { "Properties": { "PolicyDocument": { "Statement": [ @@ -1887,7 +1952,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsLambdaTaskDefinitionD9609861", + "Ref": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinition8FFEB633", }, }, { @@ -1919,7 +1984,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsLambdaTaskDefinitionExecutionRole9469B1E9", + "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRole390812B0", "Arn", ], }, @@ -1929,7 +1994,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsLambda20C12A7D", + "servicecatalogueTESTtaskAwsDelegatedToSecurityAccountEDD7C370", "Arn", ], }, @@ -1937,16 +2002,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsLambdaTaskDefinitionEventsRoleDefaultPolicyCA44BFE7", + "PolicyName": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionEventsRoleDefaultPolicy33EB3CCB", "Roles": [ { - "Ref": "CloudquerySourceAwsLambdaTaskDefinitionEventsRole4D97B167", + "Ref": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionEventsRole84A3EC34", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsLambdaTaskDefinitionExecutionRole9469B1E9": { + "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRole390812B0": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -1971,7 +2036,7 @@ spec: }, { "Key": "Name", - "Value": "AwsLambda", + "Value": "AwsDelegatedToSecurityAccount", }, { "Key": "Stack", @@ -1985,7 +2050,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsLambdaTaskDefinitionExecutionRoleDefaultPolicy8F1E97F4": { + "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRoleDefaultPolicy36910251": { "Properties": { "PolicyDocument": { "Statement": [ @@ -2017,7 +2082,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsLambdaTaskDefinitionCloudquerySourceAwsLambdaFirelensLogGroupBB74129A", + "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionCloudquerySourceAwsDelegatedToSecurityAccountFirelensLogGroupE468B6F3", "Arn", ], }, @@ -2025,18 +2090,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsLambdaTaskDefinitionExecutionRoleDefaultPolicy8F1E97F4", + "PolicyName": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRoleDefaultPolicy36910251", "Roles": [ { - "Ref": "CloudquerySourceAwsLambdaTaskDefinitionExecutionRole9469B1E9", + "Ref": "CloudquerySourceAwsDelegatedToSecurityAccountTaskDefinitionExecutionRole390812B0", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsListOrgsScheduledEventRuleE0997086": { + "CloudquerySourceAwsLambdaScheduledEventRuleC1529FB1": { "Properties": { - "ScheduleExpression": "rate(1 day)", + "ScheduleExpression": "cron(10 1 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -2067,14 +2132,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsListOrgsTaskDefinition15F8AF14", + "Ref": "CloudquerySourceAwsLambdaTaskDefinitionD9609861", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsListOrgsTaskDefinitionEventsRole2B20765E", + "CloudquerySourceAwsLambdaTaskDefinitionEventsRole4D97B167", "Arn", ], }, @@ -2083,7 +2148,37 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsListOrgsTaskDefinition15F8AF14": { + "CloudquerySourceAwsLambdaTaskDefinitionCloudquerySourceAwsLambdaFirelensLogGroupBB74129A": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsLambda", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceAwsLambdaTaskDefinitionD9609861": { "Properties": { "ContainerDefinitions": [ { @@ -2096,15 +2191,16 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_organization* + - aws_lambda_* destinations: - postgresql otel_endpoint: 0.0.0.0:4318 otel_endpoint_insecure: true spec: - accounts: - - id: cq-for-000000000018 - role_arn: arn:aws:iam::000000000018:role/cloudquery-access + org: + member_role_name: cloudquery-access + organization_units: + - ou-123 ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -2121,12 +2217,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsListOrgsAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsLambdaAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsListOrgs", + "Name": "AwsLambda", "Stack": "deploy", "Stage": "TEST", }, @@ -2171,7 +2267,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsListOrgsContainer", + "Name": "CloudquerySource-AwsLambdaContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -2260,7 +2356,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsListOrgsAWSOTELCollector", + "Name": "CloudquerySource-AwsLambdaAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -2273,11 +2369,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_organization%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_lambda_%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsListOrgs", + "Name": "AwsLambda", "Stack": "deploy", "Stage": "TEST", }, @@ -2299,7 +2395,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsListOrgsPostgresContainer", + "Name": "CloudquerySource-AwsLambdaPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -2374,7 +2470,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsListOrgsTaskDefinitionCloudquerySourceAwsListOrgsFirelensLogGroup55343A0C", + "Ref": "CloudquerySourceAwsLambdaTaskDefinitionCloudquerySourceAwsLambdaFirelensLogGroupBB74129A", }, "awslogs-region": { "Ref": "AWS::Region", @@ -2389,18 +2485,18 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsListOrgsFirelens", + "Name": "CloudquerySource-AwsLambdaFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRole2EE6E631", + "CloudquerySourceAwsLambdaTaskDefinitionExecutionRole9469B1E9", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsListOrgsTaskDefinition2D3A9A60", + "Family": "ServiceCatalogueCloudquerySourceAwsLambdaTaskDefinitionDAA42514", "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ @@ -2417,7 +2513,7 @@ spec: }, { "Key": "Name", - "Value": "AwsListOrgs", + "Value": "AwsLambda", }, { "Key": "Stack", @@ -2430,7 +2526,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsListOrgsA233C9DF", + "servicecatalogueTESTtaskAwsLambda20C12A7D", "Arn", ], }, @@ -2451,37 +2547,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsListOrgsTaskDefinitionCloudquerySourceAwsListOrgsFirelensLogGroup55343A0C": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsListOrgs", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", - }, - "CloudquerySourceAwsListOrgsTaskDefinitionEventsRole2B20765E": { + "CloudquerySourceAwsLambdaTaskDefinitionEventsRole4D97B167": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -2506,7 +2572,7 @@ spec: }, { "Key": "Name", - "Value": "AwsListOrgs", + "Value": "AwsLambda", }, { "Key": "Stack", @@ -2520,7 +2586,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsListOrgsTaskDefinitionEventsRoleDefaultPolicy80C259B3": { + "CloudquerySourceAwsLambdaTaskDefinitionEventsRoleDefaultPolicyCA44BFE7": { "Properties": { "PolicyDocument": { "Statement": [ @@ -2538,7 +2604,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsListOrgsTaskDefinition15F8AF14", + "Ref": "CloudquerySourceAwsLambdaTaskDefinitionD9609861", }, }, { @@ -2570,7 +2636,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRole2EE6E631", + "CloudquerySourceAwsLambdaTaskDefinitionExecutionRole9469B1E9", "Arn", ], }, @@ -2580,7 +2646,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsListOrgsA233C9DF", + "servicecatalogueTESTtaskAwsLambda20C12A7D", "Arn", ], }, @@ -2588,16 +2654,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsListOrgsTaskDefinitionEventsRoleDefaultPolicy80C259B3", + "PolicyName": "CloudquerySourceAwsLambdaTaskDefinitionEventsRoleDefaultPolicyCA44BFE7", "Roles": [ { - "Ref": "CloudquerySourceAwsListOrgsTaskDefinitionEventsRole2B20765E", + "Ref": "CloudquerySourceAwsLambdaTaskDefinitionEventsRole4D97B167", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRole2EE6E631": { + "CloudquerySourceAwsLambdaTaskDefinitionExecutionRole9469B1E9": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -2622,7 +2688,7 @@ spec: }, { "Key": "Name", - "Value": "AwsListOrgs", + "Value": "AwsLambda", }, { "Key": "Stack", @@ -2636,7 +2702,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRoleDefaultPolicy1D72FE55": { + "CloudquerySourceAwsLambdaTaskDefinitionExecutionRoleDefaultPolicy8F1E97F4": { "Properties": { "PolicyDocument": { "Statement": [ @@ -2668,7 +2734,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsListOrgsTaskDefinitionCloudquerySourceAwsListOrgsFirelensLogGroup55343A0C", + "CloudquerySourceAwsLambdaTaskDefinitionCloudquerySourceAwsLambdaFirelensLogGroupBB74129A", "Arn", ], }, @@ -2676,18 +2742,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRoleDefaultPolicy1D72FE55", + "PolicyName": "CloudquerySourceAwsLambdaTaskDefinitionExecutionRoleDefaultPolicy8F1E97F4", "Roles": [ { - "Ref": "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRole2EE6E631", + "Ref": "CloudquerySourceAwsLambdaTaskDefinitionExecutionRole9469B1E9", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideAutoScalingGroupsScheduledEventRuleDF674875": { + "CloudquerySourceAwsListOrgsScheduledEventRuleE0997086": { "Properties": { - "ScheduleExpression": "cron(0 0 * * ? *)", + "ScheduleExpression": "rate(1 day)", "State": "ENABLED", "Targets": [ { @@ -2718,14 +2784,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionA838A372", + "Ref": "CloudquerySourceAwsListOrgsTaskDefinition15F8AF14", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionEventsRoleF9568321", + "CloudquerySourceAwsListOrgsTaskDefinitionEventsRole2B20765E", "Arn", ], }, @@ -2734,7 +2800,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionA838A372": { + "CloudquerySourceAwsListOrgsTaskDefinition15F8AF14": { "Properties": { "ContainerDefinitions": [ { @@ -2747,16 +2813,15 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_autoscaling_groups + - aws_organization* destinations: - postgresql otel_endpoint: 0.0.0.0:4318 otel_endpoint_insecure: true spec: - org: - member_role_name: cloudquery-access - organization_units: - - ou-123 + accounts: + - id: cq-for-000000000018 + role_arn: arn:aws:iam::000000000018:role/cloudquery-access ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -2773,12 +2838,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideAutoScalingGroupsAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsListOrgsAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideAutoScalingGroups", + "Name": "AwsListOrgs", "Stack": "deploy", "Stage": "TEST", }, @@ -2823,7 +2888,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideAutoScalingGroupsContainer", + "Name": "CloudquerySource-AwsListOrgsContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -2912,7 +2977,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideAutoScalingGroupsAWSOTELCollector", + "Name": "CloudquerySource-AwsListOrgsAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -2925,11 +2990,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_autoscaling_groups', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_organization%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideAutoScalingGroups", + "Name": "AwsListOrgs", "Stack": "deploy", "Stage": "TEST", }, @@ -2951,7 +3016,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideAutoScalingGroupsPostgresContainer", + "Name": "CloudquerySource-AwsListOrgsPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -3026,7 +3091,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionCloudquerySourceAwsOrgWideAutoScalingGroupsFirelensLogGroup7749AA41", + "Ref": "CloudquerySourceAwsListOrgsTaskDefinitionCloudquerySourceAwsListOrgsFirelensLogGroup55343A0C", }, "awslogs-region": { "Ref": "AWS::Region", @@ -3041,18 +3106,18 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideAutoScalingGroupsFirelens", + "Name": "CloudquerySource-AwsListOrgsFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleCAD7367A", + "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRole2EE6E631", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinition59C11D9A", + "Family": "ServiceCatalogueCloudquerySourceAwsListOrgsTaskDefinition2D3A9A60", "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ @@ -3069,7 +3134,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideAutoScalingGroups", + "Value": "AwsListOrgs", }, { "Key": "Stack", @@ -3082,7 +3147,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideAutoScalingGroups721C2374", + "servicecatalogueTESTtaskAwsListOrgsA233C9DF", "Arn", ], }, @@ -3103,7 +3168,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionCloudquerySourceAwsOrgWideAutoScalingGroupsFirelensLogGroup7749AA41": { + "CloudquerySourceAwsListOrgsTaskDefinitionCloudquerySourceAwsListOrgsFirelensLogGroup55343A0C": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -3118,7 +3183,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideAutoScalingGroups", + "Value": "AwsListOrgs", }, { "Key": "Stack", @@ -3133,7 +3198,46 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionEventsRoleDefaultPolicy1404945D": { + "CloudquerySourceAwsListOrgsTaskDefinitionEventsRole2B20765E": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "events.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsListOrgs", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAwsListOrgsTaskDefinitionEventsRoleDefaultPolicy80C259B3": { "Properties": { "PolicyDocument": { "Statement": [ @@ -3151,7 +3255,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionA838A372", + "Ref": "CloudquerySourceAwsListOrgsTaskDefinition15F8AF14", }, }, { @@ -3183,7 +3287,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleCAD7367A", + "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRole2EE6E631", "Arn", ], }, @@ -3193,7 +3297,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideAutoScalingGroups721C2374", + "servicecatalogueTESTtaskAwsListOrgsA233C9DF", "Arn", ], }, @@ -3201,55 +3305,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionEventsRoleDefaultPolicy1404945D", + "PolicyName": "CloudquerySourceAwsListOrgsTaskDefinitionEventsRoleDefaultPolicy80C259B3", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionEventsRoleF9568321", + "Ref": "CloudquerySourceAwsListOrgsTaskDefinitionEventsRole2B20765E", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionEventsRoleF9568321": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "events.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideAutoScalingGroups", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleCAD7367A": { + "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRole2EE6E631": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -3274,7 +3339,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideAutoScalingGroups", + "Value": "AwsListOrgs", }, { "Key": "Stack", @@ -3288,7 +3353,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleDefaultPolicy2A62C527": { + "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRoleDefaultPolicy1D72FE55": { "Properties": { "PolicyDocument": { "Statement": [ @@ -3320,7 +3385,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionCloudquerySourceAwsOrgWideAutoScalingGroupsFirelensLogGroup7749AA41", + "CloudquerySourceAwsListOrgsTaskDefinitionCloudquerySourceAwsListOrgsFirelensLogGroup55343A0C", "Arn", ], }, @@ -3328,18 +3393,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleDefaultPolicy2A62C527", + "PolicyName": "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRoleDefaultPolicy1D72FE55", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleCAD7367A", + "Ref": "CloudquerySourceAwsListOrgsTaskDefinitionExecutionRole2EE6E631", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideBackupScheduledEventRuleE834008B": { + "CloudquerySourceAwsOrgWideAutoScalingGroupsScheduledEventRuleDF674875": { "Properties": { - "ScheduleExpression": "cron(0 7 * * ? *)", + "ScheduleExpression": "cron(0 0 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -3370,14 +3435,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideBackupTaskDefinition91A7A518", + "Ref": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionA838A372", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideBackupTaskDefinitionEventsRole34C9CC1F", + "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionEventsRoleF9568321", "Arn", ], }, @@ -3386,7 +3451,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsOrgWideBackupTaskDefinition91A7A518": { + "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionA838A372": { "Properties": { "ContainerDefinitions": [ { @@ -3399,9 +3464,7 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_backup_protected_resources - - aws_backup_vaults - - aws_backup_vault_recovery_points + - aws_autoscaling_groups destinations: - postgresql otel_endpoint: 0.0.0.0:4318 @@ -3427,12 +3490,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideBackupAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsOrgWideAutoScalingGroupsAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideBackup", + "Name": "AwsOrgWideAutoScalingGroups", "Stack": "deploy", "Stage": "TEST", }, @@ -3442,7 +3505,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "819MiB", + "Value": "409MiB", }, ], "Essential": true, @@ -3477,7 +3540,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideBackupContainer", + "Name": "CloudquerySource-AwsOrgWideAutoScalingGroupsContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -3566,7 +3629,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideBackupAWSOTELCollector", + "Name": "CloudquerySource-AwsOrgWideAutoScalingGroupsAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -3579,11 +3642,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_backup_protected_resources', 'DAILY'),('aws_backup_vaults', 'DAILY'),('aws_backup_vault_recovery_points', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_autoscaling_groups', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideBackup", + "Name": "AwsOrgWideAutoScalingGroups", "Stack": "deploy", "Stage": "TEST", }, @@ -3605,7 +3668,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideBackupPostgresContainer", + "Name": "CloudquerySource-AwsOrgWideAutoScalingGroupsPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -3680,7 +3743,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideBackupTaskDefinitionCloudquerySourceAwsOrgWideBackupFirelensLogGroup336D3581", + "Ref": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionCloudquerySourceAwsOrgWideAutoScalingGroupsFirelensLogGroup7749AA41", }, "awslogs-region": { "Ref": "AWS::Region", @@ -3695,19 +3758,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideBackupFirelens", + "Name": "CloudquerySource-AwsOrgWideAutoScalingGroupsFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRole55614D8C", + "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleCAD7367A", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideBackupTaskDefinitionC269FE04", - "Memory": "1024", + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinition59C11D9A", + "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -3723,7 +3786,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideBackup", + "Value": "AwsOrgWideAutoScalingGroups", }, { "Key": "Stack", @@ -3736,7 +3799,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideBackupB0D1DA08", + "servicecatalogueTESTtaskAwsOrgWideAutoScalingGroups721C2374", "Arn", ], }, @@ -3757,7 +3820,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsOrgWideBackupTaskDefinitionCloudquerySourceAwsOrgWideBackupFirelensLogGroup336D3581": { + "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionCloudquerySourceAwsOrgWideAutoScalingGroupsFirelensLogGroup7749AA41": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -3772,7 +3835,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideBackup", + "Value": "AwsOrgWideAutoScalingGroups", }, { "Key": "Stack", @@ -3787,46 +3850,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsOrgWideBackupTaskDefinitionEventsRole34C9CC1F": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "events.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideBackup", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceAwsOrgWideBackupTaskDefinitionEventsRoleDefaultPolicyFA9FAC00": { + "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionEventsRoleDefaultPolicy1404945D": { "Properties": { "PolicyDocument": { "Statement": [ @@ -3844,7 +3868,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsOrgWideBackupTaskDefinition91A7A518", + "Ref": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionA838A372", }, }, { @@ -3876,7 +3900,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRole55614D8C", + "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleCAD7367A", "Arn", ], }, @@ -3886,7 +3910,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideBackupB0D1DA08", + "servicecatalogueTESTtaskAwsOrgWideAutoScalingGroups721C2374", "Arn", ], }, @@ -3894,16 +3918,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideBackupTaskDefinitionEventsRoleDefaultPolicyFA9FAC00", + "PolicyName": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionEventsRoleDefaultPolicy1404945D", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideBackupTaskDefinitionEventsRole34C9CC1F", + "Ref": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionEventsRoleF9568321", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRole55614D8C": { + "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionEventsRoleF9568321": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -3911,7 +3935,7 @@ spec: "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { - "Service": "ecs-tasks.amazonaws.com", + "Service": "events.amazonaws.com", }, }, ], @@ -3928,7 +3952,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideBackup", + "Value": "AwsOrgWideAutoScalingGroups", }, { "Key": "Stack", @@ -3942,13 +3966,52 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRoleDefaultPolicy51C80930": { + "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleCAD7367A": { "Properties": { - "PolicyDocument": { + "AssumeRolePolicyDocument": { "Statement": [ { - "Action": [ - "secretsmanager:GetSecretValue", + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideAutoScalingGroups", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleDefaultPolicy2A62C527": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", "secretsmanager:DescribeSecret", ], "Effect": "Allow", @@ -3974,7 +4037,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideBackupTaskDefinitionCloudquerySourceAwsOrgWideBackupFirelensLogGroup336D3581", + "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionCloudquerySourceAwsOrgWideAutoScalingGroupsFirelensLogGroup7749AA41", "Arn", ], }, @@ -3982,18 +4045,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRoleDefaultPolicy51C80930", + "PolicyName": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleDefaultPolicy2A62C527", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRole55614D8C", + "Ref": "CloudquerySourceAwsOrgWideAutoScalingGroupsTaskDefinitionExecutionRoleCAD7367A", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideCertificatesScheduledEventRule7D3B1FA3": { + "CloudquerySourceAwsOrgWideBackupScheduledEventRuleE834008B": { "Properties": { - "ScheduleExpression": "cron(0 1 * * ? *)", + "ScheduleExpression": "cron(0 7 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -4024,14 +4087,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionD275457C", + "Ref": "CloudquerySourceAwsOrgWideBackupTaskDefinition91A7A518", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionEventsRole8BB088B7", + "CloudquerySourceAwsOrgWideBackupTaskDefinitionEventsRole34C9CC1F", "Arn", ], }, @@ -4040,37 +4103,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionCloudquerySourceAwsOrgWideCertificatesFirelensLogGroup91D2F11E": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideCertificates", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", - }, - "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionD275457C": { + "CloudquerySourceAwsOrgWideBackupTaskDefinition91A7A518": { "Properties": { "ContainerDefinitions": [ { @@ -4083,7 +4116,9 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_acm* + - aws_backup_protected_resources + - aws_backup_vaults + - aws_backup_vault_recovery_points destinations: - postgresql otel_endpoint: 0.0.0.0:4318 @@ -4109,12 +4144,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideCertificatesAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsOrgWideBackupAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideCertificates", + "Name": "AwsOrgWideBackup", "Stack": "deploy", "Stage": "TEST", }, @@ -4124,7 +4159,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "409MiB", + "Value": "819MiB", }, ], "Essential": true, @@ -4159,7 +4194,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideCertificatesContainer", + "Name": "CloudquerySource-AwsOrgWideBackupContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -4248,7 +4283,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideCertificatesAWSOTELCollector", + "Name": "CloudquerySource-AwsOrgWideBackupAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -4261,11 +4296,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_acm%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_backup_protected_resources', 'DAILY'),('aws_backup_vaults', 'DAILY'),('aws_backup_vault_recovery_points', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideCertificates", + "Name": "AwsOrgWideBackup", "Stack": "deploy", "Stage": "TEST", }, @@ -4287,7 +4322,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideCertificatesPostgresContainer", + "Name": "CloudquerySource-AwsOrgWideBackupPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -4362,7 +4397,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionCloudquerySourceAwsOrgWideCertificatesFirelensLogGroup91D2F11E", + "Ref": "CloudquerySourceAwsOrgWideBackupTaskDefinitionCloudquerySourceAwsOrgWideBackupFirelensLogGroup336D3581", }, "awslogs-region": { "Ref": "AWS::Region", @@ -4377,19 +4412,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideCertificatesFirelens", + "Name": "CloudquerySource-AwsOrgWideBackupFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleA5032537", + "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRole55614D8C", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideCertificatesTaskDefinition24CEF41E", - "Memory": "512", + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideBackupTaskDefinitionC269FE04", + "Memory": "1024", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -4405,7 +4440,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideCertificates", + "Value": "AwsOrgWideBackup", }, { "Key": "Stack", @@ -4418,7 +4453,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideCertificates496AB720", + "servicecatalogueTESTtaskAwsOrgWideBackupB0D1DA08", "Arn", ], }, @@ -4439,7 +4474,37 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionEventsRole8BB088B7": { + "CloudquerySourceAwsOrgWideBackupTaskDefinitionCloudquerySourceAwsOrgWideBackupFirelensLogGroup336D3581": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideBackup", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceAwsOrgWideBackupTaskDefinitionEventsRole34C9CC1F": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -4464,7 +4529,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideCertificates", + "Value": "AwsOrgWideBackup", }, { "Key": "Stack", @@ -4478,7 +4543,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionEventsRoleDefaultPolicyB2781632": { + "CloudquerySourceAwsOrgWideBackupTaskDefinitionEventsRoleDefaultPolicyFA9FAC00": { "Properties": { "PolicyDocument": { "Statement": [ @@ -4496,7 +4561,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionD275457C", + "Ref": "CloudquerySourceAwsOrgWideBackupTaskDefinition91A7A518", }, }, { @@ -4528,7 +4593,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleA5032537", + "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRole55614D8C", "Arn", ], }, @@ -4538,7 +4603,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideCertificates496AB720", + "servicecatalogueTESTtaskAwsOrgWideBackupB0D1DA08", "Arn", ], }, @@ -4546,16 +4611,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionEventsRoleDefaultPolicyB2781632", + "PolicyName": "CloudquerySourceAwsOrgWideBackupTaskDefinitionEventsRoleDefaultPolicyFA9FAC00", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionEventsRole8BB088B7", + "Ref": "CloudquerySourceAwsOrgWideBackupTaskDefinitionEventsRole34C9CC1F", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleA5032537": { + "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRole55614D8C": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -4580,7 +4645,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideCertificates", + "Value": "AwsOrgWideBackup", }, { "Key": "Stack", @@ -4594,7 +4659,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleDefaultPolicyADA104C9": { + "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRoleDefaultPolicy51C80930": { "Properties": { "PolicyDocument": { "Statement": [ @@ -4626,7 +4691,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionCloudquerySourceAwsOrgWideCertificatesFirelensLogGroup91D2F11E", + "CloudquerySourceAwsOrgWideBackupTaskDefinitionCloudquerySourceAwsOrgWideBackupFirelensLogGroup336D3581", "Arn", ], }, @@ -4634,18 +4699,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleDefaultPolicyADA104C9", + "PolicyName": "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRoleDefaultPolicy51C80930", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleA5032537", + "Ref": "CloudquerySourceAwsOrgWideBackupTaskDefinitionExecutionRole55614D8C", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideCloudFormationScheduledEventRule4E86DEC8": { + "CloudquerySourceAwsOrgWideCertificatesScheduledEventRule7D3B1FA3": { "Properties": { - "ScheduleExpression": "rate(3 hours)", + "ScheduleExpression": "cron(0 1 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -4676,14 +4741,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionFE550760", + "Ref": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionD275457C", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionEventsRole2327379D", + "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionEventsRole8BB088B7", "Arn", ], }, @@ -4692,7 +4757,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionCloudquerySourceAwsOrgWideCloudFormationFirelensLogGroup275B0945": { + "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionCloudquerySourceAwsOrgWideCertificatesFirelensLogGroup91D2F11E": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -4707,7 +4772,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideCloudFormation", + "Value": "AwsOrgWideCertificates", }, { "Key": "Stack", @@ -4722,257 +4787,53 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionEventsRole2327379D": { + "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionD275457C": { "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "events.amazonaws.com", + "ContainerDefinitions": [ + { + "Command": [ + "/bin/sh", + "-c", + "printf 'kind: source +spec: + name: aws + path: cloudquery/aws + version: v27.5.0 + tables: + - aws_acm* + destinations: + - postgresql + otel_endpoint: 0.0.0.0:4318 + otel_endpoint_insecure: true + spec: + org: + member_role_name: cloudquery-access + organization_units: + - ou-123 +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination +spec: + name: postgresql + registry: github + path: cloudquery/postgresql + version: v7.2.0 + migrate_mode: forced + spec: + connection_string: >- + user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 + dbname=postgres sslmode=verify-full +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", + ], + "DependsOn": [ + { + "Condition": "HEALTHY", + "ContainerName": "CloudquerySource-AwsOrgWideCertificatesAWSOTELCollector", }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideCloudFormation", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionEventsRoleDefaultPolicy21B918CD": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": "ecs:RunTask", - "Condition": { - "ArnEquals": { - "ecs:cluster": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], - }, - }, - }, - "Effect": "Allow", - "Resource": { - "Ref": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionFE550760", - }, - }, - { - "Action": "ecs:TagResource", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":ecs:", - { - "Ref": "AWS::Region", - }, - ":*:task/", - { - "Ref": "servicecatalogueCluster5FC34DC5", - }, - "/*", - ], - ], - }, - }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRole7AB90409", - "Arn", - ], - }, - }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideCloudFormationEFD12D82", - "Arn", - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionEventsRoleDefaultPolicy21B918CD", - "Roles": [ - { - "Ref": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionEventsRole2327379D", - }, - ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRole7AB90409": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ecs-tasks.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideCloudFormation", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRoleDefaultPolicy688A215A": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - }, - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "cloudqueryapikeyCCF82F53", - }, - }, - { - "Action": [ - "logs:CreateLogStream", - "logs:PutLogEvents", - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionCloudquerySourceAwsOrgWideCloudFormationFirelensLogGroup275B0945", - "Arn", - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRoleDefaultPolicy688A215A", - "Roles": [ - { - "Ref": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRole7AB90409", - }, - ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionFE550760": { - "Properties": { - "ContainerDefinitions": [ - { - "Command": [ - "/bin/sh", - "-c", - "printf 'kind: source -spec: - name: aws - path: cloudquery/aws - version: v27.5.0 - tables: - - aws_cloudformation_* - destinations: - - postgresql - otel_endpoint: 0.0.0.0:4318 - otel_endpoint_insecure: true - spec: - org: - member_role_name: cloudquery-access - organization_units: - - ou-123 -' > /usr/share/cloudquery/source.yaml;printf 'kind: destination -spec: - name: postgresql - registry: github - path: cloudquery/postgresql - version: v7.2.0 - migrate_mode: forced - spec: - connection_string: >- - user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 - dbname=postgres sslmode=verify-full -' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", - ], - "DependsOn": [ - { - "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideCloudFormationAWSOTELCollector", - }, - ], - "DockerLabels": { - "App": "service-catalogue", - "Name": "AwsOrgWideCloudFormation", - "Stack": "deploy", - "Stage": "TEST", + ], + "DockerLabels": { + "App": "service-catalogue", + "Name": "AwsOrgWideCertificates", + "Stack": "deploy", + "Stage": "TEST", }, "EntryPoint": [ "", @@ -4980,7 +4841,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "819MiB", + "Value": "409MiB", }, ], "Essential": true, @@ -5015,7 +4876,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideCloudFormationContainer", + "Name": "CloudquerySource-AwsOrgWideCertificatesContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -5104,7 +4965,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideCloudFormationAWSOTELCollector", + "Name": "CloudquerySource-AwsOrgWideCertificatesAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -5117,11 +4978,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_cloudformation_%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_acm%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideCloudFormation", + "Name": "AwsOrgWideCertificates", "Stack": "deploy", "Stage": "TEST", }, @@ -5143,7 +5004,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideCloudFormationPostgresContainer", + "Name": "CloudquerySource-AwsOrgWideCertificatesPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -5218,7 +5079,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionCloudquerySourceAwsOrgWideCloudFormationFirelensLogGroup275B0945", + "Ref": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionCloudquerySourceAwsOrgWideCertificatesFirelensLogGroup91D2F11E", }, "awslogs-region": { "Ref": "AWS::Region", @@ -5233,19 +5094,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideCloudFormationFirelens", + "Name": "CloudquerySource-AwsOrgWideCertificatesFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRole7AB90409", + "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleA5032537", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideCloudFormationTaskDefinition3044E3AC", - "Memory": "1024", + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideCertificatesTaskDefinition24CEF41E", + "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -5261,7 +5122,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideCloudFormation", + "Value": "AwsOrgWideCertificates", }, { "Key": "Stack", @@ -5274,7 +5135,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideCloudFormationEFD12D82", + "servicecatalogueTESTtaskAwsOrgWideCertificates496AB720", "Arn", ], }, @@ -5295,9 +5156,213 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsOrgWideCloudwatchAlarmsScheduledEventRule443F1BD5": { + "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionEventsRole8BB088B7": { "Properties": { - "ScheduleExpression": "cron(0 2 * * ? *)", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "events.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideCertificates", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionEventsRoleDefaultPolicyB2781632": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ecs:RunTask", + "Condition": { + "ArnEquals": { + "ecs:cluster": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + }, + }, + "Effect": "Allow", + "Resource": { + "Ref": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionD275457C", + }, + }, + { + "Action": "ecs:TagResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":ecs:", + { + "Ref": "AWS::Region", + }, + ":*:task/", + { + "Ref": "servicecatalogueCluster5FC34DC5", + }, + "/*", + ], + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleA5032537", + "Arn", + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAwsOrgWideCertificates496AB720", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionEventsRoleDefaultPolicyB2781632", + "Roles": [ + { + "Ref": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionEventsRole8BB088B7", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleA5032537": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideCertificates", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleDefaultPolicyADA104C9": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "cloudqueryapikeyCCF82F53", + }, + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionCloudquerySourceAwsOrgWideCertificatesFirelensLogGroup91D2F11E", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleDefaultPolicyADA104C9", + "Roles": [ + { + "Ref": "CloudquerySourceAwsOrgWideCertificatesTaskDefinitionExecutionRoleA5032537", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceAwsOrgWideCloudFormationScheduledEventRule4E86DEC8": { + "Properties": { + "ScheduleExpression": "rate(3 hours)", "State": "ENABLED", "Targets": [ { @@ -5328,14 +5393,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinition1E4F26F4", + "Ref": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionFE550760", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEventsRole23623738", + "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionEventsRole2327379D", "Arn", ], }, @@ -5344,380 +5409,11 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinition1E4F26F4": { + "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionCloudquerySourceAwsOrgWideCloudFormationFirelensLogGroup275B0945": { + "DeletionPolicy": "Retain", "Properties": { - "ContainerDefinitions": [ - { - "Command": [ - "/bin/sh", - "-c", - "printf 'kind: source -spec: - name: aws - path: cloudquery/aws - version: v27.5.0 - tables: - - aws_cloudwatch_alarms - destinations: - - postgresql - otel_endpoint: 0.0.0.0:4318 - otel_endpoint_insecure: true - spec: - org: - member_role_name: cloudquery-access - organization_units: - - ou-123 -' > /usr/share/cloudquery/source.yaml;printf 'kind: destination -spec: - name: postgresql - registry: github - path: cloudquery/postgresql - version: v7.2.0 - migrate_mode: forced - spec: - connection_string: >- - user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 - dbname=postgres sslmode=verify-full -' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", - ], - "DependsOn": [ - { - "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideCloudwatchAlarmsAWSOTELCollector", - }, - ], - "DockerLabels": { - "App": "service-catalogue", - "Name": "AwsOrgWideCloudwatchAlarms", - "Stack": "deploy", - "Stage": "TEST", - }, - "EntryPoint": [ - "", - ], - "Environment": [ - { - "Name": "GOMEMLIMIT", - "Value": "409MiB", - }, - ], - "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:stable", - "LogConfiguration": { - "LogDriver": "awsfirelens", - "Options": { - "Name": "kinesis_streams", - "region": { - "Ref": "AWS::Region", - }, - "retry_limit": "2", - "stream": { - "Ref": "LoggingStreamName", - }, - }, - }, - "MountPoints": [ - { - "ContainerPath": "/usr/share/cloudquery", - "ReadOnly": false, - "SourceVolume": "config-volume", - }, - { - "ContainerPath": "/app/.cq", - "ReadOnly": false, - "SourceVolume": "cloudquery-volume", - }, - { - "ContainerPath": "/tmp", - "ReadOnly": false, - "SourceVolume": "tmp-volume", - }, - ], - "Name": "CloudquerySource-AwsOrgWideCloudwatchAlarmsContainer", - "ReadonlyRootFilesystem": true, - "Secrets": [ - { - "Name": "DB_USERNAME", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":username::", - ], - ], - }, - }, - { - "Name": "DB_HOST", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":host::", - ], - ], - }, - }, - { - "Name": "DB_PASSWORD", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":password::", - ], - ], - }, - }, - { - "Name": "CLOUDQUERY_API_KEY", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "cloudqueryapikeyCCF82F53", - }, - ":api-key::", - ], - ], - }, - }, - ], - }, - { - "Command": [ - "--config=/etc/ecs/ecs-xray.yaml", - ], - "Essential": true, - "HealthCheck": { - "Command": [ - "CMD", - "/healthcheck", - ], - "Interval": 5, - "Retries": 3, - "Timeout": 5, - }, - "Image": "public.ecr.aws/aws-observability/aws-otel-collector:v0.35.0", - "LogConfiguration": { - "LogDriver": "awsfirelens", - "Options": { - "Name": "kinesis_streams", - "region": { - "Ref": "AWS::Region", - }, - "retry_limit": "2", - "stream": { - "Ref": "LoggingStreamName", - }, - }, - }, - "Name": "CloudquerySource-AwsOrgWideCloudwatchAlarmsAWSOTELCollector", - "PortMappings": [ - { - "ContainerPort": 4318, - "Protocol": "tcp", - }, - ], - "ReadonlyRootFilesystem": true, - }, - { - "Command": [ - "/bin/sh", - "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_cloudwatch_alarms', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", - ], - "DockerLabels": { - "App": "service-catalogue", - "Name": "AwsOrgWideCloudwatchAlarms", - "Stack": "deploy", - "Stage": "TEST", - }, - "EntryPoint": [ - "", - ], - "Essential": false, - "Image": "public.ecr.aws/docker/library/postgres:16-alpine", - "LogConfiguration": { - "LogDriver": "awsfirelens", - "Options": { - "Name": "kinesis_streams", - "region": { - "Ref": "AWS::Region", - }, - "retry_limit": "2", - "stream": { - "Ref": "LoggingStreamName", - }, - }, - }, - "Name": "CloudquerySource-AwsOrgWideCloudwatchAlarmsPostgresContainer", - "ReadonlyRootFilesystem": true, - "Secrets": [ - { - "Name": "PGUSER", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":username::", - ], - ], - }, - }, - { - "Name": "PGHOST", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":host::", - ], - ], - }, - }, - { - "Name": "PGPASSWORD", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":password::", - ], - ], - }, - }, - ], - }, - { - "Environment": [ - { - "Name": "STACK", - "Value": "deploy", - }, - { - "Name": "STAGE", - "Value": "TEST", - }, - { - "Name": "APP", - "Value": "service-catalogue", - }, - { - "Name": "GU_REPO", - "Value": "guardian/service-catalogue", - }, - ], - "Essential": true, - "FirelensConfiguration": { - "Type": "fluentbit", - }, - "Image": "ghcr.io/guardian/devx-logs:2", - "LogConfiguration": { - "LogDriver": "awslogs", - "Options": { - "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionCloudquerySourceAwsOrgWideCloudwatchAlarmsFirelensLogGroup953FDC49", - }, - "awslogs-region": { - "Ref": "AWS::Region", - }, - "awslogs-stream-prefix": "deploy/TEST/service-catalogue", - }, - }, - "MountPoints": [ - { - "ContainerPath": "/init", - "ReadOnly": false, - "SourceVolume": "firelens-volume", - }, - ], - "Name": "CloudquerySource-AwsOrgWideCloudwatchAlarmsFirelens", - "ReadonlyRootFilesystem": true, - }, - ], - "Cpu": "256", - "ExecutionRoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRole7DD198FC", - "Arn", - ], - }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEDC746AE", - "Memory": "512", - "NetworkMode": "awsvpc", - "RequiresCompatibilities": [ - "FARGATE", - ], - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideCloudwatchAlarms", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - "TaskRoleArn": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideCloudwatchAlarms5ED2B988", - "Arn", - ], - }, - "Volumes": [ - { - "Name": "config-volume", - }, - { - "Name": "cloudquery-volume", - }, - { - "Name": "tmp-volume", - }, - { - "Name": "firelens-volume", - }, - ], - }, - "Type": "AWS::ECS::TaskDefinition", - }, - "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionCloudquerySourceAwsOrgWideCloudwatchAlarmsFirelensLogGroup953FDC49": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ + "RetentionInDays": 1, + "Tags": [ { "Key": "gu:cdk:version", "Value": "TEST", @@ -5728,7 +5424,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideCloudwatchAlarms", + "Value": "AwsOrgWideCloudFormation", }, { "Key": "Stack", @@ -5743,7 +5439,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEventsRole23623738": { + "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionEventsRole2327379D": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -5768,7 +5464,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideCloudwatchAlarms", + "Value": "AwsOrgWideCloudFormation", }, { "Key": "Stack", @@ -5782,7 +5478,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEventsRoleDefaultPolicyAC0B7FE1": { + "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionEventsRoleDefaultPolicy21B918CD": { "Properties": { "PolicyDocument": { "Statement": [ @@ -5800,7 +5496,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinition1E4F26F4", + "Ref": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionFE550760", }, }, { @@ -5832,7 +5528,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRole7DD198FC", + "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRole7AB90409", "Arn", ], }, @@ -5842,7 +5538,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideCloudwatchAlarms5ED2B988", + "servicecatalogueTESTtaskAwsOrgWideCloudFormationEFD12D82", "Arn", ], }, @@ -5850,16 +5546,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEventsRoleDefaultPolicyAC0B7FE1", + "PolicyName": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionEventsRoleDefaultPolicy21B918CD", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEventsRole23623738", + "Ref": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionEventsRole2327379D", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRole7DD198FC": { + "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRole7AB90409": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -5884,7 +5580,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideCloudwatchAlarms", + "Value": "AwsOrgWideCloudFormation", }, { "Key": "Stack", @@ -5898,7 +5594,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRoleDefaultPolicy6B41E6E0": { + "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRoleDefaultPolicy688A215A": { "Properties": { "PolicyDocument": { "Statement": [ @@ -5930,7 +5626,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionCloudquerySourceAwsOrgWideCloudwatchAlarmsFirelensLogGroup953FDC49", + "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionCloudquerySourceAwsOrgWideCloudFormationFirelensLogGroup275B0945", "Arn", ], }, @@ -5938,65 +5634,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRoleDefaultPolicy6B41E6E0", + "PolicyName": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRoleDefaultPolicy688A215A", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRole7DD198FC", + "Ref": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRole7AB90409", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideDynamoDBScheduledEventRule3C36042D": { - "Properties": { - "ScheduleExpression": "cron(0 5 * * ? *)", - "State": "ENABLED", - "Targets": [ - { - "Arn": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], - }, - "EcsParameters": { - "LaunchType": "FARGATE", - "NetworkConfiguration": { - "AwsVpcConfiguration": { - "AssignPublicIp": "DISABLED", - "SecurityGroups": [ - { - "Fn::GetAtt": [ - "PostgresAccessSecurityGroupServicecatalogue03C78F14", - "GroupId", - ], - }, - ], - "Subnets": { - "Ref": "PrivateSubnets", - }, - }, - }, - "PropagateTags": "TASK_DEFINITION", - "TaskCount": 1, - "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinition8C3ACD16", - }, - }, - "Id": "Target0", - "Input": "{}", - "RoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionEventsRoleA777C8FA", - "Arn", - ], - }, - }, - ], - }, - "Type": "AWS::Events::Rule", - }, - "CloudquerySourceAwsOrgWideDynamoDBTaskDefinition8C3ACD16": { + "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionFE550760": { "Properties": { "ContainerDefinitions": [ { @@ -6009,7 +5656,7 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_dynamodb* + - aws_cloudformation_* destinations: - postgresql otel_endpoint: 0.0.0.0:4318 @@ -6018,188 +5665,43 @@ spec: org: member_role_name: cloudquery-access organization_units: - - ou-123 -' > /usr/share/cloudquery/source.yaml;printf 'kind: destination -spec: - name: postgresql - registry: github - path: cloudquery/postgresql - version: v7.2.0 - migrate_mode: forced - spec: - connection_string: >- - user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 - dbname=postgres sslmode=verify-full -' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", - ], - "DependsOn": [ - { - "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideDynamoDBAWSOTELCollector", - }, - ], - "DockerLabels": { - "App": "service-catalogue", - "Name": "AwsOrgWideDynamoDB", - "Stack": "deploy", - "Stage": "TEST", - }, - "EntryPoint": [ - "", - ], - "Environment": [ - { - "Name": "GOMEMLIMIT", - "Value": "409MiB", - }, - ], - "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:stable", - "LogConfiguration": { - "LogDriver": "awsfirelens", - "Options": { - "Name": "kinesis_streams", - "region": { - "Ref": "AWS::Region", - }, - "retry_limit": "2", - "stream": { - "Ref": "LoggingStreamName", - }, - }, - }, - "MountPoints": [ - { - "ContainerPath": "/usr/share/cloudquery", - "ReadOnly": false, - "SourceVolume": "config-volume", - }, - { - "ContainerPath": "/app/.cq", - "ReadOnly": false, - "SourceVolume": "cloudquery-volume", - }, - { - "ContainerPath": "/tmp", - "ReadOnly": false, - "SourceVolume": "tmp-volume", - }, - ], - "Name": "CloudquerySource-AwsOrgWideDynamoDBContainer", - "ReadonlyRootFilesystem": true, - "Secrets": [ - { - "Name": "DB_USERNAME", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":username::", - ], - ], - }, - }, - { - "Name": "DB_HOST", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":host::", - ], - ], - }, - }, - { - "Name": "DB_PASSWORD", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":password::", - ], - ], - }, - }, - { - "Name": "CLOUDQUERY_API_KEY", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "cloudqueryapikeyCCF82F53", - }, - ":api-key::", - ], - ], - }, - }, - ], - }, - { - "Command": [ - "--config=/etc/ecs/ecs-xray.yaml", - ], - "Essential": true, - "HealthCheck": { - "Command": [ - "CMD", - "/healthcheck", - ], - "Interval": 5, - "Retries": 3, - "Timeout": 5, - }, - "Image": "public.ecr.aws/aws-observability/aws-otel-collector:v0.35.0", - "LogConfiguration": { - "LogDriver": "awsfirelens", - "Options": { - "Name": "kinesis_streams", - "region": { - "Ref": "AWS::Region", - }, - "retry_limit": "2", - "stream": { - "Ref": "LoggingStreamName", - }, - }, - }, - "Name": "CloudquerySource-AwsOrgWideDynamoDBAWSOTELCollector", - "PortMappings": [ + - ou-123 +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination +spec: + name: postgresql + registry: github + path: cloudquery/postgresql + version: v7.2.0 + migrate_mode: forced + spec: + connection_string: >- + user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 + dbname=postgres sslmode=verify-full +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", + ], + "DependsOn": [ { - "ContainerPort": 4318, - "Protocol": "tcp", + "Condition": "HEALTHY", + "ContainerName": "CloudquerySource-AwsOrgWideCloudFormationAWSOTELCollector", }, ], - "ReadonlyRootFilesystem": true, - }, - { - "Command": [ - "/bin/sh", - "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_dynamodb%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", - ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideDynamoDB", + "Name": "AwsOrgWideCloudFormation", "Stack": "deploy", "Stage": "TEST", }, "EntryPoint": [ "", ], - "Essential": false, - "Image": "public.ecr.aws/docker/library/postgres:16-alpine", + "Environment": [ + { + "Name": "GOMEMLIMIT", + "Value": "819MiB", + }, + ], + "Essential": true, + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:stable", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -6213,11 +5715,28 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideDynamoDBPostgresContainer", + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], + "Name": "CloudquerySource-AwsOrgWideCloudFormationContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { - "Name": "PGUSER", + "Name": "DB_USERNAME", "ValueFrom": { "Fn::Join": [ "", @@ -6231,7 +5750,7 @@ spec: }, }, { - "Name": "PGHOST", + "Name": "DB_HOST", "ValueFrom": { "Fn::Join": [ "", @@ -6245,7 +5764,7 @@ spec: }, }, { - "Name": "PGPASSWORD", + "Name": "DB_PASSWORD", "ValueFrom": { "Fn::Join": [ "", @@ -6255,325 +5774,199 @@ spec: }, ":password::", ], - ], - }, - }, - ], - }, - { - "Environment": [ - { - "Name": "STACK", - "Value": "deploy", - }, - { - "Name": "STAGE", - "Value": "TEST", - }, - { - "Name": "APP", - "Value": "service-catalogue", - }, - { - "Name": "GU_REPO", - "Value": "guardian/service-catalogue", - }, - ], - "Essential": true, - "FirelensConfiguration": { - "Type": "fluentbit", - }, - "Image": "ghcr.io/guardian/devx-logs:2", - "LogConfiguration": { - "LogDriver": "awslogs", - "Options": { - "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionCloudquerySourceAwsOrgWideDynamoDBFirelensLogGroup7F40CCBC", - }, - "awslogs-region": { - "Ref": "AWS::Region", - }, - "awslogs-stream-prefix": "deploy/TEST/service-catalogue", - }, - }, - "MountPoints": [ - { - "ContainerPath": "/init", - "ReadOnly": false, - "SourceVolume": "firelens-volume", - }, - ], - "Name": "CloudquerySource-AwsOrgWideDynamoDBFirelens", - "ReadonlyRootFilesystem": true, - }, - ], - "Cpu": "256", - "ExecutionRoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleE7903999", - "Arn", - ], - }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideDynamoDBTaskDefinition5F181D12", - "Memory": "512", - "NetworkMode": "awsvpc", - "RequiresCompatibilities": [ - "FARGATE", - ], - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideDynamoDB", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - "TaskRoleArn": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideDynamoDBB67BC817", - "Arn", - ], - }, - "Volumes": [ - { - "Name": "config-volume", - }, - { - "Name": "cloudquery-volume", - }, - { - "Name": "tmp-volume", - }, - { - "Name": "firelens-volume", - }, - ], - }, - "Type": "AWS::ECS::TaskDefinition", - }, - "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionCloudquerySourceAwsOrgWideDynamoDBFirelensLogGroup7F40CCBC": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideDynamoDB", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", - }, - "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionEventsRoleA777C8FA": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "events.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideDynamoDB", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionEventsRoleDefaultPolicyAA01E9CA": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": "ecs:RunTask", - "Condition": { - "ArnEquals": { - "ecs:cluster": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], - }, + ], }, }, - "Effect": "Allow", - "Resource": { - "Ref": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinition8C3ACD16", + { + "Name": "CLOUDQUERY_API_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "cloudqueryapikeyCCF82F53", + }, + ":api-key::", + ], + ], + }, }, + ], + }, + { + "Command": [ + "--config=/etc/ecs/ecs-xray.yaml", + ], + "Essential": true, + "HealthCheck": { + "Command": [ + "CMD", + "/healthcheck", + ], + "Interval": 5, + "Retries": 3, + "Timeout": 5, }, - { - "Action": "ecs:TagResource", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":ecs:", - { - "Ref": "AWS::Region", - }, - ":*:task/", - { - "Ref": "servicecatalogueCluster5FC34DC5", - }, - "/*", - ], - ], + "Image": "public.ecr.aws/aws-observability/aws-otel-collector:v0.35.0", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, }, }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleE7903999", - "Arn", - ], + "Name": "CloudquerySource-AwsOrgWideCloudFormationAWSOTELCollector", + "PortMappings": [ + { + "ContainerPort": 4318, + "Protocol": "tcp", }, + ], + "ReadonlyRootFilesystem": true, + }, + { + "Command": [ + "/bin/sh", + "-c", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_cloudformation_%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + ], + "DockerLabels": { + "App": "service-catalogue", + "Name": "AwsOrgWideCloudFormation", + "Stack": "deploy", + "Stage": "TEST", }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideDynamoDBB67BC817", - "Arn", - ], + "EntryPoint": [ + "", + ], + "Essential": false, + "Image": "public.ecr.aws/docker/library/postgres:16-alpine", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, }, }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionEventsRoleDefaultPolicyAA01E9CA", - "Roles": [ - { - "Ref": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionEventsRoleA777C8FA", + "Name": "CloudquerySource-AwsOrgWideCloudFormationPostgresContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ + { + "Name": "PGUSER", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":username::", + ], + ], + }, + }, + { + "Name": "PGHOST", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":host::", + ], + ], + }, + }, + { + "Name": "PGPASSWORD", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":password::", + ], + ], + }, + }, + ], }, - ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleDefaultPolicy5027618D": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + { + "Environment": [ + { + "Name": "STACK", + "Value": "deploy", }, - }, - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "cloudqueryapikeyCCF82F53", + { + "Name": "STAGE", + "Value": "TEST", }, - }, - { - "Action": [ - "logs:CreateLogStream", - "logs:PutLogEvents", - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionCloudquerySourceAwsOrgWideDynamoDBFirelensLogGroup7F40CCBC", - "Arn", - ], + { + "Name": "APP", + "Value": "service-catalogue", + }, + { + "Name": "GU_REPO", + "Value": "guardian/service-catalogue", }, + ], + "Essential": true, + "FirelensConfiguration": { + "Type": "fluentbit", }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleDefaultPolicy5027618D", - "Roles": [ - { - "Ref": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleE7903999", - }, - ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleE7903999": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ecs-tasks.amazonaws.com", + "Image": "ghcr.io/guardian/devx-logs:2", + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionCloudquerySourceAwsOrgWideCloudFormationFirelensLogGroup275B0945", + }, + "awslogs-region": { + "Ref": "AWS::Region", + }, + "awslogs-stream-prefix": "deploy/TEST/service-catalogue", }, }, + "MountPoints": [ + { + "ContainerPath": "/init", + "ReadOnly": false, + "SourceVolume": "firelens-volume", + }, + ], + "Name": "CloudquerySource-AwsOrgWideCloudFormationFirelens", + "ReadonlyRootFilesystem": true, + }, + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideCloudFormationTaskDefinitionExecutionRole7AB90409", + "Arn", ], - "Version": "2012-10-17", }, + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideCloudFormationTaskDefinition3044E3AC", + "Memory": "1024", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE", + ], "Tags": [ { "Key": "gu:cdk:version", @@ -6585,7 +5978,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideDynamoDB", + "Value": "AwsOrgWideCloudFormation", }, { "Key": "Stack", @@ -6596,12 +5989,32 @@ spec: "Value": "TEST", }, ], + "TaskRoleArn": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAwsOrgWideCloudFormationEFD12D82", + "Arn", + ], + }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + { + "Name": "firelens-volume", + }, + ], }, - "Type": "AWS::IAM::Role", + "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsOrgWideEc2ScheduledEventRule2C1BD783": { + "CloudquerySourceAwsOrgWideCloudwatchAlarmsScheduledEventRule443F1BD5": { "Properties": { - "ScheduleExpression": "rate(30 minutes)", + "ScheduleExpression": "cron(0 2 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -6632,14 +6045,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideEc2TaskDefinition8CC129B6", + "Ref": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinition1E4F26F4", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideEc2TaskDefinitionEventsRole536EB5E7", + "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEventsRole23623738", "Arn", ], }, @@ -6648,7 +6061,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsOrgWideEc2TaskDefinition8CC129B6": { + "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinition1E4F26F4": { "Properties": { "ContainerDefinitions": [ { @@ -6661,9 +6074,7 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_ec2_instances - - aws_ec2_security_groups - - aws_ec2_images + - aws_cloudwatch_alarms destinations: - postgresql otel_endpoint: 0.0.0.0:4318 @@ -6689,16 +6100,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideEc2AWSOTELCollector", - }, - { - "Condition": "SUCCESS", - "ContainerName": "CloudquerySource-AwsOrgWideEc2AwsCli", + "ContainerName": "CloudquerySource-AwsOrgWideCloudwatchAlarmsAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideEc2", + "Name": "AwsOrgWideCloudwatchAlarms", "Stack": "deploy", "Stage": "TEST", }, @@ -6708,7 +6115,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "819MiB", + "Value": "409MiB", }, ], "Essential": true, @@ -6743,7 +6150,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideEc2Container", + "Name": "CloudquerySource-AwsOrgWideCloudwatchAlarmsContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -6832,7 +6239,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideEc2AWSOTELCollector", + "Name": "CloudquerySource-AwsOrgWideCloudwatchAlarmsAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -6841,42 +6248,15 @@ spec: ], "ReadonlyRootFilesystem": true, }, - { - "Command": [ - "/bin/bash", - "-c", - "ECS_CLUSTER=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.Cluster');ECS_FAMILY=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.Family');ECS_TASK_ARN=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.TaskARN');RUNNING=$(aws ecs list-tasks --cluster $ECS_CLUSTER --family $ECS_FAMILY | jq '.taskArns | length');[[ \${RUNNING} > 1 ]] && exit 114 || exit 0", - ], - "EntryPoint": [ - "", - ], - "Essential": false, - "Image": "ghcr.io/guardian/service-catalogue/singleton:stable", - "LogConfiguration": { - "LogDriver": "awsfirelens", - "Options": { - "Name": "kinesis_streams", - "region": { - "Ref": "AWS::Region", - }, - "retry_limit": "2", - "stream": { - "Ref": "LoggingStreamName", - }, - }, - }, - "Name": "CloudquerySource-AwsOrgWideEc2AwsCli", - "ReadonlyRootFilesystem": true, - }, { "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_ec2_instances', 'DAILY'),('aws_ec2_security_groups', 'DAILY'),('aws_ec2_images', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_cloudwatch_alarms', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideEc2", + "Name": "AwsOrgWideCloudwatchAlarms", "Stack": "deploy", "Stage": "TEST", }, @@ -6898,7 +6278,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideEc2PostgresContainer", + "Name": "CloudquerySource-AwsOrgWideCloudwatchAlarmsPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -6973,7 +6353,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideEc2TaskDefinitionCloudquerySourceAwsOrgWideEc2FirelensLogGroupE01DEFEE", + "Ref": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionCloudquerySourceAwsOrgWideCloudwatchAlarmsFirelensLogGroup953FDC49", }, "awslogs-region": { "Ref": "AWS::Region", @@ -6988,19 +6368,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideEc2Firelens", + "Name": "CloudquerySource-AwsOrgWideCloudwatchAlarmsFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleD0810570", + "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRole7DD198FC", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideEc2TaskDefinitionBBF19A7E", - "Memory": "1024", + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEDC746AE", + "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -7016,7 +6396,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideEc2", + "Value": "AwsOrgWideCloudwatchAlarms", }, { "Key": "Stack", @@ -7029,7 +6409,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideEc2D13594C5", + "servicecatalogueTESTtaskAwsOrgWideCloudwatchAlarms5ED2B988", "Arn", ], }, @@ -7050,7 +6430,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsOrgWideEc2TaskDefinitionCloudquerySourceAwsOrgWideEc2FirelensLogGroupE01DEFEE": { + "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionCloudquerySourceAwsOrgWideCloudwatchAlarmsFirelensLogGroup953FDC49": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -7065,7 +6445,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideEc2", + "Value": "AwsOrgWideCloudwatchAlarms", }, { "Key": "Stack", @@ -7080,7 +6460,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsOrgWideEc2TaskDefinitionEventsRole536EB5E7": { + "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEventsRole23623738": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -7105,7 +6485,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideEc2", + "Value": "AwsOrgWideCloudwatchAlarms", }, { "Key": "Stack", @@ -7119,7 +6499,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideEc2TaskDefinitionEventsRoleDefaultPolicy5D07AAE8": { + "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEventsRoleDefaultPolicyAC0B7FE1": { "Properties": { "PolicyDocument": { "Statement": [ @@ -7137,7 +6517,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsOrgWideEc2TaskDefinition8CC129B6", + "Ref": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinition1E4F26F4", }, }, { @@ -7169,7 +6549,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleD0810570", + "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRole7DD198FC", "Arn", ], }, @@ -7179,7 +6559,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideEc2D13594C5", + "servicecatalogueTESTtaskAwsOrgWideCloudwatchAlarms5ED2B988", "Arn", ], }, @@ -7187,16 +6567,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideEc2TaskDefinitionEventsRoleDefaultPolicy5D07AAE8", + "PolicyName": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEventsRoleDefaultPolicyAC0B7FE1", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideEc2TaskDefinitionEventsRole536EB5E7", + "Ref": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionEventsRole23623738", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleD0810570": { + "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRole7DD198FC": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -7221,7 +6601,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideEc2", + "Value": "AwsOrgWideCloudwatchAlarms", }, { "Key": "Stack", @@ -7235,7 +6615,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleDefaultPolicy15E52B2D": { + "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRoleDefaultPolicy6B41E6E0": { "Properties": { "PolicyDocument": { "Statement": [ @@ -7267,7 +6647,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideEc2TaskDefinitionCloudquerySourceAwsOrgWideEc2FirelensLogGroupE01DEFEE", + "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionCloudquerySourceAwsOrgWideCloudwatchAlarmsFirelensLogGroup953FDC49", "Arn", ], }, @@ -7275,18 +6655,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleDefaultPolicy15E52B2D", + "PolicyName": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRoleDefaultPolicy6B41E6E0", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleD0810570", + "Ref": "CloudquerySourceAwsOrgWideCloudwatchAlarmsTaskDefinitionExecutionRole7DD198FC", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideIamCredentialReportsScheduledEventRule17238EC8": { + "CloudquerySourceAwsOrgWideDynamoDBScheduledEventRule3C36042D": { "Properties": { - "ScheduleExpression": "rate(4 hours)", + "ScheduleExpression": "cron(0 5 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -7317,14 +6697,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinition7300D1E2", + "Ref": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinition8C3ACD16", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionEventsRole20CD424D", + "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionEventsRoleA777C8FA", "Arn", ], }, @@ -7333,7 +6713,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinition7300D1E2": { + "CloudquerySourceAwsOrgWideDynamoDBTaskDefinition8C3ACD16": { "Properties": { "ContainerDefinitions": [ { @@ -7346,7 +6726,7 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_iam_credential_reports + - aws_dynamodb* destinations: - postgresql otel_endpoint: 0.0.0.0:4318 @@ -7372,12 +6752,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideIamCredentialReportsAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsOrgWideDynamoDBAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideIamCredentialReports", + "Name": "AwsOrgWideDynamoDB", "Stack": "deploy", "Stage": "TEST", }, @@ -7387,7 +6767,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "819MiB", + "Value": "409MiB", }, ], "Essential": true, @@ -7422,7 +6802,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideIamCredentialReportsContainer", + "Name": "CloudquerySource-AwsOrgWideDynamoDBContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -7511,7 +6891,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideIamCredentialReportsAWSOTELCollector", + "Name": "CloudquerySource-AwsOrgWideDynamoDBAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -7524,11 +6904,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_iam_credential_reports', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_dynamodb%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideIamCredentialReports", + "Name": "AwsOrgWideDynamoDB", "Stack": "deploy", "Stage": "TEST", }, @@ -7550,7 +6930,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideIamCredentialReportsPostgresContainer", + "Name": "CloudquerySource-AwsOrgWideDynamoDBPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -7625,7 +7005,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionCloudquerySourceAwsOrgWideIamCredentialReportsFirelensLogGroupB7D88F87", + "Ref": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionCloudquerySourceAwsOrgWideDynamoDBFirelensLogGroup7F40CCBC", }, "awslogs-region": { "Ref": "AWS::Region", @@ -7640,19 +7020,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideIamCredentialReportsFirelens", + "Name": "CloudquerySource-AwsOrgWideDynamoDBFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRole02E86B04", + "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleE7903999", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinition3443A1FA", - "Memory": "1024", + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideDynamoDBTaskDefinition5F181D12", + "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -7668,7 +7048,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideIamCredentialReports", + "Value": "AwsOrgWideDynamoDB", }, { "Key": "Stack", @@ -7681,7 +7061,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideIamCredentialReportsD1A28FD6", + "servicecatalogueTESTtaskAwsOrgWideDynamoDBB67BC817", "Arn", ], }, @@ -7702,7 +7082,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionCloudquerySourceAwsOrgWideIamCredentialReportsFirelensLogGroupB7D88F87": { + "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionCloudquerySourceAwsOrgWideDynamoDBFirelensLogGroup7F40CCBC": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -7717,7 +7097,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideIamCredentialReports", + "Value": "AwsOrgWideDynamoDB", }, { "Key": "Stack", @@ -7732,7 +7112,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionEventsRole20CD424D": { + "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionEventsRoleA777C8FA": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -7757,7 +7137,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideIamCredentialReports", + "Value": "AwsOrgWideDynamoDB", }, { "Key": "Stack", @@ -7771,7 +7151,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionEventsRoleDefaultPolicyEDC4DC94": { + "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionEventsRoleDefaultPolicyAA01E9CA": { "Properties": { "PolicyDocument": { "Statement": [ @@ -7789,7 +7169,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinition7300D1E2", + "Ref": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinition8C3ACD16", }, }, { @@ -7821,7 +7201,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRole02E86B04", + "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleE7903999", "Arn", ], }, @@ -7831,63 +7211,24 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideIamCredentialReportsD1A28FD6", + "servicecatalogueTESTtaskAwsOrgWideDynamoDBB67BC817", "Arn", ], }, }, ], "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionEventsRoleDefaultPolicyEDC4DC94", - "Roles": [ - { - "Ref": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionEventsRole20CD424D", - }, - ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRole02E86B04": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ecs-tasks.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideIamCredentialReports", - }, - { - "Key": "Stack", - "Value": "deploy", - }, + }, + "PolicyName": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionEventsRoleDefaultPolicyAA01E9CA", + "Roles": [ { - "Key": "Stage", - "Value": "TEST", + "Ref": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionEventsRoleA777C8FA", }, ], }, - "Type": "AWS::IAM::Role", + "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRoleDefaultPolicy0BBE996D": { + "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleDefaultPolicy5027618D": { "Properties": { "PolicyDocument": { "Statement": [ @@ -7919,7 +7260,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionCloudquerySourceAwsOrgWideIamCredentialReportsFirelensLogGroupB7D88F87", + "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionCloudquerySourceAwsOrgWideDynamoDBFirelensLogGroup7F40CCBC", "Arn", ], }, @@ -7927,18 +7268,57 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRoleDefaultPolicy0BBE996D", + "PolicyName": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleDefaultPolicy5027618D", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRole02E86B04", + "Ref": "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleE7903999", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideInspectorScheduledEventRule127B8502": { + "CloudquerySourceAwsOrgWideDynamoDBTaskDefinitionExecutionRoleE7903999": { "Properties": { - "ScheduleExpression": "cron(0 3 * * ? *)", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideDynamoDB", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAwsOrgWideEc2ScheduledEventRule2C1BD783": { + "Properties": { + "ScheduleExpression": "rate(30 minutes)", "State": "ENABLED", "Targets": [ { @@ -7969,14 +7349,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionBABA9F5D", + "Ref": "CloudquerySourceAwsOrgWideEc2TaskDefinition8CC129B6", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideInspectorTaskDefinitionEventsRoleD22CD481", + "CloudquerySourceAwsOrgWideEc2TaskDefinitionEventsRole536EB5E7", "Arn", ], }, @@ -7985,7 +7365,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsOrgWideInspectorTaskDefinitionBABA9F5D": { + "CloudquerySourceAwsOrgWideEc2TaskDefinition8CC129B6": { "Properties": { "ContainerDefinitions": [ { @@ -7998,8 +7378,9 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_inspector_findings - - aws_inspector2_findings + - aws_ec2_instances + - aws_ec2_security_groups + - aws_ec2_images destinations: - postgresql otel_endpoint: 0.0.0.0:4318 @@ -8025,12 +7406,16 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideInspectorAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsOrgWideEc2AWSOTELCollector", + }, + { + "Condition": "SUCCESS", + "ContainerName": "CloudquerySource-AwsOrgWideEc2AwsCli", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideInspector", + "Name": "AwsOrgWideEc2", "Stack": "deploy", "Stage": "TEST", }, @@ -8075,7 +7460,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideInspectorContainer", + "Name": "CloudquerySource-AwsOrgWideEc2Container", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -8164,7 +7549,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideInspectorAWSOTELCollector", + "Name": "CloudquerySource-AwsOrgWideEc2AWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -8173,15 +7558,42 @@ spec: ], "ReadonlyRootFilesystem": true, }, + { + "Command": [ + "/bin/bash", + "-c", + "ECS_CLUSTER=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.Cluster');ECS_FAMILY=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.Family');ECS_TASK_ARN=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.TaskARN');RUNNING=$(aws ecs list-tasks --cluster $ECS_CLUSTER --family $ECS_FAMILY | jq '.taskArns | length');[[ \${RUNNING} > 1 ]] && exit 114 || exit 0", + ], + "EntryPoint": [ + "", + ], + "Essential": false, + "Image": "ghcr.io/guardian/service-catalogue/singleton:stable", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, + }, + }, + "Name": "CloudquerySource-AwsOrgWideEc2AwsCli", + "ReadonlyRootFilesystem": true, + }, { "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_inspector_findings', 'DAILY'),('aws_inspector2_findings', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_ec2_instances', 'DAILY'),('aws_ec2_security_groups', 'DAILY'),('aws_ec2_images', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideInspector", + "Name": "AwsOrgWideEc2", "Stack": "deploy", "Stage": "TEST", }, @@ -8203,7 +7615,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideInspectorPostgresContainer", + "Name": "CloudquerySource-AwsOrgWideEc2PostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -8278,7 +7690,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionCloudquerySourceAwsOrgWideInspectorFirelensLogGroupAFBF5583", + "Ref": "CloudquerySourceAwsOrgWideEc2TaskDefinitionCloudquerySourceAwsOrgWideEc2FirelensLogGroupE01DEFEE", }, "awslogs-region": { "Ref": "AWS::Region", @@ -8293,18 +7705,18 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideInspectorFirelens", + "Name": "CloudquerySource-AwsOrgWideEc2Firelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRole89A5DE0C", + "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleD0810570", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideInspectorTaskDefinitionDC3485A7", + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideEc2TaskDefinitionBBF19A7E", "Memory": "1024", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ @@ -8321,7 +7733,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideInspector", + "Value": "AwsOrgWideEc2", }, { "Key": "Stack", @@ -8334,7 +7746,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideInspector7DFAA956", + "servicecatalogueTESTtaskAwsOrgWideEc2D13594C5", "Arn", ], }, @@ -8355,7 +7767,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsOrgWideInspectorTaskDefinitionCloudquerySourceAwsOrgWideInspectorFirelensLogGroupAFBF5583": { + "CloudquerySourceAwsOrgWideEc2TaskDefinitionCloudquerySourceAwsOrgWideEc2FirelensLogGroupE01DEFEE": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -8370,7 +7782,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideInspector", + "Value": "AwsOrgWideEc2", }, { "Key": "Stack", @@ -8385,7 +7797,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsOrgWideInspectorTaskDefinitionEventsRoleD22CD481": { + "CloudquerySourceAwsOrgWideEc2TaskDefinitionEventsRole536EB5E7": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -8410,7 +7822,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideInspector", + "Value": "AwsOrgWideEc2", }, { "Key": "Stack", @@ -8424,7 +7836,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideInspectorTaskDefinitionEventsRoleDefaultPolicyC087EC6A": { + "CloudquerySourceAwsOrgWideEc2TaskDefinitionEventsRoleDefaultPolicy5D07AAE8": { "Properties": { "PolicyDocument": { "Statement": [ @@ -8442,7 +7854,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionBABA9F5D", + "Ref": "CloudquerySourceAwsOrgWideEc2TaskDefinition8CC129B6", }, }, { @@ -8474,7 +7886,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRole89A5DE0C", + "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleD0810570", "Arn", ], }, @@ -8484,7 +7896,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideInspector7DFAA956", + "servicecatalogueTESTtaskAwsOrgWideEc2D13594C5", "Arn", ], }, @@ -8492,16 +7904,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionEventsRoleDefaultPolicyC087EC6A", + "PolicyName": "CloudquerySourceAwsOrgWideEc2TaskDefinitionEventsRoleDefaultPolicy5D07AAE8", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionEventsRoleD22CD481", + "Ref": "CloudquerySourceAwsOrgWideEc2TaskDefinitionEventsRole536EB5E7", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRole89A5DE0C": { + "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleD0810570": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -8526,7 +7938,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideInspector", + "Value": "AwsOrgWideEc2", }, { "Key": "Stack", @@ -8540,7 +7952,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRoleDefaultPolicy546CCF04": { + "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleDefaultPolicy15E52B2D": { "Properties": { "PolicyDocument": { "Statement": [ @@ -8572,7 +7984,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideInspectorTaskDefinitionCloudquerySourceAwsOrgWideInspectorFirelensLogGroupAFBF5583", + "CloudquerySourceAwsOrgWideEc2TaskDefinitionCloudquerySourceAwsOrgWideEc2FirelensLogGroupE01DEFEE", "Arn", ], }, @@ -8580,18 +7992,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRoleDefaultPolicy546CCF04", + "PolicyName": "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleDefaultPolicy15E52B2D", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRole89A5DE0C", + "Ref": "CloudquerySourceAwsOrgWideEc2TaskDefinitionExecutionRoleD0810570", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideLoadBalancersScheduledEventRuleE82A6C43": { + "CloudquerySourceAwsOrgWideIamCredentialReportsScheduledEventRule17238EC8": { "Properties": { - "ScheduleExpression": "rate(30 minutes)", + "ScheduleExpression": "rate(4 hours)", "State": "ENABLED", "Targets": [ { @@ -8622,14 +8034,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinition333F61F5", + "Ref": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinition7300D1E2", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionEventsRole4A9EDE15", + "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionEventsRole20CD424D", "Arn", ], }, @@ -8638,7 +8050,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinition333F61F5": { + "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinition7300D1E2": { "Properties": { "ContainerDefinitions": [ { @@ -8651,8 +8063,7 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_elbv1_* - - aws_elbv2_* + - aws_iam_credential_reports destinations: - postgresql otel_endpoint: 0.0.0.0:4318 @@ -8678,12 +8089,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideLoadBalancersAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsOrgWideIamCredentialReportsAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideLoadBalancers", + "Name": "AwsOrgWideIamCredentialReports", "Stack": "deploy", "Stage": "TEST", }, @@ -8693,7 +8104,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "409MiB", + "Value": "819MiB", }, ], "Essential": true, @@ -8728,7 +8139,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideLoadBalancersContainer", + "Name": "CloudquerySource-AwsOrgWideIamCredentialReportsContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -8817,7 +8228,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideLoadBalancersAWSOTELCollector", + "Name": "CloudquerySource-AwsOrgWideIamCredentialReportsAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -8830,11 +8241,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_elbv1_%', 'DAILY'),('aws_elbv2_%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_iam_credential_reports', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideLoadBalancers", + "Name": "AwsOrgWideIamCredentialReports", "Stack": "deploy", "Stage": "TEST", }, @@ -8856,7 +8267,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideLoadBalancersPostgresContainer", + "Name": "CloudquerySource-AwsOrgWideIamCredentialReportsPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -8931,7 +8342,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionCloudquerySourceAwsOrgWideLoadBalancersFirelensLogGroup52712979", + "Ref": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionCloudquerySourceAwsOrgWideIamCredentialReportsFirelensLogGroupB7D88F87", }, "awslogs-region": { "Ref": "AWS::Region", @@ -8946,19 +8357,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideLoadBalancersFirelens", + "Name": "CloudquerySource-AwsOrgWideIamCredentialReportsFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleD042731A", + "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRole02E86B04", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionA0D1BDA3", - "Memory": "512", + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinition3443A1FA", + "Memory": "1024", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -8974,7 +8385,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideLoadBalancers", + "Value": "AwsOrgWideIamCredentialReports", }, { "Key": "Stack", @@ -8987,7 +8398,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideLoadBalancersB565C247", + "servicecatalogueTESTtaskAwsOrgWideIamCredentialReportsD1A28FD6", "Arn", ], }, @@ -9008,7 +8419,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionCloudquerySourceAwsOrgWideLoadBalancersFirelensLogGroup52712979": { + "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionCloudquerySourceAwsOrgWideIamCredentialReportsFirelensLogGroupB7D88F87": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -9023,7 +8434,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideLoadBalancers", + "Value": "AwsOrgWideIamCredentialReports", }, { "Key": "Stack", @@ -9038,7 +8449,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionEventsRole4A9EDE15": { + "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionEventsRole20CD424D": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -9063,7 +8474,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideLoadBalancers", + "Value": "AwsOrgWideIamCredentialReports", }, { "Key": "Stack", @@ -9077,7 +8488,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionEventsRoleDefaultPolicy2F850C58": { + "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionEventsRoleDefaultPolicyEDC4DC94": { "Properties": { "PolicyDocument": { "Statement": [ @@ -9095,7 +8506,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinition333F61F5", + "Ref": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinition7300D1E2", }, }, { @@ -9127,7 +8538,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleD042731A", + "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRole02E86B04", "Arn", ], }, @@ -9137,7 +8548,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideLoadBalancersB565C247", + "servicecatalogueTESTtaskAwsOrgWideIamCredentialReportsD1A28FD6", "Arn", ], }, @@ -9145,16 +8556,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionEventsRoleDefaultPolicy2F850C58", + "PolicyName": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionEventsRoleDefaultPolicyEDC4DC94", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionEventsRole4A9EDE15", + "Ref": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionEventsRole20CD424D", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleD042731A": { + "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRole02E86B04": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -9179,7 +8590,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideLoadBalancers", + "Value": "AwsOrgWideIamCredentialReports", }, { "Key": "Stack", @@ -9193,7 +8604,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleDefaultPolicy7A95B87A": { + "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRoleDefaultPolicy0BBE996D": { "Properties": { "PolicyDocument": { "Statement": [ @@ -9225,7 +8636,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionCloudquerySourceAwsOrgWideLoadBalancersFirelensLogGroup52712979", + "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionCloudquerySourceAwsOrgWideIamCredentialReportsFirelensLogGroupB7D88F87", "Arn", ], }, @@ -9233,18 +8644,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleDefaultPolicy7A95B87A", + "PolicyName": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRoleDefaultPolicy0BBE996D", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleD042731A", + "Ref": "CloudquerySourceAwsOrgWideIamCredentialReportsTaskDefinitionExecutionRole02E86B04", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideRDSScheduledEventRule95E5DE51": { + "CloudquerySourceAwsOrgWideInspectorScheduledEventRule127B8502": { "Properties": { - "ScheduleExpression": "cron(0 6 * * ? *)", + "ScheduleExpression": "cron(0 3 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -9275,14 +8686,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideRDSTaskDefinitionB16F3CC7", + "Ref": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionBABA9F5D", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideRDSTaskDefinitionEventsRoleCC75E52D", + "CloudquerySourceAwsOrgWideInspectorTaskDefinitionEventsRoleD22CD481", "Arn", ], }, @@ -9291,7 +8702,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsOrgWideRDSTaskDefinitionB16F3CC7": { + "CloudquerySourceAwsOrgWideInspectorTaskDefinitionBABA9F5D": { "Properties": { "ContainerDefinitions": [ { @@ -9304,10 +8715,8 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_rds_instances - - aws_rds_clusters - - aws_rds_db_snapshots - - aws_rds_cluster_snapshots + - aws_inspector_findings + - aws_inspector2_findings destinations: - postgresql otel_endpoint: 0.0.0.0:4318 @@ -9333,12 +8742,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideRDSAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsOrgWideInspectorAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideRDS", + "Name": "AwsOrgWideInspector", "Stack": "deploy", "Stage": "TEST", }, @@ -9348,7 +8757,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "409MiB", + "Value": "819MiB", }, ], "Essential": true, @@ -9383,7 +8792,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideRDSContainer", + "Name": "CloudquerySource-AwsOrgWideInspectorContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -9472,7 +8881,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideRDSAWSOTELCollector", + "Name": "CloudquerySource-AwsOrgWideInspectorAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -9485,11 +8894,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_rds_instances', 'DAILY'),('aws_rds_clusters', 'DAILY'),('aws_rds_db_snapshots', 'DAILY'),('aws_rds_cluster_snapshots', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_inspector_findings', 'DAILY'),('aws_inspector2_findings', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideRDS", + "Name": "AwsOrgWideInspector", "Stack": "deploy", "Stage": "TEST", }, @@ -9511,7 +8920,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideRDSPostgresContainer", + "Name": "CloudquerySource-AwsOrgWideInspectorPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -9586,7 +8995,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideRDSTaskDefinitionCloudquerySourceAwsOrgWideRDSFirelensLogGroupA77166BD", + "Ref": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionCloudquerySourceAwsOrgWideInspectorFirelensLogGroupAFBF5583", }, "awslogs-region": { "Ref": "AWS::Region", @@ -9601,19 +9010,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-AwsOrgWideRDSFirelens", + "Name": "CloudquerySource-AwsOrgWideInspectorFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRole22BD57E4", + "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRole89A5DE0C", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideRDSTaskDefinition222E5C6A", - "Memory": "512", + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideInspectorTaskDefinitionDC3485A7", + "Memory": "1024", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -9629,7 +9038,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideRDS", + "Value": "AwsOrgWideInspector", }, { "Key": "Stack", @@ -9642,7 +9051,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideRDS1CE08EDD", + "servicecatalogueTESTtaskAwsOrgWideInspector7DFAA956", "Arn", ], }, @@ -9663,7 +9072,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsOrgWideRDSTaskDefinitionCloudquerySourceAwsOrgWideRDSFirelensLogGroupA77166BD": { + "CloudquerySourceAwsOrgWideInspectorTaskDefinitionCloudquerySourceAwsOrgWideInspectorFirelensLogGroupAFBF5583": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -9678,7 +9087,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideRDS", + "Value": "AwsOrgWideInspector", }, { "Key": "Stack", @@ -9693,7 +9102,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsOrgWideRDSTaskDefinitionEventsRoleCC75E52D": { + "CloudquerySourceAwsOrgWideInspectorTaskDefinitionEventsRoleD22CD481": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -9718,7 +9127,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideRDS", + "Value": "AwsOrgWideInspector", }, { "Key": "Stack", @@ -9732,7 +9141,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideRDSTaskDefinitionEventsRoleDefaultPolicy16D51A2A": { + "CloudquerySourceAwsOrgWideInspectorTaskDefinitionEventsRoleDefaultPolicyC087EC6A": { "Properties": { "PolicyDocument": { "Statement": [ @@ -9750,7 +9159,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsOrgWideRDSTaskDefinitionB16F3CC7", + "Ref": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionBABA9F5D", }, }, { @@ -9782,7 +9191,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRole22BD57E4", + "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRole89A5DE0C", "Arn", ], }, @@ -9792,7 +9201,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideRDS1CE08EDD", + "servicecatalogueTESTtaskAwsOrgWideInspector7DFAA956", "Arn", ], }, @@ -9800,16 +9209,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideRDSTaskDefinitionEventsRoleDefaultPolicy16D51A2A", + "PolicyName": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionEventsRoleDefaultPolicyC087EC6A", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideRDSTaskDefinitionEventsRoleCC75E52D", + "Ref": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionEventsRoleD22CD481", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRole22BD57E4": { + "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRole89A5DE0C": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -9834,7 +9243,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideRDS", + "Value": "AwsOrgWideInspector", }, { "Key": "Stack", @@ -9848,7 +9257,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRoleDefaultPolicy3BBDF952": { + "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRoleDefaultPolicy546CCF04": { "Properties": { "PolicyDocument": { "Statement": [ @@ -9880,7 +9289,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideRDSTaskDefinitionCloudquerySourceAwsOrgWideRDSFirelensLogGroupA77166BD", + "CloudquerySourceAwsOrgWideInspectorTaskDefinitionCloudquerySourceAwsOrgWideInspectorFirelensLogGroupAFBF5583", "Arn", ], }, @@ -9888,18 +9297,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRoleDefaultPolicy3BBDF952", + "PolicyName": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRoleDefaultPolicy546CCF04", "Roles": [ { - "Ref": "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRole22BD57E4", + "Ref": "CloudquerySourceAwsOrgWideInspectorTaskDefinitionExecutionRole89A5DE0C", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsOrgWideS3ScheduledEventRule06193C1C": { + "CloudquerySourceAwsOrgWideLoadBalancersScheduledEventRuleE82A6C43": { "Properties": { - "ScheduleExpression": "cron(0 4 * * ? *)", + "ScheduleExpression": "rate(30 minutes)", "State": "ENABLED", "Targets": [ { @@ -9927,229 +9336,85 @@ spec: }, }, }, - "PropagateTags": "TASK_DEFINITION", - "TaskCount": 1, - "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsOrgWideS3TaskDefinitionAF066748", - }, - }, - "Id": "Target0", - "Input": "{}", - "RoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideS3TaskDefinitionEventsRoleA4289298", - "Arn", - ], - }, - }, - ], - }, - "Type": "AWS::Events::Rule", - }, - "CloudquerySourceAwsOrgWideS3TaskDefinitionAF066748": { - "Properties": { - "ContainerDefinitions": [ - { - "Command": [ - "/bin/sh", - "-c", - "printf 'kind: source -spec: - name: aws - path: cloudquery/aws - version: v27.5.0 - tables: - - aws_s3* - destinations: - - postgresql - otel_endpoint: 0.0.0.0:4318 - otel_endpoint_insecure: true - spec: - org: - member_role_name: cloudquery-access - organization_units: - - ou-123 -' > /usr/share/cloudquery/source.yaml;printf 'kind: destination -spec: - name: postgresql - registry: github - path: cloudquery/postgresql - version: v7.2.0 - migrate_mode: forced - spec: - connection_string: >- - user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 - dbname=postgres sslmode=verify-full -' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", - ], - "DependsOn": [ - { - "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsOrgWideS3AWSOTELCollector", - }, - ], - "DockerLabels": { - "App": "service-catalogue", - "Name": "AwsOrgWideS3", - "Stack": "deploy", - "Stage": "TEST", - }, - "EntryPoint": [ - "", - ], - "Environment": [ - { - "Name": "GOMEMLIMIT", - "Value": "409MiB", - }, - ], - "Essential": true, - "Image": "ghcr.io/guardian/service-catalogue/cloudquery:stable", - "LogConfiguration": { - "LogDriver": "awsfirelens", - "Options": { - "Name": "kinesis_streams", - "region": { - "Ref": "AWS::Region", - }, - "retry_limit": "2", - "stream": { - "Ref": "LoggingStreamName", - }, - }, - }, - "MountPoints": [ - { - "ContainerPath": "/usr/share/cloudquery", - "ReadOnly": false, - "SourceVolume": "config-volume", - }, - { - "ContainerPath": "/app/.cq", - "ReadOnly": false, - "SourceVolume": "cloudquery-volume", - }, - { - "ContainerPath": "/tmp", - "ReadOnly": false, - "SourceVolume": "tmp-volume", - }, - ], - "Name": "CloudquerySource-AwsOrgWideS3Container", - "ReadonlyRootFilesystem": true, - "Secrets": [ - { - "Name": "DB_USERNAME", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":username::", - ], - ], - }, - }, - { - "Name": "DB_HOST", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":host::", - ], - ], - }, - }, - { - "Name": "DB_PASSWORD", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":password::", - ], - ], - }, - }, - { - "Name": "CLOUDQUERY_API_KEY", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "cloudqueryapikeyCCF82F53", - }, - ":api-key::", - ], - ], - }, - }, - ], - }, - { - "Command": [ - "--config=/etc/ecs/ecs-xray.yaml", - ], - "Essential": true, - "HealthCheck": { - "Command": [ - "CMD", - "/healthcheck", - ], - "Interval": 5, - "Retries": 3, - "Timeout": 5, - }, - "Image": "public.ecr.aws/aws-observability/aws-otel-collector:v0.35.0", - "LogConfiguration": { - "LogDriver": "awsfirelens", - "Options": { - "Name": "kinesis_streams", - "region": { - "Ref": "AWS::Region", - }, - "retry_limit": "2", - "stream": { - "Ref": "LoggingStreamName", - }, - }, - }, - "Name": "CloudquerySource-AwsOrgWideS3AWSOTELCollector", - "PortMappings": [ - { - "ContainerPort": 4318, - "Protocol": "tcp", + "PropagateTags": "TASK_DEFINITION", + "TaskCount": 1, + "TaskDefinitionArn": { + "Ref": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinition333F61F5", }, - ], - "ReadonlyRootFilesystem": true, + }, + "Id": "Target0", + "Input": "{}", + "RoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionEventsRole4A9EDE15", + "Arn", + ], + }, }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinition333F61F5": { + "Properties": { + "ContainerDefinitions": [ { "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_s3%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "printf 'kind: source +spec: + name: aws + path: cloudquery/aws + version: v27.5.0 + tables: + - aws_elbv1_* + - aws_elbv2_* + destinations: + - postgresql + otel_endpoint: 0.0.0.0:4318 + otel_endpoint_insecure: true + spec: + org: + member_role_name: cloudquery-access + organization_units: + - ou-123 +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination +spec: + name: postgresql + registry: github + path: cloudquery/postgresql + version: v7.2.0 + migrate_mode: forced + spec: + connection_string: >- + user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 + dbname=postgres sslmode=verify-full +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", + ], + "DependsOn": [ + { + "Condition": "HEALTHY", + "ContainerName": "CloudquerySource-AwsOrgWideLoadBalancersAWSOTELCollector", + }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsOrgWideS3", + "Name": "AwsOrgWideLoadBalancers", "Stack": "deploy", "Stage": "TEST", }, "EntryPoint": [ "", ], - "Essential": false, - "Image": "public.ecr.aws/docker/library/postgres:16-alpine", + "Environment": [ + { + "Name": "GOMEMLIMIT", + "Value": "409MiB", + }, + ], + "Essential": true, + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:stable", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -10163,11 +9428,28 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsOrgWideS3PostgresContainer", + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], + "Name": "CloudquerySource-AwsOrgWideLoadBalancersContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { - "Name": "PGUSER", + "Name": "DB_USERNAME", "ValueFrom": { "Fn::Join": [ "", @@ -10181,7 +9463,7 @@ spec: }, }, { - "Name": "PGHOST", + "Name": "DB_HOST", "ValueFrom": { "Fn::Join": [ "", @@ -10195,7 +9477,7 @@ spec: }, }, { - "Name": "PGPASSWORD", + "Name": "DB_PASSWORD", "ValueFrom": { "Fn::Join": [ "", @@ -10208,322 +9490,196 @@ spec: ], }, }, - ], - }, - { - "Environment": [ - { - "Name": "STACK", - "Value": "deploy", - }, - { - "Name": "STAGE", - "Value": "TEST", - }, - { - "Name": "APP", - "Value": "service-catalogue", - }, { - "Name": "GU_REPO", - "Value": "guardian/service-catalogue", - }, - ], - "Essential": true, - "FirelensConfiguration": { - "Type": "fluentbit", - }, - "Image": "ghcr.io/guardian/devx-logs:2", - "LogConfiguration": { - "LogDriver": "awslogs", - "Options": { - "awslogs-group": { - "Ref": "CloudquerySourceAwsOrgWideS3TaskDefinitionCloudquerySourceAwsOrgWideS3FirelensLogGroup4CE08508", - }, - "awslogs-region": { - "Ref": "AWS::Region", + "Name": "CLOUDQUERY_API_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "cloudqueryapikeyCCF82F53", + }, + ":api-key::", + ], + ], }, - "awslogs-stream-prefix": "deploy/TEST/service-catalogue", - }, - }, - "MountPoints": [ - { - "ContainerPath": "/init", - "ReadOnly": false, - "SourceVolume": "firelens-volume", - }, - ], - "Name": "CloudquerySource-AwsOrgWideS3Firelens", - "ReadonlyRootFilesystem": true, - }, - ], - "Cpu": "256", - "ExecutionRoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleFDE66B47", - "Arn", - ], - }, - "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideS3TaskDefinition6734443A", - "Memory": "512", - "NetworkMode": "awsvpc", - "RequiresCompatibilities": [ - "FARGATE", - ], - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideS3", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - "TaskRoleArn": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideS38AEB5180", - "Arn", - ], - }, - "Volumes": [ - { - "Name": "config-volume", - }, - { - "Name": "cloudquery-volume", - }, - { - "Name": "tmp-volume", - }, - { - "Name": "firelens-volume", - }, - ], - }, - "Type": "AWS::ECS::TaskDefinition", - }, - "CloudquerySourceAwsOrgWideS3TaskDefinitionCloudquerySourceAwsOrgWideS3FirelensLogGroup4CE08508": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideS3", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", - }, - "CloudquerySourceAwsOrgWideS3TaskDefinitionEventsRoleA4289298": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "events.amazonaws.com", }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsOrgWideS3", + ], }, { - "Key": "Stack", - "Value": "deploy", + "Command": [ + "--config=/etc/ecs/ecs-xray.yaml", + ], + "Essential": true, + "HealthCheck": { + "Command": [ + "CMD", + "/healthcheck", + ], + "Interval": 5, + "Retries": 3, + "Timeout": 5, + }, + "Image": "public.ecr.aws/aws-observability/aws-otel-collector:v0.35.0", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, + }, + }, + "Name": "CloudquerySource-AwsOrgWideLoadBalancersAWSOTELCollector", + "PortMappings": [ + { + "ContainerPort": 4318, + "Protocol": "tcp", + }, + ], + "ReadonlyRootFilesystem": true, }, { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceAwsOrgWideS3TaskDefinitionEventsRoleDefaultPolicy151FBFA4": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": "ecs:RunTask", - "Condition": { - "ArnEquals": { - "ecs:cluster": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], - }, + "Command": [ + "/bin/sh", + "-c", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_elbv1_%', 'DAILY'),('aws_elbv2_%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + ], + "DockerLabels": { + "App": "service-catalogue", + "Name": "AwsOrgWideLoadBalancers", + "Stack": "deploy", + "Stage": "TEST", + }, + "EntryPoint": [ + "", + ], + "Essential": false, + "Image": "public.ecr.aws/docker/library/postgres:16-alpine", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", }, - }, - "Effect": "Allow", - "Resource": { - "Ref": "CloudquerySourceAwsOrgWideS3TaskDefinitionAF066748", }, }, - { - "Action": "ecs:TagResource", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":ecs:", - { - "Ref": "AWS::Region", - }, - ":*:task/", - { - "Ref": "servicecatalogueCluster5FC34DC5", - }, - "/*", + "Name": "CloudquerySource-AwsOrgWideLoadBalancersPostgresContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ + { + "Name": "PGUSER", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":username::", + ], ], - ], + }, }, - }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleFDE66B47", - "Arn", - ], + { + "Name": "PGHOST", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":host::", + ], + ], + }, }, - }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsOrgWideS38AEB5180", - "Arn", - ], + { + "Name": "PGPASSWORD", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":password::", + ], + ], + }, }, - }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceAwsOrgWideS3TaskDefinitionEventsRoleDefaultPolicy151FBFA4", - "Roles": [ - { - "Ref": "CloudquerySourceAwsOrgWideS3TaskDefinitionEventsRoleA4289298", + ], }, - ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleDefaultPolicyE82AC40F": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + { + "Environment": [ + { + "Name": "STACK", + "Value": "deploy", }, - }, - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "cloudqueryapikeyCCF82F53", + { + "Name": "STAGE", + "Value": "TEST", + }, + { + "Name": "APP", + "Value": "service-catalogue", + }, + { + "Name": "GU_REPO", + "Value": "guardian/service-catalogue", }, + ], + "Essential": true, + "FirelensConfiguration": { + "Type": "fluentbit", }, - { - "Action": [ - "logs:CreateLogStream", - "logs:PutLogEvents", - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceAwsOrgWideS3TaskDefinitionCloudquerySourceAwsOrgWideS3FirelensLogGroup4CE08508", - "Arn", - ], + "Image": "ghcr.io/guardian/devx-logs:2", + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionCloudquerySourceAwsOrgWideLoadBalancersFirelensLogGroup52712979", + }, + "awslogs-region": { + "Ref": "AWS::Region", + }, + "awslogs-stream-prefix": "deploy/TEST/service-catalogue", }, }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleDefaultPolicyE82AC40F", - "Roles": [ - { - "Ref": "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleFDE66B47", + "MountPoints": [ + { + "ContainerPath": "/init", + "ReadOnly": false, + "SourceVolume": "firelens-volume", + }, + ], + "Name": "CloudquerySource-AwsOrgWideLoadBalancersFirelens", + "ReadonlyRootFilesystem": true, }, ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleFDE66B47": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ecs-tasks.amazonaws.com", - }, - }, + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleD042731A", + "Arn", ], - "Version": "2012-10-17", }, + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionA0D1BDA3", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE", + ], "Tags": [ { "Key": "gu:cdk:version", @@ -10535,7 +9691,7 @@ spec: }, { "Key": "Name", - "Value": "AwsOrgWideS3", + "Value": "AwsOrgWideLoadBalancers", }, { "Key": "Stack", @@ -10546,59 +9702,30 @@ spec: "Value": "TEST", }, ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceAwsRemainingDataScheduledEventRuleADE2D1CC": { - "Properties": { - "ScheduleExpression": "cron(0 16 ? * SAT *)", - "State": "ENABLED", - "Targets": [ + "TaskRoleArn": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAwsOrgWideLoadBalancersB565C247", + "Arn", + ], + }, + "Volumes": [ { - "Arn": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], - }, - "EcsParameters": { - "LaunchType": "FARGATE", - "NetworkConfiguration": { - "AwsVpcConfiguration": { - "AssignPublicIp": "DISABLED", - "SecurityGroups": [ - { - "Fn::GetAtt": [ - "PostgresAccessSecurityGroupServicecatalogue03C78F14", - "GroupId", - ], - }, - ], - "Subnets": { - "Ref": "PrivateSubnets", - }, - }, - }, - "PropagateTags": "TASK_DEFINITION", - "TaskCount": 1, - "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsRemainingDataTaskDefinitionF2586400", - }, - }, - "Id": "Target0", - "Input": "{}", - "RoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceAwsRemainingDataTaskDefinitionEventsRole45369093", - "Arn", - ], - }, + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + { + "Name": "firelens-volume", }, ], }, - "Type": "AWS::Events::Rule", + "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceAwsRemainingDataTaskDefinitionCloudquerySourceAwsRemainingDataFirelensLogGroupAEF7D0F2": { + "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionCloudquerySourceAwsOrgWideLoadBalancersFirelensLogGroup52712979": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -10613,7 +9740,7 @@ spec: }, { "Key": "Name", - "Value": "AwsRemainingData", + "Value": "AwsOrgWideLoadBalancers", }, { "Key": "Stack", @@ -10628,7 +9755,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsRemainingDataTaskDefinitionEventsRole45369093": { + "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionEventsRole4A9EDE15": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -10653,7 +9780,7 @@ spec: }, { "Key": "Name", - "Value": "AwsRemainingData", + "Value": "AwsOrgWideLoadBalancers", }, { "Key": "Stack", @@ -10667,7 +9794,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsRemainingDataTaskDefinitionEventsRoleDefaultPolicyE9BEA52B": { + "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionEventsRoleDefaultPolicy2F850C58": { "Properties": { "PolicyDocument": { "Statement": [ @@ -10685,7 +9812,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsRemainingDataTaskDefinitionF2586400", + "Ref": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinition333F61F5", }, }, { @@ -10717,7 +9844,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRole7F5255B8", + "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleD042731A", "Arn", ], }, @@ -10727,7 +9854,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsRemainingData673BE318", + "servicecatalogueTESTtaskAwsOrgWideLoadBalancersB565C247", "Arn", ], }, @@ -10735,16 +9862,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsRemainingDataTaskDefinitionEventsRoleDefaultPolicyE9BEA52B", + "PolicyName": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionEventsRoleDefaultPolicy2F850C58", "Roles": [ { - "Ref": "CloudquerySourceAwsRemainingDataTaskDefinitionEventsRole45369093", + "Ref": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionEventsRole4A9EDE15", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRole7F5255B8": { + "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleD042731A": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -10769,7 +9896,7 @@ spec: }, { "Key": "Name", - "Value": "AwsRemainingData", + "Value": "AwsOrgWideLoadBalancers", }, { "Key": "Stack", @@ -10783,7 +9910,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRoleDefaultPolicy3D9DFEF7": { + "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleDefaultPolicy7A95B87A": { "Properties": { "PolicyDocument": { "Statement": [ @@ -10815,7 +9942,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsRemainingDataTaskDefinitionCloudquerySourceAwsRemainingDataFirelensLogGroupAEF7D0F2", + "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionCloudquerySourceAwsOrgWideLoadBalancersFirelensLogGroup52712979", "Arn", ], }, @@ -10823,100 +9950,86 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRoleDefaultPolicy3D9DFEF7", + "PolicyName": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleDefaultPolicy7A95B87A", "Roles": [ { - "Ref": "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRole7F5255B8", + "Ref": "CloudquerySourceAwsOrgWideLoadBalancersTaskDefinitionExecutionRoleD042731A", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsRemainingDataTaskDefinitionF2586400": { + "CloudquerySourceAwsOrgWideRDSScheduledEventRule95E5DE51": { "Properties": { - "ContainerDefinitions": [ + "ScheduleExpression": "cron(0 6 * * ? *)", + "State": "ENABLED", + "Targets": [ { - "Command": [ - "/bin/sh", - "-c", - "printf 'kind: source -spec: - name: aws - path: cloudquery/aws - version: v27.5.0 - tables: - - aws_* - skip_tables: - - aws_ec2_vpc_endpoint_services - - aws_cloudtrail_events - - aws_docdb_cluster_parameter_groups - - aws_docdb_engine_versions - - aws_ec2_instance_types - - aws_elasticache_engine_versions - - aws_elasticache_parameter_groups - - aws_elasticache_reserved_cache_nodes_offerings - - aws_elasticache_service_updates - - aws_emr_supported_instance_types - - aws_neptune_cluster_parameter_groups - - aws_neptune_db_parameter_groups - - aws_rds_cluster_parameter_groups - - aws_rds_db_parameter_groups - - aws_rds_engine_versions - - aws_servicequotas_services - - aws_identitystore_users - - aws_identitystore_groups - - aws_quicksight_data_sets - - aws_quicksight_dashboards - - aws_quicksight_analyses - - aws_quicksight_users - - aws_quicksight_templates - - aws_quicksight_groups - - aws_quicksight_folders - - aws_quicksight_data_sources - - aws_amp_workspaces - - aws_ssoadmin_instances - - aws_glue_connections - - aws_computeoptimizer_ecs_service_recommendations - - aws_xray_sampling_rules - - aws_xray_resource_policies - - aws_xray_groups - - aws_wellarchitected_* - - aws_stepfunctions_map_runs - - aws_stepfunctions_map_run_executions - - aws_stepfunctions_executions - - aws_organization* - - aws_accessanalyzer_* - - aws_securityhub_* - - aws_cloudformation_* - - aws_costexplorer_* - - aws_elbv1_* - - aws_elbv2_* - - aws_autoscaling_groups - - aws_acm* - - aws_lambda_* - - aws_ssm_parameters - - aws_cloudwatch_alarms - - aws_inspector_findings - - aws_inspector2_findings - - aws_s3* - - aws_dynamodb* + "Arn": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + "EcsParameters": { + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsVpcConfiguration": { + "AssignPublicIp": "DISABLED", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "PostgresAccessSecurityGroupServicecatalogue03C78F14", + "GroupId", + ], + }, + ], + "Subnets": { + "Ref": "PrivateSubnets", + }, + }, + }, + "PropagateTags": "TASK_DEFINITION", + "TaskCount": 1, + "TaskDefinitionArn": { + "Ref": "CloudquerySourceAwsOrgWideRDSTaskDefinitionB16F3CC7", + }, + }, + "Id": "Target0", + "Input": "{}", + "RoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideRDSTaskDefinitionEventsRoleCC75E52D", + "Arn", + ], + }, + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "CloudquerySourceAwsOrgWideRDSTaskDefinitionB16F3CC7": { + "Properties": { + "ContainerDefinitions": [ + { + "Command": [ + "/bin/sh", + "-c", + "printf 'kind: source +spec: + name: aws + path: cloudquery/aws + version: v27.5.0 + tables: - aws_rds_instances - aws_rds_clusters - aws_rds_db_snapshots - aws_rds_cluster_snapshots - - aws_backup_protected_resources - - aws_backup_vaults - - aws_backup_vault_recovery_points - - aws_ec2_instances - - aws_ec2_security_groups - - aws_ec2_images - - aws_iam_credential_reports destinations: - postgresql otel_endpoint: 0.0.0.0:4318 otel_endpoint_insecure: true spec: - concurrency: 2000 org: member_role_name: cloudquery-access organization_units: @@ -10937,12 +10050,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsRemainingDataAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsOrgWideRDSAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsRemainingData", + "Name": "AwsOrgWideRDS", "Stack": "deploy", "Stage": "TEST", }, @@ -10952,7 +10065,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "2457MiB", + "Value": "409MiB", }, ], "Essential": true, @@ -10987,7 +10100,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsRemainingDataContainer", + "Name": "CloudquerySource-AwsOrgWideRDSContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -11076,7 +10189,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsRemainingDataAWSOTELCollector", + "Name": "CloudquerySource-AwsOrgWideRDSAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -11089,11 +10202,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_%', 'WEEKLY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'WEEKLY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_rds_instances', 'DAILY'),('aws_rds_clusters', 'DAILY'),('aws_rds_db_snapshots', 'DAILY'),('aws_rds_cluster_snapshots', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsRemainingData", + "Name": "AwsOrgWideRDS", "Stack": "deploy", "Stage": "TEST", }, @@ -11115,7 +10228,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsRemainingDataPostgresContainer", + "Name": "CloudquerySource-AwsOrgWideRDSPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -11129,99 +10242,304 @@ spec: }, ":username::", ], - ], + ], + }, + }, + { + "Name": "PGHOST", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":host::", + ], + ], + }, + }, + { + "Name": "PGPASSWORD", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":password::", + ], + ], + }, + }, + ], + }, + { + "Environment": [ + { + "Name": "STACK", + "Value": "deploy", + }, + { + "Name": "STAGE", + "Value": "TEST", + }, + { + "Name": "APP", + "Value": "service-catalogue", + }, + { + "Name": "GU_REPO", + "Value": "guardian/service-catalogue", + }, + ], + "Essential": true, + "FirelensConfiguration": { + "Type": "fluentbit", + }, + "Image": "ghcr.io/guardian/devx-logs:2", + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "CloudquerySourceAwsOrgWideRDSTaskDefinitionCloudquerySourceAwsOrgWideRDSFirelensLogGroupA77166BD", + }, + "awslogs-region": { + "Ref": "AWS::Region", + }, + "awslogs-stream-prefix": "deploy/TEST/service-catalogue", + }, + }, + "MountPoints": [ + { + "ContainerPath": "/init", + "ReadOnly": false, + "SourceVolume": "firelens-volume", + }, + ], + "Name": "CloudquerySource-AwsOrgWideRDSFirelens", + "ReadonlyRootFilesystem": true, + }, + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRole22BD57E4", + "Arn", + ], + }, + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideRDSTaskDefinition222E5C6A", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE", + ], + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideRDS", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + "TaskRoleArn": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAwsOrgWideRDS1CE08EDD", + "Arn", + ], + }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + { + "Name": "firelens-volume", + }, + ], + }, + "Type": "AWS::ECS::TaskDefinition", + }, + "CloudquerySourceAwsOrgWideRDSTaskDefinitionCloudquerySourceAwsOrgWideRDSFirelensLogGroupA77166BD": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideRDS", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceAwsOrgWideRDSTaskDefinitionEventsRoleCC75E52D": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "events.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideRDS", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAwsOrgWideRDSTaskDefinitionEventsRoleDefaultPolicy16D51A2A": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ecs:RunTask", + "Condition": { + "ArnEquals": { + "ecs:cluster": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, }, }, - { - "Name": "PGHOST", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":host::", - ], - ], - }, + "Effect": "Allow", + "Resource": { + "Ref": "CloudquerySourceAwsOrgWideRDSTaskDefinitionB16F3CC7", }, - { - "Name": "PGPASSWORD", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":password::", - ], + }, + { + "Action": "ecs:TagResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":ecs:", + { + "Ref": "AWS::Region", + }, + ":*:task/", + { + "Ref": "servicecatalogueCluster5FC34DC5", + }, + "/*", ], - }, - }, - ], - }, - { - "Environment": [ - { - "Name": "STACK", - "Value": "deploy", - }, - { - "Name": "STAGE", - "Value": "TEST", - }, - { - "Name": "APP", - "Value": "service-catalogue", - }, - { - "Name": "GU_REPO", - "Value": "guardian/service-catalogue", + ], }, - ], - "Essential": true, - "FirelensConfiguration": { - "Type": "fluentbit", }, - "Image": "ghcr.io/guardian/devx-logs:2", - "LogConfiguration": { - "LogDriver": "awslogs", - "Options": { - "awslogs-group": { - "Ref": "CloudquerySourceAwsRemainingDataTaskDefinitionCloudquerySourceAwsRemainingDataFirelensLogGroupAEF7D0F2", - }, - "awslogs-region": { - "Ref": "AWS::Region", - }, - "awslogs-stream-prefix": "deploy/TEST/service-catalogue", + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRole22BD57E4", + "Arn", + ], }, }, - "MountPoints": [ - { - "ContainerPath": "/init", - "ReadOnly": false, - "SourceVolume": "firelens-volume", + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAwsOrgWideRDS1CE08EDD", + "Arn", + ], }, - ], - "Name": "CloudquerySource-AwsRemainingDataFirelens", - "ReadonlyRootFilesystem": true, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceAwsOrgWideRDSTaskDefinitionEventsRoleDefaultPolicy16D51A2A", + "Roles": [ + { + "Ref": "CloudquerySourceAwsOrgWideRDSTaskDefinitionEventsRoleCC75E52D", }, ], - "Cpu": "1024", - "ExecutionRoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRole7F5255B8", - "Arn", + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRole22BD57E4": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, ], + "Version": "2012-10-17", }, - "Family": "ServiceCatalogueCloudquerySourceAwsRemainingDataTaskDefinition14D0B33A", - "Memory": "3072", - "NetworkMode": "awsvpc", - "RequiresCompatibilities": [ - "FARGATE", - ], "Tags": [ { "Key": "gu:cdk:version", @@ -11233,7 +10551,7 @@ spec: }, { "Key": "Name", - "Value": "AwsRemainingData", + "Value": "AwsOrgWideRDS", }, { "Key": "Stack", @@ -11244,32 +10562,61 @@ spec: "Value": "TEST", }, ], - "TaskRoleArn": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsRemainingData673BE318", - "Arn", + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRoleDefaultPolicy3BBDF952": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "cloudqueryapikeyCCF82F53", + }, + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideRDSTaskDefinitionCloudquerySourceAwsOrgWideRDSFirelensLogGroupA77166BD", + "Arn", + ], + }, + }, ], + "Version": "2012-10-17", }, - "Volumes": [ - { - "Name": "config-volume", - }, - { - "Name": "cloudquery-volume", - }, - { - "Name": "tmp-volume", - }, + "PolicyName": "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRoleDefaultPolicy3BBDF952", + "Roles": [ { - "Name": "firelens-volume", + "Ref": "CloudquerySourceAwsOrgWideRDSTaskDefinitionExecutionRole22BD57E4", }, ], }, - "Type": "AWS::ECS::TaskDefinition", + "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsSSMParametersScheduledEventRule0A998B82": { + "CloudquerySourceAwsOrgWideS3ScheduledEventRule06193C1C": { "Properties": { - "ScheduleExpression": "cron(20 1 * * ? *)", + "ScheduleExpression": "cron(0 4 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -11300,53 +10647,23 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceAwsSSMParametersTaskDefinitionD2F49BC2", - }, - }, - "Id": "Target0", - "Input": "{}", - "RoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceAwsSSMParametersTaskDefinitionEventsRole510C5A17", - "Arn", - ], - }, - }, - ], - }, - "Type": "AWS::Events::Rule", - }, - "CloudquerySourceAwsSSMParametersTaskDefinitionCloudquerySourceAwsSSMParametersFirelensLogGroup0210F31D": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "AwsSSMParameters", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", + "Ref": "CloudquerySourceAwsOrgWideS3TaskDefinitionAF066748", + }, + }, + "Id": "Target0", + "Input": "{}", + "RoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideS3TaskDefinitionEventsRoleA4289298", + "Arn", + ], + }, }, ], }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", + "Type": "AWS::Events::Rule", }, - "CloudquerySourceAwsSSMParametersTaskDefinitionD2F49BC2": { + "CloudquerySourceAwsOrgWideS3TaskDefinitionAF066748": { "Properties": { "ContainerDefinitions": [ { @@ -11359,7 +10676,7 @@ spec: path: cloudquery/aws version: v27.5.0 tables: - - aws_ssm_parameters + - aws_s3* destinations: - postgresql otel_endpoint: 0.0.0.0:4318 @@ -11385,12 +10702,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-AwsSSMParametersAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsOrgWideS3AWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsSSMParameters", + "Name": "AwsOrgWideS3", "Stack": "deploy", "Stage": "TEST", }, @@ -11435,7 +10752,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-AwsSSMParametersContainer", + "Name": "CloudquerySource-AwsOrgWideS3Container", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -11524,7 +10841,7 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsSSMParametersAWSOTELCollector", + "Name": "CloudquerySource-AwsOrgWideS3AWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -11537,11 +10854,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_ssm_parameters', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_s3%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "AwsSSMParameters", + "Name": "AwsOrgWideS3", "Stack": "deploy", "Stage": "TEST", }, @@ -11563,113 +10880,445 @@ spec: }, }, }, - "Name": "CloudquerySource-AwsSSMParametersPostgresContainer", - "ReadonlyRootFilesystem": true, - "Secrets": [ - { - "Name": "PGUSER", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":username::", - ], + "Name": "CloudquerySource-AwsOrgWideS3PostgresContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ + { + "Name": "PGUSER", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":username::", + ], + ], + }, + }, + { + "Name": "PGHOST", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":host::", + ], + ], + }, + }, + { + "Name": "PGPASSWORD", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":password::", + ], + ], + }, + }, + ], + }, + { + "Environment": [ + { + "Name": "STACK", + "Value": "deploy", + }, + { + "Name": "STAGE", + "Value": "TEST", + }, + { + "Name": "APP", + "Value": "service-catalogue", + }, + { + "Name": "GU_REPO", + "Value": "guardian/service-catalogue", + }, + ], + "Essential": true, + "FirelensConfiguration": { + "Type": "fluentbit", + }, + "Image": "ghcr.io/guardian/devx-logs:2", + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "CloudquerySourceAwsOrgWideS3TaskDefinitionCloudquerySourceAwsOrgWideS3FirelensLogGroup4CE08508", + }, + "awslogs-region": { + "Ref": "AWS::Region", + }, + "awslogs-stream-prefix": "deploy/TEST/service-catalogue", + }, + }, + "MountPoints": [ + { + "ContainerPath": "/init", + "ReadOnly": false, + "SourceVolume": "firelens-volume", + }, + ], + "Name": "CloudquerySource-AwsOrgWideS3Firelens", + "ReadonlyRootFilesystem": true, + }, + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleFDE66B47", + "Arn", + ], + }, + "Family": "ServiceCatalogueCloudquerySourceAwsOrgWideS3TaskDefinition6734443A", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE", + ], + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideS3", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + "TaskRoleArn": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAwsOrgWideS38AEB5180", + "Arn", + ], + }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + { + "Name": "firelens-volume", + }, + ], + }, + "Type": "AWS::ECS::TaskDefinition", + }, + "CloudquerySourceAwsOrgWideS3TaskDefinitionCloudquerySourceAwsOrgWideS3FirelensLogGroup4CE08508": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideS3", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceAwsOrgWideS3TaskDefinitionEventsRoleA4289298": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "events.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideS3", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAwsOrgWideS3TaskDefinitionEventsRoleDefaultPolicy151FBFA4": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ecs:RunTask", + "Condition": { + "ArnEquals": { + "ecs:cluster": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + }, + }, + "Effect": "Allow", + "Resource": { + "Ref": "CloudquerySourceAwsOrgWideS3TaskDefinitionAF066748", + }, + }, + { + "Action": "ecs:TagResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":ecs:", + { + "Ref": "AWS::Region", + }, + ":*:task/", + { + "Ref": "servicecatalogueCluster5FC34DC5", + }, + "/*", ], - }, + ], }, - { - "Name": "PGHOST", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":host::", - ], - ], - }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleFDE66B47", + "Arn", + ], }, - { - "Name": "PGPASSWORD", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":password::", - ], - ], - }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAwsOrgWideS38AEB5180", + "Arn", + ], }, - ], - }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceAwsOrgWideS3TaskDefinitionEventsRoleDefaultPolicy151FBFA4", + "Roles": [ { - "Environment": [ - { - "Name": "STACK", - "Value": "deploy", + "Ref": "CloudquerySourceAwsOrgWideS3TaskDefinitionEventsRoleA4289298", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleDefaultPolicyE82AC40F": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", }, - { - "Name": "STAGE", - "Value": "TEST", + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "cloudqueryapikeyCCF82F53", }, - { - "Name": "APP", - "Value": "service-catalogue", + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceAwsOrgWideS3TaskDefinitionCloudquerySourceAwsOrgWideS3FirelensLogGroup4CE08508", + "Arn", + ], }, - { - "Name": "GU_REPO", - "Value": "guardian/service-catalogue", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleDefaultPolicyE82AC40F", + "Roles": [ + { + "Ref": "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleFDE66B47", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceAwsOrgWideS3TaskDefinitionExecutionRoleFDE66B47": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", }, - ], - "Essential": true, - "FirelensConfiguration": { - "Type": "fluentbit", }, - "Image": "ghcr.io/guardian/devx-logs:2", - "LogConfiguration": { - "LogDriver": "awslogs", - "Options": { - "awslogs-group": { - "Ref": "CloudquerySourceAwsSSMParametersTaskDefinitionCloudquerySourceAwsSSMParametersFirelensLogGroup0210F31D", - }, - "awslogs-region": { - "Ref": "AWS::Region", + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsOrgWideS3", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAwsRemainingDataScheduledEventRuleADE2D1CC": { + "Properties": { + "ScheduleExpression": "cron(0 16 ? * SAT *)", + "State": "ENABLED", + "Targets": [ + { + "Arn": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + "EcsParameters": { + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsVpcConfiguration": { + "AssignPublicIp": "DISABLED", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "PostgresAccessSecurityGroupServicecatalogue03C78F14", + "GroupId", + ], + }, + ], + "Subnets": { + "Ref": "PrivateSubnets", + }, }, - "awslogs-stream-prefix": "deploy/TEST/service-catalogue", }, - }, - "MountPoints": [ - { - "ContainerPath": "/init", - "ReadOnly": false, - "SourceVolume": "firelens-volume", + "PropagateTags": "TASK_DEFINITION", + "TaskCount": 1, + "TaskDefinitionArn": { + "Ref": "CloudquerySourceAwsRemainingDataTaskDefinitionF2586400", }, - ], - "Name": "CloudquerySource-AwsSSMParametersFirelens", - "ReadonlyRootFilesystem": true, + }, + "Id": "Target0", + "Input": "{}", + "RoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceAwsRemainingDataTaskDefinitionEventsRole45369093", + "Arn", + ], + }, }, ], - "Cpu": "256", - "ExecutionRoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRole7C7189F1", - "Arn", - ], - }, - "Family": "ServiceCatalogueCloudquerySourceAwsSSMParametersTaskDefinitionA4223CD9", - "Memory": "512", - "NetworkMode": "awsvpc", - "RequiresCompatibilities": [ - "FARGATE", - ], + }, + "Type": "AWS::Events::Rule", + }, + "CloudquerySourceAwsRemainingDataTaskDefinitionCloudquerySourceAwsRemainingDataFirelensLogGroupAEF7D0F2": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, "Tags": [ { "Key": "gu:cdk:version", @@ -11681,7 +11330,7 @@ spec: }, { "Key": "Name", - "Value": "AwsSSMParameters", + "Value": "AwsRemainingData", }, { "Key": "Stack", @@ -11692,30 +11341,11 @@ spec: "Value": "TEST", }, ], - "TaskRoleArn": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsSSMParameters3BFA609D", - "Arn", - ], - }, - "Volumes": [ - { - "Name": "config-volume", - }, - { - "Name": "cloudquery-volume", - }, - { - "Name": "tmp-volume", - }, - { - "Name": "firelens-volume", - }, - ], }, - "Type": "AWS::ECS::TaskDefinition", + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceAwsSSMParametersTaskDefinitionEventsRole510C5A17": { + "CloudquerySourceAwsRemainingDataTaskDefinitionEventsRole45369093": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -11740,7 +11370,7 @@ spec: }, { "Key": "Name", - "Value": "AwsSSMParameters", + "Value": "AwsRemainingData", }, { "Key": "Stack", @@ -11754,7 +11384,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsSSMParametersTaskDefinitionEventsRoleDefaultPolicy042DAF95": { + "CloudquerySourceAwsRemainingDataTaskDefinitionEventsRoleDefaultPolicyE9BEA52B": { "Properties": { "PolicyDocument": { "Statement": [ @@ -11772,7 +11402,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceAwsSSMParametersTaskDefinitionD2F49BC2", + "Ref": "CloudquerySourceAwsRemainingDataTaskDefinitionF2586400", }, }, { @@ -11804,7 +11434,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRole7C7189F1", + "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRole7F5255B8", "Arn", ], }, @@ -11814,7 +11444,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskAwsSSMParameters3BFA609D", + "servicecatalogueTESTtaskAwsRemainingData673BE318", "Arn", ], }, @@ -11822,16 +11452,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsSSMParametersTaskDefinitionEventsRoleDefaultPolicy042DAF95", + "PolicyName": "CloudquerySourceAwsRemainingDataTaskDefinitionEventsRoleDefaultPolicyE9BEA52B", "Roles": [ { - "Ref": "CloudquerySourceAwsSSMParametersTaskDefinitionEventsRole510C5A17", + "Ref": "CloudquerySourceAwsRemainingDataTaskDefinitionEventsRole45369093", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRole7C7189F1": { + "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRole7F5255B8": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -11856,7 +11486,7 @@ spec: }, { "Key": "Name", - "Value": "AwsSSMParameters", + "Value": "AwsRemainingData", }, { "Key": "Stack", @@ -11870,7 +11500,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRoleDefaultPolicy22EC1FE1": { + "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRoleDefaultPolicy3D9DFEF7": { "Properties": { "PolicyDocument": { "Statement": [ @@ -11902,7 +11532,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceAwsSSMParametersTaskDefinitionCloudquerySourceAwsSSMParametersFirelensLogGroup0210F31D", + "CloudquerySourceAwsRemainingDataTaskDefinitionCloudquerySourceAwsRemainingDataFirelensLogGroupAEF7D0F2", "Arn", ], }, @@ -11910,95 +11540,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRoleDefaultPolicy22EC1FE1", + "PolicyName": "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRoleDefaultPolicy3D9DFEF7", "Roles": [ { - "Ref": "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRole7C7189F1", + "Ref": "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRole7F5255B8", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceFastlyServicesScheduledEventRule1F83E593": { - "Properties": { - "ScheduleExpression": "rate(1 day)", - "State": "ENABLED", - "Targets": [ - { - "Arn": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], - }, - "EcsParameters": { - "LaunchType": "FARGATE", - "NetworkConfiguration": { - "AwsVpcConfiguration": { - "AssignPublicIp": "DISABLED", - "SecurityGroups": [ - { - "Fn::GetAtt": [ - "PostgresAccessSecurityGroupServicecatalogue03C78F14", - "GroupId", - ], - }, - ], - "Subnets": { - "Ref": "PrivateSubnets", - }, - }, - }, - "PropagateTags": "TASK_DEFINITION", - "TaskCount": 1, - "TaskDefinitionArn": { - "Ref": "CloudquerySourceFastlyServicesTaskDefinitionDCCD3FD4", - }, - }, - "Id": "Target0", - "Input": "{}", - "RoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceFastlyServicesTaskDefinitionEventsRoleE821B76C", - "Arn", - ], - }, - }, - ], - }, - "Type": "AWS::Events::Rule", - }, - "CloudquerySourceFastlyServicesTaskDefinitionCloudquerySourceFastlyServicesFirelensLogGroupF5954401": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "FastlyServices", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", - }, - "CloudquerySourceFastlyServicesTaskDefinitionDCCD3FD4": { + "CloudquerySourceAwsRemainingDataTaskDefinitionF2586400": { "Properties": { "ContainerDefinitions": [ { @@ -12007,21 +11558,86 @@ spec: "-c", "printf 'kind: source spec: - name: fastly - path: cloudquery/fastly - version: v3.0.7 + name: aws + path: cloudquery/aws + version: v27.5.0 tables: - - fastly_services - - fastly_service_versions - - fastly_service_backends - - fastly_service_domains - - fastly_service_health_checks - - fastly_account_users + - aws_* + skip_tables: + - aws_ec2_vpc_endpoint_services + - aws_cloudtrail_events + - aws_docdb_cluster_parameter_groups + - aws_docdb_engine_versions + - aws_ec2_instance_types + - aws_elasticache_engine_versions + - aws_elasticache_parameter_groups + - aws_elasticache_reserved_cache_nodes_offerings + - aws_elasticache_service_updates + - aws_emr_supported_instance_types + - aws_neptune_cluster_parameter_groups + - aws_neptune_db_parameter_groups + - aws_rds_cluster_parameter_groups + - aws_rds_db_parameter_groups + - aws_rds_engine_versions + - aws_servicequotas_services + - aws_identitystore_users + - aws_identitystore_groups + - aws_quicksight_data_sets + - aws_quicksight_dashboards + - aws_quicksight_analyses + - aws_quicksight_users + - aws_quicksight_templates + - aws_quicksight_groups + - aws_quicksight_folders + - aws_quicksight_data_sources + - aws_amp_workspaces + - aws_ssoadmin_instances + - aws_glue_connections + - aws_computeoptimizer_ecs_service_recommendations + - aws_xray_sampling_rules + - aws_xray_resource_policies + - aws_xray_groups + - aws_wellarchitected_* + - aws_stepfunctions_map_runs + - aws_stepfunctions_map_run_executions + - aws_stepfunctions_executions + - aws_organization* + - aws_accessanalyzer_* + - aws_securityhub_* + - aws_cloudformation_* + - aws_costexplorer_* + - aws_elbv1_* + - aws_elbv2_* + - aws_autoscaling_groups + - aws_acm* + - aws_lambda_* + - aws_ssm_parameters + - aws_cloudwatch_alarms + - aws_inspector_findings + - aws_inspector2_findings + - aws_s3* + - aws_dynamodb* + - aws_rds_instances + - aws_rds_clusters + - aws_rds_db_snapshots + - aws_rds_cluster_snapshots + - aws_backup_protected_resources + - aws_backup_vaults + - aws_backup_vault_recovery_points + - aws_ec2_instances + - aws_ec2_security_groups + - aws_ec2_images + - aws_iam_credential_reports destinations: - postgresql + otel_endpoint: 0.0.0.0:4318 + otel_endpoint_insecure: true spec: - concurrency: 1000 - fastly_api_key: \${FASTLY_API_KEY} + concurrency: 2000 + org: + member_role_name: cloudquery-access + organization_units: + - ou-123 ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -12038,12 +11654,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-FastlyServicesAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsRemainingDataAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "FastlyServices", + "Name": "AwsRemainingData", "Stack": "deploy", "Stage": "TEST", }, @@ -12053,7 +11669,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "409MiB", + "Value": "2457MiB", }, ], "Essential": true, @@ -12088,23 +11704,9 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-FastlyServicesContainer", + "Name": "CloudquerySource-AwsRemainingDataContainer", "ReadonlyRootFilesystem": true, "Secrets": [ - { - "Name": "FASTLY_API_KEY", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "fastlycredentialsF42D3C80", - }, - ":api-key::", - ], - ], - }, - }, { "Name": "DB_USERNAME", "ValueFrom": { @@ -12191,7 +11793,7 @@ spec: }, }, }, - "Name": "CloudquerySource-FastlyServicesAWSOTELCollector", + "Name": "CloudquerySource-AwsRemainingDataAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -12204,11 +11806,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('fastly_services', 'DAILY'),('fastly_service_versions', 'DAILY'),('fastly_service_backends', 'DAILY'),('fastly_service_domains', 'DAILY'),('fastly_service_health_checks', 'DAILY'),('fastly_account_users', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_%', 'WEEKLY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'WEEKLY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "FastlyServices", + "Name": "AwsRemainingData", "Stack": "deploy", "Stage": "TEST", }, @@ -12230,7 +11832,7 @@ spec: }, }, }, - "Name": "CloudquerySource-FastlyServicesPostgresContainer", + "Name": "CloudquerySource-AwsRemainingDataPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -12305,7 +11907,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceFastlyServicesTaskDefinitionCloudquerySourceFastlyServicesFirelensLogGroupF5954401", + "Ref": "CloudquerySourceAwsRemainingDataTaskDefinitionCloudquerySourceAwsRemainingDataFirelensLogGroupAEF7D0F2", }, "awslogs-region": { "Ref": "AWS::Region", @@ -12320,19 +11922,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-FastlyServicesFirelens", + "Name": "CloudquerySource-AwsRemainingDataFirelens", "ReadonlyRootFilesystem": true, }, ], - "Cpu": "256", + "Cpu": "1024", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleEF21A3A9", + "CloudquerySourceAwsRemainingDataTaskDefinitionExecutionRole7F5255B8", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceFastlyServicesTaskDefinition92125649", - "Memory": "512", + "Family": "ServiceCatalogueCloudquerySourceAwsRemainingDataTaskDefinition14D0B33A", + "Memory": "3072", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -12348,7 +11950,7 @@ spec: }, { "Key": "Name", - "Value": "FastlyServices", + "Value": "AwsRemainingData", }, { "Key": "Stack", @@ -12361,7 +11963,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskFastlyServices33D5467F", + "servicecatalogueTESTtaskAwsRemainingData673BE318", "Arn", ], }, @@ -12370,235 +11972,21 @@ spec: "Name": "config-volume", }, { - "Name": "cloudquery-volume", - }, - { - "Name": "tmp-volume", - }, - { - "Name": "firelens-volume", - }, - ], - }, - "Type": "AWS::ECS::TaskDefinition", - }, - "CloudquerySourceFastlyServicesTaskDefinitionEventsRoleDefaultPolicy4E34F504": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": "ecs:RunTask", - "Condition": { - "ArnEquals": { - "ecs:cluster": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], - }, - }, - }, - "Effect": "Allow", - "Resource": { - "Ref": "CloudquerySourceFastlyServicesTaskDefinitionDCCD3FD4", - }, - }, - { - "Action": "ecs:TagResource", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":ecs:", - { - "Ref": "AWS::Region", - }, - ":*:task/", - { - "Ref": "servicecatalogueCluster5FC34DC5", - }, - "/*", - ], - ], - }, - }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleEF21A3A9", - "Arn", - ], - }, - }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskFastlyServices33D5467F", - "Arn", - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceFastlyServicesTaskDefinitionEventsRoleDefaultPolicy4E34F504", - "Roles": [ - { - "Ref": "CloudquerySourceFastlyServicesTaskDefinitionEventsRoleE821B76C", - }, - ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceFastlyServicesTaskDefinitionEventsRoleE821B76C": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "events.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "FastlyServices", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleDefaultPolicy1B888D16": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "fastlycredentialsF42D3C80", - }, - }, - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - }, - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "cloudqueryapikeyCCF82F53", - }, - }, - { - "Action": [ - "logs:CreateLogStream", - "logs:PutLogEvents", - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceFastlyServicesTaskDefinitionCloudquerySourceFastlyServicesFirelensLogGroupF5954401", - "Arn", - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleDefaultPolicy1B888D16", - "Roles": [ - { - "Ref": "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleEF21A3A9", - }, - ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleEF21A3A9": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ecs-tasks.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "FastlyServices", + "Name": "cloudquery-volume", }, { - "Key": "Stack", - "Value": "deploy", + "Name": "tmp-volume", }, { - "Key": "Stage", - "Value": "TEST", + "Name": "firelens-volume", }, ], }, - "Type": "AWS::IAM::Role", + "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceGalaxiesScheduledEventRuleCC774CB8": { + "CloudquerySourceAwsSSMParametersScheduledEventRule0A998B82": { "Properties": { - "ScheduleExpression": "rate(1 day)", + "ScheduleExpression": "cron(20 1 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -12629,14 +12017,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceGalaxiesTaskDefinition0777FEFC", + "Ref": "CloudquerySourceAwsSSMParametersTaskDefinitionD2F49BC2", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceGalaxiesTaskDefinitionEventsRoleD1AF9A82", + "CloudquerySourceAwsSSMParametersTaskDefinitionEventsRole510C5A17", "Arn", ], }, @@ -12645,56 +12033,59 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceGalaxiesTaskDefinition0777FEFC": { + "CloudquerySourceAwsSSMParametersTaskDefinitionCloudquerySourceAwsSSMParametersFirelensLogGroup0210F31D": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsSSMParameters", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceAwsSSMParametersTaskDefinitionD2F49BC2": { "Properties": { "ContainerDefinitions": [ { "Command": [ "/bin/sh", "-c", - { - "Fn::Join": [ - "", - [ - "printf 'kind: source + "printf 'kind: source spec: - name: galaxies - path: guardian/galaxies - registry: github - version: v1.1.8 + name: aws + path: cloudquery/aws + version: v27.5.0 + tables: + - aws_ssm_parameters destinations: - postgresql - tables: - - galaxies_people_table - - galaxies_teams_table - - galaxies_streams_table - - galaxies_people_profile_info_table + otel_endpoint: 0.0.0.0:4318 + otel_endpoint_insecure: true spec: - bucket: ", - { - "Fn::Select": [ - 0, - { - "Fn::Split": [ - "/", - { - "Fn::Select": [ - 5, - { - "Fn::Split": [ - ":", - { - "Ref": "ActionsStaticSiteBucketArnParam", - }, - ], - }, - ], - }, - ], - }, - ], - }, - " + org: + member_role_name: cloudquery-access + organization_units: + - ou-123 ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -12707,19 +12098,16 @@ spec: user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full ' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", - ], - ], - }, ], "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-GalaxiesAWSOTELCollector", + "ContainerName": "CloudquerySource-AwsSSMParametersAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "Galaxies", + "Name": "AwsSSMParameters", "Stack": "deploy", "Stage": "TEST", }, @@ -12764,7 +12152,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-GalaxiesContainer", + "Name": "CloudquerySource-AwsSSMParametersContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -12853,7 +12241,7 @@ spec: }, }, }, - "Name": "CloudquerySource-GalaxiesAWSOTELCollector", + "Name": "CloudquerySource-AwsSSMParametersAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -12866,11 +12254,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('galaxies_people_table', 'DAILY'),('galaxies_teams_table', 'DAILY'),('galaxies_streams_table', 'DAILY'),('galaxies_people_profile_info_table', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('aws_ssm_parameters', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "Galaxies", + "Name": "AwsSSMParameters", "Stack": "deploy", "Stage": "TEST", }, @@ -12892,7 +12280,7 @@ spec: }, }, }, - "Name": "CloudquerySource-GalaxiesPostgresContainer", + "Name": "CloudquerySource-AwsSSMParametersPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -12967,7 +12355,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceGalaxiesTaskDefinitionCloudquerySourceGalaxiesFirelensLogGroupB5272E0D", + "Ref": "CloudquerySourceAwsSSMParametersTaskDefinitionCloudquerySourceAwsSSMParametersFirelensLogGroup0210F31D", }, "awslogs-region": { "Ref": "AWS::Region", @@ -12982,18 +12370,18 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-GalaxiesFirelens", + "Name": "CloudquerySource-AwsSSMParametersFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDDB0DD4B", + "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRole7C7189F1", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceGalaxiesTaskDefinition6724C33F", + "Family": "ServiceCatalogueCloudquerySourceAwsSSMParametersTaskDefinitionA4223CD9", "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ @@ -13010,7 +12398,7 @@ spec: }, { "Key": "Name", - "Value": "Galaxies", + "Value": "AwsSSMParameters", }, { "Key": "Stack", @@ -13023,7 +12411,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskGalaxiesDBF8C9E4", + "servicecatalogueTESTtaskAwsSSMParameters3BFA609D", "Arn", ], }, @@ -13044,37 +12432,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceGalaxiesTaskDefinitionCloudquerySourceGalaxiesFirelensLogGroupB5272E0D": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "Galaxies", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", - }, - "CloudquerySourceGalaxiesTaskDefinitionEventsRoleD1AF9A82": { + "CloudquerySourceAwsSSMParametersTaskDefinitionEventsRole510C5A17": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -13099,7 +12457,7 @@ spec: }, { "Key": "Name", - "Value": "Galaxies", + "Value": "AwsSSMParameters", }, { "Key": "Stack", @@ -13113,7 +12471,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceGalaxiesTaskDefinitionEventsRoleDefaultPolicy022DA6EA": { + "CloudquerySourceAwsSSMParametersTaskDefinitionEventsRoleDefaultPolicy042DAF95": { "Properties": { "PolicyDocument": { "Statement": [ @@ -13131,7 +12489,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceGalaxiesTaskDefinition0777FEFC", + "Ref": "CloudquerySourceAwsSSMParametersTaskDefinitionD2F49BC2", }, }, { @@ -13163,17 +12521,105 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDDB0DD4B", + "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRole7C7189F1", + "Arn", + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskAwsSSMParameters3BFA609D", "Arn", ], }, }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceAwsSSMParametersTaskDefinitionEventsRoleDefaultPolicy042DAF95", + "Roles": [ + { + "Ref": "CloudquerySourceAwsSSMParametersTaskDefinitionEventsRole510C5A17", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRole7C7189F1": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "AwsSSMParameters", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRoleDefaultPolicy22EC1FE1": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "cloudqueryapikeyCCF82F53", + }, + }, { - "Action": "iam:PassRole", + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskGalaxiesDBF8C9E4", + "CloudquerySourceAwsSSMParametersTaskDefinitionCloudquerySourceAwsSSMParametersFirelensLogGroup0210F31D", "Arn", ], }, @@ -13181,29 +12627,68 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceGalaxiesTaskDefinitionEventsRoleDefaultPolicy022DA6EA", + "PolicyName": "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRoleDefaultPolicy22EC1FE1", "Roles": [ { - "Ref": "CloudquerySourceGalaxiesTaskDefinitionEventsRoleD1AF9A82", + "Ref": "CloudquerySourceAwsSSMParametersTaskDefinitionExecutionRole7C7189F1", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDDB0DD4B": { + "CloudquerySourceFastlyServicesScheduledEventRule1F83E593": { "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ecs-tasks.amazonaws.com", + "ScheduleExpression": "rate(1 day)", + "State": "ENABLED", + "Targets": [ + { + "Arn": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + "EcsParameters": { + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsVpcConfiguration": { + "AssignPublicIp": "DISABLED", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "PostgresAccessSecurityGroupServicecatalogue03C78F14", + "GroupId", + ], + }, + ], + "Subnets": { + "Ref": "PrivateSubnets", + }, + }, + }, + "PropagateTags": "TASK_DEFINITION", + "TaskCount": 1, + "TaskDefinitionArn": { + "Ref": "CloudquerySourceFastlyServicesTaskDefinitionDCCD3FD4", }, }, - ], - "Version": "2012-10-17", - }, + "Id": "Target0", + "Input": "{}", + "RoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceFastlyServicesTaskDefinitionEventsRoleE821B76C", + "Arn", + ], + }, + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "CloudquerySourceFastlyServicesTaskDefinitionCloudquerySourceFastlyServicesFirelensLogGroupF5954401": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, "Tags": [ { "Key": "gu:cdk:version", @@ -13215,7 +12700,7 @@ spec: }, { "Key": "Name", - "Value": "Galaxies", + "Value": "FastlyServices", }, { "Key": "Stack", @@ -13227,150 +12712,348 @@ spec: }, ], }, - "Type": "AWS::IAM::Role", + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDefaultPolicyB6D2CB7A": { + "CloudquerySourceFastlyServicesTaskDefinitionDCCD3FD4": { "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + "ContainerDefinitions": [ + { + "Command": [ + "/bin/sh", + "-c", + "printf 'kind: source +spec: + name: fastly + path: cloudquery/fastly + version: v3.0.7 + tables: + - fastly_services + - fastly_service_versions + - fastly_service_backends + - fastly_service_domains + - fastly_service_health_checks + - fastly_account_users + destinations: + - postgresql + spec: + concurrency: 1000 + fastly_api_key: \${FASTLY_API_KEY} +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination +spec: + name: postgresql + registry: github + path: cloudquery/postgresql + version: v7.2.0 + migrate_mode: forced + spec: + connection_string: >- + user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 + dbname=postgres sslmode=verify-full +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", + ], + "DependsOn": [ + { + "Condition": "HEALTHY", + "ContainerName": "CloudquerySource-FastlyServicesAWSOTELCollector", }, + ], + "DockerLabels": { + "App": "service-catalogue", + "Name": "FastlyServices", + "Stack": "deploy", + "Stage": "TEST", }, - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", + "EntryPoint": [ + "", + ], + "Environment": [ + { + "Name": "GOMEMLIMIT", + "Value": "409MiB", + }, + ], + "Essential": true, + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:stable", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, + }, + }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], + "Name": "CloudquerySource-FastlyServicesContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ + { + "Name": "FASTLY_API_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "fastlycredentialsF42D3C80", + }, + ":api-key::", + ], + ], + }, + }, + { + "Name": "DB_USERNAME", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":username::", + ], + ], + }, + }, + { + "Name": "DB_HOST", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":host::", + ], + ], + }, + }, + { + "Name": "DB_PASSWORD", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":password::", + ], + ], + }, + }, + { + "Name": "CLOUDQUERY_API_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "cloudqueryapikeyCCF82F53", + }, + ":api-key::", + ], + ], + }, + }, + ], + }, + { + "Command": [ + "--config=/etc/ecs/ecs-xray.yaml", + ], + "Essential": true, + "HealthCheck": { + "Command": [ + "CMD", + "/healthcheck", ], - "Effect": "Allow", - "Resource": { - "Ref": "cloudqueryapikeyCCF82F53", - }, + "Interval": 5, + "Retries": 3, + "Timeout": 5, }, - { - "Action": [ - "logs:CreateLogStream", - "logs:PutLogEvents", - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceGalaxiesTaskDefinitionCloudquerySourceGalaxiesFirelensLogGroupB5272E0D", - "Arn", - ], + "Image": "public.ecr.aws/aws-observability/aws-otel-collector:v0.35.0", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, }, }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDefaultPolicyB6D2CB7A", - "Roles": [ - { - "Ref": "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDDB0DD4B", + "Name": "CloudquerySource-FastlyServicesAWSOTELCollector", + "PortMappings": [ + { + "ContainerPort": 4318, + "Protocol": "tcp", + }, + ], + "ReadonlyRootFilesystem": true, }, - ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceGitHubIssuesScheduledEventRuleAF7C253C": { - "Properties": { - "ScheduleExpression": "cron(0 2 * * ? *)", - "State": "ENABLED", - "Targets": [ { - "Arn": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], + "Command": [ + "/bin/sh", + "-c", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('fastly_services', 'DAILY'),('fastly_service_versions', 'DAILY'),('fastly_service_backends', 'DAILY'),('fastly_service_domains', 'DAILY'),('fastly_service_health_checks', 'DAILY'),('fastly_account_users', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + ], + "DockerLabels": { + "App": "service-catalogue", + "Name": "FastlyServices", + "Stack": "deploy", + "Stage": "TEST", }, - "EcsParameters": { - "LaunchType": "FARGATE", - "NetworkConfiguration": { - "AwsVpcConfiguration": { - "AssignPublicIp": "DISABLED", - "SecurityGroups": [ - { - "Fn::GetAtt": [ - "PostgresAccessSecurityGroupServicecatalogue03C78F14", - "GroupId", - ], - }, + "EntryPoint": [ + "", + ], + "Essential": false, + "Image": "public.ecr.aws/docker/library/postgres:16-alpine", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, + }, + }, + "Name": "CloudquerySource-FastlyServicesPostgresContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ + { + "Name": "PGUSER", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":username::", + ], ], - "Subnets": { - "Ref": "PrivateSubnets", - }, }, }, - "PropagateTags": "TASK_DEFINITION", - "TaskCount": 1, - "TaskDefinitionArn": { - "Ref": "CloudquerySourceGitHubIssuesTaskDefinitionFA21D536", + { + "Name": "PGHOST", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":host::", + ], + ], + }, }, - }, - "Id": "Target0", - "Input": "{}", - "RoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceGitHubIssuesTaskDefinitionEventsRole056ACC7E", - "Arn", - ], - }, - }, - ], - }, - "Type": "AWS::Events::Rule", - }, - "CloudquerySourceGitHubIssuesTaskDefinitionCloudquerySourceGitHubIssuesFirelensLogGroupE9112ED9": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "GitHubIssues", - }, - { - "Key": "Stack", - "Value": "deploy", + { + "Name": "PGPASSWORD", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":password::", + ], + ], + }, + }, + ], }, { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", - }, - "CloudquerySourceGitHubIssuesTaskDefinitionEventsRole056ACC7E": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "events.amazonaws.com", + "Environment": [ + { + "Name": "STACK", + "Value": "deploy", + }, + { + "Name": "STAGE", + "Value": "TEST", + }, + { + "Name": "APP", + "Value": "service-catalogue", + }, + { + "Name": "GU_REPO", + "Value": "guardian/service-catalogue", + }, + ], + "Essential": true, + "FirelensConfiguration": { + "Type": "fluentbit", + }, + "Image": "ghcr.io/guardian/devx-logs:2", + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "CloudquerySourceFastlyServicesTaskDefinitionCloudquerySourceFastlyServicesFirelensLogGroupF5954401", + }, + "awslogs-region": { + "Ref": "AWS::Region", + }, + "awslogs-stream-prefix": "deploy/TEST/service-catalogue", }, }, + "MountPoints": [ + { + "ContainerPath": "/init", + "ReadOnly": false, + "SourceVolume": "firelens-volume", + }, + ], + "Name": "CloudquerySource-FastlyServicesFirelens", + "ReadonlyRootFilesystem": true, + }, + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleEF21A3A9", + "Arn", ], - "Version": "2012-10-17", }, + "Family": "ServiceCatalogueCloudquerySourceFastlyServicesTaskDefinition92125649", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE", + ], "Tags": [ { "Key": "gu:cdk:version", @@ -13382,7 +13065,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubIssues", + "Value": "FastlyServices", }, { "Key": "Stack", @@ -13393,10 +13076,30 @@ spec: "Value": "TEST", }, ], + "TaskRoleArn": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskFastlyServices33D5467F", + "Arn", + ], + }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + { + "Name": "firelens-volume", + }, + ], }, - "Type": "AWS::IAM::Role", + "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceGitHubIssuesTaskDefinitionEventsRoleDefaultPolicy13E8811A": { + "CloudquerySourceFastlyServicesTaskDefinitionEventsRoleDefaultPolicy4E34F504": { "Properties": { "PolicyDocument": { "Statement": [ @@ -13414,7 +13117,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceGitHubIssuesTaskDefinitionFA21D536", + "Ref": "CloudquerySourceFastlyServicesTaskDefinitionDCCD3FD4", }, }, { @@ -13446,7 +13149,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRole6BDDECB3", + "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleEF21A3A9", "Arn", ], }, @@ -13456,7 +13159,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskGitHubIssues1EFFA3D3", + "servicecatalogueTESTtaskFastlyServices33D5467F", "Arn", ], }, @@ -13464,16 +13167,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceGitHubIssuesTaskDefinitionEventsRoleDefaultPolicy13E8811A", + "PolicyName": "CloudquerySourceFastlyServicesTaskDefinitionEventsRoleDefaultPolicy4E34F504", "Roles": [ { - "Ref": "CloudquerySourceGitHubIssuesTaskDefinitionEventsRole056ACC7E", + "Ref": "CloudquerySourceFastlyServicesTaskDefinitionEventsRoleE821B76C", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRole6BDDECB3": { + "CloudquerySourceFastlyServicesTaskDefinitionEventsRoleE821B76C": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -13481,7 +13184,7 @@ spec: "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { - "Service": "ecs-tasks.amazonaws.com", + "Service": "events.amazonaws.com", }, }, ], @@ -13498,7 +13201,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubIssues", + "Value": "FastlyServices", }, { "Key": "Stack", @@ -13512,7 +13215,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRoleDefaultPolicyFA58BC48": { + "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleDefaultPolicy1B888D16": { "Properties": { "PolicyDocument": { "Statement": [ @@ -13523,7 +13226,7 @@ spec: ], "Effect": "Allow", "Resource": { - "Ref": "githubcredentialsAF453741", + "Ref": "fastlycredentialsF42D3C80", }, }, { @@ -13554,7 +13257,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceGitHubIssuesTaskDefinitionCloudquerySourceGitHubIssuesFirelensLogGroupE9112ED9", + "CloudquerySourceFastlyServicesTaskDefinitionCloudquerySourceFastlyServicesFirelensLogGroupF5954401", "Arn", ], }, @@ -13562,41 +13265,153 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRoleDefaultPolicyFA58BC48", + "PolicyName": "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleDefaultPolicy1B888D16", "Roles": [ { - "Ref": "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRole6BDDECB3", + "Ref": "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleEF21A3A9", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceGitHubIssuesTaskDefinitionFA21D536": { + "CloudquerySourceFastlyServicesTaskDefinitionExecutionRoleEF21A3A9": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "FastlyServices", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceGalaxiesScheduledEventRuleCC774CB8": { + "Properties": { + "ScheduleExpression": "rate(1 day)", + "State": "ENABLED", + "Targets": [ + { + "Arn": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + "EcsParameters": { + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsVpcConfiguration": { + "AssignPublicIp": "DISABLED", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "PostgresAccessSecurityGroupServicecatalogue03C78F14", + "GroupId", + ], + }, + ], + "Subnets": { + "Ref": "PrivateSubnets", + }, + }, + }, + "PropagateTags": "TASK_DEFINITION", + "TaskCount": 1, + "TaskDefinitionArn": { + "Ref": "CloudquerySourceGalaxiesTaskDefinition0777FEFC", + }, + }, + "Id": "Target0", + "Input": "{}", + "RoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceGalaxiesTaskDefinitionEventsRoleD1AF9A82", + "Arn", + ], + }, + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "CloudquerySourceGalaxiesTaskDefinition0777FEFC": { "Properties": { "ContainerDefinitions": [ { "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /usr/share/cloudquery/github-private-key;echo -n $GITHUB_APP_ID > /usr/share/cloudquery/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /usr/share/cloudquery/github-installation-id;printf 'kind: source + { + "Fn::Join": [ + "", + [ + "printf 'kind: source spec: - name: github - path: cloudquery/github - version: v10.0.1 - tables: - - github_issues + name: galaxies + path: guardian/galaxies + registry: github + version: v1.1.8 destinations: - postgresql + tables: + - galaxies_people_table + - galaxies_teams_table + - galaxies_streams_table + - galaxies_people_profile_info_table spec: - concurrency: 1000 - orgs: - - guardian - app_auth: - - org: guardian - private_key_path: /usr/share/cloudquery/github-private-key - app_id: \${file:/usr/share/cloudquery/github-app-id} - installation_id: \${file:/usr/share/cloudquery/github-installation-id} - include_archived_repos: true + bucket: ", + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "/", + { + "Fn::Select": [ + 5, + { + "Fn::Split": [ + ":", + { + "Ref": "ActionsStaticSiteBucketArnParam", + }, + ], + }, + ], + }, + ], + }, + ], + }, + " ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -13609,16 +13424,19 @@ spec: user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full ' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", + ], + ], + }, ], "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-GitHubIssuesAWSOTELCollector", + "ContainerName": "CloudquerySource-GalaxiesAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "GitHubIssues", + "Name": "Galaxies", "Stack": "deploy", "Stage": "TEST", }, @@ -13628,7 +13446,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "819MiB", + "Value": "409MiB", }, ], "Essential": true, @@ -13663,51 +13481,9 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-GitHubIssuesContainer", - "ReadonlyRootFilesystem": true, - "Secrets": [ - { - "Name": "GITHUB_PRIVATE_KEY", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "githubcredentialsAF453741", - }, - ":private-key::", - ], - ], - }, - }, - { - "Name": "GITHUB_APP_ID", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "githubcredentialsAF453741", - }, - ":app-id::", - ], - ], - }, - }, - { - "Name": "GITHUB_INSTALLATION_ID", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "githubcredentialsAF453741", - }, - ":installation-id::", - ], - ], - }, - }, + "Name": "CloudquerySource-GalaxiesContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ { "Name": "DB_USERNAME", "ValueFrom": { @@ -13794,7 +13570,7 @@ spec: }, }, }, - "Name": "CloudquerySource-GitHubIssuesAWSOTELCollector", + "Name": "CloudquerySource-GalaxiesAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -13807,11 +13583,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('github_issues', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('galaxies_people_table', 'DAILY'),('galaxies_teams_table', 'DAILY'),('galaxies_streams_table', 'DAILY'),('galaxies_people_profile_info_table', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "GitHubIssues", + "Name": "Galaxies", "Stack": "deploy", "Stage": "TEST", }, @@ -13833,7 +13609,7 @@ spec: }, }, }, - "Name": "CloudquerySource-GitHubIssuesPostgresContainer", + "Name": "CloudquerySource-GalaxiesPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -13908,7 +13684,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceGitHubIssuesTaskDefinitionCloudquerySourceGitHubIssuesFirelensLogGroupE9112ED9", + "Ref": "CloudquerySourceGalaxiesTaskDefinitionCloudquerySourceGalaxiesFirelensLogGroupB5272E0D", }, "awslogs-region": { "Ref": "AWS::Region", @@ -13923,19 +13699,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-GitHubIssuesFirelens", + "Name": "CloudquerySource-GalaxiesFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRole6BDDECB3", + "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDDB0DD4B", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceGitHubIssuesTaskDefinition750BB414", - "Memory": "1024", + "Family": "ServiceCatalogueCloudquerySourceGalaxiesTaskDefinition6724C33F", + "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -13951,7 +13727,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubIssues", + "Value": "Galaxies", }, { "Key": "Stack", @@ -13964,30 +13740,264 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskGitHubIssues1EFFA3D3", + "servicecatalogueTESTtaskGalaxiesDBF8C9E4", "Arn", ], }, - "Volumes": [ - { - "Name": "config-volume", - }, - { - "Name": "cloudquery-volume", - }, - { - "Name": "tmp-volume", - }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + { + "Name": "firelens-volume", + }, + ], + }, + "Type": "AWS::ECS::TaskDefinition", + }, + "CloudquerySourceGalaxiesTaskDefinitionCloudquerySourceGalaxiesFirelensLogGroupB5272E0D": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "Galaxies", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceGalaxiesTaskDefinitionEventsRoleD1AF9A82": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "events.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "Galaxies", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceGalaxiesTaskDefinitionEventsRoleDefaultPolicy022DA6EA": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ecs:RunTask", + "Condition": { + "ArnEquals": { + "ecs:cluster": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + }, + }, + "Effect": "Allow", + "Resource": { + "Ref": "CloudquerySourceGalaxiesTaskDefinition0777FEFC", + }, + }, + { + "Action": "ecs:TagResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":ecs:", + { + "Ref": "AWS::Region", + }, + ":*:task/", + { + "Ref": "servicecatalogueCluster5FC34DC5", + }, + "/*", + ], + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDDB0DD4B", + "Arn", + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskGalaxiesDBF8C9E4", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceGalaxiesTaskDefinitionEventsRoleDefaultPolicy022DA6EA", + "Roles": [ + { + "Ref": "CloudquerySourceGalaxiesTaskDefinitionEventsRoleD1AF9A82", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDDB0DD4B": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "Galaxies", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDefaultPolicyB6D2CB7A": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "cloudqueryapikeyCCF82F53", + }, + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceGalaxiesTaskDefinitionCloudquerySourceGalaxiesFirelensLogGroupB5272E0D", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDefaultPolicyB6D2CB7A", + "Roles": [ { - "Name": "firelens-volume", + "Ref": "CloudquerySourceGalaxiesTaskDefinitionExecutionRoleDDB0DD4B", }, ], }, - "Type": "AWS::ECS::TaskDefinition", + "Type": "AWS::IAM::Policy", }, - "CloudquerySourceGitHubLanguagesScheduledEventRule3F047D8E": { + "CloudquerySourceGitHubIssuesScheduledEventRuleAF7C253C": { "Properties": { - "ScheduleExpression": "rate(7 days)", + "ScheduleExpression": "cron(0 2 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -14018,14 +14028,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceGitHubLanguagesTaskDefinitionDA995E2B", + "Ref": "CloudquerySourceGitHubIssuesTaskDefinitionFA21D536", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceGitHubLanguagesTaskDefinitionEventsRoleD0480706", + "CloudquerySourceGitHubIssuesTaskDefinitionEventsRole056ACC7E", "Arn", ], }, @@ -14034,7 +14044,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceGitHubLanguagesTaskDefinitionCloudquerySourceGitHubLanguagesFirelensLogGroup38EAAC3E": { + "CloudquerySourceGitHubIssuesTaskDefinitionCloudquerySourceGitHubIssuesFirelensLogGroupE9112ED9": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -14049,7 +14059,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubLanguages", + "Value": "GitHubIssues", }, { "Key": "Stack", @@ -14064,23 +14074,246 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceGitHubLanguagesTaskDefinitionDA995E2B": { + "CloudquerySourceGitHubIssuesTaskDefinitionEventsRole056ACC7E": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "events.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "GitHubIssues", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceGitHubIssuesTaskDefinitionEventsRoleDefaultPolicy13E8811A": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ecs:RunTask", + "Condition": { + "ArnEquals": { + "ecs:cluster": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + }, + }, + "Effect": "Allow", + "Resource": { + "Ref": "CloudquerySourceGitHubIssuesTaskDefinitionFA21D536", + }, + }, + { + "Action": "ecs:TagResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":ecs:", + { + "Ref": "AWS::Region", + }, + ":*:task/", + { + "Ref": "servicecatalogueCluster5FC34DC5", + }, + "/*", + ], + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRole6BDDECB3", + "Arn", + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskGitHubIssues1EFFA3D3", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceGitHubIssuesTaskDefinitionEventsRoleDefaultPolicy13E8811A", + "Roles": [ + { + "Ref": "CloudquerySourceGitHubIssuesTaskDefinitionEventsRole056ACC7E", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRole6BDDECB3": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "GitHubIssues", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRoleDefaultPolicyFA58BC48": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "githubcredentialsAF453741", + }, + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "cloudqueryapikeyCCF82F53", + }, + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceGitHubIssuesTaskDefinitionCloudquerySourceGitHubIssuesFirelensLogGroupE9112ED9", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRoleDefaultPolicyFA58BC48", + "Roles": [ + { + "Ref": "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRole6BDDECB3", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceGitHubIssuesTaskDefinitionFA21D536": { "Properties": { "ContainerDefinitions": [ { "Command": [ "/bin/sh", "-c", - "printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /usr/share/cloudquery/github-private-key;echo -n $GITHUB_APP_ID > /usr/share/cloudquery/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /usr/share/cloudquery/github-installation-id;printf 'kind: source spec: - name: github-languages - path: guardian/github-languages - version: v0.0.5 + name: github + path: cloudquery/github + version: v10.0.1 + tables: + - github_issues destinations: - postgresql - tables: - - github_languages - registry: github + spec: + concurrency: 1000 + orgs: + - guardian + app_auth: + - org: guardian + private_key_path: /usr/share/cloudquery/github-private-key + app_id: \${file:/usr/share/cloudquery/github-app-id} + installation_id: \${file:/usr/share/cloudquery/github-installation-id} + include_archived_repos: true ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -14097,12 +14330,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-GitHubLanguagesAWSOTELCollector", + "ContainerName": "CloudquerySource-GitHubIssuesAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "GitHubLanguages", + "Name": "GitHubIssues", "Stack": "deploy", "Stage": "TEST", }, @@ -14112,7 +14345,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "409MiB", + "Value": "819MiB", }, ], "Essential": true, @@ -14147,13 +14380,49 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-GitHubLanguagesContainer", + "Name": "CloudquerySource-GitHubIssuesContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { - "Name": "GITHUB_ACCESS_TOKEN", + "Name": "GITHUB_PRIVATE_KEY", "ValueFrom": { - "Ref": "githublanguages5093EDEC", + "Fn::Join": [ + "", + [ + { + "Ref": "githubcredentialsAF453741", + }, + ":private-key::", + ], + ], + }, + }, + { + "Name": "GITHUB_APP_ID", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "githubcredentialsAF453741", + }, + ":app-id::", + ], + ], + }, + }, + { + "Name": "GITHUB_INSTALLATION_ID", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "githubcredentialsAF453741", + }, + ":installation-id::", + ], + ], }, }, { @@ -14242,7 +14511,7 @@ spec: }, }, }, - "Name": "CloudquerySource-GitHubLanguagesAWSOTELCollector", + "Name": "CloudquerySource-GitHubIssuesAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -14255,11 +14524,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('github_languages', 'WEEKLY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'WEEKLY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('github_issues', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "GitHubLanguages", + "Name": "GitHubIssues", "Stack": "deploy", "Stage": "TEST", }, @@ -14281,7 +14550,7 @@ spec: }, }, }, - "Name": "CloudquerySource-GitHubLanguagesPostgresContainer", + "Name": "CloudquerySource-GitHubIssuesPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -14348,221 +14617,46 @@ spec: }, ], "Essential": true, - "FirelensConfiguration": { - "Type": "fluentbit", - }, - "Image": "ghcr.io/guardian/devx-logs:2", - "LogConfiguration": { - "LogDriver": "awslogs", - "Options": { - "awslogs-group": { - "Ref": "CloudquerySourceGitHubLanguagesTaskDefinitionCloudquerySourceGitHubLanguagesFirelensLogGroup38EAAC3E", - }, - "awslogs-region": { - "Ref": "AWS::Region", - }, - "awslogs-stream-prefix": "deploy/TEST/service-catalogue", - }, - }, - "MountPoints": [ - { - "ContainerPath": "/init", - "ReadOnly": false, - "SourceVolume": "firelens-volume", - }, - ], - "Name": "CloudquerySource-GitHubLanguagesFirelens", - "ReadonlyRootFilesystem": true, - }, - ], - "Cpu": "256", - "ExecutionRoleArn": { - "Fn::GetAtt": [ - "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleB77C8E3D", - "Arn", - ], - }, - "Family": "ServiceCatalogueCloudquerySourceGitHubLanguagesTaskDefinitionB1DA60BF", - "Memory": "512", - "NetworkMode": "awsvpc", - "RequiresCompatibilities": [ - "FARGATE", - ], - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "GitHubLanguages", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - "TaskRoleArn": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskGitHubLanguagesED141D27", - "Arn", - ], - }, - "Volumes": [ - { - "Name": "config-volume", - }, - { - "Name": "cloudquery-volume", - }, - { - "Name": "tmp-volume", - }, - { - "Name": "firelens-volume", - }, - ], - }, - "Type": "AWS::ECS::TaskDefinition", - }, - "CloudquerySourceGitHubLanguagesTaskDefinitionEventsRoleD0480706": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "events.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "GitHubLanguages", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceGitHubLanguagesTaskDefinitionEventsRoleDefaultPolicy1B11BE73": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": "ecs:RunTask", - "Condition": { - "ArnEquals": { - "ecs:cluster": { - "Fn::GetAtt": [ - "servicecatalogueCluster5FC34DC5", - "Arn", - ], - }, - }, - }, - "Effect": "Allow", - "Resource": { - "Ref": "CloudquerySourceGitHubLanguagesTaskDefinitionDA995E2B", - }, - }, - { - "Action": "ecs:TagResource", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":ecs:", - { - "Ref": "AWS::Region", - }, - ":*:task/", - { - "Ref": "servicecatalogueCluster5FC34DC5", - }, - "/*", - ], - ], - }, - }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleB77C8E3D", - "Arn", - ], - }, + "FirelensConfiguration": { + "Type": "fluentbit", }, - { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "servicecatalogueTESTtaskGitHubLanguagesED141D27", - "Arn", - ], + "Image": "ghcr.io/guardian/devx-logs:2", + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "CloudquerySourceGitHubIssuesTaskDefinitionCloudquerySourceGitHubIssuesFirelensLogGroupE9112ED9", + }, + "awslogs-region": { + "Ref": "AWS::Region", + }, + "awslogs-stream-prefix": "deploy/TEST/service-catalogue", }, }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CloudquerySourceGitHubLanguagesTaskDefinitionEventsRoleDefaultPolicy1B11BE73", - "Roles": [ - { - "Ref": "CloudquerySourceGitHubLanguagesTaskDefinitionEventsRoleD0480706", + "MountPoints": [ + { + "ContainerPath": "/init", + "ReadOnly": false, + "SourceVolume": "firelens-volume", + }, + ], + "Name": "CloudquerySource-GitHubIssuesFirelens", + "ReadonlyRootFilesystem": true, }, ], - }, - "Type": "AWS::IAM::Policy", - }, - "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleB77C8E3D": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ecs-tasks.amazonaws.com", - }, - }, + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceGitHubIssuesTaskDefinitionExecutionRole6BDDECB3", + "Arn", ], - "Version": "2012-10-17", }, + "Family": "ServiceCatalogueCloudquerySourceGitHubIssuesTaskDefinition750BB414", + "Memory": "1024", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE", + ], "Tags": [ { "Key": "gu:cdk:version", @@ -14574,7 +14668,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubLanguages", + "Value": "GitHubIssues", }, { "Key": "Stack", @@ -14585,71 +14679,32 @@ spec: "Value": "TEST", }, ], - }, - "Type": "AWS::IAM::Role", - }, - "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleDefaultPolicy9D57CA88": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "githublanguages5093EDEC", - }, - }, - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - }, - { - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ], - "Effect": "Allow", - "Resource": { - "Ref": "cloudqueryapikeyCCF82F53", - }, - }, - { - "Action": [ - "logs:CreateLogStream", - "logs:PutLogEvents", - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "CloudquerySourceGitHubLanguagesTaskDefinitionCloudquerySourceGitHubLanguagesFirelensLogGroup38EAAC3E", - "Arn", - ], - }, - }, + "TaskRoleArn": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskGitHubIssues1EFFA3D3", + "Arn", ], - "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleDefaultPolicy9D57CA88", - "Roles": [ + "Volumes": [ { - "Ref": "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleB77C8E3D", + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + { + "Name": "firelens-volume", }, ], }, - "Type": "AWS::IAM::Policy", + "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceGitHubRepositoriesScheduledEventRuleC7F5836E": { + "CloudquerySourceGitHubLanguagesScheduledEventRule3F047D8E": { "Properties": { - "ScheduleExpression": "cron(0 0 * * ? *)", + "ScheduleExpression": "rate(7 days)", "State": "ENABLED", "Targets": [ { @@ -14680,14 +14735,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceGitHubRepositoriesTaskDefinition921DC1BC", + "Ref": "CloudquerySourceGitHubLanguagesTaskDefinitionDA995E2B", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceGitHubRepositoriesTaskDefinitionEventsRole93DB1A26", + "CloudquerySourceGitHubLanguagesTaskDefinitionEventsRoleD0480706", "Arn", ], }, @@ -14696,40 +14751,53 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceGitHubRepositoriesTaskDefinition921DC1BC": { + "CloudquerySourceGitHubLanguagesTaskDefinitionCloudquerySourceGitHubLanguagesFirelensLogGroup38EAAC3E": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "GitHubLanguages", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceGitHubLanguagesTaskDefinitionDA995E2B": { "Properties": { "ContainerDefinitions": [ { "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /usr/share/cloudquery/github-private-key;echo -n $GITHUB_APP_ID > /usr/share/cloudquery/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /usr/share/cloudquery/github-installation-id;printf 'kind: source + "printf 'kind: source spec: - name: github - path: cloudquery/github - version: v10.0.1 - tables: - - github_repositories - - github_repository_branches - - github_repository_collaborators - - github_workflows - skip_tables: - - github_releases - - github_release_assets - - github_repository_dependabot_alerts - - github_repository_dependabot_secrets + name: github-languages + path: guardian/github-languages + version: v0.0.5 destinations: - postgresql - spec: - concurrency: 1000 - orgs: - - guardian - app_auth: - - org: guardian - private_key_path: /usr/share/cloudquery/github-private-key - app_id: \${file:/usr/share/cloudquery/github-app-id} - installation_id: \${file:/usr/share/cloudquery/github-installation-id} - include_archived_repos: true + tables: + - github_languages + registry: github ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -14746,12 +14814,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-GitHubRepositoriesAWSOTELCollector", + "ContainerName": "CloudquerySource-GitHubLanguagesAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "GitHubRepositories", + "Name": "GitHubLanguages", "Stack": "deploy", "Stage": "TEST", }, @@ -14761,7 +14829,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "819MiB", + "Value": "409MiB", }, ], "Essential": true, @@ -14796,49 +14864,13 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-GitHubRepositoriesContainer", - "ReadonlyRootFilesystem": true, - "Secrets": [ - { - "Name": "GITHUB_PRIVATE_KEY", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "githubcredentialsAF453741", - }, - ":private-key::", - ], - ], - }, - }, - { - "Name": "GITHUB_APP_ID", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "githubcredentialsAF453741", - }, - ":app-id::", - ], - ], - }, - }, - { - "Name": "GITHUB_INSTALLATION_ID", - "ValueFrom": { - "Fn::Join": [ - "", - [ - { - "Ref": "githubcredentialsAF453741", - }, - ":installation-id::", - ], - ], + "Name": "CloudquerySource-GitHubLanguagesContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ + { + "Name": "GITHUB_ACCESS_TOKEN", + "ValueFrom": { + "Ref": "githublanguages5093EDEC", }, }, { @@ -14927,7 +14959,7 @@ spec: }, }, }, - "Name": "CloudquerySource-GitHubRepositoriesAWSOTELCollector", + "Name": "CloudquerySource-GitHubLanguagesAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -14940,11 +14972,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('github_repositories', 'DAILY'),('github_repository_branches', 'DAILY'),('github_repository_collaborators', 'DAILY'),('github_workflows', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('github_languages', 'WEEKLY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'WEEKLY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "GitHubRepositories", + "Name": "GitHubLanguages", "Stack": "deploy", "Stage": "TEST", }, @@ -14966,7 +14998,7 @@ spec: }, }, }, - "Name": "CloudquerySource-GitHubRepositoriesPostgresContainer", + "Name": "CloudquerySource-GitHubLanguagesPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -15041,7 +15073,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceGitHubRepositoriesTaskDefinitionCloudquerySourceGitHubRepositoriesFirelensLogGroup42A4D85D", + "Ref": "CloudquerySourceGitHubLanguagesTaskDefinitionCloudquerySourceGitHubLanguagesFirelensLogGroup38EAAC3E", }, "awslogs-region": { "Ref": "AWS::Region", @@ -15056,19 +15088,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-GitHubRepositoriesFirelens", + "Name": "CloudquerySource-GitHubLanguagesFirelens", "ReadonlyRootFilesystem": true, }, ], "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRole03A80EA4", + "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleB77C8E3D", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceGitHubRepositoriesTaskDefinition13A7DF48", - "Memory": "1024", + "Family": "ServiceCatalogueCloudquerySourceGitHubLanguagesTaskDefinitionB1DA60BF", + "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -15084,7 +15116,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubRepositories", + "Value": "GitHubLanguages", }, { "Key": "Stack", @@ -15097,7 +15129,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskGitHubRepositories83F97F25", + "servicecatalogueTESTtaskGitHubLanguagesED141D27", "Arn", ], }, @@ -15118,37 +15150,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceGitHubRepositoriesTaskDefinitionCloudquerySourceGitHubRepositoriesFirelensLogGroup42A4D85D": { - "DeletionPolicy": "Retain", - "Properties": { - "RetentionInDays": 1, - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Name", - "Value": "GitHubRepositories", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::Logs::LogGroup", - "UpdateReplacePolicy": "Retain", - }, - "CloudquerySourceGitHubRepositoriesTaskDefinitionEventsRole93DB1A26": { + "CloudquerySourceGitHubLanguagesTaskDefinitionEventsRoleD0480706": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -15173,7 +15175,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubRepositories", + "Value": "GitHubLanguages", }, { "Key": "Stack", @@ -15187,7 +15189,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceGitHubRepositoriesTaskDefinitionEventsRoleDefaultPolicy1ECEEEB2": { + "CloudquerySourceGitHubLanguagesTaskDefinitionEventsRoleDefaultPolicy1B11BE73": { "Properties": { "PolicyDocument": { "Statement": [ @@ -15205,7 +15207,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceGitHubRepositoriesTaskDefinition921DC1BC", + "Ref": "CloudquerySourceGitHubLanguagesTaskDefinitionDA995E2B", }, }, { @@ -15237,7 +15239,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRole03A80EA4", + "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleB77C8E3D", "Arn", ], }, @@ -15247,7 +15249,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskGitHubRepositories83F97F25", + "servicecatalogueTESTtaskGitHubLanguagesED141D27", "Arn", ], }, @@ -15255,16 +15257,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceGitHubRepositoriesTaskDefinitionEventsRoleDefaultPolicy1ECEEEB2", + "PolicyName": "CloudquerySourceGitHubLanguagesTaskDefinitionEventsRoleDefaultPolicy1B11BE73", "Roles": [ { - "Ref": "CloudquerySourceGitHubRepositoriesTaskDefinitionEventsRole93DB1A26", + "Ref": "CloudquerySourceGitHubLanguagesTaskDefinitionEventsRoleD0480706", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRole03A80EA4": { + "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleB77C8E3D": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -15289,7 +15291,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubRepositories", + "Value": "GitHubLanguages", }, { "Key": "Stack", @@ -15303,7 +15305,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRoleDefaultPolicy04F69199": { + "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleDefaultPolicy9D57CA88": { "Properties": { "PolicyDocument": { "Statement": [ @@ -15314,7 +15316,7 @@ spec: ], "Effect": "Allow", "Resource": { - "Ref": "githubcredentialsAF453741", + "Ref": "githublanguages5093EDEC", }, }, { @@ -15345,7 +15347,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceGitHubRepositoriesTaskDefinitionCloudquerySourceGitHubRepositoriesFirelensLogGroup42A4D85D", + "CloudquerySourceGitHubLanguagesTaskDefinitionCloudquerySourceGitHubLanguagesFirelensLogGroup38EAAC3E", "Arn", ], }, @@ -15353,18 +15355,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRoleDefaultPolicy04F69199", + "PolicyName": "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleDefaultPolicy9D57CA88", "Roles": [ { - "Ref": "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRole03A80EA4", + "Ref": "CloudquerySourceGitHubLanguagesTaskDefinitionExecutionRoleB77C8E3D", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceGitHubTeamsScheduledEventRule051F542B": { + "CloudquerySourceGitHubRepositoriesScheduledEventRuleC7F5836E": { "Properties": { - "ScheduleExpression": "cron(0 10 ? * 1 *)", + "ScheduleExpression": "cron(0 0 * * ? *)", "State": "ENABLED", "Targets": [ { @@ -15395,14 +15397,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceGitHubTeamsTaskDefinitionB01C9D3C", + "Ref": "CloudquerySourceGitHubRepositoriesTaskDefinition921DC1BC", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceGitHubTeamsTaskDefinitionEventsRole3E2A5002", + "CloudquerySourceGitHubRepositoriesTaskDefinitionEventsRole93DB1A26", "Arn", ], }, @@ -15411,7 +15413,7 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceGitHubTeamsTaskDefinitionB01C9D3C": { + "CloudquerySourceGitHubRepositoriesTaskDefinition921DC1BC": { "Properties": { "ContainerDefinitions": [ { @@ -15424,14 +15426,15 @@ spec: path: cloudquery/github version: v10.0.1 tables: - - github_organizations - - github_organization_members - - github_teams - - github_team_members - - github_team_repositories + - github_repositories + - github_repository_branches + - github_repository_collaborators + - github_workflows skip_tables: - - github_organization_dependabot_alerts - - github_organization_dependabot_secrets + - github_releases + - github_release_assets + - github_repository_dependabot_alerts + - github_repository_dependabot_secrets destinations: - postgresql spec: @@ -15460,12 +15463,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-GitHubTeamsAWSOTELCollector", + "ContainerName": "CloudquerySource-GitHubRepositoriesAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "GitHubTeams", + "Name": "GitHubRepositories", "Stack": "deploy", "Stage": "TEST", }, @@ -15475,7 +15478,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "3276MiB", + "Value": "819MiB", }, ], "Essential": true, @@ -15510,7 +15513,7 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-GitHubTeamsContainer", + "Name": "CloudquerySource-GitHubRepositoriesContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -15641,7 +15644,7 @@ spec: }, }, }, - "Name": "CloudquerySource-GitHubTeamsAWSOTELCollector", + "Name": "CloudquerySource-GitHubRepositoriesAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -15654,11 +15657,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('github_organizations', 'WEEKLY'),('github_organization_members', 'WEEKLY'),('github_teams', 'WEEKLY'),('github_team_members', 'WEEKLY'),('github_team_repositories', 'WEEKLY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'WEEKLY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('github_repositories', 'DAILY'),('github_repository_branches', 'DAILY'),('github_repository_collaborators', 'DAILY'),('github_workflows', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "GitHubTeams", + "Name": "GitHubRepositories", "Stack": "deploy", "Stage": "TEST", }, @@ -15680,7 +15683,7 @@ spec: }, }, }, - "Name": "CloudquerySource-GitHubTeamsPostgresContainer", + "Name": "CloudquerySource-GitHubRepositoriesPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -15755,7 +15758,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceGitHubTeamsTaskDefinitionCloudquerySourceGitHubTeamsFirelensLogGroup8B1FFADC", + "Ref": "CloudquerySourceGitHubRepositoriesTaskDefinitionCloudquerySourceGitHubRepositoriesFirelensLogGroup42A4D85D", }, "awslogs-region": { "Ref": "AWS::Region", @@ -15770,19 +15773,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-GitHubTeamsFirelens", + "Name": "CloudquerySource-GitHubRepositoriesFirelens", "ReadonlyRootFilesystem": true, }, ], - "Cpu": "2048", + "Cpu": "256", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRole9DEDACFD", + "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRole03A80EA4", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceGitHubTeamsTaskDefinition5CFD9707", - "Memory": "4096", + "Family": "ServiceCatalogueCloudquerySourceGitHubRepositoriesTaskDefinition13A7DF48", + "Memory": "1024", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -15798,7 +15801,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubTeams", + "Value": "GitHubRepositories", }, { "Key": "Stack", @@ -15811,7 +15814,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskGitHubTeams5756A4ED", + "servicecatalogueTESTtaskGitHubRepositories83F97F25", "Arn", ], }, @@ -15832,7 +15835,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceGitHubTeamsTaskDefinitionCloudquerySourceGitHubTeamsFirelensLogGroup8B1FFADC": { + "CloudquerySourceGitHubRepositoriesTaskDefinitionCloudquerySourceGitHubRepositoriesFirelensLogGroup42A4D85D": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -15847,7 +15850,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubTeams", + "Value": "GitHubRepositories", }, { "Key": "Stack", @@ -15862,7 +15865,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceGitHubTeamsTaskDefinitionEventsRole3E2A5002": { + "CloudquerySourceGitHubRepositoriesTaskDefinitionEventsRole93DB1A26": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -15887,7 +15890,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubTeams", + "Value": "GitHubRepositories", }, { "Key": "Stack", @@ -15901,7 +15904,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceGitHubTeamsTaskDefinitionEventsRoleDefaultPolicy5B79609C": { + "CloudquerySourceGitHubRepositoriesTaskDefinitionEventsRoleDefaultPolicy1ECEEEB2": { "Properties": { "PolicyDocument": { "Statement": [ @@ -15919,7 +15922,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceGitHubTeamsTaskDefinitionB01C9D3C", + "Ref": "CloudquerySourceGitHubRepositoriesTaskDefinition921DC1BC", }, }, { @@ -15951,7 +15954,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRole9DEDACFD", + "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRole03A80EA4", "Arn", ], }, @@ -15961,7 +15964,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskGitHubTeams5756A4ED", + "servicecatalogueTESTtaskGitHubRepositories83F97F25", "Arn", ], }, @@ -15969,16 +15972,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceGitHubTeamsTaskDefinitionEventsRoleDefaultPolicy5B79609C", + "PolicyName": "CloudquerySourceGitHubRepositoriesTaskDefinitionEventsRoleDefaultPolicy1ECEEEB2", "Roles": [ { - "Ref": "CloudquerySourceGitHubTeamsTaskDefinitionEventsRole3E2A5002", + "Ref": "CloudquerySourceGitHubRepositoriesTaskDefinitionEventsRole93DB1A26", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRole9DEDACFD": { + "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRole03A80EA4": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -16003,7 +16006,7 @@ spec: }, { "Key": "Name", - "Value": "GitHubTeams", + "Value": "GitHubRepositories", }, { "Key": "Stack", @@ -16017,7 +16020,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRoleDefaultPolicy3618CA18": { + "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRoleDefaultPolicy04F69199": { "Properties": { "PolicyDocument": { "Statement": [ @@ -16059,7 +16062,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceGitHubTeamsTaskDefinitionCloudquerySourceGitHubTeamsFirelensLogGroup8B1FFADC", + "CloudquerySourceGitHubRepositoriesTaskDefinitionCloudquerySourceGitHubRepositoriesFirelensLogGroup42A4D85D", "Arn", ], }, @@ -16067,18 +16070,18 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRoleDefaultPolicy3618CA18", + "PolicyName": "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRoleDefaultPolicy04F69199", "Roles": [ { - "Ref": "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRole9DEDACFD", + "Ref": "CloudquerySourceGitHubRepositoriesTaskDefinitionExecutionRole03A80EA4", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceImagePackagesScheduledEventRule3090919D": { + "CloudquerySourceGitHubTeamsScheduledEventRule051F542B": { "Properties": { - "ScheduleExpression": "rate(1 day)", + "ScheduleExpression": "cron(0 10 ? * 1 *)", "State": "ENABLED", "Targets": [ { @@ -16109,14 +16112,14 @@ spec: "PropagateTags": "TASK_DEFINITION", "TaskCount": 1, "TaskDefinitionArn": { - "Ref": "CloudquerySourceImagePackagesTaskDefinition619D42BC", + "Ref": "CloudquerySourceGitHubTeamsTaskDefinitionB01C9D3C", }, }, "Id": "Target0", "Input": "{}", "RoleArn": { "Fn::GetAtt": [ - "CloudquerySourceImagePackagesTaskDefinitionEventsRoleAC920ADE", + "CloudquerySourceGitHubTeamsTaskDefinitionEventsRole3E2A5002", "Arn", ], }, @@ -16125,23 +16128,39 @@ spec: }, "Type": "AWS::Events::Rule", }, - "CloudquerySourceImagePackagesTaskDefinition619D42BC": { + "CloudquerySourceGitHubTeamsTaskDefinitionB01C9D3C": { "Properties": { "ContainerDefinitions": [ { "Command": [ "/bin/sh", "-c", - "printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /usr/share/cloudquery/github-private-key;echo -n $GITHUB_APP_ID > /usr/share/cloudquery/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /usr/share/cloudquery/github-installation-id;printf 'kind: source spec: - name: image-packages - path: guardian/image-packages - version: v0.0.1 + name: github + path: cloudquery/github + version: v10.0.1 + tables: + - github_organizations + - github_organization_members + - github_teams + - github_team_members + - github_team_repositories + skip_tables: + - github_organization_dependabot_alerts + - github_organization_dependabot_secrets destinations: - postgresql - tables: - - amigo_bake_packages - registry: github + spec: + concurrency: 1000 + orgs: + - guardian + app_auth: + - org: guardian + private_key_path: /usr/share/cloudquery/github-private-key + app_id: \${file:/usr/share/cloudquery/github-app-id} + installation_id: \${file:/usr/share/cloudquery/github-installation-id} + include_archived_repos: true ' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql @@ -16158,12 +16177,12 @@ spec: "DependsOn": [ { "Condition": "HEALTHY", - "ContainerName": "CloudquerySource-ImagePackagesAWSOTELCollector", + "ContainerName": "CloudquerySource-GitHubTeamsAWSOTELCollector", }, ], "DockerLabels": { "App": "service-catalogue", - "Name": "ImagePackages", + "Name": "GitHubTeams", "Stack": "deploy", "Stage": "TEST", }, @@ -16173,7 +16192,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "409MiB", + "Value": "3276MiB", }, ], "Essential": true, @@ -16208,13 +16227,49 @@ spec: "SourceVolume": "tmp-volume", }, ], - "Name": "CloudquerySource-ImagePackagesContainer", + "Name": "CloudquerySource-GitHubTeamsContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { - "Name": "GITHUB_ACCESS_TOKEN", + "Name": "GITHUB_PRIVATE_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "githubcredentialsAF453741", + }, + ":private-key::", + ], + ], + }, + }, + { + "Name": "GITHUB_APP_ID", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "githubcredentialsAF453741", + }, + ":app-id::", + ], + ], + }, + }, + { + "Name": "GITHUB_INSTALLATION_ID", "ValueFrom": { - "Ref": "imagepackages23DCAF05", + "Fn::Join": [ + "", + [ + { + "Ref": "githubcredentialsAF453741", + }, + ":installation-id::", + ], + ], }, }, { @@ -16303,7 +16358,7 @@ spec: }, }, }, - "Name": "CloudquerySource-ImagePackagesAWSOTELCollector", + "Name": "CloudquerySource-GitHubTeamsAWSOTELCollector", "PortMappings": [ { "ContainerPort": 4318, @@ -16316,11 +16371,11 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('amigo_bake_packages', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('github_organizations', 'WEEKLY'),('github_organization_members', 'WEEKLY'),('github_teams', 'WEEKLY'),('github_team_members', 'WEEKLY'),('github_team_repositories', 'WEEKLY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'WEEKLY'"", ], "DockerLabels": { "App": "service-catalogue", - "Name": "ImagePackages", + "Name": "GitHubTeams", "Stack": "deploy", "Stage": "TEST", }, @@ -16342,7 +16397,7 @@ spec: }, }, }, - "Name": "CloudquerySource-ImagePackagesPostgresContainer", + "Name": "CloudquerySource-GitHubTeamsPostgresContainer", "ReadonlyRootFilesystem": true, "Secrets": [ { @@ -16417,7 +16472,7 @@ spec: "LogDriver": "awslogs", "Options": { "awslogs-group": { - "Ref": "CloudquerySourceImagePackagesTaskDefinitionCloudquerySourceImagePackagesFirelensLogGroup9B65F3DA", + "Ref": "CloudquerySourceGitHubTeamsTaskDefinitionCloudquerySourceGitHubTeamsFirelensLogGroup8B1FFADC", }, "awslogs-region": { "Ref": "AWS::Region", @@ -16432,19 +16487,19 @@ spec: "SourceVolume": "firelens-volume", }, ], - "Name": "CloudquerySource-ImagePackagesFirelens", + "Name": "CloudquerySource-GitHubTeamsFirelens", "ReadonlyRootFilesystem": true, }, ], - "Cpu": "256", + "Cpu": "2048", "ExecutionRoleArn": { "Fn::GetAtt": [ - "CloudquerySourceImagePackagesTaskDefinitionExecutionRole50B3FF98", + "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRole9DEDACFD", "Arn", ], }, - "Family": "ServiceCatalogueCloudquerySourceImagePackagesTaskDefinition284C9B38", - "Memory": "512", + "Family": "ServiceCatalogueCloudquerySourceGitHubTeamsTaskDefinition5CFD9707", + "Memory": "4096", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", @@ -16460,7 +16515,7 @@ spec: }, { "Key": "Name", - "Value": "ImagePackages", + "Value": "GitHubTeams", }, { "Key": "Stack", @@ -16473,7 +16528,7 @@ spec: ], "TaskRoleArn": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskImagePackages682AF7FE", + "servicecatalogueTESTtaskGitHubTeams5756A4ED", "Arn", ], }, @@ -16494,7 +16549,7 @@ spec: }, "Type": "AWS::ECS::TaskDefinition", }, - "CloudquerySourceImagePackagesTaskDefinitionCloudquerySourceImagePackagesFirelensLogGroup9B65F3DA": { + "CloudquerySourceGitHubTeamsTaskDefinitionCloudquerySourceGitHubTeamsFirelensLogGroup8B1FFADC": { "DeletionPolicy": "Retain", "Properties": { "RetentionInDays": 1, @@ -16509,7 +16564,7 @@ spec: }, { "Key": "Name", - "Value": "ImagePackages", + "Value": "GitHubTeams", }, { "Key": "Stack", @@ -16524,7 +16579,7 @@ spec: "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, - "CloudquerySourceImagePackagesTaskDefinitionEventsRoleAC920ADE": { + "CloudquerySourceGitHubTeamsTaskDefinitionEventsRole3E2A5002": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -16549,7 +16604,7 @@ spec: }, { "Key": "Name", - "Value": "ImagePackages", + "Value": "GitHubTeams", }, { "Key": "Stack", @@ -16563,7 +16618,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceImagePackagesTaskDefinitionEventsRoleDefaultPolicy1032A0C2": { + "CloudquerySourceGitHubTeamsTaskDefinitionEventsRoleDefaultPolicy5B79609C": { "Properties": { "PolicyDocument": { "Statement": [ @@ -16581,7 +16636,7 @@ spec: }, "Effect": "Allow", "Resource": { - "Ref": "CloudquerySourceImagePackagesTaskDefinition619D42BC", + "Ref": "CloudquerySourceGitHubTeamsTaskDefinitionB01C9D3C", }, }, { @@ -16613,7 +16668,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceImagePackagesTaskDefinitionExecutionRole50B3FF98", + "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRole9DEDACFD", "Arn", ], }, @@ -16623,7 +16678,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "servicecatalogueTESTtaskImagePackages682AF7FE", + "servicecatalogueTESTtaskGitHubTeams5756A4ED", "Arn", ], }, @@ -16631,16 +16686,16 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceImagePackagesTaskDefinitionEventsRoleDefaultPolicy1032A0C2", + "PolicyName": "CloudquerySourceGitHubTeamsTaskDefinitionEventsRoleDefaultPolicy5B79609C", "Roles": [ { - "Ref": "CloudquerySourceImagePackagesTaskDefinitionEventsRoleAC920ADE", + "Ref": "CloudquerySourceGitHubTeamsTaskDefinitionEventsRole3E2A5002", }, ], }, "Type": "AWS::IAM::Policy", }, - "CloudquerySourceImagePackagesTaskDefinitionExecutionRole50B3FF98": { + "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRole9DEDACFD": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -16665,7 +16720,7 @@ spec: }, { "Key": "Name", - "Value": "ImagePackages", + "Value": "GitHubTeams", }, { "Key": "Stack", @@ -16679,7 +16734,7 @@ spec: }, "Type": "AWS::IAM::Role", }, - "CloudquerySourceImagePackagesTaskDefinitionExecutionRoleDefaultPolicyF938D0DB": { + "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRoleDefaultPolicy3618CA18": { "Properties": { "PolicyDocument": { "Statement": [ @@ -16690,7 +16745,7 @@ spec: ], "Effect": "Allow", "Resource": { - "Ref": "imagepackages23DCAF05", + "Ref": "githubcredentialsAF453741", }, }, { @@ -16721,7 +16776,7 @@ spec: "Effect": "Allow", "Resource": { "Fn::GetAtt": [ - "CloudquerySourceImagePackagesTaskDefinitionCloudquerySourceImagePackagesFirelensLogGroup9B65F3DA", + "CloudquerySourceGitHubTeamsTaskDefinitionCloudquerySourceGitHubTeamsFirelensLogGroup8B1FFADC", "Arn", ], }, @@ -16729,10 +16784,10 @@ spec: ], "Version": "2012-10-17", }, - "PolicyName": "CloudquerySourceImagePackagesTaskDefinitionExecutionRoleDefaultPolicyF938D0DB", + "PolicyName": "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRoleDefaultPolicy3618CA18", "Roles": [ { - "Ref": "CloudquerySourceImagePackagesTaskDefinitionExecutionRole50B3FF98", + "Ref": "CloudquerySourceGitHubTeamsTaskDefinitionExecutionRole9DEDACFD", }, ], }, @@ -20105,6 +20160,33 @@ spec: }, "Type": "AWS::SNS::Topic", }, + "amigobakepackagesE494D60D": { + "DeletionPolicy": "Delete", + "Properties": { + "GenerateSecretString": {}, + "Name": "/TEST/deploy/service-catalogue/amigo-bake-packages", + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::SecretsManager::Secret", + "UpdateReplacePolicy": "Delete", + }, "cloudbuster8C461939": { "DependsOn": [ "cloudbusterServiceRoleDefaultPolicy173FB27F", @@ -20984,33 +21066,6 @@ spec: "Type": "AWS::SecretsManager::Secret", "UpdateReplacePolicy": "Delete", }, - "imagepackages23DCAF05": { - "DeletionPolicy": "Delete", - "Properties": { - "GenerateSecretString": {}, - "Name": "/TEST/deploy/service-catalogue/image-packages", - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::SecretsManager::Secret", - "UpdateReplacePolicy": "Delete", - }, "interactivemonitor3435C6C2": { "DependsOn": [ "interactivemonitorServiceRoleDefaultPolicy44B1B670", @@ -24198,93 +24253,256 @@ spec: "cloudwatch:namespace": "repocop", }, }, - "Effect": "Allow", - "Resource": "*", + "Effect": "Allow", + "Resource": "*", + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "repocopServiceRoleDefaultPolicyF20BF625", + "Roles": [ + { + "Ref": "repocopServiceRole757D74E8", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "repocopgithubappauthFDE18F33": { + "DeletionPolicy": "Delete", + "Properties": { + "GenerateSecretString": {}, + "Name": "/TEST/deploy/service-catalogue/repocop-github-app-secret", + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::SecretsManager::Secret", + "UpdateReplacePolicy": "Delete", + }, + "repocoprepocopcron309MONFRI042F648A2": { + "Properties": { + "ScheduleExpression": "cron(30 9 ? * MON-FRI *)", + "State": "ENABLED", + "Targets": [ + { + "Arn": { + "Fn::GetAtt": [ + "repocop20553EB8", + "Arn", + ], + }, + "Id": "Target0", + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "repocoprepocopcron309MONFRI0AllowEventRuleServiceCataloguerepocop7BEB58922EC16074": { + "Properties": { + "Action": "lambda:InvokeFunction", + "FunctionName": { + "Fn::GetAtt": [ + "repocop20553EB8", + "Arn", + ], + }, + "Principal": "events.amazonaws.com", + "SourceArn": { + "Fn::GetAtt": [ + "repocoprepocopcron309MONFRI042F648A2", + "Arn", + ], + }, + }, + "Type": "AWS::Lambda::Permission", + }, + "servicecatalogueCluster5FC34DC5": { + "Properties": { + "ClusterSettings": [ + { + "Name": "containerInsights", + "Value": "enabled", + }, + ], + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::ECS::Cluster", + }, + "servicecatalogueClusterAF65BC89": { + "Properties": { + "CapacityProviders": [ + "FARGATE", + "FARGATE_SPOT", + ], + "Cluster": { + "Ref": "servicecatalogueCluster5FC34DC5", + }, + "DefaultCapacityProviderStrategy": [], + }, + "Type": "AWS::ECS::ClusterCapacityProviderAssociations", + }, + "servicecatalogueTESTtaskAmigoBakePackagesDefaultPolicy1351FB30": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "kinesis:Describe*", + "kinesis:Put*", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":kinesis:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":stream/", + { + "Ref": "LoggingStreamName", + }, + ], + ], + }, + }, + { + "Action": [ + "dynamodb:GetItem", + "dynamodb:BatchGetItem", + "dynamodb:Query", + "dynamodb:Scan", + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:dynamodb::000000000018:table/{BASE_IMAGES_TABLE_NAME}", + "arn:aws:dynamodb::000000000018:table/{RECIPES_TABLE_NAME}", + "arn:aws:dynamodb::000000000018:table/{BAKES_TABLE_NAME}", + ], + }, + { + "Action": "s3:GetObject", + "Effect": "Allow", + "Resource": "arn:aws:s3:::\${PACKAGES_BUCKET_NAME}/packagelists/*", + }, + { + "Action": "rds-db:connect", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":rds-db:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":dbuser:", + { + "Fn::GetAtt": [ + "PostgresInstance16DE4286E", + "DbiResourceId", + ], + }, + "/{{resolve:secretsmanager:", + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":SecretString:username::}}", + ], + ], + }, }, ], "Version": "2012-10-17", }, - "PolicyName": "repocopServiceRoleDefaultPolicyF20BF625", + "PolicyName": "servicecatalogueTESTtaskAmigoBakePackagesDefaultPolicy1351FB30", "Roles": [ { - "Ref": "repocopServiceRole757D74E8", + "Ref": "servicecatalogueTESTtaskAmigoBakePackagesE3F44845", }, ], }, "Type": "AWS::IAM::Policy", }, - "repocopgithubappauthFDE18F33": { - "DeletionPolicy": "Delete", - "Properties": { - "GenerateSecretString": {}, - "Name": "/TEST/deploy/service-catalogue/repocop-github-app-secret", - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::SecretsManager::Secret", - "UpdateReplacePolicy": "Delete", - }, - "repocoprepocopcron309MONFRI042F648A2": { + "servicecatalogueTESTtaskAmigoBakePackagesE3F44845": { "Properties": { - "ScheduleExpression": "cron(30 9 ? * MON-FRI *)", - "State": "ENABLED", - "Targets": [ - { - "Arn": { - "Fn::GetAtt": [ - "repocop20553EB8", - "Arn", - ], + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, }, - "Id": "Target0", - }, - ], - }, - "Type": "AWS::Events::Rule", - }, - "repocoprepocopcron309MONFRI0AllowEventRuleServiceCataloguerepocop7BEB58922EC16074": { - "Properties": { - "Action": "lambda:InvokeFunction", - "FunctionName": { - "Fn::GetAtt": [ - "repocop20553EB8", - "Arn", - ], - }, - "Principal": "events.amazonaws.com", - "SourceArn": { - "Fn::GetAtt": [ - "repocoprepocopcron309MONFRI042F648A2", - "Arn", ], + "Version": "2012-10-17", }, - }, - "Type": "AWS::Lambda::Permission", - }, - "servicecatalogueCluster5FC34DC5": { - "Properties": { - "ClusterSettings": [ + "ManagedPolicyArns": [ { - "Name": "containerInsights", - "Value": "enabled", + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/AWSXrayWriteOnlyAccess", + ], + ], }, ], + "RoleName": "service-catalogue-TEST-task-AmigoBakePackages", "Tags": [ { "Key": "gu:cdk:version", @@ -24304,20 +24522,7 @@ spec: }, ], }, - "Type": "AWS::ECS::Cluster", - }, - "servicecatalogueClusterAF65BC89": { - "Properties": { - "CapacityProviders": [ - "FARGATE", - "FARGATE_SPOT", - ], - "Cluster": { - "Ref": "servicecatalogueCluster5FC34DC5", - }, - "DefaultCapacityProviderStrategy": [], - }, - "Type": "AWS::ECS::ClusterCapacityProviderAssociations", + "Type": "AWS::IAM::Role", }, "servicecatalogueTESTtaskAwsCostExplorer78777A06": { "Properties": { @@ -27668,137 +27873,6 @@ spec: }, "Type": "AWS::IAM::Policy", }, - "servicecatalogueTESTtaskImagePackages682AF7FE": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ecs-tasks.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "ManagedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":iam::aws:policy/AWSXrayWriteOnlyAccess", - ], - ], - }, - ], - "RoleName": "service-catalogue-TEST-task-ImagePackages", - "Tags": [ - { - "Key": "gu:cdk:version", - "Value": "TEST", - }, - { - "Key": "gu:repo", - "Value": "guardian/service-catalogue", - }, - { - "Key": "Stack", - "Value": "deploy", - }, - { - "Key": "Stage", - "Value": "TEST", - }, - ], - }, - "Type": "AWS::IAM::Role", - }, - "servicecatalogueTESTtaskImagePackagesDefaultPolicy3D263C78": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "kinesis:Describe*", - "kinesis:Put*", - ], - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":kinesis:", - { - "Ref": "AWS::Region", - }, - ":", - { - "Ref": "AWS::AccountId", - }, - ":stream/", - { - "Ref": "LoggingStreamName", - }, - ], - ], - }, - }, - { - "Action": "rds-db:connect", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":rds-db:", - { - "Ref": "AWS::Region", - }, - ":", - { - "Ref": "AWS::AccountId", - }, - ":dbuser:", - { - "Fn::GetAtt": [ - "PostgresInstance16DE4286E", - "DbiResourceId", - ], - }, - "/{{resolve:secretsmanager:", - { - "Ref": "PostgresInstance1SecretAttachmentBA0D257D", - }, - ":SecretString:username::}}", - ], - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "servicecatalogueTESTtaskImagePackagesDefaultPolicy3D263C78", - "Roles": [ - { - "Ref": "servicecatalogueTESTtaskImagePackages682AF7FE", - }, - ], - }, - "Type": "AWS::IAM::Policy", - }, "servicecatalogueTESTtaskNS1B2D0D4B7": { "Properties": { "AssumeRolePolicyDocument": {