From 39704a00b93f2a5cf24a3a84e53910c7215c5a3f Mon Sep 17 00:00:00 2001 From: roleyfoley Date: Mon, 9 May 2022 14:23:49 +1000 Subject: [PATCH] chore(datapipeline): remove support for AWS data pipeline - The AWS data pipeline service has not been updated for some time and a lot of other tools are now available to make this easier - The pipeline is no longer used in hamlet so can be removed --- aws/components/datapipeline/id.ftl | 17 -- aws/components/datapipeline/setup.ftl | 321 ------------------------ aws/components/datapipeline/state.ftl | 65 ----- aws/services/datapipeline/id.ftl | 9 - aws/services/datapipeline/policy.ftl | 106 -------- aws/services/service.ftl | 3 - awstest/inputseeders/awstest/id.ftl | 4 - awstest/modules/datapipeline/module.ftl | 106 -------- 8 files changed, 631 deletions(-) delete mode 100644 aws/components/datapipeline/id.ftl delete mode 100644 aws/components/datapipeline/setup.ftl delete mode 100644 aws/components/datapipeline/state.ftl delete mode 100644 aws/services/datapipeline/id.ftl delete mode 100644 aws/services/datapipeline/policy.ftl delete mode 100644 awstest/modules/datapipeline/module.ftl diff --git a/aws/components/datapipeline/id.ftl b/aws/components/datapipeline/id.ftl deleted file mode 100644 index 0bae4481e..000000000 --- a/aws/components/datapipeline/id.ftl +++ /dev/null @@ -1,17 +0,0 @@ -[#ftl] -[@addResourceGroupInformation - type=DATAPIPELINE_COMPONENT_TYPE - attributes=[] - provider=AWS_PROVIDER - resourceGroup=DEFAULT_RESOURCE_GROUP - services= - [ - AWS_DATA_PIPELINE_SERVICE, - AWS_ELASTIC_COMPUTE_SERVICE, - AWS_IDENTITY_SERVICE, - AWS_VIRTUAL_PRIVATE_CLOUD_SERVICE, - AWS_SIMPLE_STORAGE_SERVICE, - AWS_KEY_MANAGEMENT_SERVICE, - AWS_IMAGE_SERVICE - ] -/] diff --git a/aws/components/datapipeline/setup.ftl b/aws/components/datapipeline/setup.ftl deleted file mode 100644 index c135fe6a3..000000000 --- a/aws/components/datapipeline/setup.ftl +++ /dev/null @@ -1,321 +0,0 @@ -[#ftl] -[#macro aws_datapipeline_cf_deployment_generationcontract_application occurrence ] - [@addDefaultGenerationContract subsets=["pregeneration", "prologue", "template", "epilogue", "cli", "config"] /] -[/#macro] - -[#macro aws_datapipeline_cf_deployment_application occurrence ] - [@debug message="Entering" context=occurrence enabled=false /] - - [#local core = occurrence.Core ] - [#local solution = occurrence.Configuration.Solution ] - [#local settings = occurrence.Configuration.Settings] - [#local resources = occurrence.State.Resources ] - [#local attributes = occurrence.State.Attributes ] - [#local image = getOccurrenceImage(occurrence)] - - [#local pipelineId = resources["dataPipeline"].Id] - [#local pipelineName = resources["dataPipeline"].Name] - [#local pipelineRoleId = resources["pipelineRole"].Id] - [#local pipelineRoleName = resources["pipelineRole"].Name] - [#local resourceRoleId = resources["resourceRole"].Id] - [#local resourceRoleName = resources["resourceRole"].Name] - [#local resourceInstanceProfileId = resources["resourceInstanceProfile"].Id] - [#local resourceInstanceProfileName = resources["resourceInstanceProfile"].Name] - - [#local securityGroupId = resources["securityGroup"].Id] - [#local securityGroupName = resources["securityGroup"].Name] - - [#local ec2ProcessorProfile = getProcessor(occurrence, EC2_COMPONENT_TYPE)] - [#local emrProcessorProfile = getProcessor(occurrence, "EMR")] - [#local networkProfile = getNetworkProfile(occurrence)] - - [#-- Baseline component lookup --] - [#local baselineLinks = getBaselineLinks(occurrence, [ "OpsData", "AppData", "Encryption", "SSHKey" ] )] - [#local baselineComponentIds = getBaselineComponentIds(baselineLinks)] - [#local operationsBucket = getExistingReference(baselineComponentIds["OpsData"]) ] - [#local dataBucket = getExistingReference(baselineComponentIds["AppData"])] - [#local sshKeyPairId = baselineComponentIds["SSHKey"]!"HamletFatal: sshKeyPairId not found" ] - - [#local pipelineCreateCommand = "createPipeline"] - - [#local networkLink = getOccurrenceNetwork(occurrence).Link!{} ] - - [#local networkLinkTarget = getLinkTarget(occurrence, networkLink ) ] - - [#if ! networkLinkTarget?has_content ] - [@fatal message="Network could not be found" context=networkLink /] - [#return] - [/#if] - - [#if deploymentSubsetRequired("pregeneration", false) && image.Source == "url" ] - [@addToDefaultBashScriptOutput - content=getAWSImageFromUrlScript(image, true) - /] - [/#if] - - [#local networkConfiguration = networkLinkTarget.Configuration.Solution] - [#local networkResources = networkLinkTarget.State.Resources ] - - [#local vpcId = networkResources["vpc"].Id ] - - [#local parameterValues = { - "_AWS_REGION" : getRegion(), - "_AVAILABILITY_ZONE" : getZones()[0].AWSZone, - "_VPC_ID" : getExistingReference(vpcId), - "_SUBNET_ID" : getSubnets(core.Tier, networkResources)[0], - "_SSH_KEY_PAIR" : getExistingReference(sshKeyPairId, NAME_ATTRIBUTE_TYPE), - "_INSTANCE_TYPE_EC2" : ec2ProcessorProfile.Processor, - "_INSTANCE_IMAGE_EC2" : getRegionObject().AMIs.Centos.EC2, - "_INSTANCE_TYPE_EMR" : emrProcessorProfile.Processor, - "_INSTANCE_COUNT_EMR_CORE" : emrProcessorProfile.DesiredCorePerZone?c, - "_INSTANCE_COUNT_EMR_TASK" : emrProcessorProfile.DesiredCorePerZone?c, - "_PIPELINE_LOG_URI" : "s3://" + operationsBucket + - formatAbsolutePath( - "datapipeline", - core.FullName, - "logs"), - "_PIPELINE_CODE_URI" : "s3://" + operationsBucket + - formatAbsolutePath( - getOccurrenceSettingValue(occurrence, "SETTINGS_PREFIX"), - "pipeline" - ), - "_ROLE_PIPELINE_NAME" : pipelineRoleName, - "_ROLE_RESOURCE_NAME" : resourceRoleName - }] - - [#-- Add in container specifics including override of defaults --] - [#-- Allows for explicit policy or managed ARN's to be assigned to the user --] - [#local contextLinks = getLinkTargets(occurrence) ] - [#local _context = - { - "DefaultEnvironment" : defaultEnvironment(occurrence, contextLinks, baselineLinks), - "Environment" : {}, - "Links" : contextLinks, - "BaselineLinks" : baselineLinks, - "Policy" : iamStandardPolicies(occurrence, baselineComponentIds), - "DefaultCoreVariables" : true, - "DefaultEnvironmentVariables" : true, - "DefaultLinkVariables" : true, - "DefaultBaselineVariables" : true - } - ] - [#local _context = invokeExtensions( occurrence, _context, {}, solution.Extensions, true )] - - [#local _context += getFinalEnvironment(occurrence, _context ) ] - [#local parameterValues += _context.Environment ] - - [#list _context.Links as linkId,linkTarget] - [#local linkTargetCore = linkTarget.Core ] - [#local linkTargetConfiguration = linkTarget.Configuration ] - [#local linkTargetResources = linkTarget.State.Resources ] - [#local linkTargetAttributes = linkTarget.State.Attributes ] - [#local linkTargetRoles = linkTarget.State.Roles] - - [#if deploymentSubsetRequired(DATAPIPELINE_COMPONENT_TYPE, true)] - [@createSecurityGroupRulesFromLink - occurrence=occurrence - groupId=securityGroupId - linkTarget=linkTarget - inboundPorts=[ "ssh" ] - networkProfile=networkProfile - /] - [/#if] - - [/#list] - - [#local myParameterValues = {}] - [#list parameterValues as key,value ] - [#local myParameterValues += - { - key?ensure_starts_with("my") : value - }] - [/#list] - - [#if deploymentSubsetRequired("config", false)] - [@addToDefaultJsonOutput content={ "values" : myParameterValues } /] - [/#if] - - [#if deploymentSubsetRequired("iam", true) ] - - [#-- Create a role under which the function will run and attach required policies --] - [#-- The role is mandatory though there may be no policies attached to it --] - [#if isPartOfCurrentDeploymentUnit(pipelineRoleId) ] - [@createRole - id=pipelineRoleId - name=pipelineRoleName - trustedServices=[ - "elasticmapreduce.amazonaws.com", - "datapipeline.amazonaws.com" - ] - managedArns=["arn:aws:iam::aws:policy/service-role/AWSDataPipelineRole"] - tags=getOccurrenceTags(occurrence) - /] - [/#if] - - [#if isPartOfCurrentDeploymentUnit(resourceRoleId) ] - [@createRole - id=resourceRoleId - name=resourceRoleName - trustedServices=[ - "ec2.amazonaws.com" - ] - managedArns=["arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforDataPipelineRole"] - tags=getOccurrenceTags(occurrence) - /] - - [#if _context.Policy?has_content] - [#local policyId = formatDependentPolicyId(pipelineId)] - [@createPolicy - id=policyId - name="base" - statements=_context.Policy - roles=resourceRoleId - /] - [/#if] - - [#local linkPolicies = getLinkTargetsOutboundRoles(_context.Links) ] - - [#if linkPolicies?has_content] - [#local policyId = formatDependentPolicyId(pipelineId, "links")] - [@createPolicy - id=policyId - name="links" - statements=linkPolicies - roles=resourceRoleId - /] - [/#if] - - [/#if] - - [/#if] - - [#if deploymentSubsetRequired(DATAPIPELINE_COMPONENT_TYPE, true)] - [@cfResource - id=resourceInstanceProfileId - type="AWS::IAM::InstanceProfile" - properties= - { - "Path" : "/", - "Roles" : [ getReference(resourceRoleId) ], - "InstanceProfileName" : resourceInstanceProfileName - } - outputs={} - /] - - [@createSecurityGroup - id=securityGroupId - vpcId=vpcId - name=securityGroupName - tags=getOccurrenceTags(occurrence) - /] - - [@createSecurityGroupRulesFromNetworkProfile - occurrence=occurrence - groupId=securityGroupId - networkProfile=networkProfile - inboundPorts=[ "ssh" ] - /] - - [#local ingressNetworkRule = { - "Ports" : "any", - "SecurityGroup" : securityGroupId - }] - - [@createSecurityGroupIngressFromNetworkRule - occurrence=occurrence - groupId=securityGroupId - networkRule=ingressNetworkRule - /] - [/#if] - - [#if deploymentSubsetRequired("cli", false)] - [#local pipelineCreateCliConfig = { - "name" : pipelineName, - "uniqueId" : pipelineId, - "tags" : getCFResourceTags(getOccurrenceTags(occurrence), false, 10) - }] - - [@addCliToDefaultJsonOutput - id=pipelineId - command=pipelineCreateCommand - content=pipelineCreateCliConfig - /] - [/#if] - - [#if deploymentSubsetRequired("prologue", false)] - [#-- Copy any asFiles needed by the task --] - [#local asFiles = getAsFileSettings(settings.Product) ] - [#if asFiles?has_content] - [@debug message="Asfiles" context=asFiles enabled=false /] - [@addToDefaultBashScriptOutput - content= - findAsFilesScript("filesToSync", asFiles) + - syncFilesToBucketScript( - "filesToSync", - getRegion(), - operationsBucket, - getOccurrenceSettingValue(occurrence, "SETTINGS_PREFIX") - ) /] - [/#if] - [/#if] - - [#if deploymentSubsetRequired("epilogue", false) ] - [@addToDefaultBashScriptOutput - content= - getAWSImageBuildScript( - "pipelineFiles", - getRegion(), - image - ) + - syncFilesToBucketScript( - "pipelineFiles", - getRegion(), - operationsBucket, - formatRelativePath( - getOccurrenceSettingValue(occurrence, "SETTINGS_PREFIX"), - "pipeline" - ) - ) + - getLocalFileScript( - "configFiles", - r'${CONFIG}', - "config.json" - ) + - [ - r'case "${STACK_OPERATION}" in', - r' create|update)', - r' mkdir "${tmpdir}/pipeline"', - r' unzip "${tmpdir}/pipeline.zip" -d "${tmpdir}/pipeline"', - r' # Get cli config file', - r' split_cli_file "${CLI}" "${tmpdir}" || return $?', - r' # Create Data pipeline', - r' info "Applying cli level configurtion"', - r' pipelineId="$(create_data_pipeline' + - r' "' + getRegion() + r'" ' + - r' "${tmpdir}/cli-' + - pipelineId + r'-' + pipelineCreateCommand + r'.json")"', - r' # Add Pipeline Definition', - r' info "Updating pipeline definition"', - r' update_data_pipeline' + - r' "' + getRegion() + r'" ' + - r' "${pipelineId}" ' + - r' "${tmpdir}/pipeline/pipeline-definition.json" ' + - r' "${tmpdir}/pipeline/pipeline-parameters.json" ' + - r' "${tmpdir}/config.json" ' + - r' "${STACK_NAME}" ' + - r' "' + securityGroupId + r'" || return $?' - ] + - pseudoStackOutputScript( - "Data Pipeline", - { - pipelineId : r'${pipelineId}' - }, - "creds-system" - ) + - [ - " ;;", - " esac" - ] - /] - [/#if] -[/#macro] diff --git a/aws/components/datapipeline/state.ftl b/aws/components/datapipeline/state.ftl deleted file mode 100644 index a38ef86fd..000000000 --- a/aws/components/datapipeline/state.ftl +++ /dev/null @@ -1,65 +0,0 @@ -[#ftl] - -[#macro aws_datapipeline_cf_state occurrence parent={} ] - [#local core = occurrence.Core] - [#local solution = occurrence.Configuration.Solution ] - - [#local pipelineId = formatResourceId( AWS_DATA_PIPELINE_RESOURCE_TYPE, core.Id )] - [#local securityGroupId = formatResourceId( AWS_VPC_SECURITY_GROUP_RESOURCE_TYPE, core.Id )] - - [#-- The ec2 Role and Instance profile must have the same name --] - [#local resourceRoleName = formatName(core.FullName, "resource")] - - [#assign componentState = - { - "Resources" : { - "dataPipeline" : { - "Id" : pipelineId, - "Name" : core.FullName, - "Type" : AWS_VPC_SECURITY_GROUP_RESOURCE_TYPE - }, - "pipelineRole" : { - "Id" : formatResourceId( AWS_IAM_ROLE_RESOURCE_TYPE, core.Id, "pipeline" ), - "Name" : formatName(core.FullName, "pipeline"), - "Type" : AWS_IAM_ROLE_RESOURCE_TYPE, - "IncludeInDeploymentState" : false - }, - "resourceRole" : { - "Id" : formatResourceId( AWS_IAM_ROLE_RESOURCE_TYPE, core.Id, "resource" ), - "Name" : resourceRoleName, - "Type" : AWS_IAM_ROLE_RESOURCE_TYPE, - "IncludeInDeploymentState" : false - }, - "resourceInstanceProfile" : { - "Id" : formatResourceId( AWS_EC2_INSTANCE_PROFILE_RESOURCE_TYPE, core.Id ), - "Name" : resourceRoleName, - "Type" : AWS_EC2_INSTANCE_PROFILE_RESOURCE_TYPE - }, - "securityGroup" : { - "Id" : securityGroupId, - "Name" : core.FullName, - "Type" : AWS_VPC_SECURITY_GROUP_RESOURCE_TYPE - } - }, - "Images": constructAWSImageResource(occurrence, "pipeline"), - "Attributes" : { - "ID" : getExistingReference( pipelineId ) - }, - "Roles" : { - "Inbound" : { - "networkacl" : { - "SecurityGroups" : securityGroupId, - "Description" : core.FullName - } - }, - "Outbound" : { - "networkacl" : { - "Ports" : [ "ssh" ], - "SecurityGroups" : securityGroupId, - "Description" : core.FullName - } - } - } - } - ] -[/#macro] diff --git a/aws/services/datapipeline/id.ftl b/aws/services/datapipeline/id.ftl deleted file mode 100644 index 3333b42b8..000000000 --- a/aws/services/datapipeline/id.ftl +++ /dev/null @@ -1,9 +0,0 @@ -[#ftl] - -[#-- Resources --] -[#assign AWS_DATA_PIPELINE_RESOURCE_TYPE = "datapipeline"] -[@addServiceResource - provider=AWS_PROVIDER - service=AWS_DATA_PIPELINE_SERVICE - resource=AWS_DATA_PIPELINE_RESOURCE_TYPE -/] diff --git a/aws/services/datapipeline/policy.ftl b/aws/services/datapipeline/policy.ftl deleted file mode 100644 index b57259248..000000000 --- a/aws/services/datapipeline/policy.ftl +++ /dev/null @@ -1,106 +0,0 @@ - -[#ftl] - -[#function dataPipelineGlobalAccess ] - [#return - getPolicyStatement( - [ - "cloudwatch:*", - "datapipeline:DescribeObjects", - "datapipeline:EvaluateExpression", - "dynamodb:BatchGetItem", - "dynamodb:DescribeTable", - "dynamodb:GetItem", - "dynamodb:Query", - "dynamodb:Scan", - "dynamodb:UpdateTable", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CancelSpotInstanceRequests", - "ec2:CreateSecurityGroup", - "ec2:CreateTags", - "ec2:DeleteTags", - "ec2:Describe*", - "ec2:ModifyImageAttribute", - "ec2:ModifyInstanceAttribute", - "ec2:RequestSpotInstances", - "ec2:RunInstances", - "ec2:StartInstances", - "ec2:StopInstances", - "ec2:TerminateInstances", - "ec2:AuthorizeSecurityGroupEgress", - "ec2:DeleteSecurityGroup", - "ec2:RevokeSecurityGroupEgress", - "ec2:DescribeNetworkInterfaces", - "ec2:CreateNetworkInterface", - "ec2:DeleteNetworkInterface", - "ec2:DetachNetworkInterface", - "elasticmapreduce:*", - "iam:GetInstanceProfile", - "iam:GetRole", - "iam:GetRolePolicy", - "iam:ListAttachedRolePolicies", - "iam:ListRolePolicies", - "iam:ListInstanceProfiles", - "iam:PassRole", - "rds:DescribeDBInstances", - "rds:DescribeDBSecurityGroups", - "redshift:DescribeClusters", - "redshift:DescribeClusterSecurityGroups", - "s3:CreateBucket", - "s3:DeleteObject", - "s3:Get*", - "s3:List*", - "s3:Put*", - "sdb:BatchPutAttributes", - "sdb:Select*", - "sns:GetTopicAttributes", - "sns:ListTopics", - "sns:Publish", - "sns:Subscribe", - "sns:Unsubscribe", - "sqs:CreateQueue", - "sqs:Delete*", - "sqs:GetQueue*", - "sqs:PurgeQueue", - "sqs:ReceiveMessage" - ]) - ] -[/#function] - -[#function dataPipelineSerivceLinkedRole ] - [#return - getPolicyStatement( - [ "iam:CreateServiceLinkedRole" ], - "*", - "", - { - "StringLike": { - "iam:AWSServiceName": ["elasticmapreduce.amazonaws.com","spot.amazonaws.com"] - } - } - ) - ] -[/#function] - -[#function dataPipelineBaseResourceAccess ] - [#return - getPolicyStatement( - [ - "cloudwatch:*", - "datapipeline:*", - "dynamodb:*", - "ec2:Describe*", - "elasticmapreduce:AddJobFlowSteps", - "elasticmapreduce:Describe*", - "elasticmapreduce:ListInstance*", - "rds:Describe*", - "redshift:DescribeClusters", - "redshift:DescribeClusterSecurityGroups", - "s3:*", - "sdb:*", - "sns:*", - "sqs:*" - ] - ) - ] -[/#function] \ No newline at end of file diff --git a/aws/services/service.ftl b/aws/services/service.ftl index 82e389cf0..c6f37b210 100644 --- a/aws/services/service.ftl +++ b/aws/services/service.ftl @@ -40,9 +40,6 @@ [#assign AWS_COGNITO_SERVICE = "cognito"] [@addService provider=AWS_PROVIDER service=AWS_COGNITO_SERVICE /] -[#assign AWS_DATA_PIPELINE_SERVICE = "datapipeline"] -[@addService provider=AWS_PROVIDER service=AWS_DATA_PIPELINE_SERVICE /] - [#assign AWS_DIRECTORY_SERVICE = "ds"] [@addService provider=AWS_PROVIDER service=AWS_DIRECTORY_SERVICE /] diff --git a/awstest/inputseeders/awstest/id.ftl b/awstest/inputseeders/awstest/id.ftl index 917b1af92..fe4cb68c9 100644 --- a/awstest/inputseeders/awstest/id.ftl +++ b/awstest/inputseeders/awstest/id.ftl @@ -117,10 +117,6 @@ "Provider" : "awstest", "Name" : "datafeed" }, - "datapipeline" : { - "Provider" : "awstest", - "Name" : "datapipeline" - }, "dataset" : { "Provider" : "awstest", "Name" : "dataset" diff --git a/awstest/modules/datapipeline/module.ftl b/awstest/modules/datapipeline/module.ftl deleted file mode 100644 index 71c73de8a..000000000 --- a/awstest/modules/datapipeline/module.ftl +++ /dev/null @@ -1,106 +0,0 @@ -[#ftl] - -[@addModule - name="datapipeline" - description="Testing module for the aws hosting of datapipelines" - provider=AWSTEST_PROVIDER - properties=[] -/] - -[#macro awstest_module_datapipeline ] - - [#-- Data Pipeline --] - [@loadModule - settingSets=[ - { - "Type" : "Builds", - "Scope" : "Products", - "Namespace" : "mockedup-integration-app-datapipelinebase", - "Settings" : { - "COMMIT" : "123456789#MockCommit#" - } - } - ] - blueprint={ - "Tiers" : { - "app" : { - "Components" : { - "datapipelinebase" : { - "Type": "datapipeline", - "deployment:Unit": "aws-datapipeline", - "Profiles" : { - "Testing" : [ "datapipelinebase" ] - } - } - } - } - }, - "TestCases" : { - "datapipelinebasecli" : { - "OutputSuffix" : "cli.json", - "Structural" : { - "JSON" : { - "Match" : { - "pipelineName" : { - "Path" : "datapipelineXappXdatapipelinebase.createPipeline.name", - "Value" : "mockedup-integration-application-datapipelinebase" - }, - "pipelineId" : { - "Path" : "datapipelineXappXdatapipelinebase.createPipeline.uniqueId", - "Value" : "datapipelineXappXdatapipelinebase" - } - } - } - } - }, - "datapipelinebaseconfig" : { - "OutputSuffix" : "config.json", - "Structural" : { - "JSON" : { - "Match" : { - "vpcId" : { - "Path" : "values.my_VPC_ID", - "Value" : "vpc-123456789abcdef12" - }, - "PipelineName" : { - "Path" : "values.my_ROLE_PIPELINE_NAME", - "Value" : "mockedup-integration-application-datapipelinebase-pipeline" - } - } - } - } - }, - "datapipelinebasetemplate" : { - "OutputSuffix" : "template.json", - "Structural" : { - "CFN" : { - "Resource" : { - "instanceProfile" : { - "Name" : "instanceProfileXappXdatapipelinebase", - "Type" : "AWS::IAM::InstanceProfile" - }, - "securityGroup" : { - "Name" : "securityGroupXappXdatapipelinebase", - "Type" : "AWS::EC2::SecurityGroup" - } - }, - "Output" : [ - "securityGroupXappXdatapipelinebase" - ] - } - } - } - }, - "TestProfiles" : { - "datapipelinebase" : { - "datapipeline" : { - "TestCases" : [ "datapipelinebasecli" ,"datapipelinebaseconfig", "datapipelinebasetemplate" ] - }, - "*" : { - "TestCases" : [ "_cfn-lint" ] - } - } - } - } - /] -[/#macro]