From d76e341f5d92886d6d3f6aff4dea34cb1fe6c7b3 Mon Sep 17 00:00:00 2001 From: Pauline <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 27 Feb 2024 11:35:02 -0600 Subject: [PATCH 1/7] Allow hatchery to assume configured image reader role --- gen3/bin/kube-setup-hatchery.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/gen3/bin/kube-setup-hatchery.sh b/gen3/bin/kube-setup-hatchery.sh index 691fb354a..16ef820de 100644 --- a/gen3/bin/kube-setup-hatchery.sh +++ b/gen3/bin/kube-setup-hatchery.sh @@ -27,15 +27,35 @@ then gen3 job cron distribute-licenses '* * * * *' fi +# if `nextflow-global.imagebuilder-reader-role-arn` is set in hatchery config, allow hatchery +# to assume the configured role +imagebuilderRoleArn=$(g3kubectl get configmap manifest-hatchery -o jsonpath={.data.nextflow-global} | jq '."imagebuilder-reader-role-arn"') +assumeImageBuilderRolePolicyBlock="" +if [ -z "$imagebuilderRoleArn" ]; then + gen3_log_err "Info: No 'nexftlow-global.imagebuilder-reader-role-arn' configuration in Hatchery configuration, not granting AssumeRole" +else + assumeImageBuilderRolePolicyBlock="""{ + "Sid": "AssumeImageBuilderReaderRole", + "Effect": "Allow", + "Action": [ + "sts:AssumeRole" + ], + "Resource": "$imagebuilderRoleArn" + }, + """ +fi + policy=$( cat < Date: Tue, 27 Feb 2024 11:47:57 -0600 Subject: [PATCH 2/7] logs --- gen3/bin/kube-setup-hatchery.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gen3/bin/kube-setup-hatchery.sh b/gen3/bin/kube-setup-hatchery.sh index 16ef820de..55bdca564 100644 --- a/gen3/bin/kube-setup-hatchery.sh +++ b/gen3/bin/kube-setup-hatchery.sh @@ -32,8 +32,9 @@ fi imagebuilderRoleArn=$(g3kubectl get configmap manifest-hatchery -o jsonpath={.data.nextflow-global} | jq '."imagebuilder-reader-role-arn"') assumeImageBuilderRolePolicyBlock="" if [ -z "$imagebuilderRoleArn" ]; then - gen3_log_err "Info: No 'nexftlow-global.imagebuilder-reader-role-arn' configuration in Hatchery configuration, not granting AssumeRole" + gen3_log_info "No 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration, not granting AssumeRole" else + gen3_log_info "Found 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration,granting AssumeRole" assumeImageBuilderRolePolicyBlock="""{ "Sid": "AssumeImageBuilderReaderRole", "Effect": "Allow", @@ -123,8 +124,9 @@ policy=$( cat < /dev/null 2>&1; then roleName="$(gen3 api safe-name hatchery-sa)" gen3 awsrole create $roleName $saName From 1cf58990ec1f69fc6fe25d6de0a10b871a1c0b69 Mon Sep 17 00:00:00 2001 From: Pauline <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:40:50 -0600 Subject: [PATCH 3/7] fix quotes in policy --- gen3/bin/kube-setup-hatchery.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/gen3/bin/kube-setup-hatchery.sh b/gen3/bin/kube-setup-hatchery.sh index 55bdca564..3eb1834fe 100644 --- a/gen3/bin/kube-setup-hatchery.sh +++ b/gen3/bin/kube-setup-hatchery.sh @@ -35,15 +35,17 @@ if [ -z "$imagebuilderRoleArn" ]; then gen3_log_info "No 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration, not granting AssumeRole" else gen3_log_info "Found 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration,granting AssumeRole" - assumeImageBuilderRolePolicyBlock="""{ - "Sid": "AssumeImageBuilderReaderRole", - "Effect": "Allow", - "Action": [ - "sts:AssumeRole" - ], - "Resource": "$imagebuilderRoleArn" - }, - """ + assumeImageBuilderRolePolicyBlock=$( cat < /dev/null 2>&1; then From 07dd341abc150be52a8bd2d62c9ebe4423129881 Mon Sep 17 00:00:00 2001 From: Pauline <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:50:43 -0600 Subject: [PATCH 4/7] always recreate hatchery SA --- gen3/bin/kube-setup-hatchery.sh | 51 +++++++++++++++++---------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/gen3/bin/kube-setup-hatchery.sh b/gen3/bin/kube-setup-hatchery.sh index 3eb1834fe..79c5ba2e6 100644 --- a/gen3/bin/kube-setup-hatchery.sh +++ b/gen3/bin/kube-setup-hatchery.sh @@ -34,7 +34,7 @@ assumeImageBuilderRolePolicyBlock="" if [ -z "$imagebuilderRoleArn" ]; then gen3_log_info "No 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration, not granting AssumeRole" else - gen3_log_info "Found 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration,granting AssumeRole" + gen3_log_info "Found 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration, granting AssumeRole" assumeImageBuilderRolePolicyBlock=$( cat < /dev/null 2>&1; then - roleName="$(gen3 api safe-name hatchery-sa)" - gen3 awsrole create $roleName $saName - policyName="$(gen3 api safe-name hatchery-policy)" - policyInfo=$(gen3_aws_run aws iam create-policy --policy-name "$policyName" --policy-document "$policy" --description "Allow hatchery to assume csoc_adminvm role in other accounts and manage dynamodb for multi-account workspaces, and to create resources for nextflow workspaces") - if [ -n "$policyInfo" ]; then - policyArn="$(jq -e -r '.["Policy"].Arn' <<< "$policyInfo")" || { echo "Cannot get 'Policy.Arn' from output: $policyInfo"; return 1; } - else - echo "Unable to create policy '$policyName'. Assume it already exists and create a new version to update the permissions..." - policyArn=$(gen3_aws_run aws iam list-policies --query "Policies[?PolicyName=='$policyName'].Arn" --output text) +echo Service account name: $saName +echo Policy document: $policy - # there can only be up to 5 versions, so delete old versions (except the current default one) - versions="$(gen3_aws_run aws iam list-policy-versions --policy-arn $policyArn | jq -r '.Versions[] | select(.IsDefaultVersion != true) | .VersionId')" - versions=(${versions}) # string to array - for v in "${versions[@]}"; do - echo "Deleting old version '$v'" - gen3_aws_run aws iam delete-policy-version --policy-arn $policyArn --version-id $v - done +# we attempt to create the SA / update the policy every time we run kube-setup-hatchery, because the policy +# has changed and must be updated +roleName="$(gen3 api safe-name hatchery-sa)" +gen3 awsrole create $roleName $saName +policyName="$(gen3 api safe-name hatchery-policy)" +policyInfo=$(gen3_aws_run aws iam create-policy --policy-name "$policyName" --policy-document "$policy" --description "Allow hatchery to assume csoc_adminvm role in other accounts and manage dynamodb for multi-account workspaces, and to create resources for nextflow workspaces") +if [ -n "$policyInfo" ]; then + policyArn="$(jq -e -r '.["Policy"].Arn' <<< "$policyInfo")" || { echo "Cannot get 'Policy.Arn' from output: $policyInfo"; return 1; } +else + echo "Unable to create policy '$policyName'. Assume it already exists and create a new version to update the permissions..." + policyArn=$(gen3_aws_run aws iam list-policies --query "Policies[?PolicyName=='$policyName'].Arn" --output text) - # create the new version - gen3_aws_run aws iam create-policy-version --policy-arn "$policyArn" --policy-document "$policy" --set-as-default - fi + # there can only be up to 5 versions, so delete old versions (except the current default one) + versions="$(gen3_aws_run aws iam list-policy-versions --policy-arn $policyArn | jq -r '.Versions[] | select(.IsDefaultVersion != true) | .VersionId')" + versions=(${versions}) # string to array + for v in "${versions[@]}"; do + echo "Deleting old version '$v'" + gen3_aws_run aws iam delete-policy-version --policy-arn $policyArn --version-id $v + done - gen3_log_info "Attaching policy '${policyName}' to role '${roleName}'" - gen3 awsrole attach-policy ${policyArn} --role-name ${roleName} --force-aws-cli || exit 1 - gen3 awsrole attach-policy "arn:aws:iam::aws:policy/AWSResourceAccessManagerFullAccess" --role-name ${roleName} --force-aws-cli || exit 1 + # create the new version + gen3_aws_run aws iam create-policy-version --policy-arn "$policyArn" --policy-document "$policy" --set-as-default fi +gen3_log_info "Attaching policy '${policyName}' to role '${roleName}'" +gen3 awsrole attach-policy ${policyArn} --role-name ${roleName} --force-aws-cli || exit 1 +gen3 awsrole attach-policy "arn:aws:iam::aws:policy/AWSResourceAccessManagerFullAccess" --role-name ${roleName} --force-aws-cli || exit 1 if [[ -f "$(gen3_secrets_folder)/prisma/apikey.json" ]]; then ACCESSKEYID=$(jq -r .AccessKeyID "$(gen3_secrets_folder)/prisma/apikey.json") From f680a51f20cb3b1b28bb6c12f56e0813fe943d5a Mon Sep 17 00:00:00 2001 From: Pauline <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:56:01 -0600 Subject: [PATCH 5/7] fix quotes in policy --- gen3/bin/kube-setup-hatchery.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gen3/bin/kube-setup-hatchery.sh b/gen3/bin/kube-setup-hatchery.sh index 79c5ba2e6..d2efef75b 100644 --- a/gen3/bin/kube-setup-hatchery.sh +++ b/gen3/bin/kube-setup-hatchery.sh @@ -29,7 +29,7 @@ fi # if `nextflow-global.imagebuilder-reader-role-arn` is set in hatchery config, allow hatchery # to assume the configured role -imagebuilderRoleArn=$(g3kubectl get configmap manifest-hatchery -o jsonpath={.data.nextflow-global} | jq '."imagebuilder-reader-role-arn"') +imagebuilderRoleArn=$(g3kubectl get configmap manifest-hatchery -o jsonpath={.data.nextflow-global} | jq -r '."imagebuilder-reader-role-arn"') assumeImageBuilderRolePolicyBlock="" if [ -z "$imagebuilderRoleArn" ]; then gen3_log_info "No 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration, not granting AssumeRole" From 5f92641ca6b6f5679ef1a5b428df2e585a518423 Mon Sep 17 00:00:00 2001 From: Pauline <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:16:10 -0500 Subject: [PATCH 6/7] ecr-access job: remove 'set -e', fix slack notifications --- kube/services/jobs/ecr-access-job.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/kube/services/jobs/ecr-access-job.yaml b/kube/services/jobs/ecr-access-job.yaml index 11979a123..89bb49d6d 100644 --- a/kube/services/jobs/ecr-access-job.yaml +++ b/kube/services/jobs/ecr-access-job.yaml @@ -65,8 +65,6 @@ spec: args: - "-c" - | - set -e - cd cloud-automation/files/scripts/ echo Installing requirements... pip3 install -r ecr-access-job-requirements.txt From 4698f43775a2a5e1fb16953d219742da6ffd2275 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Fri, 15 Mar 2024 10:34:57 -0500 Subject: [PATCH 7/7] add 'if' back - fix setup in jenkins --- gen3/bin/kube-setup-hatchery.sh | 52 +++++++++++++++++---------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/gen3/bin/kube-setup-hatchery.sh b/gen3/bin/kube-setup-hatchery.sh index a3c84747d..bdcff8ed0 100644 --- a/gen3/bin/kube-setup-hatchery.sh +++ b/gen3/bin/kube-setup-hatchery.sh @@ -180,32 +180,34 @@ saName=$(echo "hatchery-service-account" | head -c63) echo Service account name: $saName echo Policy document: $policy -# we attempt to create the SA / update the policy every time we run kube-setup-hatchery, because the policy -# has changed and must be updated -roleName="$(gen3 api safe-name hatchery-sa)" -gen3 awsrole create $roleName $saName -policyName="$(gen3 api safe-name hatchery-policy)" -policyInfo=$(gen3_aws_run aws iam create-policy --policy-name "$policyName" --policy-document "$policy" --description "Allow hatchery to assume csoc_adminvm role in other accounts and manage dynamodb for multi-account workspaces, and to create resources for nextflow workspaces") -if [ -n "$policyInfo" ]; then - policyArn="$(jq -e -r '.["Policy"].Arn' <<< "$policyInfo")" || { echo "Cannot get 'Policy.Arn' from output: $policyInfo"; return 1; } -else - echo "Unable to create policy '$policyName'. Assume it already exists and create a new version to update the permissions..." - policyArn=$(gen3_aws_run aws iam list-policies --query "Policies[?PolicyName=='$policyName'].Arn" --output text) - - # there can only be up to 5 versions, so delete old versions (except the current default one) - versions="$(gen3_aws_run aws iam list-policy-versions --policy-arn $policyArn | jq -r '.Versions[] | select(.IsDefaultVersion != true) | .VersionId')" - versions=(${versions}) # string to array - for v in "${versions[@]}"; do - echo "Deleting old version '$v'" - gen3_aws_run aws iam delete-policy-version --policy-arn $policyArn --version-id $v - done - - # create the new version - gen3_aws_run aws iam create-policy-version --policy-arn "$policyArn" --policy-document "$policy" --set-as-default +# if the policy has changed and must be updated, run: +# `kubectl delete sa hatchery-service-account && gen3 kube-setup-hatchery` +if ! g3kubectl get sa "$saName" -o json | jq -e '.metadata.annotations | ."eks.amazonaws.com/role-arn"' > /dev/null 2>&1; then + roleName="$(gen3 api safe-name hatchery-sa)" + gen3 awsrole create $roleName $saName + policyName="$(gen3 api safe-name hatchery-policy)" + policyInfo=$(gen3_aws_run aws iam create-policy --policy-name "$policyName" --policy-document "$policy" --description "Allow hatchery to assume csoc_adminvm role in other accounts and manage dynamodb for multi-account workspaces, and to create resources for nextflow workspaces") + if [ -n "$policyInfo" ]; then + policyArn="$(jq -e -r '.["Policy"].Arn' <<< "$policyInfo")" || { echo "Cannot get 'Policy.Arn' from output: $policyInfo"; return 1; } + else + echo "Unable to create policy '$policyName'. Assume it already exists and create a new version to update the permissions..." + policyArn=$(gen3_aws_run aws iam list-policies --query "Policies[?PolicyName=='$policyName'].Arn" --output text) + + # there can only be up to 5 versions, so delete old versions (except the current default one) + versions="$(gen3_aws_run aws iam list-policy-versions --policy-arn $policyArn | jq -r '.Versions[] | select(.IsDefaultVersion != true) | .VersionId')" + versions=(${versions}) # string to array + for v in "${versions[@]}"; do + echo "Deleting old version '$v'" + gen3_aws_run aws iam delete-policy-version --policy-arn $policyArn --version-id $v + done + + # create the new version + gen3_aws_run aws iam create-policy-version --policy-arn "$policyArn" --policy-document "$policy" --set-as-default + fi + gen3_log_info "Attaching policy '${policyName}' to role '${roleName}'" + gen3 awsrole attach-policy ${policyArn} --role-name ${roleName} --force-aws-cli || exit 1 + gen3 awsrole attach-policy "arn:aws:iam::aws:policy/AWSResourceAccessManagerFullAccess" --role-name ${roleName} --force-aws-cli || exit 1 fi -gen3_log_info "Attaching policy '${policyName}' to role '${roleName}'" -gen3 awsrole attach-policy ${policyArn} --role-name ${roleName} --force-aws-cli || exit 1 -gen3 awsrole attach-policy "arn:aws:iam::aws:policy/AWSResourceAccessManagerFullAccess" --role-name ${roleName} --force-aws-cli || exit 1 if [[ -f "$(gen3_secrets_folder)/prisma/apikey.json" ]]; then ACCESSKEYID=$(jq -r .AccessKeyID "$(gen3_secrets_folder)/prisma/apikey.json")