diff --git a/.secrets.baseline b/.secrets.baseline index 16475e0b2..919833990 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -342,7 +342,7 @@ "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", "is_secret": false, "is_verified": false, - "line_number": 217, + "line_number": 191, "type": "Secret Keyword" } ], diff --git a/gen3/bin/awsrole.sh b/gen3/bin/awsrole.sh index 476e7d003..144b7a4fe 100644 --- a/gen3/bin/awsrole.sh +++ b/gen3/bin/awsrole.sh @@ -20,6 +20,7 @@ gen3_awsrole_help() { # NOTE: service-account to role is 1 to 1 # # @param serviceAccount to link to the role +# @param flag (optional) - specify a flag to use a different trust policy # function gen3_awsrole_ar_policy() { local serviceAccount="$1" @@ -32,6 +33,9 @@ function gen3_awsrole_ar_policy() { local issuer_url local account_id local vpc_name + shift || return 1 + local flag=$1 + vpc_name="$(gen3 api environment)" || return 1 issuer_url="$(aws eks describe-cluster \ --name ${vpc_name} \ @@ -42,7 +46,42 @@ function gen3_awsrole_ar_policy() { local provider_arn="arn:aws:iam::${account_id}:oidc-provider/${issuer_url}" - cat - < config.tfvars @@ -199,6 +247,13 @@ EOF gen3_log_err $errMsg return 1 fi + shift || return 1 + local flag="" + # Check if the "all_namespaces" flag is provided + if [[ "$1" == "-f" || "$1" == "--flag" ]]; then + flag="$2" + shift 2 + fi # check if the name is already used by another entity local entity_type @@ -216,7 +271,7 @@ EOF fi TF_IN_AUTOMATION="true" - if ! _tfplan_role $rolename $saName $namespace; then + if ! _tfplan_role $rolename $saName $namespace -f $flag; then return 1 fi if ! _tfapply_role $rolename; then diff --git a/gen3/bin/kube-setup-argo.sh b/gen3/bin/kube-setup-argo.sh index c7243d3da..ff2438833 100644 --- a/gen3/bin/kube-setup-argo.sh +++ b/gen3/bin/kube-setup-argo.sh @@ -28,7 +28,10 @@ function setup_argo_buckets { # try to come up with a unique but composable bucket name bucketName="gen3-argo-${accountNumber}-${environment//_/-}" - userName="gen3-argo-${environment//_/-}-user" + nameSpace="$(gen3 db namespace)" + roleName="gen3-argo-${environment//_/-}-role" + bucketPolicy="argo-bucket-policy-${nameSpace}" + internalBucketPolicy="argo-internal-bucket-policy-${nameSpace}" if [[ ! -z $(g3k_config_lookup '."s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) || ! -z $(g3k_config_lookup '.argo."s3-bucket"') ]]; then if [[ ! -z $(g3k_config_lookup '."s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) ]]; then gen3_log_info "Using S3 bucket found in manifest: ${bucketName}" @@ -114,70 +117,41 @@ EOF ] } EOF - if ! secret="$(g3kubectl get secret argo-s3-creds -n argo 2> /dev/null)"; then - gen3_log_info "setting up bucket $bucketName" - - if aws s3 ls --page-size 1 "s3://${bucketName}" > /dev/null 2>&1; then - gen3_log_info "${bucketName} s3 bucket already exists" - # continue on ... - elif ! aws s3 mb "s3://${bucketName}"; then - gen3_log_err "failed to create bucket ${bucketName}" - fi - - gen3_log_info "Creating IAM user ${userName}" - if ! aws iam get-user --user-name ${userName} > /dev/null 2>&1; then - aws iam create-user --user-name ${userName} || true - else - gen3_log_info "IAM user ${userName} already exits.." - fi - - secret=$(aws iam create-access-key --user-name ${userName}) - if ! g3kubectl get namespace argo > /dev/null 2>&1; then - gen3_log_info "Creating argo namespace" - g3kubectl create namespace argo || true - g3kubectl label namespace argo app=argo || true - g3kubectl create rolebinding argo-admin --clusterrole=admin --serviceaccount=argo:default -n argo || true - fi - else - # Else we want to recreate the argo-s3-creds secret so make a temp file with the current creds and delete argo-s3-creds secret - gen3_log_info "Argo S3 setup already completed" - local secretFile="$XDG_RUNTIME_DIR/temp_key_file_$$.json" - cat > "$secretFile" < /dev/null 2>&1; then + gen3_log_info "${bucketName} s3 bucket already exists" + # continue on ... + elif ! aws s3 mb "s3://${bucketName}"; then + gen3_log_err "failed to create bucket ${bucketName}" fi - - gen3_log_info "Creating s3 creds secret in argo namespace" - if [[ "$ctxNamespace" == "default" || "$ctxNamespace" == "null" ]]; then - if [[ -z $internalBucketName ]]; then - g3kubectl delete secret -n argo argo-s3-creds || true - g3kubectl create secret -n argo generic argo-s3-creds --from-literal=AccessKeyId=$(echo $secret | jq -r .AccessKey.AccessKeyId) --from-literal=SecretAccessKey=$(echo $secret | jq -r .AccessKey.SecretAccessKey) --from-literal=bucketname=${bucketName} || true - g3kubectl create secret generic argo-s3-creds --from-literal=AccessKeyId=$(echo $secret | jq -r .AccessKey.AccessKeyId) --from-literal=SecretAccessKey=$(echo $secret | jq -r .AccessKey.SecretAccessKey) --from-literal=bucketname=${bucketName} || true - else - g3kubectl delete secret -n argo argo-s3-creds || true - g3kubectl create secret -n argo generic argo-s3-creds --from-literal=AccessKeyId=$(echo $secret | jq -r .AccessKey.AccessKeyId) --from-literal=SecretAccessKey=$(echo $secret | jq -r .AccessKey.SecretAccessKey) --from-literal=bucketname=${bucketName} --from-literal=internalbucketname=${internalBucketName} || true - g3kubectl create secret generic argo-s3-creds --from-literal=AccessKeyId=$(echo $secret | jq -r .AccessKey.AccessKeyId) --from-literal=SecretAccessKey=$(echo $secret | jq -r .AccessKey.SecretAccessKey) --from-literal=bucketname=${bucketName} || true - fi + if ! g3kubectl get namespace argo > /dev/null 2>&1; then + gen3_log_info "Creating argo namespace" + g3kubectl create namespace argo || true + g3kubectl label namespace argo app=argo || true + # Grant admin access within the argo namespace to the default SA in the argo namespace + g3kubectl create rolebinding argo-admin --clusterrole=admin --serviceaccount=argo:default -n argo || true + fi + gen3_log_info "Creating IAM role ${roleName}" + if aws iam get-role --role-name "${roleName}" > /dev/null 2>&1; then + gen3_log_info "IAM role ${roleName} already exists.." + roleArn=$(aws iam get-role --role-name "${roleName}" --query 'Role.Arn' --output text) + gen3_log_info "Role annotate" + g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} -n argo + g3kubectl annotate serviceaccount argo eks.amazonaws.com/role-arn=${roleArn} -n $nameSpace else - g3kubectl create sa argo || true - # Grant admin access within the current namespace to the argo SA in the current namespace - g3kubectl create rolebinding argo-admin --clusterrole=admin --serviceaccount=$(gen3 db namespace):argo -n $(gen3 db namespace) || true - aws iam put-user-policy --user-name ${userName} --policy-name argo-bucket-policy --policy-document file://$policyFile || true - if [[ -z $internalBucketName ]]; then - aws iam put-user-policy --user-name ${userName} --policy-name argo-internal-bucket-policy --policy-document file://$internalBucketPolicyFile || true - g3kubectl create secret generic argo-s3-creds --from-literal=AccessKeyId=$(echo $secret | jq -r .AccessKey.AccessKeyId) --from-literal=SecretAccessKey=$(echo $secret | jq -r .AccessKey.SecretAccessKey) --from-literal=bucketname=${bucketName} || true - else - g3kubectl create secret generic argo-s3-creds --from-literal=AccessKeyId=$(echo $secret | jq -r .AccessKey.AccessKeyId) --from-literal=SecretAccessKey=$(echo $secret | jq -r .AccessKey.SecretAccessKey) --from-literal=bucketname=${bucketName} --from-literal=internalbucketname=${internalBucketName} || true - - fi + gen3 awsrole create $roleName argo $nameSpace -f all_namespaces + roleArn=$(aws iam get-role --role-name "${roleName}" --query 'Role.Arn' --output text) + g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} -n argo fi + # Grant admin access within the current namespace to the argo SA in the current namespace + g3kubectl create rolebinding argo-admin --clusterrole=admin --serviceaccount=$nameSpace:argo -n $nameSpace || true + aws iam put-role-policy --role-name ${roleName} --policy-name ${bucketPolicy} --policy-document file://$policyFile || true + if [[ -z $internalBucketName ]]; then + aws iam put-role-policy --role-name ${roleName} --policy-name ${internalBucketPolicy} --policy-document file://$internalBucketPolicyFile || true + fi ## if new bucket then do the following # Get the aws keys from secret @@ -189,9 +163,9 @@ EOF aws s3api put-bucket-lifecycle --bucket ${bucketName} --lifecycle-configuration file://$bucketLifecyclePolicyFile # Always update the policy, in case manifest buckets change - aws iam put-user-policy --user-name ${userName} --policy-name argo-bucket-policy --policy-document file://$policyFile + aws iam put-role-policy --role-name ${roleName} --policy-name ${bucketPolicy} --policy-document file://$policyFile if [[ ! -z $internalBucketPolicyFile ]]; then - aws iam put-user-policy --user-name ${userName} --policy-name argo-internal-bucket-policy --policy-document file://$internalBucketPolicyFile + aws iam put-role-policy --role-name ${roleName} --policy-name ${internalBucketPolicy} --policy-document file://$internalBucketPolicyFile fi if [[ ! -z $(g3k_config_lookup '.indexd_admin_user' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) || ! -z $(g3k_config_lookup '.argo.indexd_admin_user') ]]; then if [[ ! -z $(g3k_config_lookup '.indexd_admin_user' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) ]]; then @@ -231,11 +205,12 @@ if [[ "$ctxNamespace" == "default" || "$ctxNamespace" == "null" ]]; then if (! helm status argo -n argo > /dev/null 2>&1 ) || [[ "$1" == "--force" ]]; then DBHOST=$(kubectl get secrets -n argo argo-db-creds -o json | jq -r .data.db_host | base64 -d) DBNAME=$(kubectl get secrets -n argo argo-db-creds -o json | jq -r .data.db_database | base64 -d) - if [[ -z $(kubectl get secrets -n argo argo-s3-creds -o json | jq -r .data.internalbucketname | base64 -d) ]]; then - BUCKET=$(kubectl get secrets -n argo argo-s3-creds -o json | jq -r .data.bucketname | base64 -d) + if [[ -z $internalBucketName ]]; then + BUCKET=$bucketName else - BUCKET=$(kubectl get secrets -n argo argo-s3-creds -o json | jq -r .data.internalbucketname | base64 -d) + BUCKET=$internalBucketName fi + valuesFile="$XDG_RUNTIME_DIR/values_$$.yaml" valuesTemplate="${GEN3_HOME}/kube/services/argo/values.yaml" diff --git a/kube/services/argo/workflows/fence-usersync-cron.yaml b/kube/services/argo/workflows/fence-usersync-cron.yaml new file mode 100644 index 000000000..4723ce10f --- /dev/null +++ b/kube/services/argo/workflows/fence-usersync-cron.yaml @@ -0,0 +1,10 @@ +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: fence-usersync-cron +spec: + serviceAccountName: argo + schedule: "*/30 * * * *" + workflowSpec: + workflowTemplateRef: + name: fence-usersync-workflow diff --git a/kube/services/argo/workflows/fence-usersync-wf.yaml b/kube/services/argo/workflows/fence-usersync-wf.yaml new file mode 100644 index 000000000..d7f56a2ce --- /dev/null +++ b/kube/services/argo/workflows/fence-usersync-wf.yaml @@ -0,0 +1,257 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: fence-usersync-workflow +spec: + volumeClaimTemplates: + - metadata: + name: shared-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + serviceAccountName: argo + entrypoint: fence-usersync + arguments: + parameters: + - name: ADD_DBGAP + value: "false" + - name: ONLY_DBGAP + value: "false" + templates: + - name: fence-usersync + steps: + - - name: wait-for-fence + template: wait-for-fence + - - name: awshelper + template: awshelper + - - name: usersyncer + template: usersyncer + + - name: wait-for-fence + container: + image: curlimages/curl:latest + command: ["/bin/sh","-c"] + args: ["while [ $(curl -sw '%{http_code}' http://fence-service -o /dev/null) -ne 200 ]; do sleep 5; echo 'Waiting for fence...'; done"] + + - name: awshelper + container: + image: quay.io/cdis/awshelper:master + imagePullPolicy: Always + securityContext: + runAsUser: 0 + env: + - name: gen3Env + valueFrom: + configMapKeyRef: + name: global + key: hostname + - name: userYamlS3Path + valueFrom: + configMapKeyRef: + name: manifest-global + key: useryaml_s3path + - name: slackWebHook + value: None + volumeMounts: + - name: shared-data + mountPath: /mnt/shared + command: ["/bin/bash"] + args: + - "-c" + - | + GEN3_HOME=/home/ubuntu/cloud-automation + source "${GEN3_HOME}/gen3/lib/utils.sh" + gen3_load "gen3/gen3setup" + + if [ "${userYamlS3Path}" = 'none' ]; then + # echo "using local user.yaml" + # cp /var/www/fence/user.yaml /mnt/shared/user.yaml + echo "s3 yaml not provided - bailing out" + exit 1 + else + # ----------------- + echo "awshelper downloading ${userYamlS3Path} to /mnt/shared/user.yaml" + n=0 + until [ $n -ge 5 ]; do + echo "Download attempt $n" + aws s3 cp "${userYamlS3Path}" /mnt/shared/user.yaml && break + n=$[$n+1] + sleep 2 + done + fi + if [[ ! -f /mnt/shared/user.yaml ]]; then + echo "awshelper failed to retrieve /mnt/shared/user.yaml" + exit 1 + fi + #----------- + echo "awshelper updating etl configmap" + if ! gen3 gitops etl-convert < /mnt/shared/user.yaml > /tmp/user.yaml; then + echo "ERROR: failed to generate ETL config" + exit 1 + fi + # kubectl delete configmap fence > /dev/null 2>&1 + # kubectl create configmap fence --from-file=/tmp/user.yaml + if [ "${slackWebHook}" != 'None' ]; then + curl -X POST --data-urlencode "payload={\"text\": \"AWSHelper: Syncing users on ${gen3Env}\"}" "${slackWebHook}" + fi + echo "Helper exit ok" + + - name: usersyncer + volumes: + - name: yaml-merge + configMap: + name: "fence-yaml-merge" + - name: config-volume + secret: + secretName: "fence-config" + - name: creds-volume + secret: + secretName: "fence-creds" + - name: fence-google-app-creds-secret-volume + secret: + secretName: "fence-google-app-creds-secret" + - name: fence-google-storage-creds-secret-volume + secret: + secretName: "fence-google-storage-creds-secret" + - name: fence-ssh-keys + secret: + secretName: "fence-ssh-keys" + defaultMode: 0400 + - name: fence-sshconfig + configMap: + name: "fence-sshconfig" + - name: projects + configMap: + name: "projects" + container: + image: quay.io/cdis/fence:master + imagePullPolicy: Always + env: + - name: PYTHONPATH + value: /var/www/fence + - name: SYNC_FROM_DBGAP + valueFrom: + configMapKeyRef: + name: manifest-global + key: sync_from_dbgap + - name: ADD_DBGAP + value: "{{workflow.parameters.ADD_DBGAP}}" + - name: ONLY_DBGAP + value: "{{workflow.parameters.ONLY_DBGAP}}" + - name: SLACK_SEND_DBGAP + valueFrom: + configMapKeyRef: + name: manifest-global + key: slack_send_dbgap + optional: true + - name: slackWebHook + valueFrom: + configMapKeyRef: + name: global + key: slack_webhook + optional: true + - name: gen3Env + valueFrom: + configMapKeyRef: + name: global + key: hostname + - name: FENCE_PUBLIC_CONFIG + valueFrom: + configMapKeyRef: + name: manifest-fence + key: fence-config-public.yaml + optional: true + volumeMounts: + - name: shared-data + mountPath: /mnt/shared + - name: "config-volume" + readOnly: true + mountPath: "/var/www/fence/fence-config.yaml" + subPath: fence-config.yaml + - name: "creds-volume" + readOnly: true + mountPath: "/var/www/fence/creds.json" + - name: "yaml-merge" + readOnly: true + mountPath: "/var/www/fence/yaml_merge.py" + - name: "fence-google-app-creds-secret-volume" + readOnly: true + mountPath: "/var/www/fence/fence_google_app_creds_secret.json" + subPath: fence_google_app_creds_secret.json + - name: "fence-google-storage-creds-secret-volume" + readOnly: true + mountPath: "/var/www/fence/fence_google_storage_creds_secret.json" + subPath: fence_google_storage_creds_secret.json + - name: "fence-ssh-keys" + mountPath: "/root/.ssh/id_rsa" + subPath: "id_rsa" + - name: "fence-ssh-keys" + mountPath: "/root/.ssh/id_rsa.pub" + subPath: "id_rsa.pub" + - name: "fence-sshconfig" + mountPath: "/root/.ssh/config" + subPath: "config" + - name: "projects" + mountPath: "/var/www/fence/projects.yaml" + subPath: "projects.yaml" + command: ["/bin/bash"] + args: + - "-c" + # Script always succeeds if it runs (echo exits with 0) + - | + echo "${ADD_DBGAP}" + echo "${ONLY_DBGAP}" + echo "${FENCE_PUBLIC_CONFIG:-""}" > "/var/www/fence/fence-config-public.yaml" + python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml + echo 'options use-vc' >> /etc/resolv.conf + let count=0 + while [[ ! -f /mnt/shared/user.yaml && $count -lt 50 ]]; do + echo "fence container waiting for /mnt/shared/user.yaml"; + sleep 2 + let count=$count+1 + done + if [[ "$SYNC_FROM_DBGAP" != True && "$ADD_DBGAP" != "true" ]]; then + if [[ -f /mnt/shared/user.yaml ]]; then + echo "running fence-create" + time fence-create sync --arborist http://arborist-service --yaml /mnt/shared/user.yaml + else + echo "/mnt/shared/user.yaml did not appear within timeout :-(" + false # non-zero exit code + fi + exitcode=$? + else + output=$(mktemp "/tmp/fence-create-output_XXXXXX") + if [[ -f /mnt/shared/user.yaml && "$ONLY_DBGAP" != "true" ]]; then + echo "Running fence-create dbgap-sync with user.yaml - see $output" + time fence-create sync --arborist http://arborist-service --sync_from_dbgap "True" --projects /var/www/fence/projects.yaml --yaml /mnt/shared/user.yaml 2>&1 | tee "$output" + else + echo "Running fence-create dbgap-sync without user.yaml - see $output" + time fence-create sync --arborist http://arborist-service --sync_from_dbgap "True" --projects /var/www/fence/projects.yaml 2>&1 | tee "$output" + fi + exitcode="${PIPESTATUS[0]}" + echo "$output" + # Echo what files we are seeing on dbgap ftp to Slack + # We only do this step every 12 hours and not on weekends to reduce noise + if [[ -n "$SLACK_SEND_DBGAP" && "$SLACK_SEND_DBGAP" = True ]]; then + files=$(grep "Reading file" "$output") + let hour=$(date -u +10#%H) + let dow=$(date -u +10#%u) + if ! (( hour % 12 )) && (( dow < 6 )); then + if [ "${slackWebHook}" != 'None' ]; then + curl -X POST --data-urlencode "payload={\"text\": \"FenceHelper: \n\`\`\`\n${files}\n\`\`\`\"}" "${slackWebHook}" + fi + fi + fi + fi + if [[ $exitcode -ne 0 && "${slackWebHook}" != 'None' ]]; then + emptyfile=$(grep "EnvironmentError:" "$output") + if [ ! -z "$emptyfile" ]; then + curl -X POST --data-urlencode "payload={\"text\": \"JOBSKIPPED: User sync skipped on ${gen3Env} ${emptyfile}\"}" "${slackWebHook}"; + else + curl -X POST --data-urlencode "payload={\"text\": \"JOBFAIL: User sync failed on ${gen3Env}\"}" "${slackWebHook}" + fi + fi + echo "Exit code: $exitcode" + exit "$exitcode" \ No newline at end of file