TES-290 Added helpers scripts in AWS AMI

moved files

Edited comments

edited start graphdb

edited ebs_volume.sh

Fix variables and IF statement

modules/user_data/templates/create_backup.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Configure the GraphDB backup cron job
+
+cat <<-EOF > /usr/bin/graphdb_backup
+#!/bin/bash
+
+set -euxo pipefail
+
+GRAPHDB_ADMIN_PASSWORD="\$(aws --cli-connect-timeout 300 ssm get-parameter --region ${region} --name "/${name}/graphdb/admin_password" --with-decryption | jq -r .Parameter.Value)"
+NODE_STATE="\$(curl --silent --fail --user "admin:\$GRAPHDB_ADMIN_PASSWORD" localhost:7201/rest/cluster/node/status | jq -r .nodeState)"
+
+if [ "\$NODE_STATE" != "LEADER" ]; then
+  echo "current node is not a leader, but \$NODE_STATE"
+  exit 0
+fi
+
+function trigger_backup {
+  local backup_name="\$(date +'%Y-%m-%d_%H-%M-%S').tar"
+
+  curl \
+    -vvv --fail \
+    --user "admin:\$GRAPHDB_ADMIN_PASSWORD" \
+    --url localhost:7201/rest/recovery/cloud-backup \
+    --header 'Content-Type: application/json' \
+    --header 'Accept: application/json' \
+    --data-binary @- <<-DATA
+    {
+      "backupOptions": { "backupSystemData": true },
+      "bucketUri": "s3:///${backup_bucket_name}/\$backup_name?region=${region}"
+    }
+DATA
+}
+
+function rotate_backups {
+  all_files="\$(aws --cli-connect-timeout 300 s3api list-objects --bucket ${backup_bucket_name} --query 'Contents' | jq .)"
+  count="\$(echo \$all_files | jq length)"
+  delete_count="\$((count - ${backup_retention_count} - 1))"
+
+  for i in \$(seq 0 \$delete_count); do
+    key="\$(echo \$all_files | jq -r .[\$i].Key)"
+
+    aws --cli-connect-timeout 300 s3 rm s3://${backup_bucket_name}/\$key
+  done
+}
+
+if ! trigger_backup; then
+  echo "failed to create backup"
+  exit 1
+fi
+
+rotate_backups
+
+EOF
+
+chmod +x /usr/bin/graphdb_backup
+# This cron expression schedules the job to run every day at midnight, without any restrictions on the day of the month, month, or day of the week.
+echo "0 0 * * * root ${backup_name} graphdb /usr/bin/graphdb_backup" | sudo tee /etc/cron.d/graphdb_backup

modules/user_data/templates/ebs_volume.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+until ping -c 1 google.com &> /dev/null; do
+  echo "waiting for outbound connectivity"
+  sleep 5
+done
+
+# Set common variables used throughout the script.
+imds_token=$( curl -Ss -H "X-aws-ec2-metadata-token-ttl-seconds: 300" -XPUT 169.254.169.254/latest/api/token )
+instance_id=$( curl -Ss -H "X-aws-ec2-metadata-token: $imds_token" 169.254.169.254/latest/meta-data/instance-id )
+availability_zone=$( curl -Ss -H "X-aws-ec2-metadata-token: $imds_token" 169.254.169.254/latest/meta-data/placement/availability-zone )
+volume_id=""
+
+# Search for an available EBS volume to attach to the instance. Wait one minute for a volume to become available,
+# if no volume is found - create new one, attach, format and mount the volume.
+
+for i in $(seq 1 6); do
+
+  volume_id=$(
+    aws --cli-connect-timeout 300 ec2 describe-volumes \
+      --filters "Name=status,Values=available" "Name=availability-zone,Values=$availability_zone" "Name=tag:Name,Values=${name}-graphdb-data" \
+      --query "Volumes[*].{ID:VolumeId}" \
+      --output text | \
+      sed '/^$/d'
+  )
+
+  if [ -z "${volume_id:-}" ]; then
+    echo 'ebs volume not yet available'
+    sleep 10
+  else
+    break
+  fi
+done
+
+if [ -z "${volume_id:-}" ]; then
+
+# If the volume type is GP3, then we need to provide a value for the "throughput" parameter.
+ if [ "${ebs_volume_type}" == "gp3"] || ["${ebs_volume_type}" == "gp2" ]; then
+      volume_id=$(
+        aws --cli-connect-timeout 300 ec2 create-volume \
+          --availability-zone "$availability_zone" \
+          --encrypted \
+          --kms-key-id "${ebs_kms_key_arn}" \
+          --volume-type "${ebs_volume_type}" \
+          --size "${ebs_volume_size}" \
+          --iops "${ebs_volume_iops}" \
+          --throughput "${ebs_volume_throughput}" \
+          --tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=${name}-graphdb-data}]" | \
+          jq -r .VolumeId
+      )
+    else
+      volume_id=$(
+        aws --cli-connect-timeout 300 ec2 create-volume \
+          --availability-zone "$availability_zone" \
+          --encrypted \
+          --kms-key-id "${ebs_kms_key_arn}" \
+          --volume-type "${ebs_volume_type}" \
+          --size "${ebs_volume_size}" \
+          --iops "${ebs_volume_iops}" \
+          --tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=${name}-graphdb-data}]" | \
+          jq -r .VolumeId
+      )
+    fi
+
+  aws --cli-connect-timeout 300 ec2 wait volume-available --volume-ids "$volume_id"
+fi
+
+aws --cli-connect-timeout 300 ec2 attach-volume \
+  --volume-id "$volume_id" \
+  --instance-id "$instance_id" \
+  --device "${device_name}"
+
+# Handle the EBS volume used for the GraphDB data directory
+# beware, here be dragons...
+# read these articles:
+# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
+# https://github.com/oogali/ebs-automatic-nvme-mapping/blob/master/README.md
+
+# this variable comes from terraform, and it's what we specified in the launch template as the device mapping for the ebs
+# because this might be attached under a different name, we'll need to search for it
+device_mapping_full="${device_name}"
+device_mapping_short="$(echo $device_mapping_full | cut -d'/' -f3)"
+
+graphdb_device=""
+
+# the device might not be available immediately, wait a while
+for i in $(seq 1 12); do
+  for volume in $(find /dev | grep -i 'nvme[0-21]n1$'); do
+    # extract the specified device from the vendor-specific data
+    # read https://github.com/oogali/ebs-automatic-nvme-mapping/blob/master/README.md, for more information
+    real_device=$(nvme id-ctrl --raw-binary $volume | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g')
+    if [ "$device_mapping_full" = "$real_device" ] || [ "$device_mapping_short" = "$real_device" ]; then
+      graphdb_device="$volume"
+      break
+    fi
+  done
+
+  if [ -n "$graphdb_device" ]; then
+    break
+  fi
+  sleep 5
+done
+
+# create a file system if there isn't any
+if [ "$graphdb_device: data" = "$(file -s $graphdb_device)" ]; then
+  mkfs -t ext4 $graphdb_device
+fi
+
+disk_mount_point="/var/opt/graphdb"
+
+# Check if the disk is already mounted
+if ! mount | grep -q "$graphdb_device"; then
+  echo "The disk at $graphdb_device is not mounted."
+
+  # Create the mount point if it doesn't exist
+  if [ ! -d "$disk_mount_point" ]; then
+    mkdir -p "$disk_mount_point"
+  fi
+
+  # Add an entry to the fstab file to automatically mount the disk
+  if ! grep -q "$graphdb_device" /etc/fstab; then
+    echo "$graphdb_device $disk_mount_point ext4 defaults 0 2" >> /etc/fstab
+  fi
+
+  # Mount the disk
+  mount "$disk_mount_point"
+  echo "The disk at $graphdb_device is now mounted at $disk_mount_point."
+else
+  echo "The disk at $graphdb_device is already mounted."
+fi
+
+# Ensure data folders exist
+mkdir -p $disk_mount_point/node $disk_mount_point/cluster-proxy
+
+# this is needed because after the disc attachment folder owner is reverted
+chown -R graphdb:graphdb $disk_mount_point

modules/user_data/templates/register_route53.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+until ping -c 1 google.com &> /dev/null; do
+  echo "waiting for outbound connectivity"
+  sleep 5
+done
+
+# Register the instance in Route 53, using the volume id for the sub-domain
+
+subdomain="$( echo -n "$volume_id" | sed 's/^vol-//' )"
+node_dns="$subdomain.${zone_dns_name}"
+
+aws --cli-connect-timeout 300 route53 change-resource-record-sets \
+  --hosted-zone-id "${zone_id}" \
+  --change-batch '{"Changes": [{"Action": "UPSERT","ResourceRecordSet": {"Name": "'"$node_dns"'","Type": "A","TTL": 60,"ResourceRecords": [{"Value": "'"$local_ipv4"'"}]}}]}'
+
+hostnamectl set-hostname "$node_dns"
+
+echo "Instance registered in Route 53 with DNS name: $node_dns"