Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tes 290 include helpers scripts #20

Closed
wants to merge 9 commits into from
Closed

Tes 290 include helpers scripts #20

Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
59a7ad1
TES-290 Added helpers scripts in AWS AMI
Oct 20, 2023
d94f43c
Removed sub-scripts create_backup.sh, ebs_volume.sh, register_route53.sh
Nov 1, 2023
bd1296c
Added variable for connector port
Nov 6, 2023
aaed834
Edited script paths
Nov 23, 2023
76b223c
Switched from source to args
Nov 28, 2023
0bd9a29
fix script path
Nov 29, 2023
c35f57a
Capturing the node_dns variable.
Nov 30, 2023
5f97f06
Removed variabled volume_id
Nov 30, 2023
4a47e81
Edite backup schedule variable
Dec 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 38 additions & 168 deletions modules/user_data/templates/start_graphdb.sh.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,130 +14,56 @@ imds_token=$( curl -Ss -H "X-aws-ec2-metadata-token-ttl-seconds: 300" -XPUT 169.
local_ipv4=$( curl -Ss -H "X-aws-ec2-metadata-token: $imds_token" 169.254.169.254/latest/meta-data/local-ipv4 )
instance_id=$( curl -Ss -H "X-aws-ec2-metadata-token: $imds_token" 169.254.169.254/latest/meta-data/instance-id )
availability_zone=$( curl -Ss -H "X-aws-ec2-metadata-token: $imds_token" 169.254.169.254/latest/meta-data/placement/availability-zone )
volume_id=""

# Search for an available EBS volume to attach to the instance. Wait one minute for a volume to become available,
# if no volume is found - create new one, attach, format and mount the volume.

for i in $(seq 1 6); do

volume_id=$(
aws --cli-connect-timeout 300 ec2 describe-volumes \
--filters "Name=status,Values=available" "Name=availability-zone,Values=$availability_zone" "Name=tag:Name,Values=${name}-graphdb-data" \
--query "Volumes[*].{ID:VolumeId}" \
--output text | \
sed '/^$/d'
)

if [ -z "$${volume_id:-}" ]; then
echo 'ebs volume not yet available'
sleep 10
else
break
fi
done

if [ -z "$${volume_id:-}" ]; then

volume_id=$(
aws --cli-connect-timeout 300 ec2 create-volume \
--availability-zone "$availability_zone" \
--encrypted \
--kms-key-id "${ebs_kms_key_arn}" \
--volume-type "${ebs_volume_type}" \
--size "${ebs_volume_size}" \
--iops "${ebs_volume_iops}" \
--throughput "${ebs_volume_throughput}" \
--tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=${name}-graphdb-data}]" | \
jq -r .VolumeId
)

aws --cli-connect-timeout 300 ec2 wait volume-available --volume-ids "$volume_id"
fi

aws --cli-connect-timeout 300 ec2 attach-volume \
--volume-id "$volume_id" \
--instance-id "$instance_id" \
--device "${device_name}"

# Handle the EBS volume used for the GraphDB data directory
# beware, here be dragons...
# read these articles:
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
# https://github.com/oogali/ebs-automatic-nvme-mapping/blob/master/README.md

# this variable comes from terraform, and it's what we specified in the launch template as the device mapping for the ebs
# because this might be attached under a different name, we'll need to search for it
device_mapping_full="${device_name}"
device_mapping_short="$(echo $device_mapping_full | cut -d'/' -f3)"

graphdb_device=""

# the device might not be available immediately, wait a while
for i in $(seq 1 12); do
for volume in $(find /dev | grep -i 'nvme[0-21]n1$'); do
# extract the specified device from the vendor-specific data
# read https://github.com/oogali/ebs-automatic-nvme-mapping/blob/master/README.md, for more information
real_device=$(nvme id-ctrl --raw-binary $volume | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g')
if [ "$device_mapping_full" = "$real_device" ] || [ "$device_mapping_short" = "$real_device" ]; then
graphdb_device="$volume"
break
fi
done

if [ -n "$graphdb_device" ]; then
break
fi
sleep 5
done

# create a file system if there isn't any
if [ "$graphdb_device: data" = "$(file -s $graphdb_device)" ]; then
mkfs -t ext4 $graphdb_device
fi

disk_mount_point="/var/opt/graphdb"
# Defining variables by interpolating Terraform variables

# Check if the disk is already mounted
if ! mount | grep -q "$graphdb_device"; then
echo "The disk at $graphdb_device is not mounted."
region="${region}"
name="${name}"
device_name="${device_name}"

# Create the mount point if it doesn't exist
if [ ! -d "$disk_mount_point" ]; then
mkdir -p "$disk_mount_point"
fi
backup_schedule="${backup_schedule}"
backup_retention_count="${backup_retention_count}"
backup_bucket_name="${backup_bucket_name}"

# Add an entry to the fstab file to automatically mount the disk
if ! grep -q "$graphdb_device" /etc/fstab; then
echo "$graphdb_device $disk_mount_point ext4 defaults 0 2" >> /etc/fstab
fi
ebs_volume_type="${ebs_volume_type}"
ebs_volume_size="${ebs_volume_size}"
ebs_volume_iops="${ebs_volume_iops}"
ebs_volume_throughput="${ebs_volume_throughput}"
ebs_kms_key_arn="${ebs_kms_key_arn}"

# Mount the disk
mount "$disk_mount_point"
echo "The disk at $graphdb_device is now mounted at $disk_mount_point."
else
echo "The disk at $graphdb_device is already mounted."
fi
zone_dns_name="${zone_dns_name}"
zone_id="${zone_id}"

# Ensure data folders exist
mkdir -p $disk_mount_point/node $disk_mount_point/cluster-proxy
jvm_max_memory="${jvm_max_memory}"
resource_name_prefix="${resource_name_prefix}"

# this is needed because after the disc attachment folder owner is reverted
chown -R graphdb:graphdb $disk_mount_point
GRAPHDB_CONNECTOR_PORT=""

# Register the instance in Route 53, using the volume id for the sub-domain
# Search for an available EBS volume to attach to the instance. If no volume is found - create new one, attach, format and mount the volume.
./opt/helper-scripts/ebs_volume.sh \
--name ${name} \
--ebs_volume_type ${ebs_volume_type} \
--ebs_volume_throughput ${ebs_volume_throughput} \
--ebs_kms_key_arn ${ebs_kms_key_arn} \
--ebs_volume_size ${ebs_volume_size} \
--ebs_volume_iops ${ebs_volume_iops} \
--device_name ${device_name}

subdomain="$( echo -n "$volume_id" | sed 's/^vol-//' )"
node_dns="$subdomain.${zone_dns_name}"
# Register the instance in Route 53, using the volume id for the sub-domain. Capturing the node_dns variable to be used in the rest of the script.
node_dns=$(./opt/helper-scripts/register_route53.sh \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At the end of this script there's this:

echo "Instance registered in Route 53 with DNS name:"
echo "$node_dns"

so, you'll have a lot more than just the DNS name.
Since, you have the zone name and the volume id, you can construct the FQDN. Otherwise, the script should echo just the FQDN.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed.

--name ${name} \
--zone_dns_name ${zone_dns_name} \
--zone_id ${zone_id})

aws --cli-connect-timeout 300 route53 change-resource-record-sets \
--hosted-zone-id "${zone_id}" \
--change-batch '{"Changes": [{"Action": "UPSERT","ResourceRecordSet": {"Name": "'"$node_dns"'","Type": "A","TTL": 60,"ResourceRecords": [{"Value": "'"$local_ipv4"'"}]}}]}'

hostnamectl set-hostname "$node_dns"
# Configure the GraphDB backup cron job
./opt/helper-scripts/create_backup.sh \
--region ${region} \
--name ${name} \
--backup_bucket_name ${backup_bucket_name} \
--backup_retention_count ${backup_retention_count} \
--backup_schedule ${backup_schedule}

# Configure GraphDB

aws --cli-connect-timeout 300 ssm get-parameter --region ${region} --name "/${name}/graphdb/license" --with-decryption | \
jq -r .Parameter.Value | \
base64 -d > /etc/graphdb/graphdb.license
Expand Down Expand Up @@ -169,62 +95,6 @@ cat << EOF > /etc/systemd/system/graphdb.service.d/overrides.conf
Environment="GDB_HEAP_SIZE=${jvm_max_memory}g"
EOF

# Configure the GraphDB backup cron job

cat <<-EOF > /usr/bin/graphdb_backup
#!/bin/bash

set -euxo pipefail

GRAPHDB_ADMIN_PASSWORD="\$(aws --cli-connect-timeout 300 ssm get-parameter --region ${region} --name "/${name}/graphdb/admin_password" --with-decryption | jq -r .Parameter.Value)"
NODE_STATE="\$(curl --silent --fail --user "admin:\$GRAPHDB_ADMIN_PASSWORD" localhost:7201/rest/cluster/node/status | jq -r .nodeState)"

if [ "\$NODE_STATE" != "LEADER" ]; then
echo "current node is not a leader, but \$NODE_STATE"
exit 0
fi

function trigger_backup {
local backup_name="\$(date +'%Y-%m-%d_%H-%M-%S').tar"

curl \
-vvv --fail \
--user "admin:\$GRAPHDB_ADMIN_PASSWORD" \
--url localhost:7201/rest/recovery/cloud-backup \
--header 'Content-Type: application/json' \
--header 'Accept: application/json' \
--data-binary @- <<-DATA
{
"backupOptions": { "backupSystemData": true },
"bucketUri": "s3:///${backup_bucket_name}/\$backup_name?region=${region}"
}
DATA
}

function rotate_backups {
all_files="\$(aws --cli-connect-timeout 300 s3api list-objects --bucket ${backup_bucket_name} --query 'Contents' | jq .)"
count="\$(echo \$all_files | jq length)"
delete_count="\$((count - ${backup_retention_count} - 1))"

for i in \$(seq 0 \$delete_count); do
key="\$(echo \$all_files | jq -r .[\$i].Key)"

aws --cli-connect-timeout 300 s3 rm s3://${backup_bucket_name}/\$key
done
}

if ! trigger_backup; then
echo "failed to create backup"
exit 1
fi

rotate_backups

EOF

chmod +x /usr/bin/graphdb_backup
echo "${backup_schedule} graphdb /usr/bin/graphdb_backup" > /etc/cron.d/graphdb_backup

# https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#connection-idle-timeout
echo 'net.ipv4.tcp_keepalive_time = 120' | tee -a /etc/sysctl.conf
echo 'fs.file-max = 262144' | tee -a /etc/sysctl.conf
Expand Down