Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: configure secondary ips for protocols #95

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: configure secondary ips for protocols #95

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion modules/protocol_gateways/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ locals {
frontend_container_cores_num = var.frontend_container_cores_num
share_name = var.smb_share_name
region = local.region

secondary_ips_number = var.secondary_ips_per_nic
})

protocol_script = var.protocol == "NFS" ? local.setup_nfs_protocol_script : local.setup_smb_protocol_script
Expand Down
45 changes: 41 additions & 4 deletions modules/protocol_gateways/setup_nfs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,22 @@
weka local ps

current_mngmnt_ip=$(weka local resources | grep 'Management IPs' | awk '{print $NF}')
gateway=$(ip r | grep default | awk '{print $3}')
# get device to use
port=$(ip -o -f inet addr show | grep "$current_mngmnt_ip/"| awk '{print $2}')
subnet_mask=$(ifconfig $port | grep 'inet ' | awk '{print $4}')
# get array of secondary ips
secondary_ips=($(ip -o -4 addr show dev $port | grep 'secondary' | awk '{print $4}' | grep -oE '([0-9]{1,3}\.){3}[0-9]{1,3}'))

echo "$(date -u): secondary_ips: $${secondary_ips[@]}"

function create_interface_group() {
if weka nfs interface-group | grep ${interface_group_name}; then
echo "$(date -u): interface group ${interface_group_name} already exists"
return
fi
echo "$(date -u): creating interface group"
weka nfs interface-group add ${interface_group_name} NFS
weka nfs interface-group add ${interface_group_name} NFS --subnet $subnet_mask --gateway $gateway
echo "$(date -u): interface group ${interface_group_name} created"
}

Expand Down Expand Up @@ -76,13 +86,40 @@ if [ -z "$container_id" ]; then
exit 1
fi

# get device to use
port=$(ip -o -f inet addr show | grep "$current_mngmnt_ip/"| awk '{print $2}')

# add “port” to the interface group - basically it means adding a host and its net device to the group
weka nfs interface-group port add ${interface_group_name} $container_id $port
# show interface group
weka nfs interface-group


function wait_for_nfs_interface_group(){
max_retries=12 # 12 * 10 = 2 minutes
for ((i=0; i<max_retries; i++)); do
status=$(weka nfs interface-group -J | jq -r '.[] | select(.name == "'${interface_group_name}'").status')
if [ "$status" == "OK" ]; then
echo "$(date -u): interface group status: $status"
break
fi
echo "$(date -u): waiting for interface group status to be OK, current status: $status"
sleep 10
done
if [ "$status" != "OK" ]; then
echo "$(date -u): failed to wait for the interface group status to be OK"
return 1
fi
}

# add secondary IPs for the group to use - these IPs will be used in order to mount
for seconday_ip in "$${secondary_ips[@]}"; do
wait_for_nfs_interface_group || exit 1
# add secondary ip to the interface group
retry_command "weka nfs interface-group ip-range add ${interface_group_name} $seconday_ip"

wait_for_nfs_interface_group || exit 1
done

weka nfs interface-group

# create client group if not exists and add rules / premissions
create_client_group || true

Expand Down
58 changes: 26 additions & 32 deletions modules/protocol_gateways/setup_smb.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
echo "$(date -u): running smb script"
weka local ps

current_mngmnt_ip=$(weka local resources | grep 'Management IPs' | awk '{print $NF}')
cluster_size="${gateways_number}"
secondary_ips_num="${secondary_ips_number}"

function wait_for_weka_fs(){
filesystem_name="default"
max_retries=30 # 30 * 10 = 5 minutes
Expand Down Expand Up @@ -33,9 +37,7 @@ if (( i > max_retries )); then
exit 1
fi

cluster_size="${gateways_number}"

current_mngmnt_ip=$(weka local resources | grep 'Management IPs' | awk '{print $NF}')
# get container id
for ((i=0; i<20; i++)); do
container_id=$(weka cluster container | grep frontend0 | grep $HOSTNAME | grep $current_mngmnt_ip | grep UP | awk '{print $1}')
Expand Down Expand Up @@ -76,7 +78,7 @@ for (( retry=1; retry<=max_retries; retry++ )); do
all_container_ids=$(weka cluster container | grep frontend0 | grep $gw_ips | grep UP | awk '{print $1}')
# if number of all_container_ids < cluster_size, do nothing
all_container_ids_number=$(echo "$all_container_ids" | wc -l)
if (( all_container_ids_number < cluster_size )); then
if (( all_container_ids_number < cluster_size )); then
echo "$(date -u): not all containers are ready - do retry $retry of $max_retries"
sleep 20
else
Expand All @@ -103,10 +105,8 @@ if (( all_hosts > 0 && not_ready_hosts == 0 && all_hosts == cluster_size )); the
fi

if (( all_hosts > 0 && not_ready_hosts == 0 && all_hosts < cluster_size )); then
echo "$(date -u): SMB cluster already exists, adding current container to it"
echo "$(date -u): SMB cluster already exists, you can add new container manually"

weka smb cluster containers add --container-ids $container_id
weka smb cluster wait
weka smb cluster status
exit 0
fi
Expand All @@ -115,38 +115,32 @@ echo "$(date -u): weka SMB cluster does not exist, creating it"
# get all protocol gateways fromtend container ids separated by comma
all_container_ids_str=$(echo "$all_container_ids" | tr '\n' ',' | sed 's/,$//')

# all secondary ips of all GW hosts
all_secondary_ips=($(aws ec2 describe-instances \
--filters "Name=tag:Name,Values=${gateways_name}" "Name=tag:weka_hostgroup_type,Values=gateways-protocol" \
--query 'Reservations[*].Instances[*].NetworkInterfaces[*].PrivateIpAddresses[?Primary==`false`].PrivateIpAddress' \
--region ${region} \
--output json | jq -r 'flatten[]'))
# if number of all_secondary_ips < cluster_size * secondary_ips_num, do nothing
# length of all_secondary_ips array should be cluster_size * secondary_ips_num
all_secondary_ips_number=$${#all_secondary_ips[@]}
if (( all_secondary_ips_number < cluster_size * secondary_ips_num )); then
echo "$(date -u): not all secondary ips are in the list - do nothing"
exit 1
else
echo "$(date -u): all containers are ready"
fi

# get all secondary ips separated by comma
secondary_ips_str=$(IFS=,; printf "%s" "$${all_secondary_ips[*]}")

# if smbw_enabled is true, enable SMBW by adding --smbw flag
smbw_cmd_extention=""
if [[ ${smbw_enabled} == true ]]; then
smbw_cmd_extention="--smbw --config-fs-name .config_fs"
fi


function retry_create_smb_cluster {
retry_max=60
retry_sleep=30
count=$retry_max

while [ $count -gt 0 ]; do
weka smb cluster create ${cluster_name} ${domain_name} $smbw_cmd_extention --container-ids $all_container_ids_str && break
count=$(($count - 1))
echo "Retrying create SMB cluster in $retry_sleep seconds..."
sleep $retry_sleep
done
[ $count -eq 0 ] && {
echo "create SMB cluster command failed after $retry_max attempts"
echo "$(date -u): create SMB cluster failed"
return 1
}
return 0
}

echo "$(date -u): Retrying create SMB cluster..."

retry_create_smb_cluster

echo "$(date -u): Successfully create SMB cluster..."

weka smb cluster create ${cluster_name} ${domain_name} $smbw_cmd_extention --container-ids $all_container_ids_str --smb-ips-pool $secondary_ips_str
weka smb cluster wait

# add an SMB share if share_name is not empty
Expand Down
Loading