Develop to release-1.0.1-GA (#136) (#137)

.gitignore

@@ -15,6 +15,7 @@ crash.*.log
 # to change depending on the environment.
 *.tfvars
 *.tfvars.json
+!overrides.tfvars
 
 # Ignore override files as they are usually used to override resources locally and so
 # are not checked in

automation-scripts/infra-setup/setup.conf

@@ -0,0 +1,23 @@
+# setup.conf
+AWS_ACCESS_KEY_ID="access_key"
+AWS_SECRET_ACCESS_KEY="secret_key"
+AWS_DEFAULT_REGION="region"
+KUBE_CONFIG_PATH="$HOME/.kube/config"
+AWS_TERRAFORM_BACKEND_BUCKET_NAME="bucket_name"
+AWS_TERRAFORM_BACKEND_BUCKET_REGION="region" # eu-west-1
+# Other prompt configuration
+EKS_NODES_SUBNET_IDS='[""]'
+EKS_MASTER_SUBNET_IDS='[""]'
+VELERO_AWS_ACCESS_KEY_ID="access_key"
+VELERO_AWS_SECRET_ACCESS_KEY="secret_key"
+SERVICE_TYPE="LoadBalancer"
+VPC_ID=""
+AVAILABILITY_ZONES='[""]' # ["eu-west-1a","eu-west-1b","eu-west-1c"]
+BUILDING_BLOCK="" # obsrv
+ENV="" # dev
+REGION="" # eu-west-1
+TIMEZONE="UTC"
+ALLOW_VPC_CREATION=false # It will creates VPC, Subnets.
+ALLOW_VELERO_USER_CREATION=false # It will create velero user
+ALLOW_KONG_INGRESS_SETUP=false # It will setup kong ingress
+# Add more variables as needed

automation-scripts/infra-setup/setup.md

@@ -0,0 +1,53 @@
+
+# Obsrv Infrastructure Setup Instructions
+
+## Configuration
+
+Define the necessary configurations in the `setup.conf` file:
+
+### setup.conf
+
+```bash
+AWS_ACCESS_KEY_ID="$access_key"
+AWS_SECRET_ACCESS_KEY="$secret_key"
+AWS_DEFAULT_REGION="$region"
+KUBE_CONFIG_PATH="$HOME/.kube"
+AWS_TERRAFORM_BACKEND_BUCKET_NAME="$bucket_name"
+AWS_TERRAFORM_BACKEND_BUCKET_REGION="$region"
+
+# Add more variables as needed
+```
+
+Replace placeholders (`$access_key`, `$secret_key`, `$region`, `$bucket_name`, etc.) with actual values.
+
+## Tool Installation
+
+Ensure the installation of the following tools:
+
+| Tool        | Version      |
+|-------------|--------------|
+| aws         | >=2.13.8     |
+| helm        | >=3.10.2     |
+| terraform   | >=1.5.7      |
+| terrahelp   | >=0.7.5      |
+| terragrunt  | >=0.45.6     |
+
+## Setup Process
+
+Before executing the `setup` shell script, ensure that the `curl` and `unzip` utilities are present on your system. If they are not installed, you can use the following commands to install them. Execute the provided `setup.sh`` script as a root user to avoid potential permission issues:
+
+**Prerequisites**
+```bash
+sudo apt-get update
+sudo apt-get install -y curl
+sudo apt-get install -y unzip
+```
+
+Once the `curl` and `unzip` utility is successfully installed, proceed with the setup by running the following command:
+
+```bash
+sh setup.sh ./setup.conf
+```
+
+This command utilizes the configurations specified in the `setup.conf` file to initiate the setup process.
+

automation-scripts/infra-setup/setup.sh

@@ -0,0 +1,186 @@
+#!/bin/bash
+
+version_compare() {
+    local version1="$1"
+    local version2="$2"
+    IFS='.' read v1_major v1_minor v1_patch <<< "$version1"
+    IFS='.' read v2_major v2_minor v2_patch <<< "$version2"
+
+    if [ "$v1_major" -gt "$v2_major" ] || [ "$v1_major" -eq "$v2_major" -a "$v1_minor" -gt "$v2_minor" ] || [ "$v1_major" -eq "$v2_major" -a "$v1_minor" -eq "$v2_minor" -a "$v1_patch" -ge "$v2_patch" ]; then
+        echo true  
+    else
+        echo false
+    fi
+}
+
+# Function to get installed version
+get_installed_version() {
+    local version_command="$1"
+    if [ -n "$version_command" ]; then
+        installed_version=$(eval "$version_command")
+        echo "$installed_version"
+    else
+        echo "0.0.0" 
+    fi
+}
+
+# Function to install a tool
+install_tool() {
+    local tool_name="$1"
+    local install_command="$2"
+    local version_command="$3"
+    local required_version="$4"
+
+    installed_version=$(get_installed_version "$version_command")
+    compare_result=$(version_compare "$installed_version" "$required_version")
+    if [ "$compare_result" == "true" ]; then
+        echo "$tool_name is already installed with supported version $installed_version"
+        return
+    else
+        echo "$tool_name tool version $required_version is missing, but the installed version is $installed_version. Would you like to install the stable version of $tool_name? (yes/no)"
+        read -r response
+
+        if [ "$response" == "yes" ]; then
+            echo "Installing $tool_name..."
+            eval "$install_command"
+
+            # Check if the installation was successful
+            if [ $? -eq 0 ]; then
+                echo "$tool_name installed successfully."
+                if [ -n "$version_command" ]; then
+                    installed_version=$(get_installed_version "$version_command")
+                    echo "Version: $installed_version"
+                fi
+            else
+                echo "Error: Failed to install $tool_name. Please install it manually before proceeding."
+                exit 1
+            fi
+        else
+            echo "Skipping installation of $tool_name."
+        fi
+    fi    
+}
+
+
+# Validate and install required tools
+validate_tools() {
+    # Define all the required tools with version
+    tool_versions=(
+        "aws:2.13.8"
+        "helm:3.10.2"
+        "terraform:1.5.7"
+        "terrahelp:0.7.5"
+        "terragrunt:0.45.6"
+    )
+
+    for tool_version in "${tool_versions[@]}"; do
+        IFS=':' read -r tool required_version <<< "$tool_version"
+        case $tool in
+            "aws")
+                aws_version="aws --version | awk 'NR==1{print \$1}' | cut -d'/' -f2"
+                install_tool "$tool" 'curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && sudo ./aws/install -i /usr/local/aws-cli -b /usr/local/bin' "$aws_version" "$required_version"
+                ;;
+            "helm")
+                helm_version="helm version --short | awk -F'[v+]' '/v/{print \$2}'"
+                install_tool "$tool" 'curl https://get.helm.sh/helm-v3.10.2-linux-amd64.tar.gz -o helm.tar.gz && tar -zxvf helm.tar.gz && sudo mv linux-amd64/helm /usr/local/bin/' "$helm_version" "$required_version"
+                ;;
+            "terraform")
+                terraform_version='terraform version | awk '\''/Terraform/{gsub(/[^0-9.]/, "", $2); print $2}'\'''
+                install_tool "terraform" 'curl -LO "https://releases.hashicorp.com/terraform/1.5.7/terraform_1.5.7_linux_amd64.zip" -o "terraform.zip" && unzip terraform_1.5.7_linux_amd64.zip && sudo mv terraform /usr/local/bin/' "$terraform_version" "$required_version"
+                ;;
+            "terrahelp")
+                terrahelp_version='terrahelp --version | awk '\''/terrahelp version/ {print $3}'\'''
+                install_tool "$tool" 'curl -OL https://github.com/opencredo/terrahelp/releases/download/v0.4.3/terrahelp-linux-amd64 && mv terrahelp-linux-amd64 /usr/local/bin/terrahelp && chmod +x /usr/local/bin/terrahelp' "$terrahelp_version" "$required_version"
+                ;;
+            "terragrunt")
+                terragrunt_version='terragrunt --version | awk '\''/terragrunt version/ {gsub(/v/, "", $3); print $3}'\'''
+                install_tool "$tool" 'curl -OL https://github.com/gruntwork-io/terragrunt/releases/download/v0.45.8/terragrunt_linux_amd64 && mv terragrunt_linux_amd64 /usr/local/bin/terragrunt && chmod +x /usr/local/bin/terragrunt' "$terragrunt_version" "$required_version"
+                ;;
+
+        esac
+    done
+
+    echo "All required tools are installed. Proceeding with the rest of the script..."
+}
+
+
+
+# Kube_config directory setup and takes the backup of existing kubeconfig if exists
+setup_kube_config() {
+    kube_config_path=$KUBE_CONFIG_PATH 
+    config_file="$kube_config_path/config"
+
+    # Check if the ~/.kube directory exists
+    if [ ! -d "$kube_config_path" ]; then
+        mkdir -p "$kube_config_path"
+        echo "Created $kube_config_path directory."
+    fi
+
+    if [ -e "$config_file" ]; then
+        backup_file="$kube_config_path/config_backup_$(date +'%Y%m%d%H%M%S').bak"
+        cp "$config_file" "$backup_file"
+        echo "Backup created: $backup_file"
+    else
+        touch "$config_file"
+        echo "Created an empty config file: $config_file"
+    fi
+}
+
+# Check if the config file is provided as a command-line argument
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <config_file>"
+    exit 1
+fi
+
+# Store the configuration file path
+config_file="$1"
+
+# Check if the config file exists
+if [ ! -f "$config_file" ]; then
+    echo "Error: Config file '$config_file' not found."
+    exit 1
+fi
+
+# Read and set variables from the config file
+source "$config_file"
+
+# Set up AWS environment variables
+echo "Setup Infra configurations"
+export AWS_TERRAFORM_BACKEND_BUCKET_NAME=$AWS_TERRAFORM_BACKEND_BUCKET_NAME
+export AWS_TERRAFORM_BACKEND_BUCKET_REGION=$AWS_TERRAFORM_BACKEND_BUCKET_REGION
+export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+export AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION
+export KUBE_CONFIG_PATH=$KUBE_CONFIG_PATH
+tfvars_file="../../terraform/aws/vars/overrides.tfvars"
+
+# Create the tfvars file
+cat <<EOF > "$tfvars_file"
+eks_nodes_subnet_ids = $EKS_NODES_SUBNET_IDS
+eks_master_subnet_ids = $EKS_MASTER_SUBNET_IDS
+velero_aws_access_key_id = "$VELERO_AWS_ACCESS_KEY_ID"
+velero_aws_secret_access_key = "$VELERO_AWS_SECRET_ACCESS_KEY"
+service_type = "$SERVICE_TYPE"
+vpc_id = "$VPC_ID"
+availability_zones = $AVAILABILITY_ZONES
+building_block  = "$BUILDING_BLOCK"
+env = "$ENV"
+region = "$REGION"
+timezone = "$TIMEZONE"
+create_vpc = "$ALLOW_VPC_CREATION"
+create_velero_user = "$ALLOW_VELERO_USER_CREATION"
+create_kong_ingress = "$ALLOW_KONG_INGRESS_SETUP"
+EOF
+
+echo "terraform.tfvars file created successfully at $tfvars_file."
+
+validate_tools
+#setup_kube_config - TODO - Required to verify 
+
+# Script related to terraform and deployment will start from here
+cd ../../terraform/aws
+terragrunt init
+terragrunt apply -target module.eks -var "create_vpc=$ALLOW_VPC_CREATION" -var "create_velero_user=$ALLOW_VELERO_USER_CREATION"  -var-file=vars/dev.tfvars -var-file=vars/overrides.tfvars -auto-approve
+terragrunt apply -target module.get_kubeconfig -var "create_vpc=$ALLOW_VPC_CREATION" -var "create_velero_user=$ALLOW_VELERO_USER_CREATION"  -var-file=vars/dev.tfvars -var-file=vars/overrides.tfvars -auto-approve
+terragrunt apply  -var "create_vpc=$ALLOW_VPC_CREATION" -var "create_velero_user=$ALLOW_VELERO_USER_CREATION"  -var-file=vars/dev.tfvars -var-file=vars/overrides.tfvars -auto-approve
+

terraform/aws/main.tf

@@ -30,6 +30,7 @@ provider "helm" {
 module "vpc" {
   source             = "../modules/aws/vpc"
   env                = var.env
+  count              = var.create_vpc ? 1 : 0
   building_block     = var.building_block
   region             = var.region
   availability_zones = var.availability_zones
@@ -39,14 +40,16 @@ module "eks" {
   source                = "../modules/aws/eks"
   env                   = var.env
   building_block        = var.building_block
-  eks_master_subnet_ids = module.vpc.multi_zone_public_subnets_ids
-  eks_nodes_subnet_ids  = module.vpc.single_zone_public_subnets_id
+  cluster_logs_enabled  = var.cluster_logs_enabled
+  eks_master_subnet_ids = var.create_vpc ? module.vpc[0].multi_zone_public_subnets_ids : var.eks_master_subnet_ids
+  eks_nodes_subnet_ids  = var.create_vpc ? module.vpc[0].single_zone_public_subnets_id : var.eks_nodes_subnet_ids
   region                = var.region
   depends_on            = [module.vpc]
 }
 
 module "iam" {
   source                = "../modules/aws/iam"
+  count                 = var.create_velero_user ? 1 : 0
   env                   = var.env
   building_block        = var.building_block
   velero_storage_bucket = module.s3.velero_storage_bucket
@@ -90,6 +93,7 @@ module "superset" {
   redis_namespace                   = module.redis_dedup.redis_namespace
   redis_release_name                = module.redis_dedup.redis_release_name
   postgresql_service_name           = module.postgresql.postgresql_service_name
+  service_type                      = var.service_type
 }
 
 module "grafana_configs" {
@@ -164,6 +168,7 @@ module "druid_raw_cluster" {
   druid_raw_user_password            = module.postgresql.postgresql_druid_raw_user_password
   druid_raw_sa_annotations           = "eks.amazonaws.com/role-arn: ${module.eks.druid_raw_sa_iam_role}"
   druid_cluster_namespace            = module.eks.druid_raw_namespace
+  service_type                       = var.service_type
 }
 
 module "druid_operator" {
@@ -217,6 +222,7 @@ module "dataset_api" {
   dedup_redis_release_name           = module.redis_dedup.redis_release_name
   dataset_api_namespace              = module.eks.dataset_api_namespace
   s3_bucket                          = module.s3.s3_bucket
+  service_type                       = var.service_type
 }
 
 module "secor" {
@@ -248,8 +254,8 @@ module "velero" {
   cloud_provider               = "aws"
   velero_backup_bucket         = module.s3.velero_storage_bucket
   velero_backup_bucket_region  = var.region
-  velero_aws_access_key_id     = module.iam.velero_user_access_key
-  velero_aws_secret_access_key = module.iam.velero_user_secret_key
+  velero_aws_access_key_id     = var.create_velero_user ?  module.iam[0].velero_user_access_key : var.velero_aws_access_key_id
+  velero_aws_secret_access_key = var.create_velero_user ? module.iam[0].velero_user_secret_key : var.velero_aws_secret_access_key 
 }
 
 module "alert_rules" {
@@ -265,6 +271,7 @@ module "web_console" {
   depends_on                       = [module.eks, module.monitoring]
   web_console_image_repository     = var.web_console_image_repository
   web_console_image_tag            = var.web_console_image_tag
+  service_type                     = var.service_type
 }
 
 module "get_kubeconfig" {

terraform/aws/variables.tf

@@ -23,6 +23,44 @@ variable "timezone" {
   description = "Timezone property to backup the data"
 }
 
+variable "create_vpc" {
+  type = bool
+  description = "Toggle to create to a vpc"
+}
+variable "eks_nodes_subnet_ids" {
+  type = list(string)
+  description = "The VPC's subnet id which will be used to create the EKS node groups"
+  default = [""]
+}
+variable "eks_master_subnet_ids" {
+  type = list(string)
+  description = "The VPC's subnet id which will be used to create the EKS cluster"
+  default = [""]
+}
+
+variable "create_velero_user" {
+  type = bool
+  description = "Toggle to create a velero user"
+}
+variable "velero_aws_access_key_id" {
+  type = string
+  description = "AWS Access key to access bucket"
+  default = ""
+}
+variable "velero_aws_secret_access_key" {
+  type = string
+  description = "AWs Secret access key to access bucket"
+  default = "" 
+}
+variable "service_type" {
+  type = string
+  description = "Kubernetes service type either NodePort or LoadBalancer. It is NodePort by default"
+  default = "LoadBalancer"
+}
+variable "cluster_logs_enabled" {
+  type = bool
+  description = "Toggle to enable eks cluster logs"
+}
 variable "flink_checkpoint_store_type" {
   type        = string
   description = "Flink checkpoint store type."