EDSF-535 Add prefix to tarball/agent s3 bucket

.github/workflows/dsf_poc_cli.yml

@@ -132,11 +132,6 @@ jobs:
       run: |
         find ./examples/ -type f -exec sed -i -f sed.expr {} \;
 
-    - name: Sets env vars for environment
-      run: |
-        echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV
-      if: github.ref != 'refs/heads/"master"'
-
     - name: Create terraform backend file
       run: |
         cat << EOF > $EXAMPLE_DIR/backend.tf

.github/workflows/dsf_poc_cli_azure.yml

@@ -129,11 +129,6 @@ jobs:
       run: | 
         find $EXAMPLE_DIR -type f -exec sed -i -f sed.expr {} \;
 
-    - name: Sets env vars for environment
-      run: |
-        echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV
-      if: github.ref != 'refs/heads/"master"'
-
     - name: Create terraform backend file
       run: |
         cat << EOF > $EXAMPLE_DIR/backend.tf

.github/workflows/dsf_single_account_cli.yml

@@ -98,11 +98,6 @@ jobs:
       run: |
         find ./examples/ -type f -exec sed -i -f sed.expr {} \;
 
-    - name: Sets env vars for environment
-      run: |
-        echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV
-      if: inputs.branch != 'master'
-
     - name: Set Workspace
       run: |
         if [ ${{ inputs.branch }} == 'master' ] ||  [ ${{ inputs.branch }} == 'dev' ]; then

.github/workflows/nightly_sonar_poc_basic_cli.yml

@@ -66,11 +66,6 @@ jobs:
         echo '${{ toJSON(github) }}'
         find ./examples/ -type f -exec sed -i -f sed.expr {} \;
 
-    - name: Sets env vars for environment
-      run: |
-        echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV
-      if: github.ref != 'refs/heads/"master"'
-
     - name: Create terraform backend file
       run: |
         cat << EOF > $EXAMPLE_DIR/backend.tf

.github/workflows/sonar_multi_account_cli.yml

@@ -65,11 +65,6 @@ jobs:
       run: |
         find ./examples/ -type f -exec sed -i -f sed.expr {} \;
 
-    - name: Sets env vars for environment
-      run: |
-        echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV
-      if: inputs.branch != 'master'
-
     - name: Create terraform backend file
       run: |
         cat << EOF > $EXAMPLE_DIR/backend.tf

.github/workflows/sonar_poc_cli.yml

@@ -104,11 +104,6 @@ jobs:
       run: | 
         find $EXAMPLE_DIR -type f -exec sed -i -f sed.expr {} \;
 
-    - name: Sets env vars for environment
-      run: |
-        echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV
-      if: github.ref != 'refs/heads/"master"'
-
     - name: Create terraform backend file
       run: |
         cat << EOF > $EXAMPLE_DIR/backend.tf

.github/workflows/sonar_poc_cloud.yml

@@ -47,11 +47,6 @@ jobs:
       run: | 
         find ./examples/ -type f -exec sed -i -f sed.expr {} \;
 
-    - name: Sets env vars for environment
-      run: |
-        echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV
-      if: github.ref != 'refs/heads/"master"'
-
     # Install the latest version of Terraform CLI and configure the Terraform CLI configuration file with a Terraform Cloud user API token
     - name: Setup Terraform
       uses: hashicorp/setup-terraform@v2

.github/workflows/sonar_upgrade.yml

@@ -101,11 +101,6 @@ jobs:
         run: |
           find ${{ env.DEPLOYMENT_EXAMPLE_DIR }} -type f -exec sed -i -f sed.expr {} \;
 
-      - name: Sets env vars for environment
-        run: |
-          echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV
-        if: github.ref != 'refs/heads/"master"'
-
       - name: Create terraform backend file for the deployment
         run: |
           cat << EOF > $DEPLOYMENT_EXAMPLE_DIR/backend.tf

modules/aws/agentless-gw/variables.tf

@@ -126,7 +126,7 @@ variable "binaries_location" {
     s3_region = string
     s3_key    = string
   })
-  description = "S3 DSF installation location. If tarball_url not set, binaries_location is used"
+  description = "S3 DSF installation location. If tarball_url not set, binaries_location is used. For example, { s3_bucket = 'my-bucket', s3_region = 'us-west-2', s3_key = 'installation-files/my-file'}. This means that the path to the installation file is s3://my-bucket/installation-files/my-file."
   default = {
     s3_bucket = ""
     s3_region = ""

modules/aws/core/globals/main.tf

@@ -25,8 +25,8 @@ locals {
   }
   sonar_supported_versions       = keys(local.sonar_tarball_s3_key_map)
   sonar_fully_supported_versions = setsubtract(local.sonar_supported_versions, ["4.9", "4.10.0.0", "4.10.0.1", "4.10"])
-  s3_object                      = var.tarball_s3_key != null ? var.tarball_s3_key : local.sonar_tarball_s3_key_map[var.sonar_version]
-  s3_object_version              = regex("\\d\\.\\d*", local.s3_object)
+  sonar_installation_s3_key      = var.installation_s3_key != null ? var.installation_s3_key : join("/", [local.sonar_installation_s3_prefix, local.sonar_tarball_s3_key_map[var.sonar_version]])
+  sonar_s3_object_version        = regex("\\d\\.\\d*", local.sonar_installation_s3_key)
 }
 
 locals {
@@ -50,6 +50,11 @@ locals {
   dra_version            = lookup(local.dra_version_map, var.dra_version, var.dra_version)
 }
 
+locals {
+  sonar_installation_s3_prefix = "sonar"
+  dam_agent_installation_s3_prefix = "dam-agent"
+}
+
 resource "random_id" "salt" {
   byte_length = 2
 }

modules/aws/core/globals/outputs.tf

@@ -42,18 +42,19 @@ output "tags" {
 
 output "tarball_location" {
   value = {
-    s3_bucket = var.tarball_s3_bucket.bucket
-    s3_region = var.tarball_s3_bucket.region
-    s3_key    = local.s3_object
-    version   = local.s3_object_version
+    s3_bucket = var.installation_s3_bucket.bucket
+    s3_region = var.installation_s3_bucket.region
+    s3_key    = local.sonar_installation_s3_key
+    version   = local.sonar_s3_object_version
   }
 }
 
 output "dam_agent_installation_location" {
   value = {
-    s3_bucket = var.tarball_s3_bucket.bucket
-    s3_region = var.tarball_s3_bucket.region
-    s3_key    = null
+    s3_bucket = var.installation_s3_bucket.bucket
+    s3_region = var.installation_s3_bucket.region
+    s3_prefix = local.dam_agent_installation_s3_prefix
+    s3_object = null
   }
 }
 

modules/aws/core/globals/variables.tf

@@ -4,7 +4,7 @@ variable "sonar_version" {
   description = "The Sonar version to install. Supported versions are: 4.9 and up. Both long and short version formats are supported, for example, 4.12.0.10 or 4.12. The short format maps to the latest patch."
 }
 
-variable "tarball_s3_bucket" {
+variable "installation_s3_bucket" {
   type = object({
     bucket = string
     region = string
@@ -13,13 +13,13 @@ variable "tarball_s3_bucket" {
     bucket = "1ef8de27-ed95-40ff-8c08-7969fc1b7901"
     region = "us-east-1"
   }
-  description = "S3 bucket containing the installation tarballs. Use default to get Imperva's bucket"
+  description = "S3 bucket containing the installation files for the EDF Hub, Agentless Gateway or POC DAM Agent. Use default to get Imperva's bucket."
 }
 
-variable "tarball_s3_key" {
+variable "installation_s3_key" {
   type        = string
+  description = "Name and prefix of the installation file of the EDF Hub, Agentless Gateway or POC DAM Agent in the S3 bucket. Keep empty if you prefer to use the sonar_version variable."
   default     = null
-  description = "Name of the installation file in s3 bucket. Keep empty if you prefer to use the sonar_version variable"
 }
 
 variable "dra_version" {

modules/aws/db-with-agent/iam_role.tf

@@ -22,15 +22,13 @@ locals {
     "Version" : "2012-10-17",
     "Statement" : [
       {
-        "Sid" : "VisualEditor0",
+        "Sid" : "getFileFromS3BucketPrefix",
         "Effect" : "Allow",
         "Action" : [
-          "s3:GetObject",
-          "s3:ListBucket"
+          "s3:GetObject"
         ]
         "Resource" : [
-          "arn:aws:s3:::${local.binaries_location.s3_bucket}",
-          "arn:aws:s3:::${local.binaries_location.s3_bucket}/*",
+          "arn:aws:s3:::${local.binaries_location.s3_bucket_and_prefix}/*",
         ]
       }
     ]

modules/aws/db-with-agent/main.tf

@@ -7,7 +7,9 @@ locals {
   binaries_location = {
     s3_bucket = var.binaries_location.s3_bucket
     s3_region = var.binaries_location.s3_region
-    s3_key    = var.binaries_location.s3_key != null ? var.binaries_location.s3_key : local.os_params[local.os_type].image_name
+    s3_object = var.binaries_location.s3_object != null ? var.binaries_location.s3_object : local.os_params[local.os_type].installation_filename
+    s3_key    = var.binaries_location.s3_prefix != null ? join("/", var.binaries_location.s3_prefix, var.binaries_location.s3_object) : local.binaries_location.s3_object
+    s3_bucket_and_prefix = var.binaries_location.s3_prefix != null ? join("/", local.binaries_location.s3_bucket, var.binaries_location.s3_prefix) : local.binaries_location.s3_bucket
   }
 }
 

modules/aws/db-with-agent/os_params.tf

@@ -5,7 +5,7 @@ locals {
       ami_name               = "RHEL-8.6.0_HVM-2022*-x86_64-2-Hourly2-GP2"
       ami_ssh_user           = "ec2-user"
       agent_installation_dir = "/opt/imperva",
-      image_name = "Imperva-ragent-RHEL-v8-kSMP-px86_64-b14.6.0.60.0.637577.bsx"
+      installation_filename = "Imperva-ragent-RHEL-v8-kSMP-px86_64-b14.6.0.60.0.637577.bsx"
       package_install = <<-EOF
         yum update -y
         yum install unzip -y
@@ -37,7 +37,7 @@ locals {
       ami_name               = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"
       ami_ssh_user           = "ubuntu"
       agent_installation_dir = "/usr/imperva",
-      image_name = "Imperva-ragent-UBN-px86_64-b14.6.0.60.0.636085.bsx"
+      installation_filename = "Imperva-ragent-UBN-px86_64-b14.6.0.60.0.636085.bsx"
       package_install = <<-EOF
         apt update -y
         apt install unzip

modules/aws/db-with-agent/userdata.tf

@@ -16,11 +16,12 @@ locals {
       sudo ./aws/install
       export PATH=$PATH:/usr/local/bin:/usr/local/bin
       echo "Downloading agent:"
-      aws s3 cp s3://${local.binaries_location.s3_bucket}/${local.binaries_location.s3_key} .
-      chmod +x ./${local.binaries_location.s3_key}
+      INSTALLATION_FILE=${local.binaries_location.s3_object}
+      aws s3 cp s3://${local.binaries_location.s3_bucket}/${local.binaries_location.s3_key} . --region ${local.binaries_location.s3_region}
+      chmod +x ./"$INSTALLATION_FILE"
       echo "Installing agent:"
-      ./${local.binaries_location.s3_key} -n -d ${local.agent_installation_dir}
-      rm ${local.binaries_location.s3_key}
+      ./"$INSTALLATION_FILE" -n -d ${local.agent_installation_dir}
+      rm "$INSTALLATION_FILE"
       echo "Registering agent:"
       ${local.agent_installation_dir}/ragent/bin/cli --dcfg ${local.agent_installation_dir}/ragent/etc --dtarget ${local.agent_installation_dir}/ragent/etc --dlog ${local.agent_installation_dir}/ragent/etc/logs/cli registration advanced-register registration-type=Primary is-db-agent=true tunnel-protocol=TCP gw-ip=${var.registration_params.agent_gateway_host} gw-port=443 manual-settings-activation=Automatic monitor-network-channels=Both password="${var.registration_params.secure_password}" ragent-name="${join("-", [var.friendly_name, random_id.salt.hex])}" site='${var.registration_params.site}' server-group="${var.registration_params.server_group}";
       echo "Starting agent:"

modules/aws/db-with-agent/variables.tf

@@ -79,9 +79,10 @@ variable "binaries_location" {
   type = object({
     s3_bucket = string
     s3_region = string
-    s3_key    = string
+    s3_prefix = string
+    s3_object = string
   })
-  description = "S3 DSF DAM agent installation location"
+  description = "S3 DSF DAM agent installation location. For example, { s3_bucket = 'my-bucket', s3_region = 'us-west-2', s3_prefix = 'installation-files', s3_object = 'my-file' }. This means that the path to the installation file is s3://my-bucket/installation-files/my-file. If you don't have a prefix or an object, keep them null."
   nullable    = false
 }
 

modules/aws/hub/variables.tf

@@ -146,7 +146,7 @@ variable "binaries_location" {
     s3_region = string
     s3_key    = string
   })
-  description = "S3 DSF installation location. If tarball_url not set, binaries_location is used"
+  description = "S3 DSF installation location. If tarball_url not set, binaries_location is used. For example, { s3_bucket = 'my-bucket', s3_region = 'us-west-2', s3_key = 'installation-files/my-file'}. This means that the path to the installation file is s3://my-bucket/installation-files/my-file."
   default = {
     s3_bucket = ""
     s3_region = ""

modules/aws/sonar-base-instance/iam_role.tf

@@ -43,15 +43,13 @@ locals {
     "Version" : "2012-10-17",
     "Statement" : [
       {
-        "Sid" : "VisualEditor0",
+        "Sid" : "getFileFromS3BucketPrefix",
         "Effect" : "Allow",
         "Action" : [
-          "s3:GetObject",
-          "s3:ListBucket"
+          "s3:GetObject"
         ]
         "Resource" : [
-          "arn:aws:s3:::${var.binaries_location.s3_bucket}",
-          "arn:aws:s3:::${var.binaries_location.s3_bucket}/*",
+          "arn:aws:s3:::${local.binaries_location.s3_bucket_and_prefix}/*",
         ]
       }
     ]

modules/aws/sonar-base-instance/main.tf

@@ -17,6 +17,14 @@ locals {
   security_group_ids = concat(
     [for sg in aws_security_group.dsf_base_sg : sg.id],
   var.security_group_ids)
+
+  binaries_location = {
+    # For example, if the binaries_location.s3_key is "path/to/file.zip", then the s3_bucket_prefix will be "path/to"
+    # If the binaries_location.s3_key is "file.zip", then the s3_bucket_prefix will be null
+    s3_bucket_prefix     = try(regex("^(.*)/[^/]+", var.binaries_location.s3_key), null)
+    s3_key               = local.binaries_location.s3_bucket_prefix != null ? join("/", local.binaries_location.s3_bucket_prefix, var.binaries_location.s3_key) : var.binaries_location.s3_key
+    s3_bucket_and_prefix = local.binaries_location.s3_bucket_prefix != null ? join("/", var.binaries_location.s3_bucket, local.binaries_location.s3_bucket_prefix) : var.binaries_location.s3_bucket
+  }
 }
 
 resource "aws_eip" "dsf_instance_eip" {

modules/aws/sonar-base-instance/variables.tf

@@ -139,7 +139,7 @@ variable "binaries_location" {
     s3_region = string
     s3_key    = string
   })
-  description = "S3 DSF installation location. If tarball_url not set, binaries_location is used."
+  description = "S3 DSF installation location. If tarball_url not set, binaries_location is used. For example, { s3_bucket = 'my-bucket', s3_region = 'us-west-2', s3_key = 'installation-files/my-file'}. This means that the path to the installation file is s3://my-bucket/installation-files/my-file."
   default = {
     s3_bucket = ""
     s3_region = ""

modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py

@@ -18,6 +18,8 @@
 POSTFLIGHT_VALIDATIONS_SCRIPT_NAME = "run_postflight_validations.py"
 CLEAN_OLD_DEPLOYMENTS_SCRIPT_NAME = "clean_old_deployments.sh"
 
+SONAR_INSTALLATION_S3_PREFIX = "sonar"
+
 UNDEFINED_PYTHON_LOCATION = "UNDEFINED_PYTHON_LOCATION"
 
 # Globals
@@ -732,7 +734,7 @@ def get_upgrade_script_args(target_version, tarball_location):
 
 
 def get_tarball_s3_key(target_version):
-    return f"jsonar-{target_version}.tar.gz"
+    return f"{SONAR_INSTALLATION_S3_PREFIX}/jsonar-{target_version}.tar.gz"
 
 
 def maybe_run_postflight_validations(extended_node, target_version, script_file_name, stop_on_failure,