Dam on azure (#330)

* add mx machine without ftl * add custom script to run ftl command * add readiness * add statistics, dam model instance type, storage variable, fix dam random password for 14.7 * change custom scripts variable type to map * add worakaround for Provider produced inconsistent final plan error * add agent gw, fix azure dam models and other fixes * add agent gw cluster * add mx-hub association * update gw ftl command * fix sg for udp and fix ftl * dev rebase and remove dns variables * add db with agent for postgres * protect db-wth-agent custom_script from rerun * add mysql * small changes and add dam to automation * add dam variables to plan action * update github action name for poc cli azure * add readme * code review fixes * remove ftl command change as trigger for custom_script * reorder variables to be same as in aws [skip actions] * rebase with dev * remove new version for rebase [skip actions] * change storage account for dsf installation files * add dam modules to deploy_module.yml for release * update release date
imperva · Jan 9, 2024 · 3482a49 · 3482a49
1 parent f297a6c
commit 3482a49
Show file tree

Hide file tree

Showing 53 changed files with 2,691 additions and 83 deletions.
diff --git a/.github/workflows/deploy_module.yml b/.github/workflows/deploy_module.yml
@@ -80,6 +80,20 @@ jobs:
           - source_module: "aws/db-with-agent"
             destination_repo: "terraform-aws-dsf-db-with-agent"
             begin_tag: 1.0.0
+
+          ## azurerm provider
+          - source_module: "azurerm/mx"
+            destination_repo: "terraform-azurerm-dsf-mx"
+            hidden_submodules: "azurerm/dam-base-instance azurerm/statistics null/statistics"
+            begin_tag: 1.7.4
+          - source_module: "azurerm/agent-gw"
+            destination_repo: "terraform-azurerm-dsf-agent-gw"
+            hidden_submodules: "azurerm/dam-base-instance azurerm/statistics null/statistics"
+            begin_tag: 1.7.4
+          - source_module: "azurerm/db-with-agent"
+            destination_repo: "terraform-azurerm-dsf-db-with-agent"
+            begin_tag: 1.7.4
+
           ## null provider
           - source_module: "null/agent-gw-cluster-setup"
             destination_repo: "terraform-null-dsf-agent-gw-cluster-setup"

diff --git a/.github/workflows/dsf_poc_cli_azure.yml b/.github/workflows/dsf_poc_cli_azure.yml
@@ -18,6 +18,8 @@ on:
         required: true
       ARM_CLIENT_SECRET:
         required: true
+      DAM_LICENSE:
+        required: true
 
   workflow_dispatch:
     inputs:
@@ -43,13 +45,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: DSF POC - SONAR
-            workspace: azure_cli-
+          - name: DSF POC
+            workspace: azure_cli-all-
+            enable_sonar: true
+            enable_dam: true
 
     name: '${{ matrix.name }}'
     runs-on: ubuntu-latest
     env:
       EXAMPLE_DIR: ./examples/azure/poc/dsf_deployment
+      TF_VAR_enable_sonar: ${{ matrix.enable_sonar }}
+      TF_VAR_enable_dam: ${{ matrix.enable_dam }}
     environment: test
 
     # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
@@ -109,11 +115,18 @@ jobs:
         cat << EOF > $EXAMPLE_DIR/terraform.tfvars
         resource_group_location = "East US"
         tarball_location = {
-          az_resource_group = "eytan-resource-group"
-          az_storage_account = "eytanstorageaccount"
+          az_resource_group = "dsf-rg"
+          az_storage_account = "dsfinstallation"
           az_container = "sonar"
           az_blob = "jsonar-4.13.0.10.0.tar.gz"
         }
+        dam_agent_installation_location = {
+          az_resource_group = "dsf-rg"
+          az_storage_account = "dsfinstallation"
+          az_container = "dam"
+          az_blob = "Imperva-ragent-UBN-px86_64-b14.6.0.60.0.636085.bsx"
+        }
+        simulation_db_types_for_agent=["PostgreSql", "MySql"]
         EOF
 
     # Install the latest version of Terraform CLI and configure the Terraform CLI configuration file with a Terraform Cloud user API token
@@ -126,6 +139,13 @@ jobs:
     - name: Setup jq
       uses: sergeysova/jq-action@v2
 
+    - name: Create License File
+      env:
+        MY_SECRET: ${{ secrets.DAM_LICENSE }}
+      run: |
+        echo "${{ secrets.DAM_LICENSE }}" | base64 -d > $EXAMPLE_DIR/license.mprv
+        cat $EXAMPLE_DIR/license.mprv
+
     # Initialize a new or existing Terraform working directory by creating initial files, loading any remote state, downloading modules, etc.
     - name: Terraform Init
       run: terraform -chdir=$EXAMPLE_DIR init
@@ -140,7 +160,7 @@ jobs:
           mv $EXAMPLE_DIR/sonar.tf{,_}
           mv $EXAMPLE_DIR/networking.tf{,_}
           ls -la $EXAMPLE_DIR
-          terraform -chdir=$EXAMPLE_DIR destroy -auto-approve
+          terraform -chdir=$EXAMPLE_DIR destroy -var dam_license=license.mprv -auto-approve
           mv $EXAMPLE_DIR/main.tf{_,}
           mv $EXAMPLE_DIR/outputs.tf{_,}
           mv $EXAMPLE_DIR/sonar.tf{_,}
@@ -154,14 +174,14 @@ jobs:
     - name: Terraform Plan
       run: |
         terraform -chdir=$EXAMPLE_DIR workspace list
-        terraform -chdir=$EXAMPLE_DIR plan
+        terraform -chdir=$EXAMPLE_DIR plan -var dam_license=license.mprv
 
       # On push to "main", build or change infrastructure according to Terraform configuration files
       # Note: It is recommended to set up a required "strict" status check in your repository for "Terraform Cloud". See the documentation on "strict" required status checks for more information: https://help.github.com/en/github/administering-a-repository/types-of-required-status-checks
     - name: Terraform Apply
       id: apply
       # if: github.ref == 'refs/heads/"master"' && github.event_name == 'push' || github.event_name == 'workflow_dispatch'
-      run: terraform -chdir=$EXAMPLE_DIR apply -auto-approve
+      run: terraform -chdir=$EXAMPLE_DIR apply -var dam_license=license.mprv -auto-approve
 
     - name: Terraform Output
       if: always()
@@ -180,7 +200,7 @@ jobs:
       if: always()
       run: |
         if [ '${{ steps.apply.conclusion }}' == 'success' ] || [ ${{ github.event_name }} != 'schedule' ]; then
-          terraform -chdir=$EXAMPLE_DIR destroy -auto-approve
+          terraform -chdir=$EXAMPLE_DIR destroy -var dam_license=license.mprv -auto-approve
         fi
 
     - name: Terraform Delete Workspace

diff --git a/.github/workflows/plan_cli.yml b/.github/workflows/plan_cli.yml
@@ -71,6 +71,13 @@ jobs:
                 az_container = "dummy-container"
                 az_blob = "dummy-blob"
               }
+              dam_agent_installation_location = {
+                az_resource_group = "dummy-resource-group"
+                az_storage_account = "dummy-torage-account"
+                az_container = "dummy-container"
+                az_blob = "dummy-blob"
+              }
+              dam_license="license.mprv"
           - name: AWS - POC - DSF
             example: ./examples/aws/poc/dsf_deployment
             terraformvars: |

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -205,13 +205,13 @@ jobs:
       ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
       DAM_LICENSE: ${{ secrets.DAM_LICENSE }}
 
-  test_apply:
-    needs: test_plan
-    uses: ./.github/workflows/sonar_poc_cli.yml
-    with:
-      use_modules_from_terraform_registry: true
-      explicit_ref: master
-    secrets:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+#  test_apply:
+#    needs: test_plan
+#    uses: ./.github/workflows/sonar_poc_cli.yml
+#    with:
+#      use_modules_from_terraform_registry: true
+#      explicit_ref: master
+#    secrets:
+#      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+#      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+#      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
diff --git a/README.md b/README.md
@@ -391,6 +391,15 @@ The following table lists the _latest_ eDSF Kit releases, their release date and
        Improvements and bug fixes.
     </td>
   </tr>
+  <tr>
+    <td>9 Jan 2024
+    </td>
+    <td>1.7.4
+    </td>
+    <td>
+       Added support for DAM in Azure.
+    </td>
+  </tr>
 
 </table>
 
@@ -433,9 +442,11 @@ Before using eDSF Kit to deploy DSF, it is necessary to satisfy a set of prerequ
 
 ### Azure Prerequisites
 
-1. [establish an Azure App Registration](https://learn.microsoft.com/en-us/azure/healthcare-apis/register-application) and [assign it the necessary role](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=delegate-condition) 
+1. [Establish an Azure App Registration](https://learn.microsoft.com/en-us/azure/healthcare-apis/register-application) and [assign it the necessary role](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=delegate-condition) 
    for the associated subscription. Note, Assign the Owner role to the app registration on a temporary basis. More specific permissions will be provided in the future.
-2. The deployment requires access to the Sonar binaries. Establish an Azure Storage account along with a container, and proceed to upload the Sonar binaries to this storage location as a blob.
+2. Configure programmatic deployment for the desired version of Imperva DAM by [enabling it on the relevant DAM image from the Azure Marketplace](https://portal.azure.com/#view/Microsoft_Azure_Marketplace/LegalTermsSkuProgrammaticAccessBlade/legalTermsSkuProgrammaticAccessData~/%7B%22product%22%3A%7B%22publisherId%22%3A%22imperva%22%2C%22offerId%22%3A%22imperva-dam-v14%22%2C%22planId%22%3A%22securesphere-imperva-dam-14%22%2C%22standardContractAmendmentsRevisionId%22%3Anull%2C%22isCspEnabled%22%3Atrue%7D%7D). For DAM LTS version, use [DAM LTS Azure Marketplace image](https://portal.azure.com/#view/Microsoft_Azure_Marketplace/LegalTermsSkuProgrammaticAccessBlade/legalTermsSkuProgrammaticAccessData~/%7B%22product%22%3A%7B%22publisherId%22%3A%22imperva%22%2C%22offerId%22%3A%22imperva-dam-v14-lts%22%2C%22planId%22%3A%22securesphere-imperva-dam-14%22%2C%22standardContractAmendmentsRevisionId%22%3Anull%2C%22isCspEnabled%22%3Atrue%7D%7D).
+For the POC example, configure programmatic deployment also for [Ubuntu Pro 20.04 LTS image](https://portal.azure.com/#view/Microsoft_Azure_Marketplace/LegalTermsSkuProgrammaticAccessBlade/legalTermsSkuProgrammaticAccessData~/%7B%22product%22%3A%7B%22publisherId%22%3A%22canonical%22%2C%22offerId%22%3A%220001-com-ubuntu-pro-focal%22%2C%22planId%22%3A%22pro-20_04-lts%22%2C%22standardContractAmendmentsRevisionId%22%3Anull%2C%22isCspEnabled%22%3Atrue%7D%7D).
+3. The deployment requires access to the Sonar and DAM Agent installation binaries. Establish an Azure Storage account along with a container, and proceed to upload the Sonar and DAM Agent installation binaries to this storage location as a blob.
 
 
 ## Choosing the Example/Recipe that Fits Your Use Case
@@ -554,7 +565,7 @@ For more details about each example, click on the example name.
      </td>
      <td>Lab/POC
      </td>
-     <td>A DSF deployment with a DSF Hub, an Agentless Gateway, DSF Hub and Agentless Gateway HADR, federation and networking.
+     <td>A DSF deployment with a DSF Hub, an Agentless Gateway, DSF Hub and Agentless Gateway HADR, federation and networking, DAM (MX and Agent Gateways), and Agent audit sources.
      </td>
      <td><a href="https://github.com/imperva/dsfkit/tree/1.7.3/examples/aws/poc/dsf_deployment/dsf_deployment_1_7_3.zip">dsf_deployment_1_7_3.zip</a>
      </td>
@@ -1359,6 +1370,15 @@ Below is a list of possible issues and troubleshooting remediations.
    Increase the quota using the link provided in your own error message.
    </td>
   </tr>
+  <tr>
+   <td>Image legal terms not accepted
+   </td>
+   <td>compute.VirtualMachinesClient#CreateOrUpdate: Failure sending request: StatusCode=400 -- Original Error: Code="ResourcePurchaseValidationFailed" Message="User failed validation to purchase resources. Error message: 'You have not accepted the legal terms on this subscription
+   </td>
+   <td>
+   Configure programmatic deployment for the desired image. Follow the instructions in the <a href="https://github.com/imperva/dsfkit/tree/master#azure-prerequisites">Azure Prerequisites</a>.
+   </td>
+  </tr>
  </table>
 </details>
 

diff --git a/examples/azure/poc/dsf_deployment/README.md b/examples/azure/poc/dsf_deployment/README.md
@@ -1,7 +1,7 @@
 # DSF Deployment example
 [![GitHub tag](https://img.shields.io/github/v/tag/imperva/dsfkit.svg)](https://github.com/imperva/dsfkit/tags)
 
-This example provides DSF (Data Security Fabric) deployment with DSF Hub, and Agentless Gateways.
+This example provides DSF (Data Security Fabric) deployment with DSF Hub, Agentless Gateways, DAM (Database Activity Monitoring) and Agent audit sources.
 
 ## Modularity
 The deployment is modular and allows users to deploy one or more of the following modules:
@@ -12,26 +12,66 @@ The deployment is modular and allows users to deploy one or more of the followin
    - DSF Hub DR HADR (High Availability Disaster Recovery) node
    - Agentless Gateways
    - Agentless Gateways DR HADR (High Availability Disaster Recovery) nodes
+3. DAM
+   - MX
+   - Agent Gateways
+4. Audit sources
+   - Agent audit sources (virtual machine instances)
+
+### Deploying Specific Modules
+
+To deploy specific modules, you can customize the deployment by setting the corresponding variables in your Terraform configuration. Here are the instructions to deploy the following specific modules:
+
+#### 1. DAM Only Deployment
+
+To deploy only the DAM module, set the following variables in your Terraform configuration:
+```
+enable_dam = true
+enable_sonar = false
+```
+
+This configuration will enable the DAM module while disabling the Sonar module.
+
+#### 2. Sonar Only Deployment
+
+To deploy only the Sonar module, set the following variables in your Terraform configuration:
+```
+enable_dam = false
+enable_sonar = true
+```
+
+This configuration will enable the Sonar module, including the DSF Hub, while disabling the DAM module.
+
+Feel free to customize your deployment by setting the appropriate variables based on your requirements.
 
 ## Variables
 Several variables in the `variables.tf` file are important for configuring the deployment. The following variables dictate the deployment content and should be paid more attention to:
+
+### Sub-Products
 - `enable_sonar`: Enable Sonar sub-product
+- `enable_dam`: Enable DAM sub-product
+
+### Server Count
+- `agentless_gw_count`: Number of Agentless Gateways
 - `agent_gw_count`: Number of Agent Gateways
+
+### High Availability (HADR)
 - `hub_hadr`: Enable DSF Hub High Availability Disaster Recovery (HADR)
 - `agentless_gw_hadr`: Enable Agentless Gateway High Availability Disaster Recovery (HADR)
 
+### Audit Sources for Simulation Purposes
+- `simulation_db_types_for_agent`: Types of databases to provision for Agent Gateways
+
 ## Mandatory Variables
 Before initiating the Terraform deployment, it is essential to set up the following variables:
 - `resource_group_location`: The region of the resource group to which all DSF components will be associated.
 - `tarball_location`: Storage account and container location of the DSF installation software. az_blob is the full path to the tarball file within the storage account container. 
-
-
-### Networking
-- `subnet_ids`: IDs of the subnets for the deployment. If not specified, a new vpc is created.
+- `dam_agent_installation_location`: Storage account and container location of the DAM Agent installation software. az_blob is the full path to the installation file within the storage account container.
+- `dam_license`: DAM license file path.
 
 ## Default Example
 To perform the default deployment, run the following command:
 
 ```bash
-terraform apply -var="resource_group_location=${region}" -var='tarball_location={"az_resource_group": "${storage-resource-group}", "az_storage_account":"${storage_account_name}","az_container":"${container_name}","az_blob":"jsonar-4.13.0.10.0.tar.gz"}' -auto-approve
+terraform apply -var="resource_group_location=${region}" -var='tarball_location={"az_resource_group": "${storage-resource-group}", "az_storage_account":"${storage_account_name}","az_container":"${container_name}","az_blob":"jsonar-4.13.0.10.0.tar.gz"}' -var='dam_agent_installation_location={"az_resource_group": "${storage-resource-group}", "az_storage_account":"${storage_account_name}","az_container":"${container_name}","az_blob":"Imperva-ragent-UBN-px86_64-b14.6.0.60.0.636085.bsx"}' -var="dam_license=/path/to/license/file" -auto-approve
 ```
diff --git a/examples/azure/poc/dsf_deployment/agent_sources.tf b/examples/azure/poc/dsf_deployment/agent_sources.tf
@@ -0,0 +1,30 @@
+locals {
+  db_types_for_agent = local.agent_gw_count > 0 ? var.simulation_db_types_for_agent : []
+}
+
+module "db_with_agent" {
+  source  = "../../../../modules/azurerm/db-with-agent"
+  count   = length(local.db_types_for_agent)
+
+  friendly_name     = join("-", [local.deployment_name_salted, "db", "with", "agent", count.index])
+  resource_group    = local.resource_group
+  binaries_location = var.dam_agent_installation_location
+  db_type           = local.db_types_for_agent[count.index]
+  subnet_id         = module.network[0].vnet_subnets[0]
+  ssh_key = {
+    ssh_public_key            = tls_private_key.ssh_key.public_key_openssh
+    ssh_private_key_file_path = local_sensitive_file.ssh_key.filename
+  }
+  allowed_ssh_cidrs = concat([format("%s/32", module.mx[0].private_ip)], module.network[0].vnet_address_space)
+
+  registration_params = {
+    agent_gateway_host = module.agent_gw[0].private_ip
+    secure_password    = local.password
+    server_group       = module.mx[0].configuration.default_server_group
+    site               = module.mx[0].configuration.default_site
+  }
+  tags = local.tags
+  depends_on = [
+    module.agent_gw_cluster_setup
+  ]
+}