Merge branch 'main' into marrobi/issue3177

.devcontainer/Dockerfile

@@ -37,7 +37,7 @@ RUN apt-get update && apt-get install -y ca-certificates curl gnupg lsb-release
     && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \
     && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \
     | tee /etc/apt/sources.list.d/docker.list > /dev/null \
-    && apt-get update && apt-get install -y docker-ce="5:23.0.3-1~debian.11~bullseye" docker-ce-cli="5:23.0.3-1~debian.11~bullseye" containerd.io="1.6.20-1" docker-buildx-plugin --no-install-recommends \
+    && apt-get update && apt-get install -y docker-ce="5:24.0.0-1~debian.11~bullseye" docker-ce-cli="5:24.0.0-1~debian.11~bullseye" docker-compose-plugin="2.21.0-1~debian.11~bullseye" containerd.io="1.6.24-1" docker-buildx-plugin --no-install-recommends \
     && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
 # Install Certbot
@@ -75,7 +75,7 @@ COPY ["airlock_processor/requirements.txt", "/tmp/pip-tmp/airlock_processor/"]
 RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt
 
 # Install azure-cli
-ARG AZURE_CLI_VERSION=2.50.0-1~bullseye
+ARG AZURE_CLI_VERSION=2.57.0-1~bullseye
 COPY .devcontainer/scripts/azure-cli.sh /tmp/
 RUN export AZURE_CLI_VERSION=${AZURE_CLI_VERSION} \
     && /tmp/azure-cli.sh

.devcontainer/devcontainer.json

@@ -88,7 +88,10 @@
               "console": "integratedTerminal",
               "preLaunchTask": "Copy_env_file_for_api_debug",
               "cwd": "${workspaceFolder}/api_app",
-              "envFile": "${workspaceFolder}/api_app/.env"
+              "envFile": "${workspaceFolder}/api_app/.env",
+              "env": {
+                "OTEL_RESOURCE_ATTRIBUTES": "service.name=api,service.instance.id=local_debug,service.version=dev"
+              }
             },
             {
               "name": "E2E Extended",
@@ -190,8 +193,10 @@
               "cwd": "${workspaceFolder}/resource_processor",
               "envFile": "${workspaceFolder}/core/private.env",
               "env": {
-                "PYTHONPATH": "."
-              }
+                "PYTHONPATH": ".",
+                "OTEL_RESOURCE_ATTRIBUTES": "service.name=resource_processor,service.instance.id=local_debug,service.version=dev"
+              },
+              "justMyCode": false
             },
             {
               "name": "Debug Python file",

.github/ISSUE_TEMPLATE/bug_report.md

@@ -15,8 +15,8 @@ A clear and concise description of what the bug is.
 **Steps to reproduce**
 
 1.
-1.
-1.
+2.
+3.
 
 **Azure TRE release version (e.g. v0.14.0 or main):**
 

.github/actions/devcontainer_run_command/action.yml

@@ -121,6 +121,18 @@ inputs:
     description: "JSON string containing key/value pairs to injet into the Resource Processor as ENV vars"
     required: false
     default: ""
+  KV_PURGE_PROTECTION_ENABLED:
+    description: "A boolean indicating if the purge protection will be enabled on the core keyvault."
+    required: false
+    default: "true"
+  FIREWALL_SKU:
+    description: "Firewall SKU"
+    required: false
+    default: ""
+  APP_GATEWAY_SKU:
+    description: "Application Gateway SKU"
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -132,7 +144,7 @@ runs:
         echo "AZURE_ENVIRONMENT=$azure_env" >> $GITHUB_ENV
 
     - name: Azure Login
-      uses: azure/login@v1
+      uses: azure/login@v2
       if: contains(inputs.COMMAND, 'bootstrap') != true
       with:
         creds: ${{ inputs.AZURE_CREDENTIALS }}
@@ -174,6 +186,9 @@ runs:
           -e TF_INPUT="0" \
           -e TF_IN_AUTOMATION="1" \
           -e USE_ENV_VARS_NOT_FILES="true" \
+          -e ARM_STORAGE_USE_AZUREAD="true" \
+          -e ARM_USE_AZUREAD="true" \
+          -e ARM_USE_OIDC="true" \
           -e BUNDLE_TYPE="${{ inputs.BUNDLE_TYPE }}" \
           -e WORKSPACE_SERVICE_NAME="${{ inputs.WORKSPACE_SERVICE_NAME }}" \
           -e ARM_ENVIRONMENT="${{ env.ARM_ENVIRONMENT }}" \
@@ -219,6 +234,7 @@ runs:
           -e IS_API_SECURED=${{ inputs.IS_API_SECURED }} \
           -e DOCKER_BUILDKIT=1 \
           -e TF_VAR_stateful_resources_locked=${{ inputs.STATEFUL_RESOURCES_LOCKED }} \
+          -e TF_VAR_kv_purge_protection_enabled="${{ inputs.KV_PURGE_PROTECTION_ENABLED }}" \
           -e TF_VAR_enable_airlock_malware_scanning=${{ inputs.ENABLE_AIRLOCK_MALWARE_SCANNING }} \
           -e CI_CACHE_ACR_NAME="${{ inputs.CI_CACHE_ACR_NAME }}" \
           -e TF_VAR_core_app_service_plan_sku="${{ (inputs.CORE_APP_SERVICE_PLAN_SKU != ''
@@ -229,6 +245,8 @@ runs:
             && inputs.RP_BUNDLE_VALUES) || '{}' }}' \
           -e TF_VAR_resource_processor_number_processes_per_instance="${{ (inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE != ''
             && inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE) || 5 }}" \
+          -e TF_VAR_firewall_sku=${{ inputs.FIREWALL_SKU }} \
+          -e TF_VAR_app_gateway_sku=${{ inputs.APP_GATEWAY_SKU }} \
           -e E2E_TESTS_NUMBER_PROCESSES="${{ inputs.E2E_TESTS_NUMBER_PROCESSES }}" \
           '${{ inputs.CI_CACHE_ACR_NAME }}${{ env.ACR_DOMAIN_SUFFIX }}/tredev:${{ inputs.DEVCONTAINER_TAG }}' \
         bash -c "${{ inputs.COMMAND }}"

.github/dependabot.yml

@@ -55,3 +55,12 @@ updates:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
     open-pull-requests-limit: 0
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch"]
+    open-pull-requests-limit: 0

.github/linters/.hadolint.yaml

@@ -3,3 +3,4 @@ ignored:
   # sometimes pinned versions are removed from the package source so we decided to ignore this rule.
   - DL3008
   - DL3018
+  - DL3029

.github/linters/.tflint.hcl

@@ -30,8 +30,3 @@ rule "terraform_naming_convention" {
 rule "terraform_standard_module_structure" {
   enabled = true
 }
-
-rule "azurerm_resource_missing_tags" {
-  enabled = true
-  tags = ["tre_id"]
-}

.github/linters/.tflint_core.hcl

@@ -0,0 +1,19 @@
+# This is used for TRE tags validation only.
+
+config {
+  module = true
+  force = false
+}
+
+plugin "azurerm" {
+    enabled = true
+}
+
+rule "terraform_typed_variables" {
+  enabled = false
+}
+
+rule "azurerm_resource_missing_tags" {
+  enabled = true
+  tags = ["tre_id"]
+}

.github/scripts/build.js

@@ -5,6 +5,7 @@
 // These tests can be run from the dev container using the run-tests.sh script
 //
 const { createHash } = require('crypto');
+const { create } = require('domain');
 
 async function getCommandFromComment({ core, context, github }) {
   const commentUsername = context.payload.comment.user.login;
@@ -292,11 +293,12 @@ function getRefIdForBranch(branchName) {
   return createShortHash(`refs/heads/${branchName}\n`);
 }
 function createShortHash(ref) {
-  const hash = createHash('sha1').update(ref, 'utf8').digest('hex')
+  const hash = createHash('sha512').update(ref, 'utf8').digest('hex');
   return hash.substring(0, 8);
 }
 
 module.exports = {
   getCommandFromComment,
-  labelAsExternalIfAuthorDoesNotHaveWriteAccess
+  labelAsExternalIfAuthorDoesNotHaveWriteAccess,
+  createShortHash
 }

.github/scripts/build.test.js

@@ -1,4 +1,4 @@
-const { getCommandFromComment, labelAsExternalIfAuthorDoesNotHaveWriteAccess } = require('./build.js')
+const { getCommandFromComment, labelAsExternalIfAuthorDoesNotHaveWriteAccess, createShortHash } = require('./build.js')
 const { createGitHubContext, PR_NUMBER, outputFor, toHaveComment } = require('./test-helpers.js')
 
 expect.extend({
@@ -146,7 +146,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
-            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `cbce50da`\)/,
+            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `291ae84f`\)/,
           });
         });
       });
@@ -326,7 +326,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.FORK_NON_DOCS_CHANGES,
-            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `6db070b1`\)/,
+            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `607c7437`\)/,
           });
         });
       })
@@ -355,7 +355,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.FORK_NON_DOCS_CHANGES,
-            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `6db070b1`\)/,
+            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `607c7437`\)/,
           });
         });
       })
@@ -381,7 +381,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
-            bodyMatcher: /Running extended tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `cbce50da`\)/,
+            bodyMatcher: /Running extended tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `291ae84f`\)/,
           });
         });
       });
@@ -407,7 +407,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
-            bodyMatcher: /Running extended AAD tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `cbce50da`\)/,
+            bodyMatcher: /Running extended AAD tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `291ae84f`\)/,
           });
         });
       });
@@ -433,7 +433,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
-            bodyMatcher: /Running shared service tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `cbce50da`\)/,
+            bodyMatcher: /Running shared service tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `291ae84f`\)/,
           });
         });
       });
@@ -549,7 +549,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.FORK_NON_DOCS_CHANGES,
-            bodyMatcher: /Running extended tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `6db070b1`\)/,
+            bodyMatcher: /Running extended tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `607c7437`\)/,
           });
         });
       })
@@ -679,13 +679,13 @@ goes here`,
       });
 
       test('should set prRefId output', async () => {
-        // Using a PR number of 123 should give a refid of 'cbce50da'
+        // Using a PR number of 123 should give a refid of '291ae84f'
         // Based on running `echo "refs/pull/123/merge" | shasum | cut -c1-8` (as per the original bash scripts)
         const context = createCommentContext({
           pullRequestNumber: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES
         });
         await getCommandFromComment({ core, context, github });
-        expect(outputFor(mockCoreSetOutput, 'prRefId')).toBe('cbce50da');
+        expect(outputFor(mockCoreSetOutput, 'prRefId')).toBe('291ae84f');
       });
 
       test('should not set branchRefId output for PR from forked repo', async () => {
@@ -701,13 +701,13 @@ goes here`,
 
       test('should set branchRefId for PR from upstream repo', async () => {
         // Using PR 123 which is faked as a PR from the upstream repo
-        // The Using a PR number of 123 should give a refid of '71f7c907'
+        // The Using a PR number of 123 should give a refid of '6b751c8f'
         // Based on running `echo "refs/heads/pr-head-ref" | shasum | cut -c1-8` (as per the original bash scripts)
         const context = createCommentContext({
           pullRequestNumber: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES
         });
         await getCommandFromComment({ core, context, github });
-        expect(outputFor(mockCoreSetOutput, 'branchRefId')).toBe('71f7c907');
+        expect(outputFor(mockCoreSetOutput, 'branchRefId')).toBe('6b751c8f');
       });
 
       test('should set prHeadSha output', async () => {
@@ -770,4 +770,12 @@ goes here`,
     });
   });
 
+  describe('createShortHash creates a short hash from a long hash', () => {
+    test('should return the first 8 characters of the hash', () => {
+      const longHash = '0123456789abcdef';
+      const shortHash = '1c043fbe';
+      expect(createShortHash(longHash)).toBe(shortHash);
+    }
+    );
+  });
 });

.github/scripts/yarn.lock

@@ -774,12 +774,12 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
-braces@^3.0.2:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
-  integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==
+braces@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789"
+  integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==
   dependencies:
-    fill-range "^7.0.1"
+    fill-range "^7.1.1"
 
 browser-process-hrtime@^1.0.0:
   version "1.0.0"
@@ -1109,10 +1109,10 @@ fb-watchman@^2.0.0:
   dependencies:
     bser "2.1.1"
 
-fill-range@^7.0.1:
-  version "7.0.1"
-  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
-  integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==
+fill-range@^7.1.1:
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292"
+  integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==
   dependencies:
     to-regex-range "^5.0.1"
 
@@ -2479,9 +2479,9 @@ write-file-atomic@^3.0.0:
     typedarray-to-buffer "^3.1.5"
 
 ws@^7.4.6:
-  version "7.5.9"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591"
-  integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==
+  version "7.5.10"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9"
+  integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==
 
 xml-name-validator@^3.0.0:
   version "3.0.0"