[NR-347440] CFN template validate lambda integration logs

.github/workflows/pull_request.yaml

@@ -125,7 +125,7 @@ jobs:
           go tool cover -html=coverage/coverage.out -o coverage/coverage.html
 
       - name: Upload Coverage Report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-report
           path: src/coverage/coverage.html

.github/workflows/release-lambda-code.yaml

@@ -34,7 +34,7 @@ jobs:
           cd ..
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: log-forwarder-zip
           path: |
@@ -82,7 +82,7 @@ jobs:
 
     steps:
       - name: Download Artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: log-forwarder-zip
           path: ./build-artifacts

.github/workflows/run-e2e-tests.yaml

@@ -1,22 +1,81 @@
 name: E2E Test Workflow
 
 on:
-  pull_request:
-    branches:
-      - develop
-      - main
+  pull_request_review:
+    types:
+      - submitted
+  schedule:
+    - cron: '0 0 1 * *'
 
 jobs:
-  run-e2e-tests:
+  build-templates:
+    if: github.event.review.state == 'approved'
     runs-on: ubuntu-latest
     permissions:
       id-token: write
       contents: write
+    strategy:
+      matrix:
+        TEMPLATE_FILE: [lambda-template]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+
+      - name: Install AWS SAM CLI
+        run: |
+          pip install aws-sam-cli
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ secrets.AWS_E2E_ROLE }}
+          aws-region: us-east-1
+
+      - name: Build SAM Application
+        env:
+          S3_BUCKET: unified-lambda-e2e-test-templates
+        run: |
+          sam build -u --template-file "${{ matrix.TEMPLATE_FILE }}.yaml" --build-dir ".aws-sam/build/${{ matrix.TEMPLATE_FILE }}"
+          sam package --s3-bucket "$S3_BUCKET" --template-file ".aws-sam/build/${{ matrix.TEMPLATE_FILE }}/template.yaml" --output-template-file ".aws-sam/build/${{ matrix.TEMPLATE_FILE }}/${{ matrix.TEMPLATE_FILE }}.yaml"
 
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.TEMPLATE_FILE }}.yaml
+          path: .aws-sam/build/${{ matrix.TEMPLATE_FILE }}/${{ matrix.TEMPLATE_FILE }}.yaml
+
+      - name: Send failure notification to Slack
+        if: always()
+        uses: ravsamhq/notify-slack-action@v1
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+
+  run-e2e-tests-cloudwatch:
+    needs: [build-templates]
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    strategy:
+      matrix:
+        test-case: [test_logs_with_filter_pattern, test_logs_for_secret_manager, test_logs_for_invalid_log_group]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Download Artifact
+        uses: actions/download-artifact@v4
+        with:
+          path: build-artifacts
+
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -32,29 +91,101 @@ jobs:
           role-to-assume: ${{ secrets.AWS_E2E_ROLE }}
           aws-region: us-east-1
 
-      - name: Run e2e tests
+      - name: Run e2e tests for cloudwatch
         env:
           NEW_RELIC_USER_KEY: ${{ secrets.NEW_RELIC_USER_KEY }}
           NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-          S3_BUCKET: unified-lambda-e2e-test-templates
         run: |
-          cd e2e-tests/
-          ./build-templates.sh
-
-          echo "Running s3 and cloudwatch trigger tests parallely"
-          ./lambda-cloudwatch-trigger.sh &
-          pid1=$!
-          echo "Testing setting up cloudwatch trigger with PID: $pid1"
-
-          ./lambda-s3-trigger.sh &
-          pid2=$!
-          echo "Testing setting up s3 trigger with PID: $pid2"
-
-          wait $pid1
-          wait $pid2
+          cd e2e-tests
+          ./lambda-cloudwatch-trigger.sh ${{ matrix.test-case }}
+
+      - name: Send failure notification to Slack
+        if: always()
+        uses: ravsamhq/notify-slack-action@v1
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+
+  run-e2e-tests-s3:
+    needs: [build-templates]
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    strategy:
+      matrix:
+        test-case: [test_logs_for_prefix, test_logs_for_secret_manager, test_logs_for_invalid_bucket_name]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download Artifact
+        uses: actions/download-artifact@v4
+        with:
+          path: build-artifacts
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+
+      - name: Install AWS SAM CLI
+        run: |
+          pip install aws-sam-cli
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ secrets.AWS_E2E_ROLE }}
+          aws-region: us-east-1
+
+      - name: Run e2e tests for s3
+        env:
+          NEW_RELIC_USER_KEY: ${{ secrets.NEW_RELIC_USER_KEY }}
+          NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+        run: |
+          cd e2e-tests
+          ./lambda-s3-trigger.sh ${{ matrix.test-case }}
+
+      - name: Send failure notification to Slack
+        if: always()
+        uses: ravsamhq/notify-slack-action@v1
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+
+  clean-up:
+    needs: [run-e2e-tests-cloudwatch, run-e2e-tests-s3]
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - name: Install AWS SAM CLI
+        run: |
+          pip install aws-sam-cli
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ secrets.AWS_E2E_ROLE }}
+          aws-region: us-east-1
 
       - name: Delete Resources
         env:
           S3_BUCKET: unified-lambda-e2e-test-templates
         run:
-          aws s3 rm "s3://$S3_BUCKET" --recursive
+          aws s3 rm "s3://$S3_BUCKET" --recursive
+
+      - name: Send failure notification to Slack
+        if: always()
+        uses: ravsamhq/notify-slack-action@v1
+        with:
+          status: ${{ job.status }}
+          notify_when: 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

e2e-tests/common-scripts/entity_synthesis_param.cfg

@@ -0,0 +1,4 @@
+# Entity synthesis Parameters , aws related params are dropped in pipeline.
+instrumentation_provider=aws
+instrumentation_name=lambda
+instrumentation_version=1.0.0

e2e-tests/common-scripts/logs-scripts.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+source test-configs.cfg
+source entity_synthesis_param.cfg
+source stack-scripts.sh
+
+validate_logs_in_new_relic() {
+  user_key=$1
+  account_id=$2
+  attribute_key=$3
+  attribute_value=$4
+  log_message=$5
+
+  sleep_time=$SLEEP_TIME
+  attempt=1
+
+  while [[ $attempt -lt $MAX_RETRIES ]]; do
+    echo "Fetching logs from new relic for $attribute_key: $attribute_value"
+    sleep "$sleep_time"
+    response=$(fetch_new_relic_logs_api "$user_key" "$account_id" "$attribute_key" "$attribute_value")
+
+    if echo "$response" | grep -q "$log_message"; then
+      echo "Log event successfully found in New Relic."
+      validate_logs_meta_data "$response"
+      return 0
+    fi
+
+    if (( sleep_time < MAX_SLEEP_TIME )); then
+      sleep_time=$(( sleep_time * 2 ))
+    fi
+    echo "Log event not found in New Relic. Retrying in $sleep_time seconds..."
+    attempt=$((attempt + 1))
+  done
+
+  exit_with_error "Log event with $attribute_key: $attribute_value not found in New Relic. Error Received: $response"
+}
+
+validate_logs_not_present() {
+  user_key=$1
+  account_id=$2
+  attribute_key=$3
+  attribute_value=$4
+  log_message=$5
+
+  sleep_time=$SLEEP_TIME
+  attempt=1
+
+  while [[ $attempt -lt $MAX_RETRIES ]]; do
+    echo "Fetching logs from new relic for $attribute_key: $attribute_value"
+    sleep "$sleep_time"
+    response=$(fetch_new_relic_logs_api "$user_key" "$account_id" "$attribute_key" "$attribute_value")
+
+    if echo "$response" | grep -q "$log_message"; then
+      exit_with_error "Log event found in New Relic. Validation failed"
+    fi
+
+    echo "Log event not found in New Relic. Retrying in $sleep_time seconds..."
+    attempt=$((attempt + 1))
+  done
+
+  echo "Log event with $attribute_key: $attribute_value not found in New Relic. Validation succeeded"
+}
+
+fetch_new_relic_logs_api() {
+  user_key=$1
+  account_id=$2
+  attribute_key=$3
+  attribute_value=$4
+
+  nrql_query="SELECT * FROM Log WHERE $attribute_key LIKE '%$attribute_value%' SINCE $TIME_RANGE ago"
+  query='{"query":"query($id: Int!, $nrql: Nrql!) { actor { account(id: $id) { nrql(query: $nrql) { results } } } }","variables":{"id":'$account_id',"nrql":"'$nrql_query'"}}'
+
+  response=$(curl -s -X POST \
+    -H "Content-Type: application/json" \
+    -H "API-Key: $user_key" \
+    -d "$query" \
+    https://api.newrelic.com/graphql)
+
+  echo "$response"
+}
+
+create_log_message() {
+  log_message=$1
+  filter_pattern=$2
+
+  UUID=$(uuidgen)
+  echo "RequestId: $UUID, message: $log_message, filter: $filter_pattern"
+}
+
+validate_logs_meta_data (){
+  response=$1
+
+  # Validate custom attributes
+  if ! echo "$response" | grep -q "\"$CUSTOM_ATTRIBUTE_KEY\":\"$CUSTOM_ATTRIBUTE_VALUE\""; then
+    exit_with_error "Custom attribute $CUSTOM_ATTRIBUTE_KEY with value $CUSTOM_ATTRIBUTE_VALUE not found in New Relic logs."
+  fi
+  echo "Custom attributes {attributeKey: $CUSTOM_ATTRIBUTE_KEY, attributeValue: $CUSTOM_ATTRIBUTE_VALUE} validated successfully."
+
+  # Validate common attributes
+  while IFS='=' read -r key value; do
+    if [[ $key == instrumentation_* ]]; then
+      new_key=$(echo "$key" | sed 's/_/./g')
+      if ! echo "$response" | grep -q "\"$new_key\":\"$value\""; then
+        exit_with_error "Entity synthesis attribute $new_key with value $value not found in New Relic logs."
+      fi
+    fi
+  done < entity_synthesis_param.cfg
+
+  echo "Entity synthesis attributes validated successfully."
+}