Merge branch 'develop' into feat/DEPLOY-512-l2ep-foundry

.ct.yml

@@ -0,0 +1,5 @@
+# See: https://github.com/helm/chart-testing
+target-branch: develop
+chart-dirs: 'charts'
+check-version-increment: false
+validate-maintainers: false

.github/actions/notify-slack-jobs-result/README.md

@@ -0,0 +1,37 @@
+# Notify Slack Jobs Result
+
+Sends a Slack message to a specified channel detailing the results of one to many GHA job results using a regex. The job results will be grouped by the `github_job_name_regex` and displayed underneath the `message_title`, with the regex matching group displayed as an individual result. This is primarily designed for when you have test groups running in a matrix, and would like condensed reporting on their status by group. It's often accompanied by posting a Slack message before to start a thread, then attaching all the results to that thread like we do in the reporting section of the [live-testnet-test.yml workflow](../../workflows/live-testnet-tests.yml). Check out the example below, where we post an initial summary message, then use this action to thread together specific results:
+
+```yaml
+message_title: Optimism Goerli
+github_job_name_regex: ^Optimism Goerli (?<cap>.*?) Tests$ # Note that the regex MUST have a capturing group named "cap"
+```
+
+![example](image.png)
+
+## Inputs
+
+```yaml
+inputs:
+  github_token:
+    description: "The GitHub token to use for authentication (usually ${{ github.token }})"
+    required: true
+  github_repository:
+    description: "The GitHub owner/repository to use for authentication (usually ${{ github.repository }}))"
+    required: true
+  workflow_run_id:
+    description: "The workflow run ID to get the results from (usually ${{ github.run_id }})"
+    required: true
+  github_job_name_regex:
+    description: "The regex to use to match 1..many job name(s) to collect results from. Should include a capture group named 'cap' for the part of the job name you want to display in the Slack message (e.g. ^Client Compatability Test (?<cap>.*?)$)"
+    required: true
+  message_title:
+    description: "The title of the Slack message"
+    required: true
+  slack_channel_id:
+    description: "The Slack channel ID to post the message to"
+    required: true
+  slack_thread_ts:
+    description: "The Slack thread timestamp to post the message to, handy for keeping multiple related results in a single thread"
+    required: false
+```

.github/actions/notify-slack-jobs-result/action.yml

@@ -0,0 +1,110 @@
+name: Notify Slack Jobs Result
+description: Will send a notification in Slack for the result of a GitHub action run, typically for test results
+inputs:
+  github_token:
+    description: "The GitHub token to use for authentication (usually github.token)"
+    required: true
+  github_repository:
+    description: "The GitHub owner/repository to use for authentication (usually github.repository))"
+    required: true
+  workflow_run_id:
+    description: "The workflow run ID to get the results from (usually github.run_id)"
+    required: true
+  github_job_name_regex:
+    description: "The regex to use to match 1..many job name(s) to collect results from. Should include a capture group named 'cap' for the part of the job name you want to display in the Slack message (e.g. ^Client Compatability Test (?<cap>.*?)$)"
+    required: true
+  message_title:
+    description: "The title of the Slack message"
+    required: true
+  slack_channel_id:
+    description: "The Slack channel ID to post the message to"
+    required: true
+  slack_bot_token:
+    description: "The Slack bot token to use for authentication which needs permission and an installed app in the channel"
+    required: true
+  slack_thread_ts:
+    description: "The Slack thread timestamp to post the message to, handy for keeping multiple related results in a single thread"
+    required: false
+
+runs:
+  using: composite
+  steps:
+    - name: Get Results
+      shell: bash
+      id: test-results
+      run: |
+        # I feel like there's some clever, fully jq way to do this, but I ain't got the motivation to figure it out
+        echo "Querying test results at https://api.github.com/repos/${{inputs.github_repository}}/actions/runs/${{ inputs.workflow_run_id }}/jobs"
+
+        PARSED_RESULTS=$(curl \
+        -H "Authorization: Bearer ${{ inputs.github_token }}" \
+        'https://api.github.com/repos/${{inputs.github_repository}}/actions/runs/${{ inputs.workflow_run_id }}/jobs' \
+        | jq -r --arg pattern "${{ inputs.github_job_name_regex }}" '.jobs[]
+        | select(.name | test($pattern)) as $job
+        | $job.steps[]
+        | select(.name == "Run Tests")
+        | { conclusion: (if .conclusion == "success" then ":white_check_mark:" else ":x:" end), cap: ("*" + ($job.name | capture($pattern).cap) + "*"), html_url: $job.html_url }')
+
+        echo "Parsed Results:"
+        echo $PARSED_RESULTS
+
+        ALL_SUCCESS=true
+        echo "Checking for failures"
+        echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'
+        for row in $(echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'); do
+            ALL_SUCCESS=false
+            break
+        done
+        echo "Success: $ALL_SUCCESS"
+
+        echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT
+
+        FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[] 
+        | { 
+          conclusion: .conclusion, 
+          cap: .cap,
+          html_url: .html_url
+          }
+        ]
+        | map("{\"type\": \"section\", \"text\": {\"type\": \"mrkdwn\", \"text\": \"<\(.html_url)|\(.cap)>: \(.conclusion)\"}}")
+        | join(",")')
+
+        echo "Formatted Results:"
+        echo $FORMATTED_RESULTS
+
+        # Cleans out backslashes and quotes from jq
+        CLEAN_RESULTS=$(echo "$FORMATTED_RESULTS" | sed 's/\\\"/"/g' | sed 's/^"//;s/"$//')
+
+        echo "Clean Results"
+        echo $CLEAN_RESULTS
+
+        echo results=$CLEAN_RESULTS >> $GITHUB_OUTPUT
+    - name: Post Results
+      uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+      env:
+        SLACK_BOT_TOKEN: ${{ inputs.slack_bot_token }}
+      with:
+        channel-id: ${{ inputs.slack_channel_id }}
+        payload: |
+          {
+            "thread_ts": "${{ inputs.slack_thread_ts }}",
+            "attachments": [
+              {
+                "color": "${{ steps.test-results.outputs.all_success == 'true' && '#2E7D32' || '#C62828' }}",
+                "blocks": [
+                  {
+                    "type": "header",
+                    "text": {
+                      "type": "plain_text",
+                      "text": "${{ inputs.message_title }} ${{ steps.test-results.outputs.all_success == 'true' && ':white_check_mark:' || ':x:'}}",
+                      "emoji": true
+                    }
+                  },
+                  {
+                    "type": "divider"
+                  },
+                  ${{ steps.test-results.outputs.results }}
+                ]
+              }
+            ]
+            }

.github/tracing/README.md

@@ -48,4 +48,55 @@ This folder contains the following config files:
 
 These config files are for an OTEL collector, grafana Tempo, and a grafana UI instance to run as containers on the same network.
 `otel-collector-dev.yaml` is the configuration for dev (i.e. your local machine) environments, and forwards traces from the otel collector to the grafana tempo instance on the same network. 
-`otel-collector-ci.yaml` is the configuration for the CI runs, and exports the trace data to the artifact from the github run. 
+`otel-collector-ci.yaml` is the configuration for the CI runs, and exports the trace data to the artifact from the github run.
+
+## Adding Traces to Plugins and to core
+
+Adding traces requires identifying an observability gap in a related group of code executions or a critical path in your application. This is intuitive for the developer:
+
+- "What's the flow of component interaction in this distributed system?"
+- "What's the behavior of the JobProcessorOne component when jobs with [x, y, z] attributes are processed?"
+- "Is this critical path workflow behaving the way we expect?"
+
+The developer will measure a flow of execution from end to end in one trace. Each logically separate measure of this flow is called a span. Spans have either one or no parent span and multiple children span. The relationship between parent and child spans in agreggate will form a directed acyclic graph. The trace begins at the root of this graph.
+
+The most trivial application of a span is measuring top level performance in one critical path. There is much more you can do, including creating human readable and timestamped events within a span (useful for monitoring concurrent access to resources), recording errors, linking parent and children spans through large parts of an application, and even extending a span beyond a single process.
+
+Spans are created by `tracers` and passed through go applications by `Context`s. A tracer must be initialized first. Both core and plugin developers will initialize a tracer from the globally registered trace provider:
+
+```
+tracer := otel.GetTracerProvider().Tracer("example.com/foo")
+```
+
+The globally registered tracer provider is available for plugins after they are initialized, and available in core after configuration is processed (`initGlobals`).
+
+Add spans by:
+```
+  func interestingFunc() {
+    // Assuming there is an appropriate parentContext
+	ctx, span := tracer.Start(parentContext, "hello-span")
+	defer span.End()
+
+	// do some work to track with hello-span
+  }
+```
+As implied by the example, `span` is a child of its parent span captured by `parentContext`.
+
+
+Note that in certain situations, there are 3rd party libraries that will setup spans. For instance:
+
+```
+import (
+	"github.com/gin-gonic/gin"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"
+)
+
+router := gin.Default()
+router.Use(otelgin.Middleware("service-name"))
+```
+
+The developer aligns with best practices when they:
+- Start with critical paths
+- Measure paths from end to end (Context is wired all the way through)
+- Emphasize broadness of measurement over depth
+- Use automatic instrumentation if possible

.github/workflows/bash-scripts.yml

@@ -0,0 +1,35 @@
+name: Bash Scripts
+
+on:
+  pull_request:
+
+jobs:
+  changes:
+    name: detect changes
+    runs-on: ubuntu-latest
+    outputs:
+      bash-scripts-src: ${{ steps.bash-scripts.outputs.src }}
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
+        id: bash-scripts
+        with:
+          filters: |
+            src:
+            - 'tools/bin/**'
+            - '.github/workflows/bash-scripts.yml'
+  shellcheck:
+    name: ShellCheck Lint
+    runs-on: ubuntu-latest
+    needs: [changes]
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Run ShellCheck
+        if: needs.changes.outputs.bash-scripts-src == 'true'
+        uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # v2.0.0
+        with:
+          scandir: "./tools/bin"
+          # Consider changing this to check for warnings once all warnings are fixed.
+          severity: error

.github/workflows/build-publish-pr.yml

@@ -2,8 +2,8 @@ name: "Build and Publish from PR"
 
 ##
 # This workflow builds and publishes a Docker image for Chainlink from a PR.
-# It doesn't use an environment, has its own special IAM role, does not sign
-# the image, and publishes to a special ECR repo.
+# It has its own special IAM role, does not sign the image, and publishes to
+# a special ECR repo.
 ##
 
 on:
@@ -13,6 +13,7 @@ jobs:
   build-publish-untrusted:
     if: ${{ ! startsWith(github.ref_name, 'release/') }}
     runs-on: ubuntu-20.04
+    environment: sdlc
     permissions:
       id-token: write
       contents: read
@@ -53,6 +54,51 @@ jobs:
           dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
           dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
 
+      - name: Get PR labels
+        id: pr-labels
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          RESPONSE=$(gh pr view ${PR_NUMBER} --json labels)
+          # Check if the labels command was successful
+          if [[ $? -ne 0 ]]; then
+            echo "Error fetching labels"
+            exit 1
+          fi
+          echo "RESPONSE=${RESPONSE}"
+          LABELS=$(echo "$RESPONSE" | jq -r '.labels | map(.name) | join(", ")')
+          # Check if any labels were found
+          if [[ -z "${LABELS:-}" ]]; then
+            echo "No labels found"
+          else
+            echo "labels=${LABELS}" | tee -a "${GITHUB_OUTPUT}"
+          fi
+
+      - name: Setup GAP
+        if: contains(steps.pr-labels.outputs.labels, 'crib')
+        uses: smartcontractkit/.github/actions/setup-gap@main
+        with:
+          aws-region: ${{ secrets.AWS_REGION }}
+          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_PUBLISH_PR_ARN }}
+          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_ARGO_SAND }}
+          use-argocd: "true"
+          argocd-user: ${{ secrets.ARGOCD_USER_SAND }}
+          argocd-pass: ${{ secrets.ARGOCD_PASS_SAND }}
+
+      # Run an Argo CD sync after the image is built.
+      - name: Argo CD App Sync
+        if: contains(steps.pr-labels.outputs.labels, 'crib')
+        shell: bash
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          argocd app sync \
+            --plaintext \
+            --grpc-web \
+            --async \
+            "crib-chainlink-${PR_NUMBER}"
+
       - name: Collect Metrics
         if: always()
         id: collect-gha-metrics

.github/workflows/ci-core.yml

@@ -50,7 +50,7 @@ jobs:
       - name: Checkout the repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Setup node
-        uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
+        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
       - name: Setup NodeJS
         uses: ./.github/actions/setup-nodejs
         with:
@@ -97,7 +97,7 @@ jobs:
         working-directory: ./.github/actions/setup-postgres
       - name: Store logs artifacts
         if: always()
-        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: ${{ matrix.cmd }}_logs
           path: |
@@ -136,7 +136,7 @@ jobs:
       - name: Checkout the repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Setup node
-        uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
+        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
       - name: Setup NodeJS
         uses: ./.github/actions/setup-nodejs
         with:
@@ -154,7 +154,7 @@ jobs:
       - name: Setup DB
         run: ./chainlink.test local db preparetest
       - name: Load test outputs
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
           path: ./artifacts
       - name: Build flakey test runner