Merge remote-tracking branch 'origin/main' into yuri.l/USMON-765_http…

…2_cont_frames

.ddqa/config.toml

@@ -10,6 +10,22 @@ github_team = "agent-metrics-logs"
 github_labels = ["team/agent-metrics-logs"]
 exclude_members = ["olivielpeau"]
 
+[teams."Agent Logs"]
+jira_project = "AGNTLOG"
+jira_issue_type = "Task"
+jira_statuses = ["To Do", "In Progress", "Done"]
+github_team = "agent-logs"
+github_labels = ["team/agent-logs"]
+exclude_members = [""]
+
+[teams."Agent Metrics"]
+jira_project = "AGTMETRICS"
+jira_issue_type = "Task"
+jira_statuses = ["To Do", "In Progress", "Done"]
+github_team = "agent-metrics"
+github_labels = ["team/agent-metrics"]
+exclude_members = [""]
+
 [teams."Agent Processing and Routing"]
 jira_project = "APR"
 jira_issue_type = "Task"
@@ -26,6 +42,24 @@ github_team = "agent-shared-components"
 github_labels = ["team/agent-shared-components"]
 exclude_members = ["sgnn7", "truthbk", "cmourot"]
 
+# Those will be uncommented once the proper issue types are created in JIRA.
+#
+# [teams."Agent Runtimes"]
+# jira_project = "ARUN"
+# jira_issue_type = "QA"
+# jira_statuses = ["To Do", "In Progress", "Done"]
+# github_team = "agent-runtimes"
+# github_labels = ["team/agent-runtimes"]
+# exclude_members = ["sgnn7", "cmourot"]
+# 
+# [teams."Agent Configuration"]
+# jira_project = "ACFG"
+# jira_issue_type = "QA"
+# jira_statuses = ["To Do", "In Progress", "Done"]
+# github_team = "agent-configuration"
+# github_labels = ["team/agent-configuration"]
+# exclude_members = ["sgnn7"]
+
 [teams."Agent Developer Tools"]
 jira_project = "ADXT"
 jira_issue_type = "QA"
@@ -50,6 +84,13 @@ github_team = "agent-delivery"
 github_labels = ["team/agent-delivery"]
 exclude_members = ["KSerrania"]
 
+[teams."Agent Discovery"]
+jira_project = "DSCVR"
+jira_issue_type = "Task"
+jira_statuses = ["To Do", "In Progress", "Done"]
+github_team = "agent-discovery"
+github_labels = ["team/agent-discovery"]
+
 [teams."Universal Service Monitoring"]
 jira_project = "USMON"
 jira_issue_type = "Task"

.github/dependabot.yaml

@@ -64,7 +64,7 @@ updates:
     labels:
       - dependencies
       - dependencies-go
-      - team/agent-shared-components
+      - team/agent-configuration
       - changelog/no-changelog
     ignore:
       # Ignore internal modules

.github/workflows/buildimages-update.yml

@@ -141,7 +141,7 @@ jobs:
           TMP_PR_BODY_PATH: /tmp/pr_body
           GH_TOKEN: ${{ github.token }}
           PR_TITLE: "[automated] ${{ steps.update_build_images.outputs.MESSAGE }}"
-          PR_LABELS: "go-update,team/agent-shared-components"
+          PR_LABELS: "go-update,team/agent-runtimes"
           CURRENT_BUILDIMAGE_TAG: ${{ steps.current_buildimage_tag.outputs.BUILDIMAGE_TAG }}
           IMAGES_ID: ${{ inputs.images_id }}
           CURRENT_GO_VERSION: ${{ steps.current_go_version.outputs.GO_VERSION }}

.github/workflows/bump_integrations_core.yml

@@ -0,0 +1,40 @@
+name: Bump Integrations Core
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 4 * * 1,3' # Run on Monday, Wednesday at 4:00 UTC
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+permissions: {}
+
+jobs:
+    bump_integrations_core:
+        runs-on: ubuntu-latest
+        permissions:
+          contents: write
+          pull-requests: write
+        steps:
+            - name: Checkout the main branch
+              uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+              with:
+                fetch-depth: 0
+                persist-credentials: true
+
+            - name: Install Python
+              uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+              with:
+                python-version-file: .python-version
+                cache: "pip"
+
+            - name: Install Python dependencies
+              run: |
+                python -m pip install --upgrade pip
+                pip install -r requirements.txt
+                pip install -r tasks/libs/requirements-github.txt
+                pip install -r tasks/requirements_release_tasks.txt
+
+            - name: Bump Integrations Core
+              run: |
+                inv release.bump-integrations-core --slack-webhook ${{ secrets.BUMP_INTEGRATIONS_CORE_SLACK_WEBHOOK }}
+

.github/workflows/create_rc_pr.yml

@@ -108,8 +108,14 @@ jobs:
                   echo "CHANGES=$(inv -e release.check-for-changes -r "$MATRIX")" >> $GITHUB_OUTPUT
                 fi
 
+            - name: Check if agent 6 is in qualification phase
+              if: ${{ env.IS_AGENT6_RELEASE == 'true' }}
+              run: |
+                is_qualification=$(inv -e release.is-qualification -r 6.53.x --output)
+                echo "IS_QUALIFICATION=$is_qualification" >> $GITHUB_ENV
+
             - name: Create RC PR
-              if: ${{ steps.check_for_changes.outputs.CHANGES == 'true' || env.IS_AGENT6_RELEASE == 'true' }}
+              if: ${{ steps.check_for_changes.outputs.CHANGES == 'true' || ( env.IS_AGENT6_RELEASE == 'true' && env.IS_QUALIFICATION == 'false') }}
               env:
                 MATRIX: ${{ matrix.value }}
               run: |
@@ -118,3 +124,8 @@ jobs:
                 else
                   inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }}
                 fi
+
+            - name: Rebuild agent 6 RC pipeline if no changes and in qualification phase
+              if: ${{ env.IS_AGENT6_RELEASE == 'true' && steps.check_for_changes.outputs.CHANGES == 'false' && env.IS_QUALIFICATION == 'true' }}
+              run: |
+                inv -e release.run-rc-pipeline -r 6.53.x

.github/workflows/update_dependencies.yml

@@ -60,7 +60,7 @@ jobs:
           TMP_PR_BODY_PATH: /tmp/pr_body
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
           PR_TITLE: "[automated] Update golang.org/x/... dependencies"
-          PR_LABELS: "team/agent-shared-components,qa/done,changelog/no-changelog"
+          PR_LABELS: "team/agent-runtimes,qa/done,changelog/no-changelog"
           GITHUB_REF: ${{ github.ref }}
         run: |
           # Generate the PR description
@@ -80,4 +80,5 @@ jobs:
           --base "$GITHUB_REF" \
           --title "$PR_TITLE" \
           --body-file "$TMP_PR_BODY_PATH" \
-          --label "$PR_LABELS"
+          --label "$PR_LABELS" \
+          --reviewer "DataDog/agent-runtimes"

.gitlab-ci.yml

@@ -620,9 +620,11 @@ workflow:
   - <<: *if_main_branch
   - <<: *if_release_branch
 
-.not_on_release_branch:
+.not_on_release_branch_or_tagged_commit:
   - <<: *if_release_branch
     when: never
+  - <<: *if_tagged_commit
+    when: never
 
 .only_main:
   - <<: *if_not_main_branch
@@ -819,14 +821,24 @@ workflow:
         - test/new-e2e/tests/remote-config/**/*
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
-.on_asc_or_e2e_changes:
+.on_arun_or_e2e_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - changes:
+      paths:
+        - cmd/**/*
+        - pkg/**/*
+        - comp/**/*
+        - test/new-e2e/tests/agent-runtimes/**/*
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+
+.on_acfg_or_e2e_changes:
   - !reference [.on_e2e_main_release_or_rc]
   - changes:
       paths:
         - cmd/**/*
         - pkg/**/*
         - comp/**/*
-        - test/new-e2e/tests/agent-shared-components/**/*
+        - test/new-e2e/tests/agent-configuration//**/*
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
 .on_subcommands_or_e2e_changes:

.gitlab/JOBOWNERS

@@ -14,7 +14,7 @@ build_clang_*                        @DataDog/ebpf-platform
 build_processed_btfhub_archive       @DataDog/ebpf-platform
 
 # Deps fetch
-fetch_openjdk                        @DataDog/agent-metrics-logs
+fetch_openjdk                        @DataDog/agent-metrics
 
 # Source test
 # Notifications are handled separately for more fine-grained control on go tests
@@ -27,7 +27,7 @@ prepare_secagent_ebpf_functional_tests* @DataDog/agent-security
 protobuf_test                           @DataDog/multiple
 
 # Send count metrics about Golang dependencies
-golang_deps_send_count_metrics       @DataDog/agent-shared-components
+golang_deps_send_count_metrics       @DataDog/agent-runtimes
 # Golang dependency diff generation
 golang_deps_diff                     @DataDog/ebpf-platform
 golang_deps_commenter                @DataDog/ebpf-platform

.gitlab/binary_build/cluster_agent.yml

@@ -1,7 +1,11 @@
 ---
 .cluster_agent-build_common:
   stage: binary_build
-  needs: ["go_mod_tidy_check"]
+  rules:
+    !reference [.on_tag_or_a7]
+  needs: ["go_mod_tidy_check", "go_deps"]
+  before_script:
+    - !reference [.retrieve_linux_go_deps]
   script:
     - inv check-go-version
     - inv -e cluster-agent.build --release-version "$RELEASE_VERSION_7"
@@ -15,24 +19,14 @@
 
 cluster_agent-build_amd64:
   extends: .cluster_agent-build_common
-  rules:
-    !reference [.on_tag_or_a7]
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
-  needs: ["go_mod_tidy_check", "go_deps"]
   variables:
     ARCH: amd64
-  before_script:
-    - !reference [.retrieve_linux_go_deps]
 
 cluster_agent-build_arm64:
   extends: .cluster_agent-build_common
-  rules:
-    !reference [.on_tag_or_a7]
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:arm64"]
-  needs: ["go_mod_tidy_check", "go_deps"]
   variables:
     ARCH: arm64
-  before_script:
-    - !reference [.retrieve_linux_go_deps]

.gitlab/common/test_infra_version.yml

@@ -5,5 +5,5 @@
 
 ---
 variables:
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 9e836ad2dc14
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: f92dca10d03c
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ''

.gitlab/e2e/e2e.yml

@@ -166,14 +166,23 @@ new-e2e-remote-config:
     TARGETS: ./tests/remote-config
     TEAM: remote-config
 
-new-e2e-agent-shared-components:
+new-e2e-agent-configuration:
   extends: .new_e2e_template_needs_deb_windows_x64
   rules:
-    - !reference [.on_asc_or_e2e_changes]
+    - !reference [.on_acfg_or_e2e_changes]
     - !reference [.manual]
   variables:
-    TARGETS: ./tests/agent-shared-components
-    TEAM: agent-shared-components
+    TARGETS: ./tests/agent-configuration
+    TEAM: agent-configuration
+
+new-e2e-agent-runtimes:
+  extends: .new_e2e_template_needs_deb_windows_x64
+  rules:
+    - !reference [.on_arun_or_e2e_changes]
+    - !reference [.manual]
+  variables:
+    TARGETS: ./tests/agent-runtimes
+    TEAM: agent-runtimes
 
 new-e2e-agent-subcommands:
   extends: .new_e2e_template_needs_deb_windows_x64
@@ -182,7 +191,7 @@ new-e2e-agent-subcommands:
     - !reference [.manual]
   variables:
     TARGETS: ./tests/agent-subcommands
-    TEAM: agent-shared-components
+    TEAM: agent-configuration
   parallel:
     matrix:
       - EXTRA_PARAMS: --run "Test(Linux|Windows)StatusSuite"
@@ -203,11 +212,11 @@ new-e2e-fips-compliance-test:
     - qa_agent_fips
     - deploy_deb_testing-a7_x64
   rules:
-    - !reference [.on_asc_or_e2e_changes]
+    - !reference [.on_arun_or_e2e_changes]
     - !reference [.manual]
   variables:
     TARGETS: ./tests/fips-compliance
-    TEAM: agent-shared-components
+    TEAM: agent-runtimes
 
 new-e2e-windows-service-test:
   extends: .new_e2e_template

.gitlab/functional_test/regression_detector.yml

@@ -22,7 +22,7 @@ single-machine-performance-regression_detector:
       - outputs/decision_record.md # for posterity, this is appended to final PR comment
     when: always
   variables:
-    SMP_VERSION: 0.20.1
+    SMP_VERSION: 0.20.2
   # See 'decision_record.md' for the determination of whether this job passes or fails.
   allow_failure: false
   script:
@@ -100,7 +100,7 @@ single-machine-performance-regression_detector:
     - echo "${BASELINE_SHA} | ${BASELINE_IMAGE}"
     - COMPARISON_IMAGE=${SMP_ECR_URL}/${SMP_AGENT_TEAM_ID}-agent:${CI_COMMIT_SHA}-7-amd64
     - echo "${CI_COMMIT_SHA} | ${COMPARISON_IMAGE}"
-    - SMP_TAGS="ci_pipeline_id=${CI_PIPELINE_ID},ci_job_id=${CI_JOB_ID}"
+    - SMP_TAGS="ci_pipeline_id=${CI_PIPELINE_ID},ci_job_id=${CI_JOB_ID},ci_commit_branch=${CI_COMMIT_BRANCH}"
     - echo "Tags passed through SMP are ${SMP_TAGS}"
     - RUST_LOG="info,aws_config::profile::credentials=error"
     - RUST_LOG_DEBUG="debug,aws_config::profile::credentials=error"

.gitlab/internal_image_deploy/internal_image_deploy.yml

@@ -51,19 +51,19 @@ docker_trigger_internal-fips:
   stage: internal_image_deploy
   rules: !reference [.on_deploy_internal_or_manual]
   needs:
-    - job: docker_build_fips_agent7
+    - job: docker_build_fips_agent7_jmx
       artifacts: false
-    - job: docker_build_fips_agent7_arm64
+    - job: docker_build_fips_agent7_arm64_jmx
       artifacts: false
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   variables:
     DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo
     IMAGE_VERSION: tmpl-v11
     IMAGE_NAME: datadog-agent
-    RELEASE_TAG: ${CI_COMMIT_REF_SLUG}-fips
-    BUILD_TAG: ${CI_COMMIT_REF_SLUG}-fips
-    TMPL_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips
+    RELEASE_TAG: ${CI_COMMIT_REF_SLUG}-fips-jmx
+    BUILD_TAG: ${CI_COMMIT_REF_SLUG}-fips-jmx
+    TMPL_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx
     TMPL_SRC_REPO: ci/datadog-agent/agent
     RELEASE_STAGING: "true"
   script:

.gitlab/kernel_matrix_testing/common.yml

@@ -218,7 +218,7 @@
     # setup_env job hasn't finished. This causes instances to be leftover for more time than necessary.
     - inv kmt.wait-for-setup-job --pipeline-id $CI_PIPELINE_ID --arch $ARCH --component $TEST_COMPONENT
     - aws ec2 describe-instances --filters $FILTER_TEAM $FILTER_MANAGED $FILTER_PIPELINE $FILTER_ARCH $FILTER_INSTANCE_TYPE $FILTER_TEST_COMPONENT --output json --query $QUERY_INSTANCE_IDS | tee -a instance.json
-    - cat instance.json | jq -r 'map(.[]) | .[]' | grep -v "null" | xargs -n 1 -t aws ec2 terminate-instances --instance-ids
+    - cat instance.json | jq -r 'map(.[]) | .[]' | grep -v "null" | xargs --no-run-if-empty -n 1 -t aws ec2 terminate-instances --instance-ids
   after_script:
     - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY
     - !reference [.tag_kmt_ci_job]

.gitlab/pkg_metrics/pkg_metrics.yml

@@ -85,7 +85,7 @@ check_pkg_size:
     - if: $CI_COMMIT_BRANCH == "main"
       when: on_success
       allow_failure: true
-    - !reference [.not_on_release_branch]
+    - !reference [.not_on_release_branch_or_tagged_commit]
     - !reference [.except_mergequeue]
     - when: on_success
   needs:

.gitlab/source_test/tooling_unit_tests.yml

@@ -10,4 +10,3 @@ invoke_unit_tests:
   script:
     - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages
     - inv -e invoke-unit-tests.run
-

.gitlab/trigger_release/trigger_release.yml

@@ -75,7 +75,7 @@ generate_windows_gitlab_runner_bump_pr:
       when: never
     - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+){0,1}$/
       when: never
-    - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$/
+    - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$/
 
   script:
     # We are using the agent platform auto PR github app to access the buildenv repository (already used for macOS builds)
@@ -95,7 +95,7 @@ generate_windows_gitlab_runner_bump_pr_manual:
       when: never
     - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+){0,1}$/
       when: never
-    - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$/
+    - if: $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$/
       when: manual
   script:
     # We are using the agent platform auto PR github app to access the buildenv repository (already used for macOS builds)