diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 1b1512433c93..0b0efac5ed68 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -132,7 +132,7 @@ steps: steps: - label: ":linux: Auditbeat Ubuntu Integration Tests" key: "auditbeat-extended-integ-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ command: | set -euo pipefail cd auditbeat @@ -153,7 +153,7 @@ steps: - label: ":linux: Auditbeat Ubuntu arm64 Integration Tests" key: "auditbeat-extended-arm64-integ-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ command: | set -euo pipefail cd auditbeat @@ -194,7 +194,7 @@ steps: context: "auditbeat: Linux arm64 Unit Tests" - label: ":mac: Auditbeat macOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -214,7 +214,7 @@ steps: context: "auditbeat: macOS x86_64 Unit Tests" - label: ":mac: Auditbeat macOS arm64 ARM Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 87f1925a3722..c7ac3072a842 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -133,7 +133,7 @@ steps: steps: - label: ":mac: Filebeat macOS x86_64 Unit Tests" key: "macos-unit-tests-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -154,7 +154,7 @@ steps: - label: ":mac: Filebeat macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 136195e74b1a..cdb3959c2533 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -166,8 +166,7 @@ steps: - group: "Heartbeat Extended Testing MacOS" key: "heartbeat-extended-tests-macos" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - label: ":mac: Heartbeat MacOS Unit Tests" key: "macos-extended" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index ed9fb14f3d42..f4a04dbb0c2b 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -244,7 +244,7 @@ steps: - group: "Metricbeat Extended MacOS Tests" key: "metricbeat-extended-macos-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - label: ":mac: MacOS x64_64 Unit Tests" key: "extended-macos-x64-64-unit-tests" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index ee6e654cd3e6..74873046c56d 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -175,7 +175,7 @@ steps: steps: - label: ":mac: MacOS x86_64 Unit Tests" key: "macos-x86-64-unit-tests-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -196,7 +196,7 @@ steps: - label: ":mac: MacOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 1c541bb7896b..55affae41289 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -5,7 +5,7 @@ "pipelineSlug": "beats", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ "mergify[bot]" ], + "allowed_list": ["dependabot[bot]", "mergify[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, @@ -21,7 +21,7 @@ "pipelineSlug": "beats-xpack-elastic-agent", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_list": ["dependabot[bot]", "mergify[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 2e13a0d9b959..14a79eb76d9d 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -184,7 +184,7 @@ steps: - group: "x-pack/auditbeat MacOS Extended Tests" key: "x-pack-auditbeat-extended-tests-macos" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - label: ":mac: MacOS x86_64 Unit Tests" command: | diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index b01d6de53e2d..415c3947874a 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -7,17 +7,8 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" - IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" - IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" - IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" - IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" # Other deps ASDF_MAGE_VERSION: 1.15.0 diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 471dc32b4e59..1707bca29ecc 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -226,7 +226,7 @@ steps: key: "x-pack-filebeat-extended-tests" steps: - label: ":mac: MacOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -246,7 +246,7 @@ steps: - label: ":mac: MacOS arm64 Unit Tests" skip: "https://github.com/elastic/beats/issues/33036" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -266,7 +266,7 @@ steps: - label: ":linux: Cloud (MODULE) Tests" key: "x-pack-filebeat-extended-cloud-test" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 3a833f369e24..3ccb94bdf5e6 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -197,7 +197,7 @@ steps: - group: "x-pack/heartbeat macOS Extended Tests" key: "x-pack-heartbeat-extended-tests-macos" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - label: ":mac: x-pack/heartbeat macOS x86_64 Unit Tests" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index bc9ed7040e86..fb75291dde0d 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -208,7 +208,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|aws).*/ steps: - label: ":mac: MacOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -228,7 +228,7 @@ steps: - label: ":mac: MacOS arm64 Unit Tests" skip: "https://github.com/elastic/beats/issues/33036" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -249,7 +249,7 @@ steps: - label: ":linux: Cloud (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test" skip: "doesn't belong in a stage in Jenkins, thus skipped" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ # see link in Jenkins: https://github.com/elastic/beats/blob/ccd7b135df70358f8a02393d9bd8b716428b8048/x-pack/metricbeat/Jenkinsfile.yml#L39 # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 command: | @@ -282,7 +282,7 @@ steps: skip: "https://github.com/elastic/beats/issues/36425" # see commented out section in Jenkins: https://github.com/elastic/beats/blob/main/x-pack/metricbeat/Jenkinsfile.yml#L41-L52 # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index 7c740ecd5cb7..219bfe5910dd 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -10,7 +10,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 80bf88843d76..117824689a91 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -264,7 +264,7 @@ steps: - group: "x-pack/packetbeat MacOS Extended Tests" key: "x-pack-packetbeat-extended-macos-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - label: ":mac: MacOS Unit Tests" key: "extended-macos-unit-tests" diff --git a/.github/workflows/bump-elastic-stack-snapshot.yml b/.github/workflows/bump-elastic-stack-snapshot.yml index f4a2842fa9d0..835ead87a6a4 100644 --- a/.github/workflows/bump-elastic-stack-snapshot.yml +++ b/.github/workflows/bump-elastic-stack-snapshot.yml @@ -9,9 +9,6 @@ on: permissions: contents: read -env: - JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - jobs: filter: runs-on: ubuntu-latest @@ -29,14 +26,16 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.filter.outputs.matrix) }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current with: vaultUrl: ${{ secrets.VAULT_ADDR }} vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - pipeline: ./.ci/bump-elastic-stack-snapshot.yml + pipeline: .github/workflows/updatecli.d/bump-elastic-stack-snapshot.yml + values: .github/workflows/updatecli.d/scm.yml + command: '--experimental apply' notifySlackChannel: "#ingest-notifications" messageIfFailure: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@update-me-with-the-slack-team-to-be-poked` please look what's going on <${{ env.JOB_URL }}|here>" env: diff --git a/.github/workflows/bump-golang.yml b/.github/workflows/bump-golang.yml index 393c57f89a07..751b8612571e 100644 --- a/.github/workflows/bump-golang.yml +++ b/.github/workflows/bump-golang.yml @@ -9,34 +9,35 @@ on: permissions: contents: read -env: - JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - jobs: bump-main: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current with: vaultUrl: ${{ secrets.VAULT_ADDR }} vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - pipeline: ./.ci/bump-golang.yml + pipeline: .github/workflows/updatecli.d/bump-golang.yml + values: .github/workflows/updatecli.d/scm.yml + command: '--experimental apply' notifySlackChannel: "#ingest-notifications" messageIfFailure: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@update-me-with-the-slack-team-to-be-poked` please look what's going on <${{ env.JOB_URL }}|here>" bump-7-17: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current with: vaultUrl: ${{ secrets.VAULT_ADDR }} vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - pipeline: ./.ci/bump-golang-7.17.yml + pipeline: .github/workflows/updatecli.d/bump-golang-7.17.yml + values: .github/workflows/updatecli.d/scm.yml + command: '--experimental apply' notifySlackChannel: "#ingest-notifications" messageIfFailure: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@update-me-with-the-slack-team-to-be-poked` please look what's going on <${{ env.JOB_URL }}|here>" diff --git a/.github/workflows/opentelemetry.yml b/.github/workflows/opentelemetry.yml deleted file mode 100644 index 84a6209ff2c9..000000000000 --- a/.github/workflows/opentelemetry.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Look up results at https://ela.st/oblt-ci-cd-stats. -# There will be one service per GitHub repository, including the org name, and one Transaction per Workflow. -name: OpenTelemetry Export Trace - -on: - workflow_run: - workflows: [ "*" ] - types: [completed] - -permissions: - contents: read - -jobs: - otel-export-trace: - runs-on: ubuntu-latest - steps: - - uses: elastic/apm-pipeline-library/.github/actions/opentelemetry@current - with: - vaultUrl: ${{ secrets.VAULT_ADDR }} - vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} - vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.ci/bump-elastic-stack-snapshot.yml b/.github/workflows/updatecli.d/bump-elastic-stack-snapshot.yml similarity index 80% rename from .ci/bump-elastic-stack-snapshot.yml rename to .github/workflows/updatecli.d/bump-elastic-stack-snapshot.yml index f9060400c3ff..f679e7924f39 100644 --- a/.ci/bump-elastic-stack-snapshot.yml +++ b/.github/workflows/updatecli.d/bump-elastic-stack-snapshot.yml @@ -6,6 +6,7 @@ actions: default: title: '[updatecli] update elastic stack version for testing {{ source "latestVersion" }}' kind: github/pullrequest + scmid: default spec: labels: - automation @@ -13,21 +14,17 @@ actions: - backport-skip - build-monitoring - Team:Beats-On-Call - description: | - Generated automatically with {{ requiredEnv "JOB_URL" }} - scmid: default scms: default: kind: github spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' - owner: elastic - repository: beats + owner: '{{ .scm.owner }}' + repository: '{{ .scm.repository }}' + user: '{{ requiredEnv "GITHUB_ACTOR" }}' token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' branch: '{{ requiredEnv "BRANCH" }}' + commitusingapi: true sources: latestVersion: diff --git a/.ci/bump-golang-7.17.yml b/.github/workflows/updatecli.d/bump-golang-7.17.yml similarity index 96% rename from .ci/bump-golang-7.17.yml rename to .github/workflows/updatecli.d/bump-golang-7.17.yml index 5b6619bc453f..d6405aa1a8de 100644 --- a/.ci/bump-golang-7.17.yml +++ b/.github/workflows/updatecli.d/bump-golang-7.17.yml @@ -6,13 +6,12 @@ scms: githubConfig: kind: github spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' - owner: elastic - repository: beats + owner: '{{ .scm.owner }}' + repository: '{{ .scm.repository }}' + user: '{{ requiredEnv "GITHUB_ACTOR" }}' token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' branch: "7.17" + commitusingapi: true actions: beats: @@ -21,12 +20,9 @@ actions: sourceid: latestGoVersion title: '[Automation][7.17] Bump Golang version to {{ source "latestGoVersion" }}' spec: - automerge: false labels: - dependencies - backport-skip - description: | - Generated automatically with {{ requiredEnv "JOB_URL" }} sources: minor: diff --git a/.ci/bump-golang.yml b/.github/workflows/updatecli.d/bump-golang.yml similarity index 95% rename from .ci/bump-golang.yml rename to .github/workflows/updatecli.d/bump-golang.yml index 2957acd62236..9c30136f4445 100644 --- a/.ci/bump-golang.yml +++ b/.github/workflows/updatecli.d/bump-golang.yml @@ -6,13 +6,12 @@ scms: githubConfig: kind: github spec: - user: '{{ requiredEnv "GIT_USER" }}' - email: '{{ requiredEnv "GIT_EMAIL" }}' - owner: elastic - repository: beats + owner: '{{ .scm.owner }}' + repository: '{{ .scm.repository }}' + user: '{{ requiredEnv "GITHUB_ACTOR" }}' token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GIT_USER" }}' branch: main + commitusingapi: true actions: beats: @@ -21,12 +20,9 @@ actions: sourceid: latestGoVersion title: '[Automation] Bump Golang version to {{ source "latestGoVersion" }}' spec: - automerge: false labels: - dependencies - backport-skip - description: | - Generated automatically with {{ requiredEnv "JOB_URL" }} sources: minor: diff --git a/.github/workflows/updatecli.d/scm.yml b/.github/workflows/updatecli.d/scm.yml new file mode 100644 index 000000000000..fd532f00f616 --- /dev/null +++ b/.github/workflows/updatecli.d/scm.yml @@ -0,0 +1,4 @@ +--- +scm: + owner: elastic + repository: beats diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2b571875e347..2b870c03f990 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -266,6 +266,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Ensure all responses sent by HTTP Endpoint are HTML-escaped. {pull}39329[39329] - Update CEL mito extensions to v1.11.0 to improve type checking. {pull}39460[39460] - Improve logging of request and response with request trace logging in error conditions. {pull}39455[39455] +- Add HTTP metrics to CEL input. {issue}39501[39501] {pull}39503[39503] *Auditbeat* @@ -308,6 +309,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Winlogbeat* +- Use fixed size buffer at first pass for event parsing, improving throughput {issue}39530[39530] {pull}39544[39544] *Functionbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 4e58644e883b..b25883debde2 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -5256,11 +5256,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2@v1.26 -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/config -Version: v1.17.7 +Version: v1.27.11 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/config@v1.17.7/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/config@v1.27.11/LICENSE.txt: Apache License @@ -5468,11 +5468,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/confi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/credentials -Version: v1.12.20 +Version: v1.17.11 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/credentials@v1.12.20/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/credentials@v1.17.11/LICENSE.txt: Apache License @@ -5680,11 +5680,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/crede -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/feature/ec2/imds -Version: v1.12.17 +Version: v1.16.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/feature/ec2/imds@v1.12.17/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/feature/ec2/imds@v1.16.1/LICENSE.txt: Apache License @@ -5892,11 +5892,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/featu -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/feature/s3/manager -Version: v1.11.33 +Version: v1.16.15 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/feature/s3/manager@v1.11.33/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/feature/s3/manager@v1.16.15/LICENSE.txt: Apache License @@ -6104,11 +6104,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/featu -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/cloudformation -Version: v1.20.4 +Version: v1.50.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/cloudformation@v1.20.4/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/cloudformation@v1.50.0/LICENSE.txt: Apache License @@ -6316,11 +6316,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/cloudwatch -Version: v1.26.0 +Version: v1.38.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/cloudwatch@v1.26.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/cloudwatch@v1.38.0/LICENSE.txt: Apache License @@ -6528,11 +6528,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs -Version: v1.15.5 +Version: v1.35.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs@v1.15.5/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs@v1.35.1/LICENSE.txt: Apache License @@ -6740,11 +6740,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/costexplorer -Version: v1.18.4 +Version: v1.38.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/costexplorer@v1.18.4/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/costexplorer@v1.38.0/LICENSE.txt: Apache License @@ -6952,11 +6952,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/ec2 -Version: v1.36.1 +Version: v1.160.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/ec2@v1.36.1/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/ec2@v1.160.0/LICENSE.txt: Apache License @@ -7164,11 +7164,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 -Version: v1.18.4 +Version: v1.30.5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2@v1.18.4/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2@v1.30.5/LICENSE.txt: Apache License @@ -7376,11 +7376,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/health -Version: v1.17.0 +Version: v1.24.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/health@v1.17.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/health@v1.24.4/LICENSE.txt: Apache License @@ -7588,11 +7588,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/iam -Version: v1.18.4 +Version: v1.32.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/iam@v1.18.4/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/iam@v1.32.0/LICENSE.txt: Apache License @@ -7800,11 +7800,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/kinesis -Version: v1.15.8 +Version: v1.27.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/kinesis@v1.15.8/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/kinesis@v1.27.4/LICENSE.txt: Apache License @@ -8012,11 +8012,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/organizations -Version: v1.15.2 +Version: v1.27.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/organizations@v1.15.2/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/organizations@v1.27.3/LICENSE.txt: Apache License @@ -8224,11 +8224,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/rds -Version: v1.20.1 +Version: v1.78.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/rds@v1.20.1/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/rds@v1.78.0/LICENSE.txt: Apache License @@ -8436,11 +8436,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi -Version: v1.13.5 +Version: v1.21.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi@v1.13.5/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi@v1.21.4/LICENSE.txt: Apache License @@ -8648,11 +8648,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/s3 -Version: v1.27.11 +Version: v1.53.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/s3@v1.27.11/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/s3@v1.53.1/LICENSE.txt: Apache License @@ -8860,11 +8860,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/sqs -Version: v1.18.4 +Version: v1.31.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/sqs@v1.18.4/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/sqs@v1.31.4/LICENSE.txt: Apache License @@ -9072,11 +9072,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/sts -Version: v1.16.19 +Version: v1.28.6 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/sts@v1.16.19/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/sts@v1.28.6/LICENSE.txt: Apache License @@ -12969,11 +12969,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.9.4 +Version: v0.9.7 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.7/LICENSE: Apache License Version 2.0, January 2004 @@ -25256,11 +25256,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : go.uber.org/zap -Version: v1.26.0 +Version: v1.27.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.26.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/zap@v1.27.0/LICENSE: Copyright (c) 2016-2017 Uber Technologies, Inc. @@ -34577,11 +34577,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go@v1.38.60 -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream -Version: v1.4.8 +Version: v1.6.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream@v1.4.8/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream@v1.6.2/LICENSE.txt: Apache License @@ -35213,11 +35213,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/inter -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/internal/ini -Version: v1.3.24 +Version: v1.8.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/internal/ini@v1.3.24/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/internal/ini@v1.8.0/LICENSE.txt: Apache License @@ -35425,11 +35425,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/inter -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/internal/v4a -Version: v1.0.14 +Version: v1.3.5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/internal/v4a@v1.0.14/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/internal/v4a@v1.3.5/LICENSE.txt: Apache License @@ -35849,11 +35849,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/internal/checksum -Version: v1.1.18 +Version: v1.3.7 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/internal/checksum@v1.1.18/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/internal/checksum@v1.3.7/LICENSE.txt: Apache License @@ -36273,11 +36273,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/internal/s3shared -Version: v1.13.17 +Version: v1.17.5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/internal/s3shared@v1.13.17/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/internal/s3shared@v1.17.5/LICENSE.txt: Apache License @@ -36485,11 +36485,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/sso -Version: v1.11.23 +Version: v1.20.5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/sso@v1.11.23/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/sso@v1.20.5/LICENSE.txt: Apache License @@ -36697,11 +36697,11 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/ssooidc -Version: v1.13.5 +Version: v1.23.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/ssooidc@v1.13.5/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/ssooidc@v1.23.4/LICENSE.txt: Apache License @@ -38885,6 +38885,36 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0 limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/pkcs8 +Version: v1.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/pkcs8@v1.0.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 youmark + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + -------------------------------------------------------------------------------- Dependency : github.com/elazarl/goproxy Version: v0.0.0-20180725130230-947c36da3153 @@ -54355,11 +54385,11 @@ Contents of probable licence file $GOMODCACHE/go.opentelemetry.io/otel/trace@v1. -------------------------------------------------------------------------------- Dependency : go.uber.org/goleak -Version: v1.2.0 +Version: v1.3.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.2.0/LICENSE: +Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.3.0/LICENSE: The MIT License (MIT) diff --git a/go.mod b/go.mod index c8ebb6668642..5772bdffc937 100644 --- a/go.mod +++ b/go.mod @@ -30,20 +30,20 @@ require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/aws/aws-lambda-go v1.44.0 github.com/aws/aws-sdk-go-v2 v1.26.1 - github.com/aws/aws-sdk-go-v2/config v1.17.7 - github.com/aws/aws-sdk-go-v2/credentials v1.12.20 - github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.26.0 - github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.15.5 - github.com/aws/aws-sdk-go-v2/service/costexplorer v1.18.4 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.36.1 - github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.18.4 - github.com/aws/aws-sdk-go-v2/service/iam v1.18.4 - github.com/aws/aws-sdk-go-v2/service/organizations v1.15.2 - github.com/aws/aws-sdk-go-v2/service/rds v1.20.1 - github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.13.5 - github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 - github.com/aws/aws-sdk-go-v2/service/sqs v1.18.4 - github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 + github.com/aws/aws-sdk-go-v2/config v1.27.11 + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 + github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.0 + github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.1 + github.com/aws/aws-sdk-go-v2/service/costexplorer v1.38.0 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 + github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.30.5 + github.com/aws/aws-sdk-go-v2/service/iam v1.32.0 + github.com/aws/aws-sdk-go-v2/service/organizations v1.27.3 + github.com/aws/aws-sdk-go-v2/service/rds v1.78.0 + github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.21.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 + github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4 + github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 github.com/awslabs/goformation/v4 v4.1.0 github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 github.com/bsm/sarama-cluster v2.1.14-0.20180625083203-7e67d87a6b3f+incompatible @@ -196,11 +196,11 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/Azure/go-autorest/autorest/adal v0.9.21 github.com/apache/arrow/go/v14 v14.0.2 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 - github.com/aws/aws-sdk-go-v2/service/cloudformation v1.20.4 - github.com/aws/aws-sdk-go-v2/service/health v1.17.0 - github.com/aws/aws-sdk-go-v2/service/kinesis v1.15.8 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 + github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 + github.com/aws/aws-sdk-go-v2/service/health v1.24.4 + github.com/aws/aws-sdk-go-v2/service/kinesis v1.27.4 github.com/aws/smithy-go v1.20.2 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 @@ -264,17 +264,17 @@ require ( github.com/apache/thrift v0.19.0 // indirect github.com/armon/go-radix v1.0.0 // indirect github.com/aws/aws-sdk-go v1.38.60 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cilium/ebpf v0.13.2 // indirect diff --git a/go.sum b/go.sum index 8d16569f65ca..8e07e1d229fe 100644 --- a/go.sum +++ b/go.sum @@ -287,90 +287,89 @@ github.com/aws/aws-sdk-go v1.38.60 h1:MgyEsX0IMwivwth1VwEnesBpH0vxbjp5a0w1lurMOX github.com/aws/aws-sdk-go v1.38.60/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3/go.mod h1:gNsR5CaXKmQSSzrmGxmwmct/r+ZBfbxorAuXYsj/M5Y= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 h1:tcFliCWne+zOuUfKNRn8JdFBuWPDuISDH08wD2ULkhk= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot91steJeti4ryyu/tLd4Sk84O5W22L7O2EQU= -github.com/aws/aws-sdk-go-v2/config v1.17.7 h1:odVM52tFHhpqZBKNjVW5h+Zt1tKHbhdTQRb+0WHrNtw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= github.com/aws/aws-sdk-go-v2/config v1.17.7/go.mod h1:dN2gja/QXxFF15hQreyrqYhLBaQo1d9ZKe/v/uplQoI= -github.com/aws/aws-sdk-go-v2/credentials v1.12.20 h1:9+ZhlDY7N9dPnUmf7CDfW9In4sW5Ff3bh7oy4DzS1IE= +github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= +github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= github.com/aws/aws-sdk-go-v2/credentials v1.12.20/go.mod h1:UKY5HyIux08bbNA7Blv4PcXQ8cTkGh7ghHMFklaviR4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 h1:r08j4sbZu/RVi+BNxkBJwPMUYY3P8mgSDuKkZ/ZN1lE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 h1:fAoVmNGhir6BR+RU0/EI+6+D7abM+MCwWf8v4ip5jNI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33/go.mod h1:84XgODVR8uRhmOnUkKGUZKqIMxmjmLOR8Uyp7G/TPwc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10/go.mod h1:F+EZtuIwjlv35kRJPyBGcsA4f7bnSoz15zOQ2lJq1Z4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13/go.mod h1:wLLesU+LdMZDM3U0PP9vZXJW39zmD/7L4nY2pSrYZ/g= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4/go.mod h1:8glyUqVIM4AmeenIsPo0oVh3+NUwnsQml2OFupfQW+0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 h1:wj5Rwc05hvUSvKuOF29IYb9QrCLjU+rHAy/x/o0DK2c= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14 h1:ZSIPAkAsCCjYrhqfw2+lNzWDzxzHXEckFkTePL5RSWQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14/go.mod h1:AyGgqiKv9ECM6IZeNQtdT8NnMvUb3/2wokeq2Fgryto= -github.com/aws/aws-sdk-go-v2/service/cloudformation v1.20.4 h1:faP794ma9ZY/24XAV8cm/lkQzRFSg3zBHCi5Nc8+CaM= -github.com/aws/aws-sdk-go-v2/service/cloudformation v1.20.4/go.mod h1:ybjChNDMfPtc7f8ILTb+ov6CpE/KtAae9fD8HHtYfzU= -github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.26.0 h1:sSzrsKQULJmPtmu6By4wR6g0701nGqonssKOy35uOd0= -github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.26.0/go.mod h1:t5mizLPjCYafXoHCXOHJU7z4OvLbY70Echvb1ciBTV4= -github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.15.5 h1:aPK8IBVKeozo/pNGshT8xOJ2V3Y7ykOM49QcY0vhUSM= -github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.15.5/go.mod h1:ErjxucZaraVbYm66xxub00qmGBw7md2RFqy6624KbR8= -github.com/aws/aws-sdk-go-v2/service/costexplorer v1.18.4 h1:jbfG3cbq1kiK1/OAfUh4zf1ADtAU8KoeOPfF94S96pU= -github.com/aws/aws-sdk-go-v2/service/costexplorer v1.18.4/go.mod h1:yC5cDNa3xzSh5NIU5x0NBBo6QkcsaM0tuPNCczeUPoU= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.36.1 h1:FS8Ja6LuLDVHcX+rmoNpOXqYb52N2A5DwQy7Dgduq4Q= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.36.1/go.mod h1:KOy1O7Fc2+GRgsbn/Kjr15vYDVXMEQALBaPRia3twSY= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.18.4 h1:ZBYifRGfN3dOKzvk0+XJiUKOFzqoJddYqCVsN5quCh4= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.18.4/go.mod h1:9wKR88sRRyxrUAw5iVSDTfcCz90BLEFcAiyzP4v39uY= -github.com/aws/aws-sdk-go-v2/service/health v1.17.0 h1:DlG9888p6n8Fizx8Vuw1qalBOBtjoDk70UzqyilQ7+s= -github.com/aws/aws-sdk-go-v2/service/health v1.17.0/go.mod h1:z7JTQWRaBIdYYxK8TqDi4MKYYl04uI+jvTJuMEKIsL0= -github.com/aws/aws-sdk-go-v2/service/iam v1.18.4 h1:E41guA79mjEbwJdh0zXz1d8+Zt4zxRr+b1ipiVbKXzs= -github.com/aws/aws-sdk-go-v2/service/iam v1.18.4/go.mod h1:FpNvAfCZyIQ3qeNJUOw4CShKvdizHblXqAvSk0qmyL4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0= +github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 h1:Ap5tOJfeAH1hO2UQc3X3uMlwP7uryFeZXMvZCXIlLSE= +github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0/go.mod h1:/v2KYdCW4BaHKayenaWEXOOdxItIwEA3oU0XzuQY3F0= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.0 h1:vAfGwYFCcPDS9Bg7ckfMBer6olJLOHsOAVoKWpPIirs= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.0/go.mod h1:U12sr6Lt14X96f16t+rR52+2BdqtydwN7DjEEHRMjO0= +github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.1 h1:suWu59CRsDNhw2YXPpa6drYEetIUUIMUhkzHmucbCf8= +github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.1/go.mod h1:tZiRxrv5yBRgZ9Z4OOOxwscAZRFk5DgYhEcjX1QpvgI= +github.com/aws/aws-sdk-go-v2/service/costexplorer v1.38.0 h1:0q4pClt2ckd6awhQYEysexryCmA7q2HMI0O5dBrA5B8= +github.com/aws/aws-sdk-go-v2/service/costexplorer v1.38.0/go.mod h1:uLOg0o57AyQQhZGtUKIlcBJOKE53mO9bXKyrM9dFhy4= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0 h1:ooy0OFbrdSwgk32OFGPnvBwry5ySYCKkgTEbQ2hejs8= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.160.0/go.mod h1:xejKuuRDjz6z5OqyeLsz01MlOqqW7CqpAB4PabNvpu8= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.30.5 h1:/x2u/TOx+n17U+gz98TOw1HKJom0EOqrhL4SjrHr0cQ= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.30.5/go.mod h1:e1McVqsud0JOERidvppLEHnuCdh/X6MRyL5L0LseAUk= +github.com/aws/aws-sdk-go-v2/service/health v1.24.4 h1:5QROeJylnNdBQxxYn4BPpbgoo3nXT+SMG3KvFd71O4s= +github.com/aws/aws-sdk-go-v2/service/health v1.24.4/go.mod h1:p489k/dsudsm+FK8MSFJYk0kMqY4h7tTE2YU/s6VN6E= +github.com/aws/aws-sdk-go-v2/service/iam v1.32.0 h1:ZNlfPdw849gBo/lvLFbEEvpTJMij0LXqiNWZ+lIamlU= +github.com/aws/aws-sdk-go-v2/service/iam v1.32.0/go.mod h1:aXWImQV0uTW35LM0A/T4wEg6R1/ReXUu4SM6/lUHYK0= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18 h1:BBYoNQt2kUZUUK4bIPsKrCcjVPUMNsgQpNAwhznK/zo= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18/go.mod h1:NS55eQ4YixUJPTC+INxi2/jCqe1y2Uw3rnh9wEOVJxY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4/go.mod h1:uKkN7qmSIsNJVyMtxNQoCEYMvFEXbOg9fwCJPdfp2u8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 h1:HfVVR1vItaG6le+Bpw6P4midjBDMKnjMyZnw9MXYUcE= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17/go.mod h1:YqMdV+gEKCQ59NrB7rzrJdALeBIsYiVi8Inj3+KcqHI= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ= github.com/aws/aws-sdk-go-v2/service/kinesis v1.6.0/go.mod h1:9O7UG2pELnP0hq35+Gd7XDjOLBkg7tmgRQ0y14ZjoJI= -github.com/aws/aws-sdk-go-v2/service/kinesis v1.15.8 h1:iXRv1ZOF6riNcy5UR6LPTaGa64wZQH8tKj5LHsiajY8= -github.com/aws/aws-sdk-go-v2/service/kinesis v1.15.8/go.mod h1:oWvoK8MyYnXi6ZxSpgU7kFxIPGX8EfbCrdQCNgPnhCc= -github.com/aws/aws-sdk-go-v2/service/organizations v1.15.2 h1:lwVNtW6wmwa9iIH017Y9qMoGCcEtvDYJQGUO/1jlRBc= -github.com/aws/aws-sdk-go-v2/service/organizations v1.15.2/go.mod h1:QV/cuhF5g2FEc7178E+mpmiqf7sS2aHCDGLNkVgHf2o= -github.com/aws/aws-sdk-go-v2/service/rds v1.20.1 h1:5PrsAmuF3r9bvZMxKxHnJlHSh0IYDAWEzpRRnDlE7nM= -github.com/aws/aws-sdk-go-v2/service/rds v1.20.1/go.mod h1:PBfhG/hYU+oCP1uT7fNfaqaAvxQGbB0POqh1GE/7OdM= -github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.13.5 h1:nAHv/rx0pSqpECdrNtmKKb7RzYOpqXda+Dt8xBJifiM= -github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.13.5/go.mod h1:LQ8mizR4n/TdYBwmU4ZXfbKceeBSrdIzZBM7jZqMK0U= -github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 h1:3/gm/JTX9bX8CpzTgIlrtYpB3EVBDxyg/GY/QdcIEZw= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.27.4 h1:Oe8awBiS/iitcsRJB5+DHa3iCxoA0KwJJf0JNrYMINY= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.27.4/go.mod h1:RCZCSFbieSgNG1RKegO26opXV4EXyef/vNBVJsUyHuw= +github.com/aws/aws-sdk-go-v2/service/organizations v1.27.3 h1:CnPWlONzFX9/yO6IGuKg9sWUE8WhKztYRFbhmOHXjJI= +github.com/aws/aws-sdk-go-v2/service/organizations v1.27.3/go.mod h1:hUHSXe9HFEmLfHrXndAX5e69rv0nBsg22VuNQYl0JLM= +github.com/aws/aws-sdk-go-v2/service/rds v1.78.0 h1:EfurrcA19HaB9gZYd157DiozoPfkX2CH5/QnDZqNFrY= +github.com/aws/aws-sdk-go-v2/service/rds v1.78.0/go.mod h1:Rw15qGaGWu3jO0dOz7JyvdOEjgae//YrJxVWLYGynvg= +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.21.4 h1:c1jtPWZSmgMmPkCgwv67GE0ugdEgnLVo/BHR1wl3Dm0= +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.21.4/go.mod h1:FWw+Jnx+SlpsrU/NQ/f7f+1RdixTApZiU2o9FOubiDQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11/go.mod h1:fmgDANqTUCxciViKl9hb/zD5LFbvPINFRgWhDbR+vZo= -github.com/aws/aws-sdk-go-v2/service/sqs v1.18.4 h1:/O5+Nzs3k9gVx7gGUblbGf7rHZz71tYaOq9czgBaQZs= -github.com/aws/aws-sdk-go-v2/service/sqs v1.18.4/go.mod h1:j65jgKI0Gnc6SO25l2q0qV+X3b9S40571AOZ53bEXRI= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 h1:pwvCchFUEnlceKIgPUouBJwK81aCkQ8UDMORfeFtW10= +github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o= +github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4 h1:mE2ysZMEeQ3ulHWs4mmc4fZEhOfeY1o6QXAfDqjbSgw= +github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4/go.mod h1:lCN2yKnj+Sp9F6UzpoPPTir+tSaC9Jwf6LcmTqnXFZw= github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 h1:GUnZ62TevLqIoDyHeiWj2P7EqaosgakBKVvWriIdLQY= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5/go.mod h1:csZuQY65DAdFBt1oIjO5hhBR49kQqop4+lcuCjf2arA= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 h1:9pPi0PsFNAGILFfPCk8Y0iyEBGc6lu6OQ97U7hmdesg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/awslabs/goformation/v3 v3.1.0/go.mod h1:hQ5RXo3GNm2laHWKizDzU5DsDy+yNcenSca2UxN0850= diff --git a/winlogbeat/sys/wineventlog/format_message.go b/winlogbeat/sys/wineventlog/format_message.go index e6502d384fae..9c1cf8254ace 100644 --- a/winlogbeat/sys/wineventlog/format_message.go +++ b/winlogbeat/sys/wineventlog/format_message.go @@ -75,23 +75,39 @@ func evtFormatMessage(metadataHandle EvtHandle, eventHandle EvtHandle, messageID valuesPtr = &values[0] } - // Determine the buffer size needed (given in WCHARs). - var bufferUsed uint32 - err := _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, 0, nil, &bufferUsed) - if err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno. - return "", fmt.Errorf("failed in EvtFormatMessage: %w", err) - } + // best guess render buffer size, 16KB, to avoid rendering message twice in most cases + const bestGuessRenderBufferSize = 1 << 14 + + // EvtFormatMessage operates with WCHAR buffer, assuming the size of the buffer in characters. + // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage + var bufferNeeded uint32 + bufferSize := uint32(bestGuessRenderBufferSize / 2) // Get a buffer from the pool and adjust its length. bb := sys.NewPooledByteBuffer() defer bb.Free() - // The documentation for EventFormatMessage specifies that the buffer is - // requested "in characters", and the buffer itself is LPWSTR, meaning the - // characters are WCHAR so double the value. - // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage - bb.Reserve(int(bufferUsed * 2)) + bb.Reserve(int(bufferSize * 2)) + + err := _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, bufferSize, bb.PtrAt(0), &bufferNeeded) + switch err { //nolint:errorlint // This is an errno or nil. + case nil: // OK + return sys.UTF16BytesToString(bb.Bytes()) + + // Ignore some errors so it can tolerate missing or mismatched parameter values. + case windows.ERROR_EVT_UNRESOLVED_VALUE_INSERT, + windows.ERROR_EVT_UNRESOLVED_PARAMETER_INSERT, + windows.ERROR_EVT_MAX_INSERTS_REACHED: + return sys.UTF16BytesToString(bb.Bytes()) + + case windows.ERROR_INSUFFICIENT_BUFFER: + bb.Reserve(int(bufferNeeded * 2)) + bufferSize = bufferNeeded + + default: + return "", fmt.Errorf("failed in EvtFormatMessage: %w", err) + } - err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, bufferUsed, bb.PtrAt(0), &bufferUsed) + err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, bufferSize, bb.PtrAt(0), &bufferNeeded) switch err { //nolint:errorlint // This is an errno or nil. case nil: // OK diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows.go b/winlogbeat/sys/wineventlog/wineventlog_windows.go index 6b4abfaf5d1f..22495f6bda2e 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows.go @@ -239,15 +239,9 @@ func RenderEvent( // Only a single string is returned when rendering XML. err = FormatEventString(EvtFormatMessageXml, - eventHandle, providerName, EvtHandle(publisherHandle), lang, out) + eventHandle, providerName, EvtHandle(publisherHandle), lang, renderBuf, out) // Recover by rendering the XML without the RenderingInfo (message string). if err != nil { - // Do not try to recover from InsufficientBufferErrors because these - // can be retried with a larger buffer. - if errors.Is(err, sys.InsufficientBufferError{}) { - return err - } - err = RenderEventXML(eventHandle, renderBuf, out) } @@ -256,8 +250,8 @@ func RenderEvent( // Message reads the event data associated with the EvtHandle and renders // and returns the message only. -func Message(h EvtHandle, buf []byte, pubHandleProvider func(string) sys.MessageFiles) (message string, err error) { - providerName, err := evtRenderProviderName(buf, h) +func Message(h EvtHandle, renderBuf []byte, pubHandleProvider func(string) sys.MessageFiles) (message string, err error) { + providerName, err := evtRenderProviderName(renderBuf, h) if err != nil { return "", err } @@ -386,12 +380,15 @@ func Close(h EvtHandle) error { // publisherHandle is a handle to the publisher's metadata as provided by // EvtOpenPublisherMetadata. // lang is the language ID. +// renderBuf is a scratch buffer to render the message, if not provided or of +// insufficient size then a buffer from a system pool will be used func FormatEventString( messageFlag EvtFormatMessageFlag, eventHandle EvtHandle, publisher string, publisherHandle EvtHandle, lang uint32, + renderBuf []byte, out io.Writer, ) error { // Open a publisher handle if one was not provided. @@ -405,29 +402,42 @@ func FormatEventString( defer _EvtClose(ph) //nolint:errcheck // This is just a resource release. } - // Determine the buffer size needed (given in WCHARs). - var bufferUsed uint32 - err := _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, 0, nil, &bufferUsed) - if err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno. + var bufferPtr *byte + if renderBuf != nil { + bufferPtr = &renderBuf[0] + } + + // EvtFormatMessage operates with WCHAR buffer, assuming the size of the buffer in characters. + // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage + var bufferNeeded uint32 + bufferSize := uint32(len(renderBuf) / 2) + + err := _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, bufferSize, bufferPtr, &bufferNeeded) + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // This is an errno. return fmt.Errorf("failed in EvtFormatMessage: %w", err) + } else if err == nil { + // Windows API returns a null terminated WCHAR C-style string in the buffer. bufferNeeded applies + // only when ERROR_INSUFFICIENT_BUFFER is returned. Luckily the UTF16ToUTF8Bytes/UTF16ToString + // functions stop at null termination. Note, as signaled in a comment at the end of this function, + // this behavior is bad for EvtFormatMessageKeyword as then the API returns a list of null terminated + // strings in the buffer (it's fine for now as we don't use this parameter value). + return common.UTF16ToUTF8Bytes(renderBuf, out) } // Get a buffer from the pool and adjust its length. bb := sys.NewPooledByteBuffer() defer bb.Free() - // The documentation for EvtFormatMessage specifies that the buffer is - // requested "in characters", and the buffer itself is LPWSTR, meaning the - // characters are WCHAR so double the value. - // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage - bb.Reserve(int(bufferUsed * 2)) - err = _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, bufferUsed, bb.PtrAt(0), &bufferUsed) + bb.Reserve(int(bufferNeeded * 2)) + bufferSize = bufferNeeded + + err = _EvtFormatMessage(ph, eventHandle, 0, 0, nil, messageFlag, bufferSize, bb.PtrAt(0), &bufferNeeded) if err != nil { return fmt.Errorf("failed in EvtFormatMessage: %w", err) } // This assumes there is only a single string value to read. This will - // not work to read keys (when messageFlag == EvtFormatMessageKeyword). + // not work to read keys (when messageFlag == EvtFormatMessageKeyword) return common.UTF16ToUTF8Bytes(bb.Bytes(), out) } diff --git a/x-pack/filebeat/docs/inputs/input-cel.asciidoc b/x-pack/filebeat/docs/inputs/input-cel.asciidoc index a6adf5e4ad9f..7ec869e42cc7 100644 --- a/x-pack/filebeat/docs/inputs/input-cel.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-cel.asciidoc @@ -765,15 +765,36 @@ observe the activity of the input. [options="header"] |======= -| Metric | Description -| `resource` | URL or path of the input resource. -| `cel_executions` | Number times the CEL program has been executed. -| `batches_received_total` | Number of event arrays received. -| `events_received_total` | Number of events received. -| `batches_published_total` | Number of event arrays published. -| `events_published_total` | Number of events published. -| `cel_processing_time` | Histogram of the elapsed successful CEL program processing times in nanoseconds. -| `batch_processing_time` | Histogram of the elapsed successful batch processing times in nanoseconds (time of receipt to time of ACK for non-empty batches). +| Metric | Description +| `resource` | URL or path of the input resource. +| `cel_executions` | Number times the CEL program has been executed. +| `batches_received_total` | Number of event arrays received. +| `events_received_total` | Number of events received. +| `batches_published_total` | Number of event arrays published. +| `events_published_total` | Number of events published. +| `cel_processing_time` | Histogram of the elapsed successful CEL program processing times in nanoseconds. +| `batch_processing_time` | Histogram of the elapsed successful batch processing times in nanoseconds (time of receipt to time of ACK for non-empty batches). +| `http_request_total` | Total number of processed requests. +| `http_request_errors_total` | Total number of request errors. +| `http_request_delete_total` | Total number of `DELETE` requests. +| `http_request_get_total` | Total number of `GET` requests. +| `http_request_head_total` | Total number of `HEAD` requests. +| `http_request_options_total` | Total number of `OPTIONS` requests. +| `http_request_patch_total` | Total number of `PATCH` requests. +| `http_request_post_total` | Total number of `POST` requests. +| `http_request_put_total` | Total number of `PUT` requests. +| `http_request_body_bytes_total` | Total of the requests body size. +| `http_request_body_bytes` | Histogram of the requests body size. +| `http_response_total` | Total number of responses received. +| `http_response_errors_total` | Total number of response errors. +| `http_response_1xx_total` | Total number of `1xx` responses. +| `http_response_2xx_total` | Total number of `2xx` responses. +| `http_response_3xx_total` | Total number of `3xx` responses. +| `http_response_4xx_total` | Total number of `4xx` responses. +| `http_response_5xx_total` | Total number of `5xx` responses. +| `http_response_body_bytes_total` | Total of the responses body size. +| `http_response_body_bytes` | Histogram of the responses body size. +| `http_round_trip_time` | Histogram of the round trip time. |======= ==== Developer tools diff --git a/x-pack/filebeat/input/awss3/input_test.go b/x-pack/filebeat/input/awss3/input_test.go index 83015c1661be..432bd360bfc6 100644 --- a/x-pack/filebeat/input/awss3/input_test.go +++ b/x-pack/filebeat/input/awss3/input_test.go @@ -5,7 +5,6 @@ package awss3 import ( - "context" "errors" "testing" @@ -155,12 +154,3 @@ func TestRegionSelection(t *testing.T) { }) } } - -func newV2Context() (v2.Context, func()) { - ctx, cancel := context.WithCancel(context.Background()) - return v2.Context{ - Logger: logp.NewLogger("awss3_test"), - ID: "test_id", - Cancelation: ctx, - }, cancel -} diff --git a/x-pack/filebeat/input/awss3/s3_objects_test.go b/x-pack/filebeat/input/awss3/s3_objects_test.go index d0b4021c7f87..df50726823fd 100644 --- a/x-pack/filebeat/input/awss3/s3_objects_test.go +++ b/x-pack/filebeat/input/awss3/s3_objects_test.go @@ -38,8 +38,10 @@ func newS3Object(t testing.TB, filename, contentType string) (s3EventV2, *s3.Get func newS3GetObjectResponse(filename string, data []byte, contentType string) *s3.GetObjectOutput { r := bytes.NewReader(data) + contentLength := int64(r.Len()) + getObjectOutput := s3.GetObjectOutput{} - getObjectOutput.ContentLength = int64(r.Len()) + getObjectOutput.ContentLength = &contentLength getObjectOutput.Body = io.NopCloser(r) if contentType != "" { getObjectOutput.ContentType = &contentType diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index 88d7a20b458b..759809e6e80c 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -42,6 +42,7 @@ import ( "github.com/elastic/beats/v7/libbeat/monitoring/inputmon" "github.com/elastic/beats/v7/libbeat/version" "github.com/elastic/beats/v7/x-pack/filebeat/input/internal/httplog" + "github.com/elastic/beats/v7/x-pack/filebeat/input/internal/httpmon" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/monitoring" @@ -122,7 +123,7 @@ func (i input) run(env v2.Context, src *source, cursor map[string]interface{}, p cfg := src.cfg log := env.Logger.With("input_url", cfg.Resource.URL) - metrics := newInputMetrics(env.ID) + metrics, reg := newInputMetrics(env.ID) defer metrics.Close() ctx := ctxtool.FromCanceller(env.Cancelation) @@ -132,7 +133,7 @@ func (i input) run(env v2.Context, src *source, cursor map[string]interface{}, p cfg.Resource.Tracer.Filename = strings.ReplaceAll(cfg.Resource.Tracer.Filename, "*", id) } - client, trace, err := newClient(ctx, cfg, log) + client, trace, err := newClient(ctx, cfg, log, reg) if err != nil { return err } @@ -686,7 +687,7 @@ func getLimit(which string, rateLimit map[string]interface{}, log *logp.Logger) return limit, true } -func newClient(ctx context.Context, cfg config, log *logp.Logger) (*http.Client, *httplog.LoggingRoundTripper, error) { +func newClient(ctx context.Context, cfg config, log *logp.Logger, reg *monitoring.Registry) (*http.Client, *httplog.LoggingRoundTripper, error) { if !wantClient(cfg) { return nil, nil, nil } @@ -729,6 +730,10 @@ func newClient(ctx context.Context, cfg config, log *logp.Logger) (*http.Client, c.Transport = trace } + if reg != nil { + c.Transport = httpmon.NewMetricsRoundTripper(c.Transport, reg) + } + c.CheckRedirect = checkRedirect(cfg.Resource, log) if cfg.Resource.Retry.getMaxAttempts() > 1 { @@ -1070,7 +1075,7 @@ type inputMetrics struct { batchProcessingTime metrics.Sample // histogram of the elapsed successful batch processing times in nanoseconds (time of receipt to time of ACK for non-empty batches). } -func newInputMetrics(id string) *inputMetrics { +func newInputMetrics(id string) (*inputMetrics, *monitoring.Registry) { reg, unreg := inputmon.NewInputRegistry(inputName, id, nil) out := &inputMetrics{ unregister: unreg, @@ -1088,7 +1093,7 @@ func newInputMetrics(id string) *inputMetrics { _ = adapter.NewGoMetrics(reg, "batch_processing_time", adapter.Accept). Register("histogram", metrics.NewHistogram(out.batchProcessingTime)) - return out + return out, reg } func (m *inputMetrics) Close() { diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index caf695f1cb9f..4bcf0eee296e 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -57,7 +57,7 @@ func GetListMetricsOutput(namespace string, regionName string, period time.Durat listMetricsInput := &cloudwatch.ListMetricsInput{ NextToken: nextToken, - IncludeLinkedAccounts: includeLinkedAccounts, + IncludeLinkedAccounts: &includeLinkedAccounts, } // To filter the results to show only metrics that have had data points published diff --git a/x-pack/osquerybeat/beater/action_handler.go b/x-pack/osquerybeat/beater/action_handler.go index c4650ee9f165..a2a86bdf8dc2 100644 --- a/x-pack/osquerybeat/beater/action_handler.go +++ b/x-pack/osquerybeat/beater/action_handler.go @@ -21,6 +21,10 @@ var ( ErrNoQueryExecutor = errors.New("no query executor configures") ) +type actionResultPublisher interface { + PublishActionResult(req map[string]interface{}, res map[string]interface{}) +} + type publisher interface { Publish(index, actionID, responseID string, meta map[string]interface{}, hits []map[string]interface{}, ecsm ecs.Mapping, reqData interface{}) } diff --git a/x-pack/osquerybeat/beater/osquerybeat.go b/x-pack/osquerybeat/beater/osquerybeat.go index bb82525a5d59..b4fe30a47d88 100644 --- a/x-pack/osquerybeat/beater/osquerybeat.go +++ b/x-pack/osquerybeat/beater/osquerybeat.go @@ -177,7 +177,7 @@ func (bt *osquerybeat) Run(b *beat.Beat) error { } // Set reseable action handler - rah := newResetableActionHandler(bt.log) + rah := newResetableActionHandler(bt.pub, bt.log) defer rah.Clear() g, ctx := errgroup.WithContext(ctx) diff --git a/x-pack/osquerybeat/beater/resetable_action_handler.go b/x-pack/osquerybeat/beater/resetable_action_handler.go index 1b6bb20e1db4..a7daba113130 100644 --- a/x-pack/osquerybeat/beater/resetable_action_handler.go +++ b/x-pack/osquerybeat/beater/resetable_action_handler.go @@ -31,6 +31,8 @@ var ( // // The lifetime of this should the a scope of the beat Run type resetableActionHandler struct { + pub actionResultPublisher + log *logp.Logger ah client.Action @@ -43,8 +45,9 @@ type resetableActionHandler struct { type optionFunc func(a *resetableActionHandler) -func newResetableActionHandler(log *logp.Logger, opts ...optionFunc) *resetableActionHandler { +func newResetableActionHandler(pub actionResultPublisher, log *logp.Logger, opts ...optionFunc) *resetableActionHandler { a := &resetableActionHandler{ + pub: pub, log: log, timeout: defaultTimeout, } @@ -69,6 +72,9 @@ func (a *resetableActionHandler) Execute(ctx context.Context, req map[string]int res = renderResult(res, err) err = nil } + if a.pub != nil { + a.pub.PublishActionResult(req, res) + } }() res, err = a.execute(ctx, req) diff --git a/x-pack/osquerybeat/beater/resetable_action_handler_test.go b/x-pack/osquerybeat/beater/resetable_action_handler_test.go index eafeb758f679..d44985b3ac76 100644 --- a/x-pack/osquerybeat/beater/resetable_action_handler_test.go +++ b/x-pack/osquerybeat/beater/resetable_action_handler_test.go @@ -40,6 +40,15 @@ func (a *mockActionHandler) Name() string { return "osquery" } +type mockActionResultPublisher struct { + req, res map[string]interface{} +} + +func (p *mockActionResultPublisher) PublishActionResult(req map[string]interface{}, res map[string]interface{}) { + p.req = req + p.res = res +} + func TestResetableActionHandler(t *testing.T) { ctx, cn := context.WithCancel(context.Background()) defer cn() @@ -78,7 +87,8 @@ func TestResetableActionHandler(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - rah := newResetableActionHandler(log, resetableActionHandlerWithTimeout(testActionHandlerTimeout)) + pub := &mockActionResultPublisher{} + rah := newResetableActionHandler(pub, log, resetableActionHandlerWithTimeout(testActionHandlerTimeout)) defer rah.Clear() if tc.ah != nil { diff --git a/x-pack/osquerybeat/cmd/root.go b/x-pack/osquerybeat/cmd/root.go index 73584ec06f12..9c02433169ef 100644 --- a/x-pack/osquerybeat/cmd/root.go +++ b/x-pack/osquerybeat/cmd/root.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/spf13/cobra" + "google.golang.org/protobuf/types/known/structpb" "github.com/elastic/elastic-agent-client/v7/pkg/client" "github.com/elastic/elastic-agent-client/v7/pkg/proto" @@ -81,6 +82,92 @@ func genVerifyCmd(_ instance.Settings) *cobra.Command { } func osquerybeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + // For the older stack there were no streams, creating one + if len(rawIn.GetStreams()) == 0 { + return osquerybeatCfgNoStreams(rawIn, agentInfo) + } + return osquerybeatCfgFromStreams(rawIn, agentInfo) +} + +func osquerybeatCfgFromStreams(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { + + streams := make([]*proto.Stream, 0, len(rawIn.Streams)) + + // Attach osquery configuration to the osquery_manager.result stream and set it as a first stream + for _, stream := range rawIn.Streams { + if stream.DataStream != nil && stream.DataStream.Dataset == config.DefaultDataset { + if stream.Source == nil { + // If for any reason the stream source is missing completely, use datastream source as before + stream.Source = rawIn.Source + } else { + // Set osquery configuration value + fieldsSrc := rawIn.Source.Fields + fieldsDst := stream.Source.Fields + var osqVal *structpb.Value + if fieldsSrc != nil { + osqVal = fieldsSrc["osquery"] + } + if osqVal != nil { + fieldsDst["osquery"] = osqVal + } + // Setting id to the source because it is being picked up from there in shared management.CreateInputsFromStreams + vId, ok := fieldsDst["id"] + shouldSet := false + if !ok || vId == nil { + shouldSet = true + } else { + if _, ok := vId.GetKind().(*structpb.Value_NullValue); ok { + shouldSet = true + } + } + if shouldSet { + fieldsDst["id"] = structpb.NewStringValue(rawIn.Id) + } + } + streams = append([]*proto.Stream{stream}, streams...) + continue + } + streams = append(streams, stream) + } + rawIn.Streams = streams + + streamList, err := management.CreateInputsFromStreams(rawIn, "logs", agentInfo) + if err != nil { + return nil, fmt.Errorf("error creating input list from raw expected config: %w", err) + } + + var ns string + if rawIn.DataStream != nil { + ns = rawIn.DataStream.Namespace + if ns == "" { + ns = config.DefaultNamespace + } + } + + for iter := range streamList { + if _, ok := streamList[iter]["type"]; !ok { + streamList[iter]["type"] = rawIn.Type + } + if v, ok := streamList[iter]["data_stream"]; ok { + if m, ok := v.(map[string]interface{}); ok { + if _, ok := m["namespace"]; !ok { + m["namespace"] = ns + } + } + } + } + + // format for the reloadable list needed by the cm.Reload() method + configList, err := management.CreateReloadConfigFromInputs(streamList) + if err != nil { + return nil, fmt.Errorf("error creating config for reloader: %w", err) + } + + return configList, nil +} + +// This is needed for compatibility with the legacy implementation where kibana set empty streams array [] into the policy +func osquerybeatCfgNoStreams(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo) ([]*reload.ConfigWithMeta, error) { // Convert to streams, osquerybeat doesn't use streams streams := make([]*proto.Stream, 1) @@ -113,7 +200,7 @@ func osquerybeatCfg(rawIn *proto.UnitExpectedConfig, agentInfo *client.AgentInfo modules[iter]["type"] = "log" } - // format for the reloadable list needed bythe cm.Reload() method + // format for the reloadable list needed by the cm.Reload() method configList, err := management.CreateReloadConfigFromInputs(modules) if err != nil { return nil, fmt.Errorf("error creating config for reloader: %w", err) diff --git a/x-pack/osquerybeat/cmd/root_test.go b/x-pack/osquerybeat/cmd/root_test.go new file mode 100644 index 000000000000..5d0df4df0a6d --- /dev/null +++ b/x-pack/osquerybeat/cmd/root_test.go @@ -0,0 +1,98 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/elastic/beats/v7/libbeat/common/reload" + + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestOsquerybeatCfg(t *testing.T) { + matches, err := filepath.Glob("testdata/osquerycfg/*.in.json") + if err != nil { + t.Fatal(err) + } + + for _, match := range matches { + dir := filepath.Dir(match) + key := strings.TrimSuffix(filepath.Base(match), `.in.json`) + + out := filepath.Join(dir, key+".out.json") + t.Run(key, func(in, out string) func(t *testing.T) { + return func(t *testing.T) { + var rawIn proto.UnitExpectedConfig + err := readRawIn(in, &rawIn) + if err != nil { + t.Fatal(err) + } + + want, err := readOut(out) + if err != nil { + t.Fatal(err) + } + + cfg, err := osquerybeatCfg(&rawIn, &client.AgentInfo{ID: "abc7d0a8-ce04-4663-95da-ff6d537c268f", Version: "8.13.1"}) + if err != nil { + t.Fatal(err) + } + got, err := cfgToArrMap(cfg) + if err != nil { + t.Fatal(err) + } + + diff := cmp.Diff(want, got) + if diff != "" { + t.Fatal(diff) + } + } + }(match, out)) + } +} + +func readRawIn(filename string, rawIn *proto.UnitExpectedConfig) error { + b, err := os.ReadFile(filename) + if err != nil { + return err + } + err = json.Unmarshal(b, rawIn) + return err +} + +func readOut(filename string) (cfg []map[string]interface{}, err error) { + b, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + err = json.Unmarshal(b, &cfg) + if err != nil { + return nil, err + } + return cfg, err +} + +func cfgToArrMap(cfg []*reload.ConfigWithMeta) ([]map[string]interface{}, error) { + res := make([]map[string]interface{}, 0, len(cfg)) + for _, c := range cfg { + var m mapstr.M + err := c.Config.Unpack(&m) + if err != nil { + return nil, err + } + res = append(res, map[string]interface{}(m)) + } + return res, nil +} diff --git a/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy.in.json b/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy.in.json new file mode 100644 index 000000000000..f358b4fdf6e7 --- /dev/null +++ b/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy.in.json @@ -0,0 +1,51 @@ +{ + "source": { + "data_stream": { + "namespace": "default" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "meta": { + "package": { + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "name": "osquery_manager-1", + "package_policy_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "policy": { + "revision": 2 + }, + "revision": 1, + "streams": [ + ], + "type": "osquery" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "type": "osquery", + "name": "osquery_manager-1", + "revision": 1, + "meta": { + "source": { + "package": { + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "package": { + "source": { + "name": "osquery_manager", + "version": "1.12.1" + }, + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "data_stream": { + "source": { + "namespace": "default" + }, + "namespace": "default" + }, + "streams": [ + ] +} \ No newline at end of file diff --git a/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy.out.json b/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy.out.json new file mode 100644 index 000000000000..2ec760a08f81 --- /dev/null +++ b/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy.out.json @@ -0,0 +1,77 @@ +[ + { + "data_stream": { + "namespace": "default" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "index": "logs-osquery_manager.result-default", + "meta": { + "package": { + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "name": "osquery_manager-1", + "package_policy_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "policy": { + "revision": 2 + }, + "processors": [ + { + "add_fields": { + "fields": { + "input_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c" + }, + "target": "@metadata" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.result", + "namespace": "default", + "type": "logs" + }, + "target": "data_stream" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.result" + }, + "target": "event" + } + }, + { + "add_fields": { + "fields": { + "stream_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c" + }, + "target": "@metadata" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f", + "snapshot": false, + "version": "8.13.1" + }, + "target": "elastic_agent" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f" + }, + "target": "agent" + } + } + ], + "revision": 1, + "streams": [], + "type": "log" + } +] \ No newline at end of file diff --git a/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy_with_osquery.in.json b/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy_with_osquery.in.json new file mode 100644 index 000000000000..c3bb5d4e3802 --- /dev/null +++ b/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy_with_osquery.in.json @@ -0,0 +1,56 @@ +{ + "source": { + "data_stream": { + "namespace": "default" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "meta": { + "package": { + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "name": "osquery_manager-1", + "osquery": { + "options": { + "host_identifier": "hostname" + } + }, + "package_policy_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "policy": { + "revision": 3 + }, + "revision": 2, + "streams": [ + ], + "type": "osquery" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "type": "osquery", + "name": "osquery_manager-1", + "revision": 2, + "meta": { + "source": { + "package": { + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "package": { + "source": { + "name": "osquery_manager", + "version": "1.12.1" + }, + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "data_stream": { + "source": { + "namespace": "default" + }, + "namespace": "default" + }, + "streams": [ + ] +} \ No newline at end of file diff --git a/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy_with_osquery.out.json b/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy_with_osquery.out.json new file mode 100644 index 000000000000..7568395785ea --- /dev/null +++ b/x-pack/osquerybeat/cmd/testdata/osquerycfg/legacy_with_osquery.out.json @@ -0,0 +1,82 @@ +[ + { + "data_stream": { + "namespace": "default" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "index": "logs-osquery_manager.result-default", + "meta": { + "package": { + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "name": "osquery_manager-1", + "osquery": { + "options": { + "host_identifier": "hostname" + } + }, + "package_policy_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "policy": { + "revision": 3 + }, + "processors": [ + { + "add_fields": { + "fields": { + "input_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c" + }, + "target": "@metadata" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.result", + "namespace": "default", + "type": "logs" + }, + "target": "data_stream" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.result" + }, + "target": "event" + } + }, + { + "add_fields": { + "fields": { + "stream_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c" + }, + "target": "@metadata" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f", + "snapshot": false, + "version": "8.13.1" + }, + "target": "elastic_agent" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f" + }, + "target": "agent" + } + } + ], + "revision": 2, + "streams": [], + "type": "log" + } +] \ No newline at end of file diff --git a/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams.in.json b/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams.in.json new file mode 100644 index 000000000000..8081cb2c4845 --- /dev/null +++ b/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams.in.json @@ -0,0 +1,104 @@ +{ + "source": { + "data_stream": { + "namespace": "default" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "meta": { + "package": { + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "name": "osquery_manager-1", + "package_policy_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "policy": { + "revision": 2 + }, + "revision": 1, + "streams": [ + { + "data_stream": { + "dataset": "osquery_manager.action.responses", + "type": "logs" + }, + "id": "osquery-osquery_manager.action.responses-74c7d0a8-ce04-4663-95da-ff6d537c268c", + "query": null + }, + { + "data_stream": { + "dataset": "osquery_manager.result", + "type": "logs" + }, + "id": null, + "query": null + } + ], + "type": "osquery" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "type": "osquery", + "name": "osquery_manager-1", + "revision": 1, + "meta": { + "source": { + "package": { + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "package": { + "source": { + "name": "osquery_manager", + "version": "1.12.1" + }, + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "data_stream": { + "source": { + "namespace": "default" + }, + "namespace": "default" + }, + "streams": [ + { + "source": { + "data_stream": { + "dataset": "osquery_manager.action.responses", + "type": "logs" + }, + "id": "osquery-osquery_manager.action.responses-74c7d0a8-ce04-4663-95da-ff6d537c268c", + "query": null + }, + "id": "osquery-osquery_manager.action.responses-74c7d0a8-ce04-4663-95da-ff6d537c268c", + "data_stream": { + "source": { + "dataset": "osquery_manager.action.responses", + "type": "logs" + }, + "dataset": "osquery_manager.action.responses", + "type": "logs" + } + }, + { + "source": { + "data_stream": { + "dataset": "osquery_manager.result", + "type": "logs" + }, + "id": null, + "query": null + }, + "data_stream": { + "source": { + "dataset": "osquery_manager.result", + "type": "logs" + }, + "dataset": "osquery_manager.result", + "type": "logs" + } + } + ] +} \ No newline at end of file diff --git a/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams.out.json b/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams.out.json new file mode 100644 index 000000000000..b691078a3f61 --- /dev/null +++ b/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams.out.json @@ -0,0 +1,122 @@ +[ + { + "data_stream": { + "dataset": "osquery_manager.result", + "namespace": "default", + "type": "logs" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "index": "logs-osquery_manager.result-default", + "processors": [ + { + "add_fields": { + "fields": { + "input_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c" + }, + "target": "@metadata" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.result", + "namespace": "default", + "type": "logs" + }, + "target": "data_stream" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.result" + }, + "target": "event" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f", + "snapshot": false, + "version": "8.13.1" + }, + "target": "elastic_agent" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f" + }, + "target": "agent" + } + } + ], + "type": "osquery" + }, + { + "data_stream": { + "dataset": "osquery_manager.action.responses", + "namespace": "default", + "type": "logs" + }, + "id": "osquery-osquery_manager.action.responses-74c7d0a8-ce04-4663-95da-ff6d537c268c", + "index": "logs-osquery_manager.action.responses-default", + "processors": [ + { + "add_fields": { + "fields": { + "input_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c" + }, + "target": "@metadata" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.action.responses", + "namespace": "default", + "type": "logs" + }, + "target": "data_stream" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.action.responses" + }, + "target": "event" + } + }, + { + "add_fields": { + "fields": { + "stream_id": "osquery-osquery_manager.action.responses-74c7d0a8-ce04-4663-95da-ff6d537c268c" + }, + "target": "@metadata" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f", + "snapshot": false, + "version": "8.13.1" + }, + "target": "elastic_agent" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f" + }, + "target": "agent" + } + } + ], + "type": "osquery" + } +] \ No newline at end of file diff --git a/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams_with_osquery.in.json b/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams_with_osquery.in.json new file mode 100644 index 000000000000..f6703263e6ad --- /dev/null +++ b/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams_with_osquery.in.json @@ -0,0 +1,109 @@ +{ + "source": { + "data_stream": { + "namespace": "default" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "meta": { + "package": { + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "name": "osquery_manager-1", + "osquery": { + "options": { + "host_identifier": "hostname" + } + }, + "package_policy_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "policy": { + "revision": 3 + }, + "revision": 2, + "streams": [ + { + "data_stream": { + "dataset": "osquery_manager.action.responses", + "type": "logs" + }, + "id": "osquery-osquery_manager.action.responses-74c7d0a8-ce04-4663-95da-ff6d537c268c", + "query": null + }, + { + "data_stream": { + "dataset": "osquery_manager.result", + "type": "logs" + }, + "id": null, + "query": null + } + ], + "type": "osquery" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "type": "osquery", + "name": "osquery_manager-1", + "revision": 2, + "meta": { + "source": { + "package": { + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "package": { + "source": { + "name": "osquery_manager", + "version": "1.12.1" + }, + "name": "osquery_manager", + "version": "1.12.1" + } + }, + "data_stream": { + "source": { + "namespace": "default" + }, + "namespace": "default" + }, + "streams": [ + { + "source": { + "data_stream": { + "dataset": "osquery_manager.action.responses", + "type": "logs" + }, + "id": "osquery-osquery_manager.action.responses-74c7d0a8-ce04-4663-95da-ff6d537c268c", + "query": null + }, + "id": "osquery-osquery_manager.action.responses-74c7d0a8-ce04-4663-95da-ff6d537c268c", + "data_stream": { + "source": { + "dataset": "osquery_manager.action.responses", + "type": "logs" + }, + "dataset": "osquery_manager.action.responses", + "type": "logs" + } + }, + { + "source": { + "data_stream": { + "dataset": "osquery_manager.result", + "type": "logs" + }, + "id": null, + "query": null + }, + "data_stream": { + "source": { + "dataset": "osquery_manager.result", + "type": "logs" + }, + "dataset": "osquery_manager.result", + "type": "logs" + } + } + ] +} \ No newline at end of file diff --git a/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams_with_osquery.out.json b/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams_with_osquery.out.json new file mode 100644 index 000000000000..aa4a70a74efe --- /dev/null +++ b/x-pack/osquerybeat/cmd/testdata/osquerycfg/two_streams_with_osquery.out.json @@ -0,0 +1,127 @@ +[ + { + "data_stream": { + "dataset": "osquery_manager.result", + "namespace": "default", + "type": "logs" + }, + "id": "74c7d0a8-ce04-4663-95da-ff6d537c268c", + "index": "logs-osquery_manager.result-default", + "osquery": { + "options": { + "host_identifier": "hostname" + } + }, + "processors": [ + { + "add_fields": { + "fields": { + "input_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c" + }, + "target": "@metadata" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.result", + "namespace": "default", + "type": "logs" + }, + "target": "data_stream" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.result" + }, + "target": "event" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f", + "snapshot": false, + "version": "8.13.1" + }, + "target": "elastic_agent" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f" + }, + "target": "agent" + } + } + ], + "type": "osquery" + }, + { + "data_stream": { + "dataset": "osquery_manager.action.responses", + "namespace": "default", + "type": "logs" + }, + "id": "osquery-osquery_manager.action.responses-74c7d0a8-ce04-4663-95da-ff6d537c268c", + "index": "logs-osquery_manager.action.responses-default", + "processors": [ + { + "add_fields": { + "fields": { + "input_id": "74c7d0a8-ce04-4663-95da-ff6d537c268c" + }, + "target": "@metadata" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.action.responses", + "namespace": "default", + "type": "logs" + }, + "target": "data_stream" + } + }, + { + "add_fields": { + "fields": { + "dataset": "osquery_manager.action.responses" + }, + "target": "event" + } + }, + { + "add_fields": { + "fields": { + "stream_id": "osquery-osquery_manager.action.responses-74c7d0a8-ce04-4663-95da-ff6d537c268c" + }, + "target": "@metadata" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f", + "snapshot": false, + "version": "8.13.1" + }, + "target": "elastic_agent" + } + }, + { + "add_fields": { + "fields": { + "id": "abc7d0a8-ce04-4663-95da-ff6d537c268f" + }, + "target": "agent" + } + } + ], + "type": "osquery" + } +] \ No newline at end of file diff --git a/x-pack/osquerybeat/internal/config/config.go b/x-pack/osquerybeat/internal/config/config.go index 0d23af8186fc..ec873206de75 100644 --- a/x-pack/osquerybeat/internal/config/config.go +++ b/x-pack/osquerybeat/internal/config/config.go @@ -22,9 +22,10 @@ import ( // query: select * from usb_devices const ( - DefaultNamespace = "default" - DefaultDataset = "osquery_manager.result" - DefaultType = "logs" + DefaultNamespace = "default" + DefaultDataset = "osquery_manager.result" + DefaultType = "logs" + DefaultActionResponsesDataset = "osquery_manager.action.responses" ) var datastreamPrefix = fmt.Sprintf("%s-%s-", DefaultType, DefaultDataset) diff --git a/x-pack/osquerybeat/internal/pub/publisher.go b/x-pack/osquerybeat/internal/pub/publisher.go index 6c49ef060170..d336a42515f9 100644 --- a/x-pack/osquerybeat/internal/pub/publisher.go +++ b/x-pack/osquerybeat/internal/pub/publisher.go @@ -26,8 +26,13 @@ type Publisher struct { b *beat.Beat log *logp.Logger - mx sync.Mutex + mx sync.Mutex + + // client for osquery_manager.result client beat.Client + + // client for osquery_manager.action.responses + actionResponsesClient beat.Client } func New(b *beat.Beat, log *logp.Logger) *Publisher { @@ -45,27 +50,64 @@ func (p *Publisher) Configure(inputs []config.InputConfig) error { p.mx.Lock() defer p.mx.Unlock() - processors, err := p.processorsForInputsConfig(inputs) - if err != nil { - return err - } + // Setup configuration pointers to the clients and corresponding default datasets + + // The osquery_manager.result is always first + if len(inputs) > 0 { + processors, err := p.processorsForInputConfig(inputs[0], config.DefaultDataset) + if err != nil { + return err + } + + p.log.Debugf("Connect publisher for %s with processors: %d", config.DefaultDataset, len(processors.All())) + // Connect publisher + client, err := p.b.Publisher.ConnectWith(beat.ClientConfig{ + Processing: beat.ProcessingConfig{ + Processor: processors, + }, + }) + if err != nil { + return err + } + + // Swap client + oldclient := p.client + p.client = client + if oldclient != nil { + oldclient.Close() + } - p.log.Debugf("Connect publisher with processors: %d", len(processors.All())) - // Connect publisher - client, err := p.b.Publisher.ConnectWith(beat.ClientConfig{ - Processing: beat.ProcessingConfig{ - Processor: processors, - }, - }) - if err != nil { - return err } - // Swap client - oldclient := p.client - p.client = client - if oldclient != nil { - oldclient.Close() + // Attach remaining DefaultActionResultsDataset if present + if len(inputs) > 1 { + processors, err := p.processorsForInputConfig(inputs[1], config.DefaultActionResponsesDataset) + if err != nil { + return err + } + + p.log.Debugf("Connect publisher for %s with processors: %d", config.DefaultActionResponsesDataset, len(processors.All())) + // Connect publisher + client, err := p.b.Publisher.ConnectWith(beat.ClientConfig{ + Processing: beat.ProcessingConfig{ + Processor: processors, + }, + }) + if err != nil { + return err + } + + // Swap client + oldclient := p.actionResponsesClient + p.actionResponsesClient = client + if oldclient != nil { + oldclient.Close() + } + } else { + if p.actionResponsesClient != nil { + p.actionResponsesClient.Close() + p.actionResponsesClient = nil + } } return nil } @@ -91,40 +133,93 @@ func (p *Publisher) Close() { } } -func (p *Publisher) processorsForInputsConfig(inputs []config.InputConfig) (procs *processors.Processors, err error) { +func (p *Publisher) PublishActionResult(req map[string]interface{}, res map[string]interface{}) { + p.mx.Lock() + defer p.mx.Unlock() + + if p.actionResponsesClient == nil { + p.log.Info("Action responses stream is not configured. Action response is dropped.") + return + } + + fields := actionResultToEvent(req, res) + event := beat.Event{ + Timestamp: time.Now(), + Fields: fields, + } + + p.log.Debugf("Action response event is sent, fields: %#v", fields) + + p.actionResponsesClient.Publish(event) +} + +func actionResultToEvent(req, res map[string]interface{}) map[string]interface{} { + m := make(map[string]interface{}, 8) + + copyKey := func(key string, src, dst map[string]interface{}) { + if v, ok := src[key]; ok { + dst[key] = v + } + } + + copyKey("started_at", res, m) + copyKey("completed_at", res, m) + copyKey("error", res, m) + + if v, ok := res["count"]; ok { + m["action_response"] = map[string]interface{}{ + "osquery": map[string]interface{}{ + "count": v, + }, + } + } + + if v, ok := req["id"]; ok { + m["action_id"] = v + } + + if v, ok := req["input_type"]; ok { + m["action_input_type"] = v + } + + if v, ok := req["data"]; ok { + m["action_data"] = v + } + + return m +} + +func (p *Publisher) processorsForInputConfig(inCfg config.InputConfig, defaultDataset string) (procs *processors.Processors, err error) { procs = processors.NewList(nil) // Use only first input processor // Every input will have a processor that adds the elastic_agent info, we need only one // Not expecting other processors at the moment and this needs to work for 7.13 - for _, input := range inputs { - if len(input.Processors) > 0 { - // Attach the data_stream processor. This will append the data_stream attributes to the events. - // This is needed for the proper logstash auto-discovery of the destination datastream for the results. - ds := add_data_stream.DataStream{ - Namespace: input.Datastream.Namespace, - Dataset: input.Datastream.Dataset, - Type: input.Datastream.Type, - } - if ds.Namespace == "" { - ds.Namespace = config.DefaultNamespace - } - if ds.Dataset == "" { - ds.Dataset = config.DefaultDataset - } - if ds.Type == "" { - ds.Type = config.DefaultType - } - - procs.AddProcessor(add_data_stream.New(ds)) - - userProcs, err := processors.New(input.Processors) - if err != nil { - return nil, err - } - procs.AddProcessors(*userProcs) - break + if len(inCfg.Processors) > 0 { + // Attach the data_stream processor. This will append the data_stream attributes to the events. + // This is needed for the proper logstash auto-discovery of the destination datastream for the results. + ds := add_data_stream.DataStream{ + Namespace: inCfg.Datastream.Namespace, + Dataset: inCfg.Datastream.Dataset, + Type: inCfg.Datastream.Type, + } + if ds.Namespace == "" { + ds.Namespace = config.DefaultNamespace + } + if ds.Dataset == "" { + ds.Dataset = defaultDataset + } + if ds.Type == "" { + ds.Type = config.DefaultType + } + + procs.AddProcessor(add_data_stream.New(ds)) + + userProcs, err := processors.New(inCfg.Processors) + if err != nil { + return nil, err } + procs.AddProcessors(*userProcs) } return procs, nil } diff --git a/x-pack/osquerybeat/internal/pub/publisher_test.go b/x-pack/osquerybeat/internal/pub/publisher_test.go index 488516bb01a7..4c34b667ff8b 100644 --- a/x-pack/osquerybeat/internal/pub/publisher_test.go +++ b/x-pack/osquerybeat/internal/pub/publisher_test.go @@ -5,6 +5,7 @@ package pub import ( + "encoding/json" "testing" "time" @@ -112,3 +113,92 @@ func TestHitToEvent(t *testing.T) { } } } + +func TestActionResultToEvent(t *testing.T) { + + tests := []struct { + name string + req, res map[string]interface{} + want map[string]interface{} + }{ + { + name: "successful", + req: toMap(t, `{ + "data": { + "id": "a72d65d8-200a-4b43-8dbd-7bc0e9ce8e65", + "query": "select * from osquery_info" + }, + "id": "5c433f88-ab0d-41e2-af76-6ff16ae3ced8", + "input_type": "osquery", + "type": "INPUT_ACTION" + }`), + res: toMap(t, `{ + "completed_at": "2024-04-18T19:39:39.740162Z", + "count": 1, + "started_at": "2024-04-18T19:39:39.532125Z" + } `), + // "agent_id": "bf3d6036-2260-4bbf-94a3-5ccce0d75d9e", + want: toMap(t, `{ + "completed_at": "2024-04-18T19:39:39.740162Z", + "action_response": { + "osquery": { + "count": 1 + } + }, + "action_id": "5c433f88-ab0d-41e2-af76-6ff16ae3ced8", + "started_at": "2024-04-18T19:39:39.532125Z", + "action_input_type": "osquery", + "action_data": { + "id": "a72d65d8-200a-4b43-8dbd-7bc0e9ce8e65", + "query": "select * from osquery_info" + } + }`), + }, + { + name: "error", + req: toMap(t, `{ + "data": { + "id": "08995ee8-5182-423e-9527-552736411010", + "query": "select * from osquery_foo" + }, + "id": "70539d80-4082-41e9-aff4-fbb877dd752b", + "input_type": "osquery", + "type": "INPUT_ACTION" + }`), + res: toMap(t, `{ + "completed_at": "2024-04-20T14:56:34.87195Z", + "error": "query failed, code: 1, message: no such table: osquery_foo", + "started_at": "2024-04-20T14:56:34.87195Z" + }`), + // "agent_id": "bf3d6036-2260-4bbf-94a3-5ccce0d75d9e", + want: toMap(t, `{ + "completed_at": "2024-04-20T14:56:34.87195Z", + "action_id": "70539d80-4082-41e9-aff4-fbb877dd752b", + "started_at": "2024-04-20T14:56:34.87195Z", + "action_input_type": "osquery", + "error": "query failed, code: 1, message: no such table: osquery_foo", + "action_data": { + "id": "08995ee8-5182-423e-9527-552736411010", + "query": "select * from osquery_foo" + } + }`), + }, + } + + for _, tc := range tests { + got := actionResultToEvent(tc.req, tc.res) + diff := cmp.Diff(tc.want, got) + if diff != "" { + t.Error(diff) + } + } +} + +func toMap(t *testing.T, s string) map[string]interface{} { + var m map[string]interface{} + err := json.Unmarshal([]byte(s), &m) + if err != nil { + t.Fatal(err) + } + return m +}