From 831555793a66737003e4a795c0ba52e3bb7c4121 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 4 Sep 2023 14:12:12 +0200 Subject: [PATCH] Sonarqube buildkite integration (#3108) * Sonarqube buildkite integration * Sonarqube buildkite integration * Fixed git log * Run unit tests on a VM * Execute ut on main branch * Removed sonarcloud github workflow * Added retry to go download * removed CodeCoverage (build.yml) GHA workflow * Run buildkite unit tests on each pull request * Run buildkite unit tests on each pull request * Run buildkite unit tests on each pull request * Run buildkite unit tests on each pull request * Run buildkite unit tests on each pull request * Added a test file for sonar analysis * Added a test file for sonar analysis * cleanup * Merged with main * Added echo to unit tests * Added echo to unit tests * Fix common.sh logging * Try old way on go installation * Removed comments * check: added not covered file to test sonar notifications * removed the test file (cherry picked from commit 2e9a256297b85b015c388dbd5dd87248fde4d8c3) --- .buildkite/hooks/post-checkout | 4 +- .buildkite/pipeline.yml | 27 ++++++ .buildkite/scripts/common.sh | 4 +- .buildkite/scripts/steps/unit-tests.sh | 7 ++ .github/workflows/build.yml | 55 ------------ .github/workflows/sonarcloud.yml | 120 ------------------------- sonar-project.properties | 7 +- 7 files changed, 41 insertions(+), 183 deletions(-) create mode 100644 .buildkite/scripts/steps/unit-tests.sh delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/sonarcloud.yml diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 6e50f6ecd34..e10f15de7b6 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -46,8 +46,8 @@ MERGE_BRANCH="pr_merge_${PR_ID}" checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" -echo "--- Commit information" -git log --format=%B -n 1 +echo "Commit information" +git --no-pager log --format=%B -n 1 # Ensure buildkite groups are rendered echo "" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0d2db4356a5..256a351e8ff 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,6 +4,33 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" DOCKER_REGISTRY: "docker.elastic.co" steps: + - label: "Unit tests" + key: "unit-tests" + command: ".buildkite/scripts/steps/unit-tests.sh" + artifact_paths: + - "build/TEST-**" + - "build/diagnostics/*" + agents: + provider: "gcp" + retry: + manual: + allowed: true + + - label: ":sonarqube: Continuous Code Inspection" + env: + VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token" + agents: + image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" + command: + - "buildkite-agent artifact download build/TEST-go-unit.cov ." + - "buildkite-agent artifact download build/TEST-go-unit.out.json ." + - "/scan-source-code.sh" + depends_on: + - "unit-tests" + retry: + manual: + allowed: true + - label: "Integration tests" key: "integration-tests" command: ".buildkite/scripts/steps/integration_tests.sh" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 65f06f19df6..08fd8aeff49 100644 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -55,7 +55,7 @@ mage() { go version if ! [ -x "$(type -p mage | sed 's/mage is //g')" ]; then - echo "--- installing mage ${SETUP_MAGE_VERSION}" + echo "installing mage ${SETUP_MAGE_VERSION}" make mage fi pushd "$WORKSPACE" @@ -71,7 +71,7 @@ go(){ if ! [ -x "$(type -p go | sed 's/go is //g')" ]; then getOSOptions - echo "--- installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" " + echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" " local _bin="${WORKSPACE}/bin" mkdir -p "${_bin}" retry 5 curl -sL -o "${_bin}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${AGENT_OS_NAME}-${AGENT_OS_ARCH}" diff --git a/.buildkite/scripts/steps/unit-tests.sh b/.buildkite/scripts/steps/unit-tests.sh new file mode 100644 index 00000000000..b4d399801bf --- /dev/null +++ b/.buildkite/scripts/steps/unit-tests.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euxo pipefail + +source .buildkite/scripts/common.sh + +echo "--- Unit tests" +TEST_COVERAGE=true mage unitTest \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 730a4759026..00000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: CodeCoverage Build -on: - pull_request: - -jobs: - coverage: - name: CodeCoverage - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - - name: Extract Go version from .go-version - run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: "${{ env.GO_VERSION }}" - cache: true - - - name: Go cache - uses: actions/cache@v3 - with: - # In order: - # * Module download cache - # * Build cache (Linux) - path: | - ~/go/pkg/mod - key: ${{ runner.os }}-go-cache-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-cache - - - name: Run Mage - uses: magefile/mage-action@v2 - env: - TEST_COVERAGE: 'true' - with: - version: latest - args: unitTest - - - name: Save PR information - run: | - mkdir -p ./pr - echo ${{ github.event.number }} > ./build/pull_request_number - echo ${{ github.event.pull_request.base.ref }} > ./build/pull_request_base - echo ${{ github.event.pull_request.head.ref }} > ./build/pull_request_branch - - - uses: actions/upload-artifact@v3 - with: - name: test-coverage - path: | - build/TEST-* - build/pull_request* - if-no-files-found: error diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml deleted file mode 100644 index 486cb8d9093..00000000000 --- a/.github/workflows/sonarcloud.yml +++ /dev/null @@ -1,120 +0,0 @@ -name: SonarCloud - -on: - workflow_dispatch: - push: - branches: - - main - workflow_run: - workflows: ["CodeCoverage Build"] - types: - - completed - -jobs: - coverage: - name: SonarCloud - runs-on: ubuntu-latest - if: > - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - repository: '${{ github.event.workflow_run.head_repository.full_name }}' - ref: '${{ github.event.workflow_run.head_branch }}' - - - name: 'Download artifact' - uses: actions/github-script@v3.1.0 - with: - script: | - var artifacts = await github.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{github.event.workflow_run.id }}, - }); - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "test-coverage" - })[0]; - var download = await github.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/test-coverage.zip', Buffer.from(download.data)); - - run: unzip -o test-coverage.zip -d build - - - name: Retrieve the pr number - id: pr-number - uses: actions/github-script@v6 - with: - result-encoding: string - script: | - var fs = require('fs'); - var pr_number = Number(fs.readFileSync('./build/pull_request_number')); - return pr_number; - - - name: Retrieve the pr base - id: pr-base - uses: actions/github-script@v6 - with: - result-encoding: string - script: | - var fs = require('fs'); - var pr_base = fs.readFileSync('./build/pull_request_base'); - return pr_base; - - - name: Retrieve the pr branch - id: pr-branch - uses: actions/github-script@v6 - with: - result-encoding: string - script: | - var fs = require('fs'); - var pr_base = fs.readFileSync('./build/pull_request_branch'); - return pr_base; - - - name: SonarCloud Scan - uses: SonarSource/sonarcloud-github-action@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - with: - args: > - -Dsonar.scm.revision=${{ github.event.workflow_run.head_sha }} - -Dsonar.pullrequest.key=${{ steps.pr-number.outputs.result }} - -Dsonar.pullrequest.branch=${{ steps.pr-branch.outputs.result }} - -Dsonar.pullrequest.base=${{ steps.pr-base.outputs.result }} - - branch-coverage: - name: Branch SonarCloud - runs-on: ubuntu-latest - if: > - github.event_name == 'push' || - github.event_name == 'workflow_dispatch' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version-file: 'go.mod' - cache: true - - - name: Run Mage - uses: magefile/mage-action@v2 - env: - TEST_COVERAGE: 'true' - with: - version: latest - args: unitTest - - - name: SonarCloud Scan - uses: SonarSource/sonarcloud-github-action@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/sonar-project.properties b/sonar-project.properties index da643cd741e..ae93ac43218 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,6 +1,5 @@ -sonar.organization=elastic-1 -sonar.projectKey=elastic_elastic-agent -sonar.host.url=https://sonarcloud.io +sonar.projectKey=elastic_elastic-agent_AYluowg0xMq8P7b4moiZ +sonar.host.url=https://sonar.elastic.dev sonar.sources=. sonar.exclusions=**/*_test.go, .git/**, dev-tools/**, /magefile.go, changelog/**, _meta/**, deploy/**, docs/**, img/**, specs/**, pkg/testing/** @@ -13,7 +12,7 @@ sonar.go.exclusions=**/vendor/**,**/*_mock.go sonar.sourceEncoding=UTF-8 -#Prevent C analysis +# Prevent C analysis sonar.c.file.suffixes=- sonar.cpp.file.suffixes=- sonar.objc.file.suffixes=-