✨ add benchmark test for scans (#1067)

---------

Signed-off-by: Ivan Milchev <[email protected]>

.github/workflows/main-benchmark.yml

@@ -0,0 +1,63 @@
+name: Benchmark main
+
+## Only trigger tests if source is changing
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - '**.go'
+      - '**.mod'
+      - 'go.sum'
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+jobs:
+  go-bench:
+    runs-on: ubuntu-latest
+    env:
+      BRANCH_NAME: ${{ github.head_ref || github.ref_name }} 
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Import environment variables from file
+        run: cat ".github/env" >> $GITHUB_ENV
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ">=${{ env.golang-version }}"
+          cache: false
+      - name: Run benchmark
+        run: make benchmark/go | tee benchmark.txt
+
+        # Remove log statements and leave just the benchmark results
+      - name: Cleanup benchmark file
+        run: sed -i -n '/goos:/,$p' benchmark.txt
+
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache/restore@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'go'
+          # Where the output from the benchmark tool is stored
+          output-file-path: benchmark.txt
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json
+          save-data-file: false
+
+      - name: Download previous benchmark data
+        uses: actions/cache/restore@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark

.github/workflows/pr-test-lint.yml

@@ -91,6 +91,49 @@ jobs:
         name: Event File
         path: ${{ github.event_path }}
 
+  go-bench:
+    runs-on: ubuntu-latest
+    if: github.ref != 'refs/heads/main'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Import environment variables from file
+        run: cat ".github/env" >> $GITHUB_ENV
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ">=${{ env.golang-version }}"
+          cache: false
+      - name: Run benchmark
+        run: make benchmark/go | tee benchmark.txt
+
+        # Remove log statements and leave just the benchmark results
+      - name: Cleanup benchmark file
+        run: sed -i -n '/goos:/,$p' benchmark.txt
+
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache/restore@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'go'
+          # Where the output from the benchmark tool is stored
+          output-file-path: benchmark.txt
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+          summary-always: true
+          fail-on-alert: true
+          save-data-file: false
+          alert-threshold: '150%'
+
   license-check:
     runs-on: ubuntu-latest
     steps:

Makefile

@@ -113,6 +113,9 @@ test/lint: test/lint/golangci-lint/run
 
 test: test/go test/lint
 
+benchmark/go:
+	go test -bench=. -benchmem go.mondoo.com/cnspec/v10/policy/scan/benchmark
+
 test/go: cnspec/generate test/go/plain
 
 test/go/plain:

policy/scan/benchmark/benchmark_test.go

@@ -0,0 +1,121 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package benchmark
+
+import (
+	"context"
+	"testing"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+
+	"github.com/stretchr/testify/require"
+	"go.mondoo.com/cnquery/v10"
+	"go.mondoo.com/cnquery/v10/mqlc"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/inventory"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/testutils"
+	"go.mondoo.com/cnspec/v10/policy"
+	"go.mondoo.com/cnspec/v10/policy/scan"
+)
+
+func init() {
+	log.Logger = log.Logger.Level(zerolog.Disabled)
+	zerolog.SetGlobalLevel(zerolog.Disabled)
+}
+
+func BenchmarkScan_SingleAsset(b *testing.B) {
+	ctx := context.Background()
+	runtime := testutils.Local()
+	conf := mqlc.NewConfig(runtime.Schema(), cnquery.DefaultFeatures)
+	job := &scan.Job{
+		Inventory: &inventory.Inventory{
+			Spec: &inventory.InventorySpec{
+				Assets: []*inventory.Asset{
+					{
+						Connections: []*inventory.Config{
+							{
+								Type: "k8s",
+								Options: map[string]string{
+									"path": "../testdata/1pod.yaml",
+								},
+								Discover: &inventory.Discovery{
+									Targets: []string{"pods"},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	loader := policy.DefaultBundleLoader()
+	bundle, err := loader.BundleFromPaths("../testdata/kubernetes-security.mql.yaml")
+	require.NoError(b, err)
+
+	_, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{
+		CompilerConfig: conf,
+		RemoveFailing:  true,
+	})
+	require.NoError(b, err)
+
+	job.Bundle = bundle
+
+	scanner := scan.NewLocalScanner(scan.DisableProgressBar())
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		res, err := scanner.RunIncognito(ctx, job)
+		require.NoError(b, err)
+		require.NotNil(b, res)
+	}
+}
+
+func BenchmarkScan_MultipleAssets(b *testing.B) {
+	ctx := context.Background()
+	runtime := testutils.Local()
+	conf := mqlc.NewConfig(runtime.Schema(), cnquery.DefaultFeatures)
+	job := &scan.Job{
+		Inventory: &inventory.Inventory{
+			Spec: &inventory.InventorySpec{
+				Assets: []*inventory.Asset{
+					{
+						Connections: []*inventory.Config{
+							{
+								Type: "k8s",
+								Options: map[string]string{
+									"path": "../testdata/2pods.yaml",
+								},
+								Discover: &inventory.Discovery{
+									Targets: []string{"pods"},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	loader := policy.DefaultBundleLoader()
+	bundle, err := loader.BundleFromPaths("../testdata/kubernetes-security.mql.yaml")
+	require.NoError(b, err)
+
+	_, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{
+		CompilerConfig: conf,
+		RemoveFailing:  true,
+	})
+	require.NoError(b, err)
+
+	job.Bundle = bundle
+
+	scanner := scan.NewLocalScanner(scan.DisableProgressBar())
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		res, err := scanner.RunIncognito(ctx, job)
+		require.NoError(b, err)
+		require.NotNil(b, res)
+	}
+}