✨ go benchmark for scans (#3159)

* go benchmark for scans

Signed-off-by: Ivan Milchev <[email protected]>

* move benchmark module to the correct location

Signed-off-by: Ivan Milchev <[email protected]>

* allow disabling of the progress bar

Signed-off-by: Ivan Milchev <[email protected]>

* fix benchmark

Signed-off-by: Ivan Milchev <[email protected]>

* fix linter

Signed-off-by: Ivan Milchev <[email protected]>

---------

Signed-off-by: Ivan Milchev <[email protected]>

.github/workflows/main-benchmark.yml

@@ -0,0 +1,64 @@
+name: Benchmark main
+
+## Only trigger tests if source is changing
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - '**.go'
+      - '**.mod'
+      - 'go.sum'
+      - .github/workflows/main-benchmark.yml
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+jobs:
+  go-bench:
+    runs-on: ubuntu-latest
+    env:
+      BRANCH_NAME: ${{ github.head_ref || github.ref_name }} 
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Import environment variables from file
+        run: cat ".github/env" >> $GITHUB_ENV
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ">=${{ env.golang-version }}"
+          cache: false
+      - name: Run benchmark
+        run: make benchmark/go | tee benchmark.txt
+
+        # Remove log statements and leave just the benchmark results
+      - name: Cleanup benchmark file
+        run: sed -i -n '/goos:/,$p' benchmark.txt
+
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache/restore@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'go'
+          # Where the output from the benchmark tool is stored
+          output-file-path: benchmark.txt
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json
+          save-data-file: true
+
+      - name: Save benchmark data
+        uses: actions/cache/save@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark

.github/workflows/pr-test-lint.yml

@@ -90,6 +90,49 @@ jobs:
         with:
           name: test-results
           path: report.xml
+
+  go-bench:
+    runs-on: ubuntu-latest
+    if: github.ref != 'refs/heads/main'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Import environment variables from file
+        run: cat ".github/env" >> $GITHUB_ENV
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ">=${{ env.golang-version }}"
+          cache: false
+      - name: Run benchmark
+        run: make benchmark/go | tee benchmark.txt
+
+        # Remove log statements and leave just the benchmark results
+      - name: Cleanup benchmark file
+        run: sed -i -n '/goos:/,$p' benchmark.txt
+
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache/restore@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'go'
+          # Where the output from the benchmark tool is stored
+          output-file-path: benchmark.txt
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+          summary-always: true
+          fail-on-alert: true
+          save-data-file: false
+          alert-threshold: '150%'
 
   event_file:
     name: "Store event file"

Makefile

@@ -594,6 +594,9 @@ test/lint: test/lint/golangci-lint/run
 
 test: test/go test/lint
 
+benchmark/go:
+	go test -bench=. -benchmem go.mondoo.com/cnquery/v10/explorer/scan/benchmark
+
 test/go: cnquery/generate test/go/plain
 
 test/go/plain:

explorer/scan/benchmark/benchmark_test.go

@@ -0,0 +1,119 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package benchmark
+
+import (
+	"context"
+	"testing"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+
+	"github.com/stretchr/testify/require"
+	"go.mondoo.com/cnquery/v10"
+	"go.mondoo.com/cnquery/v10/explorer"
+	"go.mondoo.com/cnquery/v10/explorer/scan"
+	"go.mondoo.com/cnquery/v10/mqlc"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/inventory"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/testutils"
+)
+
+func init() {
+	log.Logger = log.Logger.Level(zerolog.Disabled)
+	zerolog.SetGlobalLevel(zerolog.Disabled)
+}
+
+func BenchmarkScan_SingleAsset(b *testing.B) {
+	ctx := context.Background()
+	runtime := testutils.Local()
+	conf := mqlc.NewConfig(runtime.Schema(), cnquery.DefaultFeatures)
+	job := &scan.Job{
+		Inventory: &inventory.Inventory{
+			Spec: &inventory.InventorySpec{
+				Assets: []*inventory.Asset{
+					{
+						Connections: []*inventory.Config{
+							{
+								Type: "k8s",
+								Options: map[string]string{
+									"path": "./testdata/1pod.yaml",
+								},
+								Discover: &inventory.Discovery{
+									Targets: []string{"pods"},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundle, err := explorer.BundleFromPaths("./testdata/mondoo-kubernetes-inventory.mql.yaml")
+	require.NoError(b, err)
+
+	_, err = bundle.CompileExt(context.Background(), explorer.BundleCompileConf{
+		CompilerConfig: conf,
+		RemoveFailing:  true,
+	})
+	require.NoError(b, err)
+
+	job.Bundle = bundle
+
+	scanner := scan.NewLocalScanner(scan.DisableProgressBar())
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		res, err := scanner.RunIncognito(ctx, job)
+		require.NoError(b, err)
+		require.NotNil(b, res)
+	}
+}
+
+func BenchmarkScan_MultipleAssets(b *testing.B) {
+	ctx := context.Background()
+	runtime := testutils.Local()
+	conf := mqlc.NewConfig(runtime.Schema(), cnquery.DefaultFeatures)
+	job := &scan.Job{
+		Inventory: &inventory.Inventory{
+			Spec: &inventory.InventorySpec{
+				Assets: []*inventory.Asset{
+					{
+						Connections: []*inventory.Config{
+							{
+								Type: "k8s",
+								Options: map[string]string{
+									"path": "./testdata/2pods.yaml",
+								},
+								Discover: &inventory.Discovery{
+									Targets: []string{"pods"},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundle, err := explorer.BundleFromPaths("./testdata/mondoo-kubernetes-inventory.mql.yaml")
+	require.NoError(b, err)
+
+	_, err = bundle.CompileExt(context.Background(), explorer.BundleCompileConf{
+		CompilerConfig: conf,
+		RemoveFailing:  true,
+	})
+	require.NoError(b, err)
+
+	job.Bundle = bundle
+
+	scanner := scan.NewLocalScanner(scan.DisableProgressBar())
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		res, err := scanner.RunIncognito(ctx, job)
+		require.NoError(b, err)
+		require.NotNil(b, res)
+	}
+}

explorer/scan/benchmark/testdata/1pod.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    admission-result: pass
+  name: passing-pod-yaml
+  namespace: default
+spec:
+  automountServiceAccountToken: false
+  containers:
+  - image: ubuntu:20.04
+    imagePullPolicy: Always
+    command: ["/bin/sh", "-c"]
+    args: ["sleep 6000"]
+    name: ubuntu

explorer/scan/benchmark/testdata/2pods.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    admission-result: pass
+  name: passing-pod-yaml
+  namespace: default
+spec:
+  automountServiceAccountToken: false
+  containers:
+  - image: ubuntu:20.04
+    imagePullPolicy: Always
+    command: ["/bin/sh", "-c"]
+    args: ["sleep 6000"]
+    name: ubuntu
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    admission-result: pass
+  name: passing-pod-yaml-2
+  namespace: default
+spec:
+  automountServiceAccountToken: false
+  containers:
+  - image: ubuntu:20.04
+    imagePullPolicy: Always
+    command: ["/bin/sh", "-c"]
+    args: ["sleep 6000"]
+    name: ubuntu