Skip to content

Commit

Permalink
⭐ Add output handler to store asset reports (#982)
Browse files Browse the repository at this point in the history
* ⭐ Add output handler to store asset reports.
* ✨ Rework output target handling.

---------

Signed-off-by: Preslav <[email protected]>
  • Loading branch information
preslavgerchev authored Dec 20, 2023
1 parent c6825f1 commit 2034124
Show file tree
Hide file tree
Showing 15 changed files with 427 additions and 103 deletions.
64 changes: 36 additions & 28 deletions apps/cnspec/cmd/scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ func init() {
scanCmd.Flags().MarkHidden("category")
scanCmd.Flags().Int("score-threshold", 0, "If any score falls below the threshold, exit 1.")
scanCmd.Flags().Bool("share", false, "create a web-based private reports when cnspec is unauthenticated. Defaults to false.")
scanCmd.Flags().String("output-target", "", "Set output target to which the asset report will be sent. Currently only supports AWS SQS topic URLs and local files")
}

var scanCmd = &cobra.Command{
Expand All @@ -79,7 +80,7 @@ To manually configure a policy, use this:
$ cnspec scan local -f bundle.mql.yaml --incognito
`,
PreRun: func(cmd *cobra.Command, args []string) {
PreRunE: func(cmd *cobra.Command, args []string) error {
// Special handling for users that want to see what output options are
// available. We have to do this before printing the help because we
// don't have a target connection or provider.
Expand Down Expand Up @@ -109,6 +110,11 @@ To manually configure a policy, use this:
viper.BindPFlag("record", cmd.Flags().Lookup("record"))

viper.BindPFlag("output", cmd.Flags().Lookup("output"))
if err := viper.BindPFlag("output-target", cmd.Flags().Lookup("output-target")); err != nil {
return err
}

return nil
},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if len(args) == 0 {
Expand All @@ -119,12 +125,13 @@ To manually configure a policy, use this:
}

var scanCmdRun = func(cmd *cobra.Command, runtime *providers.Runtime, cliRes *plugin.ParseCLIRes) {
ctx := context.Background()
conf, err := getCobraScanConfig(cmd, runtime, cliRes)
if err != nil {
log.Fatal().Err(err).Msg("failed to prepare config")
}

err = conf.loadPolicies()
err = conf.loadPolicies(ctx)
if err != nil {
log.Fatal().Err(err).Msg("failed to resolve policies")
}
Expand All @@ -135,7 +142,19 @@ var scanCmdRun = func(cmd *cobra.Command, runtime *providers.Runtime, cliRes *pl
}

logger.DebugDumpJSON("report", report)
printReports(report, conf, cmd)

handlerConf := reporter.HandlerConfig{
Format: conf.OutputFormat,
OutputTarget: conf.OutputTarget,
Incognito: conf.IsIncognito,
}
outputHandler, err := reporter.NewOutputHandler(handlerConf)
if err != nil {
log.Fatal().Err(err).Msg("failed to create an output handler")
}
if err := outputHandler.WriteReport(ctx, report); err != nil {
log.Fatal().Err(err).Msg("failed to write report to output target")
}

var shareReport bool
if viper.IsSet("share") {
Expand Down Expand Up @@ -181,15 +200,16 @@ func getPoliciesForCompletion() []string {
}

type scanConfig struct {
Features cnquery.Features
Inventory *inventory.Inventory
ReportType scan.ReportType
Output string
PolicyPaths []string
PolicyNames []string
Props map[string]string
Bundle *policy.Bundle
runtime *providers.Runtime
Features cnquery.Features
Inventory *inventory.Inventory
ReportType scan.ReportType
OutputTarget string
OutputFormat string
PolicyPaths []string
PolicyNames []string
Props map[string]string
Bundle *policy.Bundle
runtime *providers.Runtime

IsIncognito bool
ScoreThreshold int
Expand Down Expand Up @@ -245,6 +265,7 @@ func getCobraScanConfig(cmd *cobra.Command, runtime *providers.Runtime, cliRes *
Props: props,
runtime: runtime,
AgentMrn: opts.AgentMrn,
OutputTarget: viper.GetString("output-target"),
}

// if users want to get more information on available output options,
Expand All @@ -259,7 +280,7 @@ func getCobraScanConfig(cmd *cobra.Command, runtime *providers.Runtime, cliRes *
if ok, _ := cmd.Flags().GetBool("json"); ok {
output = "json"
}
conf.Output = output
conf.OutputFormat = output

// detect CI/CD runs and read labels from runtime and apply them to all assets in the inventory
runtimeEnv := execruntime.Detect()
Expand Down Expand Up @@ -313,7 +334,7 @@ func getCobraScanConfig(cmd *cobra.Command, runtime *providers.Runtime, cliRes *
return &conf, nil
}

func (c *scanConfig) loadPolicies() error {
func (c *scanConfig) loadPolicies(ctx context.Context) error {
if c.IsIncognito {
if len(c.PolicyPaths) == 0 {
return nil
Expand All @@ -327,7 +348,7 @@ func (c *scanConfig) loadPolicies() error {

bundle.ConvertQuerypacks()

_, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{
_, err = bundle.CompileExt(ctx, policy.BundleCompileConf{
Schema: c.runtime.Schema(),
// We don't care about failing queries for local runs. We may only
// process a subset of all the queries in the bundle. When we receive
Expand Down Expand Up @@ -386,19 +407,6 @@ func RunScan(config *scanConfig, scannerOpts ...scan.ScannerOption) (*policy.Rep
return res.GetFull(), nil
}

func printReports(report *policy.ReportCollection, conf *scanConfig, cmd *cobra.Command) {
// print the output using the specified output format
r, err := reporter.New(conf.Output)
if err != nil {
log.Fatal().Msg(err.Error())
}

r.IsIncognito = conf.IsIncognito
if err = r.Print(report, os.Stdout); err != nil {
log.Fatal().Err(err).Msg("failed to print")
}
}

func dedupe[T string | int](sliceList []T) []T {
allKeys := make(map[T]bool)
list := []T{}
Expand Down
12 changes: 6 additions & 6 deletions apps/cnspec/cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,12 +140,12 @@ func getServeConfig() (*scanConfig, error) {
runtime := providers.DefaultRuntime()

conf := scanConfig{
Features: opts.GetFeatures(),
DoRecord: viper.GetBool("record"),
ReportType: scan.ReportType_ERROR,
Output: "",
runtime: runtime,
AgentMrn: opts.AgentMrn,
Features: opts.GetFeatures(),
DoRecord: viper.GetBool("record"),
ReportType: scan.ReportType_ERROR,
OutputFormat: "",
runtime: runtime,
AgentMrn: opts.AgentMrn,
}

// detect CI/CD runs and read labels from runtime and apply them to all assets in the inventory
Expand Down
8 changes: 2 additions & 6 deletions apps/cnspec/cmd/vuln.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package cmd

import (
"encoding/json"
"os"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -164,13 +163,10 @@ var vulnCmdRun = func(cmd *cobra.Command, runtime *providers.Runtime, cliRes *pl

func printVulns(report *mvd.VulnReport, conf *scanConfig, target string) {
// print the output using the specified output format
r, err := reporter.New("full")
if err != nil {
log.Fatal().Msg(err.Error())
}
r := reporter.NewReporter(reporter.Full, false)

logger.DebugDumpJSON("vulnReport", report)
if err = r.PrintVulns(report, os.Stdout, target); err != nil {
if err := r.PrintVulns(report, target); err != nil {
log.Fatal().Err(err).Msg("failed to print")
}
}
57 changes: 57 additions & 0 deletions cli/reporter/aws_sqs_handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// Copyright (c) Mondoo, Inc.
// SPDX-License-Identifier: BUSL-1.1

package reporter

import (
"context"
"fmt"
"regexp"
"strings"

"github.com/rs/zerolog/log"
"go.mondoo.com/cnspec/v9/policy"
"gocloud.dev/pubsub"
_ "gocloud.dev/pubsub/awssnssqs"
)

var sqsRegex = regexp.MustCompile(`(https:\/\/|http:\/\/)?(sqs)[.][a-z]{2}[-][a-z]{3,}[-][0-9]{1}[.](amazonaws.com)[\/][0-9]{12}[\/]{1}[a-zA-Z0-9-_]*`)

type awsSqsHandler struct {
sqsQueueUrl string
format Format
}

func (h *awsSqsHandler) WriteReport(ctx context.Context, report *policy.ReportCollection) error {
// the url may be passed in with a https:// or an http:// prefix, we can trim those
trimmedUrl := strings.TrimPrefix(h.sqsQueueUrl, "https://")
trimmedUrl = strings.TrimPrefix(trimmedUrl, "http://")
topic, err := pubsub.OpenTopic(ctx, "awssqs://"+trimmedUrl)
if err != nil {
return err
}
defer topic.Shutdown(ctx) //nolint: errcheck
data, err := h.convertReport(report)
if err != nil {
return err
}
err = topic.Send(ctx, &pubsub.Message{
Body: data,
})
if err != nil {
return err
}
log.Info().Str("url", h.sqsQueueUrl).Msg("sent report to SQS queue")
return nil
}

func (h *awsSqsHandler) convertReport(report *policy.ReportCollection) ([]byte, error) {
switch h.format {
case YAML:
return reportToYaml(report)
case JSON:
return reportToJson(report)
default:
return nil, fmt.Errorf("'%s' is not supported in the aws sqs handler, please use one of the other formats", string(h.format))
}
}
Loading

0 comments on commit 2034124

Please sign in to comment.