From 02cf716507a73e54ba66b2e5a205cb4f9c8749d6 Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Fri, 15 Sep 2023 08:28:09 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20add=20configuration=20for=20prov?= =?UTF-8?q?iders=20in=20dev=20env=20(#1738)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This has now happened a few too many times: You get a PR with a change to a provider, but forgot to update your builtin.go file. After 30min you realize you have been testing the providers on disk, instead of the ones in the repo. Also configuring providers is a bit annoying (with all the things you need to change in `builtin.go`). This is now streamlined in `make providers/config`. You can configure it in `providers.yaml` in the root folder. --------- Signed-off-by: Dominik Richter --- Makefile | 7 +- docs/development.md | 34 +-- providers-sdk/v1/util/configure/configure.go | 206 +++++++++++++++++++ providers.yaml | 15 ++ providers/builtin.go | 84 +------- providers/coordinator.go | 8 + providers/providers.go | 17 ++ 7 files changed, 263 insertions(+), 108 deletions(-) create mode 100644 providers-sdk/v1/util/configure/configure.go create mode 100644 providers.yaml diff --git a/Makefile b/Makefile index 9845e9e2db..950781139e 100644 --- a/Makefile +++ b/Makefile @@ -135,7 +135,7 @@ define gomodtidyProvider endef .PHONY: providers -providers: providers/proto providers/build +providers: providers/proto providers/config providers/build .PHONY: providers/proto providers/proto: @@ -145,6 +145,11 @@ providers/proto: go generate ./providers-sdk/v1/inventory go generate ./providers-sdk/v1/plugin +.PHONY: providers/config +providers/config: + go run ./providers-sdk/v1/util/configure/configure.go -f providers.yaml -o providers/builtin.go + gofmt -w providers/builtin.go + .PHONY: providers/lr providers/lr: go build -o lr ./providers-sdk/v1/lr/cli/main.go diff --git a/docs/development.md b/docs/development.md index 4d91ec0c7a..01f1ad19f2 100644 --- a/docs/development.md +++ b/docs/development.md @@ -60,34 +60,16 @@ In v9 we introduced providers, which split up the providers into individual go m To debug a provider locally with cnquery: -1. Copy the resources json file `providers/TYPE/resources/TYPE.resources.json` to the `providers` top-level dir -2. Add the provider to `providers/builtin.go` - +1. Modify the `providers.yaml` in the root folder and add providers you want to test to the `builtin` field. Example: + ```yaml + builtin: [aws] ``` - awsconf "go.mondoo.com/cnquery/providers/aws/config" - awsp "go.mondoo.com/cnquery/providers/aws/provider" - - //go:embed aws.resources.json - var awsInfo []byte - - awsconf.Config.ID: { - Runtime: &RunningProvider{ - Name: awsconf.Config.Name, - ID: awsconf.Config.ID, - Plugin: awsp.Init(), - Schema: MustLoadSchema("aws", awsInfo), - isClosed: false, - }, - Config: &awsconf.Config, - }, +2. Build and update everything via: + ```bash + make providers/config ``` - -3. Change the local provider location in `go.mod` - - `replace go.mondoo.com/cnquery/providers/aws => ./providers/aws` - -4. Build the provider after making changes: `make providers/build/aws` -5. Run cnquery like you normally do (`go run apps/cnquery/cnquery.go shell aws`). You should see it note that it is "using builtin provider for X" +3. You can now use and debug your code. For example `make cnquery/install` or start a debugger. +4. Once done, please remember to restore `providers.yaml` (or just set back: `builtin: []`) and re-run `make providers/config`. ## Contribute changes diff --git a/providers-sdk/v1/util/configure/configure.go b/providers-sdk/v1/util/configure/configure.go new file mode 100644 index 0000000000..77eb1b50bb --- /dev/null +++ b/providers-sdk/v1/util/configure/configure.go @@ -0,0 +1,206 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package main + +import ( + "fmt" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "go.mondoo.com/cnquery/logger" + "sigs.k8s.io/yaml" +) + +type ProvidersConf struct { + Builtin []string `json:"builtin"` +} + +var rootCmd = &cobra.Command{ + Use: "configure [-f config] [-o file]", + Short: "configure providers for cnquery", + Run: func(cmd *cobra.Command, args []string) { + confPath, err := cmd.Flags().GetString("file") + if err != nil { + log.Fatal().Err(err).Msg("Can't get --file") + } + outPath, err := cmd.Flags().GetString("output") + if err != nil { + log.Fatal().Err(err).Msg("Can't get --output") + } + + raw, err := os.ReadFile(confPath) + if err != nil { + log.Fatal().Err(err).Str("path", confPath).Msg("failed to read config file") + } + + var conf ProvidersConf + err = yaml.Unmarshal(raw, &conf) + if err != nil { + log.Fatal().Err(err).Str("path", confPath).Msg("failed to parse config file") + } + + builtinGo, err := genBuiltinGo(conf) + + if err = os.WriteFile(outPath, []byte(builtinGo), 0o644); err != nil { + log.Fatal().Err(err).Str("path", outPath).Msg("failed to write output") + } + log.Info().Str("path", outPath).Strs("providers", conf.Builtin).Msg("(1/3) configured builtin providers") + + buildProviders(conf.Builtin) + log.Info().Strs("providers", conf.Builtin).Msg("(2/3) built providers") + + rewireDependencies(conf.Builtin) + log.Info().Str("path", outPath).Strs("providers", conf.Builtin).Msg("(3/3) rewired dependencies/files") + }, +} + +func genBuiltinGo(conf ProvidersConf) (string, error) { + var imports string + var infos string + var configs string + + for _, provider := range conf.Builtin { + imports += fmt.Sprintf("\t%sconf \"go.mondoo.com/cnquery/providers/%s/config\"\n", provider, provider) + imports += fmt.Sprintf("\t%s \"go.mondoo.com/cnquery/providers/%s/provider\"\n", provider, provider) + infos += fmt.Sprintf( + "//go:embed %s.resources.json\n"+ + "var %sInfo []byte\n", + provider, provider) + configs += fmt.Sprintf(` + %sconf.Config.ID: { + Runtime: &RunningProvider{ + Name: %sconf.Config.Name, + ID: %sconf.Config.ID, + Plugin: %s.Init(), + Schema: MustLoadSchema("%s", %sInfo), + isClosed: false, + }, + Config: &%sconf.Config, + }, +`, provider, provider, provider, provider, provider, provider, provider) + } + + return fmt.Sprintf(template, imports, infos, configs), nil +} + +const template = `// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 +// +// This file is auto-generated by 'make providers/config' +// and configured via 'providers.yaml' + +package providers + +// This is primarily useful for debugging purposes, if you want to +// trace into any provider without having to debug the plugin +// connection separately. + +import ( + _ "embed" + + coreconf "go.mondoo.com/cnquery/providers/core/config" + core "go.mondoo.com/cnquery/providers/core/provider" +%s) + +//go:embed core/resources/core.resources.json +var coreInfo []byte + +%s +var builtinProviders = map[string]*builtinProvider{ + coreconf.Config.ID: { + Runtime: &RunningProvider{ + Name: coreconf.Config.Name, + ID: coreconf.Config.ID, + Plugin: core.Init(), + Schema: MustLoadSchema("core", coreInfo), + isClosed: false, + }, + Config: &coreconf.Config, + }, +%s +} +` + +func buildProviders(providers []string) { + for i, provider := range providers { + cmd := exec.Command("make", "providers/build/"+provider) + log.Debug().Str("provider", provider).Msg("build provider " + strconv.Itoa(i+1) + "/" + strconv.Itoa(len(providers))) + if err := cmd.Run(); err != nil { + log.Fatal().Err(err).Str("provider", provider).Msg("failed to build provider") + } + + // inefficient copy... + src := "providers/" + provider + "/resources/" + provider + ".resources.json" + raw, err := os.ReadFile(src) + if err != nil { + log.Fatal().Err(err).Str("src", src).Msg("failed to read resources json") + } + + dst := "providers/" + provider + ".resources.json" + err = os.WriteFile(dst, raw, 0o644) + if err != nil { + log.Fatal().Err(err).Str("dst", dst).Msg("failed to write resources json") + } + } +} + +var ( + reBuiltinReplace = regexp.MustCompile(`replace go.mondoo.com/cnquery/providers/.* => ./providers/.*`) + reBuiltinDep = regexp.MustCompile(`go.mondoo.com/cnquery/providers/.*`) +) + +func rewireDependencies(providers []string) { + raw, err := os.ReadFile("go.mod") + if err != nil { + log.Fatal().Err(err).Msg("failed to read go.mod") + } + + raws := string(raw) + raws = reBuiltinReplace.ReplaceAllString(raws, "") + raws = reBuiltinDep.ReplaceAllString(raws, "") + + deps := "" + replace := "" + for _, provider := range providers { + // we don't care about the specific version for dev + deps += "\n\tgo.mondoo.com/cnquery/providers/" + provider + " v0.0.0" + replace += "\nreplace go.mondoo.com/cnquery/providers/" + provider + " => ./providers/" + provider + } + if deps != "" { + raws = strings.Replace(raws, "require (", "require ("+deps, 1) + raws = strings.Replace(raws, "module go.mondoo.com/cnquery", "module go.mondoo.com/cnquery\n"+replace, 1) + } + + err = os.WriteFile("go.mod", []byte(raws), 0o644) + if err != nil { + log.Fatal().Err(err).Msg("failed to write go.mod") + } + + cmd := exec.Command("go", "mod", "tidy") + log.Debug().Msg("go mod tidy") + if err := cmd.Run(); err != nil { + log.Fatal().Err(err).Msg("failed to go mod tidy") + } +} + +func init() { + rootCmd.Flags().StringP("file", "f", "providers.yaml", "config file for providers") + rootCmd.Flags().StringP("output", "o", "providers/builtin.go", "output builtin.go file") +} + +func main() { + logger.CliCompactLogger(logger.LogOutputWriter) + zerolog.SetGlobalLevel(zerolog.DebugLevel) + + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/providers.yaml b/providers.yaml new file mode 100644 index 0000000000..3379838386 --- /dev/null +++ b/providers.yaml @@ -0,0 +1,15 @@ +# Copyright (c) Mondoo, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# Configure builtin providers for the dev environment. +# This only covers additional builtin providers, ie those +# that are normally not builtin. Things like `core` are +# always builtin. +# +# Builtin providers are great testing. Example: if you +# review a PR where someone changed the `os` provider, +# you'll want to set it as builtin for fast testing. +# In this case it will be pulled from your local repo +# instead of using the pre-installed provider in +# your OS (like ~/.mondoo/providers/os). +builtin: [] diff --git a/providers/builtin.go b/providers/builtin.go index b05114897a..94023e95cd 100644 --- a/providers/builtin.go +++ b/providers/builtin.go @@ -1,43 +1,25 @@ // Copyright (c) Mondoo, Inc. // SPDX-License-Identifier: BUSL-1.1 +// +// This file is auto-generated by 'make providers/config' +// and configured via 'providers.yaml' package providers -// Uncomment any provider you want to load directly into the binary. // This is primarily useful for debugging purposes, if you want to // trace into any provider without having to debug the plugin // connection separately. import ( _ "embed" - "encoding/json" - osfs "os" - "go.mondoo.com/cnquery/providers-sdk/v1/plugin" - "go.mondoo.com/cnquery/providers-sdk/v1/resources" coreconf "go.mondoo.com/cnquery/providers/core/config" core "go.mondoo.com/cnquery/providers/core/provider" - // osconf "go.mondoo.com/cnquery/providers/os/config" - // os "go.mondoo.com/cnquery/providers/os/provider" ) -var BuiltinCoreID = coreconf.Config.ID - //go:embed core/resources/core.resources.json var coreInfo []byte -// //go:embed os/resources/os.resources.json -// var osInfo []byte - -// //go:embed network/resources/network.resources.json -// var networkInfo []byte - -// //go:embed k8s/resources/k8s.resources.json -// var k8sInfo []byte - -// //go:embed azure.resources.json -// var azureInfo []byte - var builtinProviders = map[string]*builtinProvider{ coreconf.Config.ID: { Runtime: &RunningProvider{ @@ -49,64 +31,4 @@ var builtinProviders = map[string]*builtinProvider{ }, Config: &coreconf.Config, }, - // osconf.Config.ID: { - // Runtime: &RunningProvider{ - // Name: osconf.Config.Name, - // ID: osconf.Config.ID, - // Plugin: os.Init(), - // Schema: MustLoadSchema("os", osInfo), - // isClosed: false, - // }, - // Config: &osconf.Config, - // }, - // networkconf.Config.ID: { - // Runtime: &RunningProvider{ - // Name: networkconf.Config.Name, - // ID: networkconf.Config.ID, - // Plugin: network.Init(), - // Schema: MustLoadSchema("network", networkInfo), - // isClosed: false, - // }, - // Config: &networkconf.Config, - // }, - // k8sconf.Config.ID: { - // Runtime: &RunningProvider{ - // Name: k8sconf.Config.Name, - // ID: k8sconf.Config.ID, - // Plugin: k8s.Init(), - // Schema: MustLoadSchema("k8s", k8sInfo), - // isClosed: false, - // }, - // Config: &k8sconf.Config, - // }, - // azureconf.Config.ID: { - // Runtime: &RunningProvider{ - // Name: azureconf.Config.Name, - // ID: azureconf.Config.ID, - // Plugin: azure.Init(), - // Schema: MustLoadSchema("azure", azureInfo), - // isClosed: false, - // }, - // Config: &azureconf.Config, -} - -type builtinProvider struct { - Runtime *RunningProvider - Config *plugin.Provider -} - -func MustLoadSchema(name string, data []byte) *resources.Schema { - var res resources.Schema - if err := json.Unmarshal(data, &res); err != nil { - panic("failed to embed schema for " + name) - } - return &res -} - -func MustLoadSchemaFromFile(name string, path string) *resources.Schema { - raw, err := osfs.ReadFile(path) - if err != nil { - panic("cannot read schema file: " + path) - } - return MustLoadSchema(name, raw) } diff --git a/providers/coordinator.go b/providers/coordinator.go index 0bd952eb06..f28976c6fb 100644 --- a/providers/coordinator.go +++ b/providers/coordinator.go @@ -15,9 +15,12 @@ import ( "github.com/rs/zerolog/log" pp "go.mondoo.com/cnquery/providers-sdk/v1/plugin" "go.mondoo.com/cnquery/providers-sdk/v1/resources" + coreconf "go.mondoo.com/cnquery/providers/core/config" "go.mondoo.com/cnquery/providers/core/resources/versions/semver" ) +var BuiltinCoreID = coreconf.Config.ID + var Coordinator = coordinator{ Running: []*RunningProvider{}, } @@ -28,6 +31,11 @@ type coordinator struct { mutex sync.Mutex } +type builtinProvider struct { + Runtime *RunningProvider + Config *pp.Provider +} + type RunningProvider struct { Name string ID string diff --git a/providers/providers.go b/providers/providers.go index cf8ee88913..79f6ccb64c 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "os" + osfs "os" "path/filepath" "runtime" "strings" @@ -567,3 +568,19 @@ func (p Providers) Add(nu *Provider) { p[nu.ID] = nu } } + +func MustLoadSchema(name string, data []byte) *resources.Schema { + var res resources.Schema + if err := json.Unmarshal(data, &res); err != nil { + panic("failed to embed schema for " + name) + } + return &res +} + +func MustLoadSchemaFromFile(name string, path string) *resources.Schema { + raw, err := osfs.ReadFile(path) + if err != nil { + panic("cannot read schema file: " + path) + } + return MustLoadSchema(name, raw) +}