From 7d127dff464521c472d58f307a70fddea911eb29 Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Fri, 2 Feb 2024 17:02:54 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=9D=20update=20docs=20(#3181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update generated code Signed-off-by: Ivan Milchev * update dev docs Signed-off-by: Ivan Milchev * add go dictionary Signed-off-by: Ivan Milchev * change Signed-off-by: Ivan Milchev * fix comments Update docs/development.md Co-authored-by: Letha Update docs/development.md Co-authored-by: Letha Update docs/development.md Co-authored-by: Letha Update docs/development.md Co-authored-by: Letha Update docs/development.md Co-authored-by: Letha Update docs/development.md Co-authored-by: Letha --------- Signed-off-by: Ivan Milchev Co-authored-by: Letha --- .github/workflows/spell-check.yaml | 1 + docs/development.md | 113 ++++++++++++++++++ explorer/cnquery_explorer.pb.go | 2 +- explorer/scan/cnquery_explorer_scan.pb.go | 2 +- llx/llx.pb.go | 2 +- providers-sdk/v1/inventory/inventory.pb.go | 2 +- providers-sdk/v1/plugin/plugin.pb.go | 2 +- providers-sdk/v1/plugin/plugin_grpc.pb.go | 2 +- providers-sdk/v1/resources/resources.pb.go | 2 +- providers-sdk/v1/upstream/health/errors.pb.go | 2 +- providers-sdk/v1/upstream/health/health.pb.go | 2 +- providers-sdk/v1/upstream/mvd/cvss/cvss.pb.go | 2 +- providers-sdk/v1/upstream/mvd/mvd.pb.go | 2 +- providers-sdk/v1/upstream/upstream.pb.go | 2 +- providers-sdk/v1/vault/vault.pb.go | 2 +- providers/aws/resources/aws.lr.go | 2 +- sbom/sbom.pb.go | 2 +- shared/proto/cnquery.pb.go | 2 +- shared/proto/cnquery_grpc.pb.go | 2 +- 19 files changed, 131 insertions(+), 17 deletions(-) diff --git a/.github/workflows/spell-check.yaml b/.github/workflows/spell-check.yaml index 57be00a2d4..1fac350ace 100644 --- a/.github/workflows/spell-check.yaml +++ b/.github/workflows/spell-check.yaml @@ -36,6 +36,7 @@ jobs: cspell:software-terms/src/software-tools.txt cspell:companies/src/companies.txt mondoo:mondoo_dictionary.txt + cspell:golang/dict/go.txt comment: name: Report diff --git a/docs/development.md b/docs/development.md index f28200c207..fe3a071302 100644 --- a/docs/development.md +++ b/docs/development.md @@ -178,6 +178,7 @@ go 1.21 use ( ./cnquery + ./cnquery/providers/atlassian ./cnquery/providers/arista ./cnquery/providers/aws ./cnquery/providers/azure @@ -200,6 +201,118 @@ use ( ) ``` +## Providers development best practices + +The more time we spend building providers, the more learn how to do better in the future. Here we describe learnings that will help you get started with providers development. + +### Referencing MQL resources +Often we have a top-level MQL resource, which we want to reference in another top-level resource. + +For example, GCP networks can be retrieved for a project. That is a top-level resource: +``` +// GCP Compute Engine +private gcp.project.computeService { + // Google Compute Engine VPC network in a project + networks() []gcp.project.computeService.network +} +``` + +However, we have a reference to a GCP network in a GCP Compute address. This allows us to quickly navigate to the network in which an address is created: +``` +private gcp.project.computeService.address { + // Static IP address + address string + + // Network in which to reserve the address + network() gcp.project.computeService.network +} +``` + +The simple way to implement the reference would be to call the GCP API every time `gcp.project.computeService.address.network` is executed. However, this would generate an excessive amount of API calls when scanning large GCP projects. If we have 10 addresses, this would mean 10 separate API calls to get the network, one for each of them. + +MQL has powerful caching capabilities that let us achieve the same end result with a single (or fewer) API calls. + +First, create an init function for `gcp.project.computeService.network`, which is the resource we are cross-referencing: +```go +func initGcpProjectComputeServiceNetwork(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[string]*llx.RawData, plugin.Resource, error) { + // Here we check that the resource isn't fully initialized yet + if len(args) > 2 { + return args, nil, nil + } + + // If no args are set, try reading them from the platform ID + if len(args) == 0 { + if ids := getAssetIdentifier(runtime); ids != nil { + args["name"] = llx.StringData(ids.name) + args["projectId"] = llx.StringData(ids.project) + } else { + return nil, nil, errors.New("no asset identifier found") + } + } + + // We create a gcp.project.computeService resource which would allow us to retrieve networks via MQL + obj, err := CreateResource(runtime, "gcp.project.computeService", map[string]*llx.RawData{ + "projectId": args["projectId"], + }) + if err != nil { + return nil, nil, err + } + + // Cast the resource to the appropriate type + computeSvc := obj.(*mqlGcpProjectComputeService) + // List the networks: equivalent to gcp.project.computeService.networks MQL query. This retrieves all networks in the project and caches them in the MQL cache. Consecutive calls to this retrieve the data from the cache and do not execute any API calls. + networks := computeSvc.GetNetworks() + if networks.Error != nil { + return nil, nil, networks.Error + } + + // Filter the networks in memory by comparing them with the input arguments + for _, n := range networks.Data { + network := n.(*mqlGcpProjectComputeServiceNetwork) + name := network.GetName() + if name.Error != nil { + return nil, nil, name.Error + } + projectId := network.GetProjectId() + if projectId.Error != nil { + return nil, nil, projectId.Error + } + + // return the resource if found + if name.Data == args["name"].Value && projectId.Data == args["projectId"].Value { + return args, network, nil + } + } + return nil, nil, fmt.Errorf("not found") +} +``` + +Then, we implement the function for retrieving the network for a GCP compute address: +```go +func (g *mqlGcpProjectComputeServiceAddress) network() (*mqlGcpProjectComputeServiceNetwork, error) { + if g.NetworkUrl.Error != nil { + return nil, g.NetworkUrl.Error + } + networkUrl := g.NetworkUrl.Data + + // Format is https://www.googleapis.com/compute/v1/projects/project1/global/networks/net-1 + params := strings.TrimPrefix(networkUrl, "https://www.googleapis.com/compute/v1/") + parts := strings.Split(params, "/") + resId := resourceId{Project: parts[1], Region: parts[2], Name: parts[4]} + + // Use the init function for the resource to find the one that we need + res, err := CreateResource(g.MqlRuntime, "gcp.project.computeService.network", map[string]*llx.RawData{ + "name": llx.StringData(resId.Name), + "projectId": llx.StringData(resId.Project), + }) + if err != nil { + return nil, err + } + return res.(*mqlGcpProjectComputeServiceNetwork), nil +} + +``` + ## Contribute changes ### Mark PRs with emojis diff --git a/explorer/cnquery_explorer.pb.go b/explorer/cnquery_explorer.pb.go index 172ed6bab6..93556d5b96 100644 --- a/explorer/cnquery_explorer.pb.go +++ b/explorer/cnquery_explorer.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: cnquery_explorer.proto package explorer diff --git a/explorer/scan/cnquery_explorer_scan.pb.go b/explorer/scan/cnquery_explorer_scan.pb.go index b28f98f067..b629648d63 100644 --- a/explorer/scan/cnquery_explorer_scan.pb.go +++ b/explorer/scan/cnquery_explorer_scan.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: cnquery_explorer_scan.proto package scan diff --git a/llx/llx.pb.go b/llx/llx.pb.go index 3434808649..20bf374d97 100644 --- a/llx/llx.pb.go +++ b/llx/llx.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: llx.proto package llx diff --git a/providers-sdk/v1/inventory/inventory.pb.go b/providers-sdk/v1/inventory/inventory.pb.go index ce79216393..989b6f7248 100644 --- a/providers-sdk/v1/inventory/inventory.pb.go +++ b/providers-sdk/v1/inventory/inventory.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: inventory.proto package inventory diff --git a/providers-sdk/v1/plugin/plugin.pb.go b/providers-sdk/v1/plugin/plugin.pb.go index 65c358e324..179a891c8c 100644 --- a/providers-sdk/v1/plugin/plugin.pb.go +++ b/providers-sdk/v1/plugin/plugin.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: plugin.proto package plugin diff --git a/providers-sdk/v1/plugin/plugin_grpc.pb.go b/providers-sdk/v1/plugin/plugin_grpc.pb.go index 4637ac6a16..1a73f61341 100644 --- a/providers-sdk/v1/plugin/plugin_grpc.pb.go +++ b/providers-sdk/v1/plugin/plugin_grpc.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.25.1 +// - protoc v4.25.2 // source: plugin.proto package plugin diff --git a/providers-sdk/v1/resources/resources.pb.go b/providers-sdk/v1/resources/resources.pb.go index 4fbae71bb1..d2e130ef46 100644 --- a/providers-sdk/v1/resources/resources.pb.go +++ b/providers-sdk/v1/resources/resources.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: resources.proto package resources diff --git a/providers-sdk/v1/upstream/health/errors.pb.go b/providers-sdk/v1/upstream/health/errors.pb.go index 52c8a91e4c..b9f8da9a03 100644 --- a/providers-sdk/v1/upstream/health/errors.pb.go +++ b/providers-sdk/v1/upstream/health/errors.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: errors.proto package health diff --git a/providers-sdk/v1/upstream/health/health.pb.go b/providers-sdk/v1/upstream/health/health.pb.go index 7ed56f6cbe..cb74ced034 100644 --- a/providers-sdk/v1/upstream/health/health.pb.go +++ b/providers-sdk/v1/upstream/health/health.pb.go @@ -18,7 +18,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: health.proto package health diff --git a/providers-sdk/v1/upstream/mvd/cvss/cvss.pb.go b/providers-sdk/v1/upstream/mvd/cvss/cvss.pb.go index c56f0b8287..ca19027d89 100644 --- a/providers-sdk/v1/upstream/mvd/cvss/cvss.pb.go +++ b/providers-sdk/v1/upstream/mvd/cvss/cvss.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: cvss.proto package cvss diff --git a/providers-sdk/v1/upstream/mvd/mvd.pb.go b/providers-sdk/v1/upstream/mvd/mvd.pb.go index 188a766a60..f1bdf92334 100644 --- a/providers-sdk/v1/upstream/mvd/mvd.pb.go +++ b/providers-sdk/v1/upstream/mvd/mvd.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: mvd.proto package mvd diff --git a/providers-sdk/v1/upstream/upstream.pb.go b/providers-sdk/v1/upstream/upstream.pb.go index 66f48aaa9f..92062800a7 100644 --- a/providers-sdk/v1/upstream/upstream.pb.go +++ b/providers-sdk/v1/upstream/upstream.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: upstream.proto package upstream diff --git a/providers-sdk/v1/vault/vault.pb.go b/providers-sdk/v1/vault/vault.pb.go index f01783b784..ce0c4bcfd0 100644 --- a/providers-sdk/v1/vault/vault.pb.go +++ b/providers-sdk/v1/vault/vault.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: vault.proto package vault diff --git a/providers/aws/resources/aws.lr.go b/providers/aws/resources/aws.lr.go index 59a3e726e1..ef86122385 100644 --- a/providers/aws/resources/aws.lr.go +++ b/providers/aws/resources/aws.lr.go @@ -199,7 +199,7 @@ func init() { Create: createAwsEfs, }, "aws.efs.filesystem": { - // to override args, implement: initAwsEfsFilesystem(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[string]*llx.RawData, plugin.Resource, error) + Init: initAwsEfsFilesystem, Create: createAwsEfsFilesystem, }, "aws.kms": { diff --git a/sbom/sbom.pb.go b/sbom/sbom.pb.go index b280e386f6..b651bab4fb 100644 --- a/sbom/sbom.pb.go +++ b/sbom/sbom.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: sbom.proto package sbom diff --git a/shared/proto/cnquery.pb.go b/shared/proto/cnquery.pb.go index 22609f5b27..4563f2c217 100644 --- a/shared/proto/cnquery.pb.go +++ b/shared/proto/cnquery.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.32.0 -// protoc v4.25.1 +// protoc v4.25.2 // source: cnquery.proto package proto diff --git a/shared/proto/cnquery_grpc.pb.go b/shared/proto/cnquery_grpc.pb.go index 0cd6c6e7bd..7cecade2ed 100644 --- a/shared/proto/cnquery_grpc.pb.go +++ b/shared/proto/cnquery_grpc.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.25.1 +// - protoc v4.25.2 // source: cnquery.proto package proto