From 46ad5cb77496fef1e21680b74b221f480045115b Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Tue, 26 Sep 2023 00:38:51 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20get=20id=20field=20from=20record?= =?UTF-8?q?ings=20+=20recording=20asset=20detection=20+=20runtime=20clonin?= =?UTF-8?q?g=20(#1932)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 get id field from recordings + recording asset detection + runtime cloning There are a few things that were wrong when replaying recordings for more discoverable services: 1. Any resource with the ID field would not correctly store or read it from recordings. The ID field can be grabbed from the resource in the recording. 2. Asset detection from recording was very inaccurate when using the Platform.Title. Fix it and make it actually usable. 3. Finally, when cloning runtimes during later stages of the discovery process, we loose all connections from the mock provider. Copy them over. Signed-off-by: Dominik Richter * 🟢 go mod tidy everything --------- Signed-off-by: Dominik Richter --- apps/cnquery/cmd/plugin.go | 3 +-- go.mod | 2 +- go.sum | 4 ++-- providers/arista/go.mod | 2 +- providers/arista/go.sum | 4 ++-- providers/aws/go.mod | 2 +- providers/aws/go.sum | 4 ++-- providers/equinix/go.mod | 2 +- providers/equinix/go.sum | 4 ++-- providers/gitlab/go.mod | 2 +- providers/gitlab/go.sum | 4 ++-- providers/opcua/go.mod | 2 +- providers/opcua/go.sum | 4 ++-- providers/recording.go | 32 ++++++++++++++++++++++++++------ providers/runtime.go | 9 +++++++++ providers/vcd/go.mod | 2 +- providers/vcd/go.sum | 4 ++-- 17 files changed, 57 insertions(+), 29 deletions(-) diff --git a/apps/cnquery/cmd/plugin.go b/apps/cnquery/cmd/plugin.go index a689dc7f79..44c049de7f 100644 --- a/apps/cnquery/cmd/plugin.go +++ b/apps/cnquery/cmd/plugin.go @@ -117,8 +117,7 @@ func (c *cnqueryPlugin) RunQuery(conf *run.RunQueryConfig, runtime *providers.Ru for i := range assets { connectAsset := assets[i] - connectAssetRuntime := providers.Coordinator.NewRuntime() - connectAssetRuntime.Recording = runtime.Recording + connectAssetRuntime := providers.Coordinator.NewRuntimeFrom(runtime) if err := connectAssetRuntime.DetectProvider(connectAsset); err != nil { return err diff --git a/go.mod b/go.mod index f6e45b31ad..189daf14bb 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.18 github.com/pkg/sftp v1.13.6 github.com/pkg/term v1.2.0-beta.2 - github.com/rs/zerolog v1.30.0 + github.com/rs/zerolog v1.31.0 github.com/segmentio/fasthash v1.0.3 github.com/segmentio/ksuid v1.0.4 github.com/sethvargo/go-password v0.2.0 diff --git a/go.sum b/go.sum index 4cab89dc93..d2cd064208 100644 --- a/go.sum +++ b/go.sum @@ -782,8 +782,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.3.0 h1:q15RT/pd6UggBXVBuLps8BXRvl5GPBcwVA7BJHMLuTw= diff --git a/providers/arista/go.mod b/providers/arista/go.mod index df6a5bc9ca..fe5d48c338 100644 --- a/providers/arista/go.mod +++ b/providers/arista/go.mod @@ -44,7 +44,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/rs/zerolog v1.30.0 // indirect + github.com/rs/zerolog v1.31.0 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/segmentio/ksuid v1.0.4 // indirect github.com/smarty/assertions v1.15.1 // indirect diff --git a/providers/arista/go.sum b/providers/arista/go.sum index 8cd8c85768..5bc0f901e7 100644 --- a/providers/arista/go.sum +++ b/providers/arista/go.sum @@ -215,8 +215,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= diff --git a/providers/aws/go.mod b/providers/aws/go.mod index 0bd42b5cd4..e1507c6945 100644 --- a/providers/aws/go.mod +++ b/providers/aws/go.mod @@ -52,7 +52,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 github.com/aws/smithy-go v1.14.2 github.com/cockroachdb/errors v1.11.1 - github.com/rs/zerolog v1.30.0 + github.com/rs/zerolog v1.31.0 github.com/spf13/afero v1.9.5 github.com/stretchr/testify v1.8.4 go.mondoo.com/cnquery v0.0.0-20230915180754-c5f61bc705cf diff --git a/providers/aws/go.sum b/providers/aws/go.sum index 65f844abfd..4d34b57af9 100644 --- a/providers/aws/go.sum +++ b/providers/aws/go.sum @@ -395,8 +395,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= diff --git a/providers/equinix/go.mod b/providers/equinix/go.mod index 7a4a36b733..2fadd6dc43 100644 --- a/providers/equinix/go.mod +++ b/providers/equinix/go.mod @@ -6,7 +6,7 @@ go 1.20 require ( github.com/packethost/packngo v0.30.0 - github.com/rs/zerolog v1.30.0 + github.com/rs/zerolog v1.31.0 go.mondoo.com/cnquery v0.0.0-20230915180754-c5f61bc705cf ) diff --git a/providers/equinix/go.sum b/providers/equinix/go.sum index 7b954cbe6a..1a87975510 100644 --- a/providers/equinix/go.sum +++ b/providers/equinix/go.sum @@ -217,8 +217,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= diff --git a/providers/gitlab/go.mod b/providers/gitlab/go.mod index e09dda8c2f..d1123c5348 100644 --- a/providers/gitlab/go.mod +++ b/providers/gitlab/go.mod @@ -5,7 +5,7 @@ replace go.mondoo.com/cnquery => ../.. go 1.21 require ( - github.com/rs/zerolog v1.30.0 + github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 github.com/xanzy/go-gitlab v0.91.1 go.mondoo.com/cnquery v0.0.0-20230920205842-55a158611de3 diff --git a/providers/gitlab/go.sum b/providers/gitlab/go.sum index 8f135b51be..d1a9ba37a9 100644 --- a/providers/gitlab/go.sum +++ b/providers/gitlab/go.sum @@ -226,8 +226,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= diff --git a/providers/opcua/go.mod b/providers/opcua/go.mod index 66e16b32e8..94df21cd9d 100644 --- a/providers/opcua/go.mod +++ b/providers/opcua/go.mod @@ -42,7 +42,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/rs/zerolog v1.30.0 // indirect + github.com/rs/zerolog v1.31.0 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/segmentio/ksuid v1.0.4 // indirect github.com/smarty/assertions v1.15.1 // indirect diff --git a/providers/opcua/go.sum b/providers/opcua/go.sum index 8512ee143c..251c6ac235 100644 --- a/providers/opcua/go.sum +++ b/providers/opcua/go.sum @@ -218,8 +218,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= diff --git a/providers/recording.go b/providers/recording.go index 51a779f8b8..043788e156 100644 --- a/providers/recording.go +++ b/providers/recording.go @@ -364,15 +364,31 @@ func (r *recording) findAssetConnID(asset *inventory.Asset, conf *inventory.Conf id = asset.Mrn } else if asset.Id != "" { id = asset.Id - } else if asset.Platform != nil { - id = asset.Platform.Title } found := -1 - for i := range r.Assets { - if r.Assets[i].Asset.ID == id { - found = i - break + + if id != "" { + for i := range r.Assets { + if r.Assets[i].Asset.ID == id { + found = i + break + } + } + if found != -1 { + return found, id + } + } + + if asset.Platform != nil { + for i := range r.Assets { + if r.Assets[i].Asset.Title == asset.Platform.Title { + found = i + break + } + } + if found != -1 { + return found, r.Assets[found].Asset.ID } } @@ -460,6 +476,10 @@ func (r *recording) GetData(connectionID uint32, resource string, id string, fie } data, ok := obj.Fields[field] + if !ok && field == "id" { + return llx.StringData(id), true + } + return data, ok } diff --git a/providers/runtime.go b/providers/runtime.go index 775c282b94..66f8c947f8 100644 --- a/providers/runtime.go +++ b/providers/runtime.go @@ -72,6 +72,15 @@ func (c *coordinator) NewRuntime() *Runtime { return res } +func (c *coordinator) NewRuntimeFrom(parent *Runtime) *Runtime { + res := c.NewRuntime() + res.Recording = parent.Recording + for k, v := range parent.providers { + res.providers[k] = v + } + return res +} + type shutdownResult struct { Response *plugin.ShutdownRes Error error diff --git a/providers/vcd/go.mod b/providers/vcd/go.mod index 2ea755c5fe..ed4d02ab9a 100644 --- a/providers/vcd/go.mod +++ b/providers/vcd/go.mod @@ -5,7 +5,7 @@ replace go.mondoo.com/cnquery => ../.. go 1.20 require ( - github.com/rs/zerolog v1.30.0 + github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 github.com/vmware/go-vcloud-director/v2 v2.21.0 go.mondoo.com/cnquery v0.0.0-00010101000000-000000000000 diff --git a/providers/vcd/go.sum b/providers/vcd/go.sum index 44318522f6..31a182c52a 100644 --- a/providers/vcd/go.sum +++ b/providers/vcd/go.sum @@ -217,8 +217,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=