From f38e2644547a346d66716629455cbcbb430d437b Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Tue, 12 Sep 2023 06:20:31 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B9=20remove=20the=20remainder=20of=20?= =?UTF-8?q?v8=20=5Fresources=20and=20=5Fmotor=20(#1694)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit just not all of them yet, waiting for all the discovery pieces to land. If you feel comfortable, feel free to remove the rest. Signed-off-by: Dominik Richter --- _motor/inventory/manager_load_test.go | 88 ----- _motor/inventory/manager_query_test.go | 112 ------ _motor/inventory/manager_test.go | 32 -- _resources/lumi.go | 6 - _resources/mock.resource.go | 61 ---- _resources/observers.go | 228 ------------ _resources/observers_test.go | 60 ---- _resources/registry.go | 244 ------------- _resources/runtime.go | 462 ------------------------- _resources/runtime_test.go | 20 -- _resources/schema.go | 22 -- 11 files changed, 1335 deletions(-) delete mode 100644 _motor/inventory/manager_load_test.go delete mode 100644 _motor/inventory/manager_query_test.go delete mode 100644 _motor/inventory/manager_test.go delete mode 100644 _resources/lumi.go delete mode 100644 _resources/mock.resource.go delete mode 100644 _resources/observers.go delete mode 100644 _resources/observers_test.go delete mode 100644 _resources/registry.go delete mode 100644 _resources/runtime.go delete mode 100644 _resources/runtime_test.go delete mode 100644 _resources/schema.go diff --git a/_motor/inventory/manager_load_test.go b/_motor/inventory/manager_load_test.go deleted file mode 100644 index 0d4d895bff..0000000000 --- a/_motor/inventory/manager_load_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package inventory - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - v1 "go.mondoo.com/cnquery/motor/inventory/v1" - "go.mondoo.com/cnquery/motor/vault/credentials_resolver" -) - -func TestInventoryLoader(t *testing.T) { - inventory, err := v1.InventoryFromFile("./v1/testdata/inventory.yaml") - require.NoError(t, err) - - im, err := New(WithInventory(inventory)) - require.NoError(t, err) - - credsResolver := credentials_resolver.New(im.GetVault(), false) - - // gather all assets and check their secrets - assetList := im.GetAssets() - require.NoError(t, err) - - for i := range assetList { - a := assetList[i] - for j := range a.Connections { - conn := a.Connections[j] - for k := range conn.Credentials { - cred := conn.Credentials[k] - _, err := credsResolver.GetCredential(cred) - assert.NoError(t, err, cred.SecretId) - } - } - } -} - -func TestAssetLoader(t *testing.T) { - _, err := New(WithAssets([]*v1.Asset{ - { - Name: "test asset", - }, - })) - require.NoError(t, err) -} - -func TestAwsInventoryLoader(t *testing.T) { - inventory, err := v1.InventoryFromFile("./v1/testdata/aws_inventory.yaml") - require.NoError(t, err) - - os.Setenv("AWS_PROFILE", "mondoo-dev") - os.Setenv("AWS_REGION", "us-east-1") - - im, err := New(WithInventory(inventory)) - require.NoError(t, err) - - credsResolver := credentials_resolver.New(im.GetVault(), false) - - // gather all assets and check their secrets - assetList := im.GetAssets() - require.NoError(t, err) - - for i := range assetList { - a := assetList[i] - for j := range a.Connections { - conn := a.Connections[j] - for k := range conn.Credentials { - cred := conn.Credentials[k] - resolvedCred, err := credsResolver.GetCredential(cred) - assert.NoError(t, err, cred.SecretId) - assert.NotNil(t, resolvedCred) - } - } - } -} - -func TestVaultInventoryLoader(t *testing.T) { - // load inventory with vault name, it will throw an error since vault is not configured fully - inventory, err := v1.InventoryFromFile("./v1/testdata/vault_inventory.yaml") - require.NoError(t, err) - - _, err = New(WithInventory(inventory)) - require.Error(t, err) -} diff --git a/_motor/inventory/manager_query_test.go b/_motor/inventory/manager_query_test.go deleted file mode 100644 index d44f4745f4..0000000000 --- a/_motor/inventory/manager_query_test.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package inventory - -import ( - "testing" - - "go.mondoo.com/cnquery/motor/vault" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - v1 "go.mondoo.com/cnquery/motor/inventory/v1" - "go.mondoo.com/cnquery/motor/vault/credentials_resolver" - mockvault "go.mondoo.com/cnquery/motor/vault/mock" -) - -func TestSecretManagerPassword(t *testing.T) { - im, err := New( - WithInventory(&v1.Inventory{ - Spec: &v1.InventorySpec{ - CredentialQuery: "{type: 'password', secret_id: 'mockPassword', user: 'test-user'}", - }, - }), - WithVault(mockvault.New()), - ) - require.NoError(t, err) - - assetObj := &v1.Asset{ - Name: "asset-name", - Platform: &v1.Platform{Name: "ubuntu"}, - Connections: []*v1.Config{ - {Type: "ssh", Insecure: true}, - }, - } - - credential, err := im.QuerySecretId(assetObj) - require.NoError(t, err) - - assert.Equal(t, vault.CredentialType_password, credential.Type) - assert.Equal(t, "test-user", credential.User) - assert.Equal(t, "mockPassword", credential.SecretId) - - // now we try to get the full credential with the secret - credsResolver := credentials_resolver.New(im.GetVault(), false) - _, err = credsResolver.GetCredential(credential) - assert.NoError(t, err) -} - -func TestSecretManagerPrivateKey(t *testing.T) { - im, err := New( - WithInventory(&v1.Inventory{ - Spec: &v1.InventorySpec{ - CredentialQuery: "{type: 'private_key', secret_id: 'mockPKey', user: 'some-user'}", - }, - }), - WithVault(mockvault.New()), - ) - require.NoError(t, err) - - assetObj := &v1.Asset{ - Name: "asset-name", - Platform: &v1.Platform{Name: "ubuntu"}, - Connections: []*v1.Config{ - {Type: "ssh", Insecure: true}, - }, - } - - credential, err := im.QuerySecretId(assetObj) - require.NoError(t, err) - - assert.Equal(t, vault.CredentialType_private_key, credential.Type) - assert.Equal(t, "some-user", credential.User) - assert.Equal(t, "mockPKey", credential.SecretId) - - // now we try to get the full credential with the secret - credsResolver := credentials_resolver.New(im.GetVault(), false) - _, err = credsResolver.GetCredential(credential) - assert.NoError(t, err) -} - -func TestSecretManagerBadKey(t *testing.T) { - im, err := New( - WithInventory(&v1.Inventory{ - Spec: &v1.InventorySpec{ - CredentialQuery: "{type: 'password', secret_id: 'bad-id', user: 'some-user'}", - }, - }), - WithVault(mockvault.New()), - ) - require.NoError(t, err) - - assetObj := &v1.Asset{ - Name: "asset-name", - Platform: &v1.Platform{Name: "ubuntu"}, - Connections: []*v1.Config{ - {Type: "ssh", Insecure: true}, - }, - } - - // NOTE: we get the secret id but the load from the vault will fail - credential, err := im.QuerySecretId(assetObj) - assert.NoError(t, err) - assert.Equal(t, vault.CredentialType_password, credential.Type) - assert.Equal(t, "some-user", credential.User) - assert.Equal(t, "bad-id", credential.SecretId) - - // now we try to get the full credential with the secret - credsResolver := credentials_resolver.New(im.GetVault(), false) - _, err = credsResolver.GetCredential(credential) - assert.Error(t, err) -} diff --git a/_motor/inventory/manager_test.go b/_motor/inventory/manager_test.go deleted file mode 100644 index 852d9568ff..0000000000 --- a/_motor/inventory/manager_test.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package inventory - -import ( - "context" - "testing" - - v1 "go.mondoo.com/cnquery/motor/inventory/v1" - - "github.com/stretchr/testify/require" -) - -func TestInventoryIdempotent(t *testing.T) { - v1inventory, err := v1.InventoryFromFile("./v1/testdata/k8s_mount.yaml") - require.NoError(t, err) - - ctx := context.Background() - - im, err := New(WithInventory(v1inventory)) - require.NoError(t, err) - - // runs resolve step, especially the creds resolution - im.Resolve(ctx) - - im, err = New(WithInventory(v1inventory)) - require.NoError(t, err) - - // runs resolve step, especially the creds resolution - im.Resolve(ctx) -} diff --git a/_resources/lumi.go b/_resources/lumi.go deleted file mode 100644 index 7db8d4e18b..0000000000 --- a/_resources/lumi.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package resources - -//go:generate protoc --proto_path=. --go_out=. --go_opt=paths=source_relative resources.proto diff --git a/_resources/mock.resource.go b/_resources/mock.resource.go deleted file mode 100644 index 4e5eec0b14..0000000000 --- a/_resources/mock.resource.go +++ /dev/null @@ -1,61 +0,0 @@ -// copyright: 2019, Dominik Richter and Christoph Hartmann -// author: Dominik Richter -// author: Christoph Hartmann - -package resources - -import ( - "errors" -) - -// MockResource helps mocking resources with fields with static data -type MockResource struct { - StaticFields map[string]*Field - StaticResource *Resource -} - -// MqlResource provides static resource information -func (m MockResource) MqlResource() *Resource { - return m.StaticResource -} - -// Fields lists all fields of the mock resource -func (m MockResource) Fields() []*Field { - res := []*Field{} - for _, f := range m.StaticFields { - res = append(res, f) - } - return res -} - -// Field retrieves the current value of a field -func (m MockResource) Field(name string) (interface{}, error) { - f, ok := m.StaticFields[name] - if !ok { - return nil, errors.New("cannot find field " + name) - } - return f, nil -} - -// Register a field and all its callbacks -func (m MockResource) Register(field string) error { - _, ok := m.StaticFields[field] - if !ok { - return errors.New("cannot find field " + field) - } - return m.StaticResource.MotorRuntime.Observers.Trigger(m.MqlResource().FieldUID(field)) -} - -// Compute a field. For mock, all fields are always computed -func (m MockResource) MqlCompute(field string) error { - _, ok := m.StaticFields[field] - if !ok { - return errors.New("cannot find field " + field) - } - return nil -} - -// Validate has nothing to do in mock, everything is valid. -func (m MockResource) Validate() error { - return nil -} diff --git a/_resources/observers.go b/_resources/observers.go deleted file mode 100644 index 4f09f9ab86..0000000000 --- a/_resources/observers.go +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package resources - -import ( - "errors" - "sync" - - "github.com/rs/zerolog/log" - "go.mondoo.com/cnquery/types" -) - -// Callback is a function without arguments -type Callback func() - -// Callbacks is a map of callbacks -type Callbacks struct{ sync.Map } - -// Store a callback -func (c *Callbacks) Store(key string, callback Callback) { - c.Map.Store(key, callback) -} - -// Load a callback by ID -func (c *Callbacks) Load(key string) (Callback, bool) { - res, ok := c.Map.Load(key) - if !ok { - return nil, ok - } - return res.(Callback), ok -} - -// List all keys with callbacks -func (c *Callbacks) List() []string { - res := []string{} - c.Range(func(k string, _ Callback) bool { - res = append(res, k) - return true - }) - return res -} - -// Delete a callback -func (c *Callbacks) Delete(key string) { - c.Map.Delete(key) -} - -// Range cycles over all callbacks -func (c *Callbacks) Range(cb func(string, Callback) bool) { - c.Map.Range(func(k, v interface{}) bool { - return cb(k.(string), v.(Callback)) - }) -} - -// CallbacksList a list of all callback maps -type CallbacksList struct{ sync.Map } - -// List all key-value lists of ID associations -func (c *CallbacksList) List() map[string][]string { - res := make(map[string][]string) - c.Map.Range(func(k, v interface{}) bool { - res[k.(string)] = v.(*Callbacks).List() - return true - }) - return res -} - -// Store a key-value combination and return if it is the initial for this key -// and if this value already exists. -// 1. true = this is the first time this key is stored -// 2. true = this key-value combination already existed -func (c *CallbacksList) Store(key, value string, cb Callback) (bool, bool) { - v, ok := c.Map.Load(key) - var callbacks *Callbacks - if !ok { - callbacks = &Callbacks{} - c.Map.Store(key, callbacks) - } else { - callbacks = v.(*Callbacks) - } - - _, exists := callbacks.Load(value) - callbacks.Store(value, cb) - - return !ok, exists -} - -// Delete a key-value callback -// Return true if the key is now empty -func (c *CallbacksList) Delete(key, value string) bool { - callbacks, ok := c.Load(key) - if !ok { - return true - } - - callbacks.Delete(value) - isEmpty := true - callbacks.Range(func(_ string, _ Callback) bool { - isEmpty = false - return false - }) - if isEmpty { - c.Map.Delete(key) - } - - return isEmpty -} - -// Load a key -func (c *CallbacksList) Load(key string) (*Callbacks, bool) { - v, ok := c.Map.Load(key) - if !ok { - return nil, false - } - return v.(*Callbacks), true -} - -// Hooks is a map of func -type Hooks struct{ sync.Map } - -func (c *Hooks) Store(k string, v func()) { - c.Map.Store(k, v) -} - -func (c *Hooks) Load(k string) (func(), bool) { - res, ok := c.Map.Load(k) - if !ok { - return nil, ok - } - return res.(func()), ok -} - -// Observers manages all the observers -type Observers struct { - list CallbacksList - reverseList *types.StringToStrings - hooks *Hooks -} - -// NewObservers creates an observers instance -func NewObservers() *Observers { - return &Observers{ - hooks: &Hooks{}, - reverseList: &types.StringToStrings{}, - } -} - -// List out all observers -func (ctx *Observers) List() (map[string][]string, map[string][]string) { - return ctx.list.List(), ctx.reverseList.List() -} - -// Watch a UID for any changes to it and call the watcher via callback if anything changes -// we return a boolean to indicate if this is the first watcher the resource field receives -// true => first time this resource is watched -// false => not the first watcher -func (ctx *Observers) Watch(resourceFieldUID string, watcherUID string, callback Callback) (bool, bool, error) { - if watcherUID == "" { - return false, false, errors.New("cannot register observer with empty watcher UID") - } - - initial, exists := ctx.list.Store(resourceFieldUID, watcherUID, callback) - ctx.reverseList.Store(watcherUID, resourceFieldUID) - - return initial, exists, nil -} - -// Unwatch a watcher from a resource field -// returns a boolean if the ist of watcher is now empty -// true => this was the last watcher, no-one is watching this resource field -// false => there are still watchers on this resource field -func (ctx *Observers) Unwatch(resourceFieldUID string, watcherUID string) (bool, error) { - if watcherUID == "" { - return false, errors.New("cannot unwatch observer with empty watcher UID") - } - - listIsEmpty := ctx.list.Delete(resourceFieldUID, watcherUID) - ctx.reverseList.Delete(watcherUID, resourceFieldUID) - - return listIsEmpty, nil -} - -// OnUnwatch will trigger the given handler once the watcher is removed -func (ctx *Observers) OnUnwatch(watcherUID string, f func()) { - ctx.hooks.Store("unwatch\x00"+watcherUID, f) -} - -// UnwatchAll the references that this watcher is looking at. If it finds -// any resourceFieldUID that isn't watched by anyone anymore it will recursively -// do the same to this watcher. -func (ctx *Observers) UnwatchAll(watcherUID string) error { - log.Trace().Str("watcher", watcherUID).Msg("observer> unwatch all") - - h, ok := ctx.hooks.Load("unwatch\x00" + watcherUID) - if ok { - h() - } - - cbs := ctx.reverseList.ListKey(watcherUID) - for _, key := range cbs { - last, err := ctx.Unwatch(key, watcherUID) - if err != nil { - return err - } - if last { - err = ctx.UnwatchAll(key) - if err != nil { - return err - } - } - } - return nil -} - -// Trigger a change to an observed ID. This means that all watchers on this -// field will be called -func (ctx *Observers) Trigger(resourceFieldUID string) error { - f, ok := ctx.list.Load(resourceFieldUID) - if !ok { - return errors.New("cannot find field " + resourceFieldUID + " to trigger its change.") - } - f.Range(func(_ string, cb Callback) bool { - cb() - return true - }) - return nil -} diff --git a/_resources/observers_test.go b/_resources/observers_test.go deleted file mode 100644 index 74dea515fc..0000000000 --- a/_resources/observers_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package resources - -import ( - "testing" - - uuid "github.com/gofrs/uuid" - "github.com/stretchr/testify/assert" - "go.mondoo.com/cnquery/types" -) - -func TestCallbacksList(t *testing.T) { - o := CallbacksList{} - a := uuid.Must(uuid.NewV4()).String() - b := uuid.Must(uuid.NewV4()).String() - cb := func() {} - - assert.Equal(t, map[string][]string{}, o.List()) - - o.Store(a, b, cb) - assert.Equal(t, map[string][]string{a: []string{b}}, o.List()) - - last := o.Delete(a, b) - assert.True(t, last) - assert.Equal(t, map[string][]string{}, o.List()) -} - -func TestObservers(t *testing.T) { - o := Observers{ - hooks: &Hooks{}, - reverseList: &types.StringToStrings{}, - } - a := uuid.Must(uuid.NewV4()).String() - b := uuid.Must(uuid.NewV4()).String() - - _, err := o.Unwatch(a, b) - assert.Nil(t, err, "no error on Unwatched id") - - err = o.UnwatchAll(a) - assert.Nil(t, err, "no error on UnwatchedAll id") - - initial, exist, err := o.Watch(a, b, func() {}) - assert.True(t, initial, "initial watcher") - assert.False(t, exist, "existing watcher") - assert.Nil(t, err, "no error on Watch") - - assert.Equal(t, map[string][]string{ - a: []string{b}}, o.list.List()) - - assert.Equal(t, map[string][]string{ - b: []string{a}}, o.reverseList.List()) - - err = o.UnwatchAll(b) - assert.Nil(t, err, "no error on UnwatchedAll id") - - assert.Equal(t, map[string][]string{}, o.list.List()) - assert.Equal(t, map[string][]string{}, o.reverseList.List()) -} diff --git a/_resources/registry.go b/_resources/registry.go deleted file mode 100644 index 146d4ee140..0000000000 --- a/_resources/registry.go +++ /dev/null @@ -1,244 +0,0 @@ -// copyright: 2019, Dominik Richter and Christoph Hartmann -// author: Dominik Richter -// author: Christoph Hartmann - -package resources - -import ( - "encoding/json" - "errors" - "strings" - - "go.mondoo.com/cnquery/utils/sortx" - "go.mondoo.com/cnquery/types" -) - -// Args for initializing resources -type Args map[string]interface{} - -type FieldFilter struct { // TODO: tbd -} - -// Registry of all initialized resources -type Registry struct { - Resources map[string]*ResourceCls -} - -// NewRegistry creates a new instance of the resource registry and cache -func NewRegistry() *Registry { - return &Registry{ - Resources: make(map[string]*ResourceCls), - } -} - -// Add all resources from another registry to this registry -func (ctx *Registry) Add(r *Registry) { - for k, v := range r.Resources { - ctx.Resources[k] = v - } -} - -// Clone creates a shallow copy of this registry, which means you can add/remove -// resources, but don't mess with their underlying configuration -func (ctx *Registry) Clone() *Registry { - res := make(map[string]*ResourceCls, len(ctx.Resources)) - for k, v := range ctx.Resources { - res[k] = v - } - return &Registry{res} -} - -// LoadJson loads a set of resource definitions from JSON into the registry -func (ctx *Registry) LoadJson(raw []byte) error { - schema := Schema{} - if err := json.Unmarshal(raw, &schema); err != nil { - return errors.New("cannot load embedded core resource schema") - } - - // since we establish the resource chain of any missing resources, - // it is important to add things in the right order (for now) - keys := sortx.Keys(schema.Resources) - - // make sure we import aliases last. aliases should always refer to existing resources - // so we want to make sure those are properly registered in the registry first before we import aliases too - aliases := []string{} - for _, k := range keys { - isAlias := k != schema.Resources[k].Id - if isAlias { - aliases = append(aliases, k) - continue - } - if err := ctx.AddResourceInfo(schema.Resources[k]); err != nil { - return errors.New("failed to add resource info: " + err.Error()) - } - } - - for _, k := range aliases { - if err := ctx.AddResourceInfo(schema.Resources[k]); err != nil { - return errors.New("failed to add resource info: " + err.Error()) - } - info := ctx.Resources[schema.Resources[k].Id] - ctx.Resources[k] = info - ctx.ensureResourceChain(k, info.Private, true) - } - - return nil -} - -// for a given resource name, make sure all parent resources exist -// e.g. sshd.config ==> make sure sshd exists -func (ctx *Registry) ensureResourceChain(name string, isPrivate, isAlias bool) { - parts := strings.Split(name, ".") - if len(parts) == 1 { - return - } - cur := parts[0] - for i := 0; i < len(parts)-1; i++ { - o, ok := ctx.Resources[cur] - // it can be that we're trying to lookup an alias that doesn't have the parent defined - if !ok && isAlias { - o, ok = ctx.Resources[parts[i]] - if ok { - ctx.Resources[cur] = o - } - } - if !ok { - o = newResourceCls(cur) - ctx.Resources[cur] = o - // parent resources get the visibility of their children by default - // any public child overwrites the rest for the parent (see below) - o.Private = isPrivate - } - - // we may need to overwrite parent resource declaration if we realize the child is public - if !isPrivate { - o.Private = false - } - next := cur + "." + parts[i+1] - - f, ok := o.Fields[parts[i+1]] - if !ok { - f = &Field{ - Name: parts[i+1], - Type: string(types.Resource(next)), - IsMandatory: false, - IsImplicitResource: true, - Refs: []string{}, - IsPrivate: isPrivate, - } - o.Fields[parts[i+1]] = f - } - // same as above: if any child is public, the field in the chain must become public - if !isPrivate { - f.IsPrivate = isPrivate - } - - cur = next - } -} - -func mergeResourceInfoPartial(a *ResourceCls, b *ResourceInfo) error { - if a.Id != b.Id { - return errors.New("could not merge resources because IDs don't match: " + a.Id + ", " + b.Id) - } - - if a.Name != b.Name { - return errors.New("could not merge resources because names differ: " + a.Id + ", " + b.Id) - } - - if a.ListType != b.ListType { - return errors.New("could not merge resources because list type doesn't match: " + a.Id + ", " + b.Id) - } - - if a.Private != b.Private { - return errors.New("could not merge resources because mismatched private modifier: " + a.Id + ", " + b.Id) - } - - if a.Title == "" { - a.Title = b.Title - } - - if a.Desc == "" { - a.Desc = b.Desc - } - - if a.MinMondooVersion == "" { - a.MinMondooVersion = b.MinMondooVersion - } - - for _, f := range b.Fields { - if _, ok := a.Fields[f.Name]; !ok { - a.Fields[f.Name] = f - } - } - - return nil -} - -func (ctx *Registry) AddResourceInfo(info *ResourceInfo) error { - name := info.Id - - // NOTE: we do not yet merge resources! So error for now. - if r, ok := ctx.Resources[name]; ok { - if err := mergeResourceInfoPartial(r, info); err != nil { - return err - } - return nil - } - - if info.Fields == nil { - info.Fields = map[string]*Field{} - } - - ctx.Resources[name] = &ResourceCls{ - ResourceInfo: *info, - } - - ctx.ensureResourceChain(name, info.Private, false) - - return nil -} - -// Add a new resource with a factory for creating an instance -func (ctx *Registry) AddFactory(name string, factory ResourceFactory) error { - if name == "" { - return errors.New("trying to add factory for a resource without a name") - } - - resource, ok := ctx.Resources[name] - if !ok { - return errors.New("resource '" + name + "' cannot be found") - } - - resource.Factory = factory - return nil -} - -// Names all resources -func (ctx *Registry) Names() []string { - res := make([]string, len(ctx.Resources)) - i := 0 - for key := range ctx.Resources { - res[i] = key - i++ - } - return res -} - -// Fields of a resource -func (ctx *Registry) Fields(name string) (map[string]*Field, error) { - r, ok := ctx.Resources[name] - if !ok { - return nil, errors.New("Failed to get fields for resource " + name + ", couldn't find a resource with that name") - } - return r.Fields, nil -} - -// Schema of all loaded resources -func (ctx *Registry) Schema() *Schema { - res := Schema{Resources: make(map[string]*ResourceInfo)} - for id, i := range ctx.Resources { - res.Resources[id] = &i.ResourceInfo - } - return &res -} diff --git a/_resources/runtime.go b/_resources/runtime.go deleted file mode 100644 index 98661b2975..0000000000 --- a/_resources/runtime.go +++ /dev/null @@ -1,462 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package resources - -import ( - "errors" - fmt "fmt" - "net/http" - "sync" - - "github.com/rs/zerolog/log" - "go.mondoo.com/cnquery/motor" - "go.mondoo.com/cnquery/motor/providers" - "go.mondoo.com/cnquery/providers-sdk/v1/inventory" - "go.mondoo.com/ranger-rpc" -) - -// NotReadyError indicates the results are not ready to be processed further -type NotReadyError struct{} - -func (n NotReadyError) Error() string { - return "NotReadyError" -} - -var NotFound = errors.New("not found") - -// CacheEntry contains cached data for resources -type CacheEntry struct { - Timestamp int64 - Valid bool - Data interface{} - Error error -} - -// Cache is a map containing CacheEntry values -type Cache struct{ sync.Map } - -// Store a new call connection -func (c *Cache) Store(key string, v *CacheEntry) { c.Map.Store(key, v) } - -// Load a call connection -func (c *Cache) Load(key string) (*CacheEntry, bool) { - res, ok := c.Map.Load(key) - if res == nil { - return nil, ok - } - return res.(*CacheEntry), ok -} - -// Delete a Cache Entry -func (c *Cache) Delete(key string) { c.Map.Delete(key) } - -// mondoo platform config so that resource scan talk upstream -// TODO: this configuration struct does not belong into the MQL package -// nevertheless the MQL runtime needs to have something that allows users -// to store additional credentials so that resource can use those for -// their resources. -type UpstreamConfig struct { - AssetMrn string - SpaceMrn string - ApiEndpoint string - Plugins []ranger.ClientPlugin - Incognito bool - HttpClient *http.Client -} - -// Runtime of all initialized resources -type Runtime struct { - Motor *motor.Motor - Registry *Registry - cache *Cache - Observers *Observers - UpstreamConfig *UpstreamConfig - children []*Runtime -} - -// NewRuntime creates a new runtime from a registry and motor backend -func NewRuntime(registry *Registry, motor *motor.Motor) *Runtime { - if registry == nil { - panic("cannot initialize MQL runtime without a registry") - } - if motor == nil { - panic("cannot initialize MQL runtime without a motor") - } - - return &Runtime{ - Registry: registry, - Observers: NewObservers(), - Motor: motor, - cache: &Cache{}, - children: []*Runtime{}, - } -} - -func args2map(args []interface{}) (*Args, error) { - if args == nil { - res := make(Args) - return &res, nil - } - - if len(args)%2 == 1 { - panic("failed to get named argument, it should be supplied as (key, values, ...) and I'm missing a value") - } - - res := make(Args) - for i := 0; i < len(args); { - name, ok := args[i].(string) - if !ok { - // TODO: can we get rid of this fmt method? - return nil, fmt.Errorf("Failed to get named argument, it is not a string field: %#v", args[0]) - } - - res[name] = args[i+1] - i += 2 - } - return &res, nil -} - -func (ctx *Runtime) cloneWithMotor(motor *motor.Motor) *Runtime { - if motor == nil { - panic("cannot initialize MQL runtime without a motor") - } - - return &Runtime{ - Registry: ctx.Registry, - Observers: ctx.Observers, - Motor: motor, - cache: &Cache{}, - children: []*Runtime{}, - } -} - -func (ctx *Runtime) createMockResource(name string, cls *ResourceCls) (ResourceType, error) { - res := MockResource{ - StaticFields: cls.Fields, - StaticResource: &Resource{ - ResourceID: ResourceID{Id: "", Name: name}, - }, - } - ctx.Set(res.MqlResource().Name, res.MqlResource().Id, &res) - return res, nil -} - -func (ctx *Runtime) lookupResource(name string) (*ResourceCls, string, error) { - for { - r := ctx.Registry.Resources[name] - if r == nil { - return nil, name, errors.New("cannot find resource '" + name + "'") - } else if r.Factory != nil { - return r, name, nil - } else if r.Name != name { - // A resource was given an alias. Look up through aliases - name = r.Name - } else { - // We found a resource with a factory - return nil, name, errors.New("cannot find resource factory for '" + name + "'") - } - } -} - -// CreateResourceWithID creates a new resource instance and force it to have a certain ID -func (ctx *Runtime) CreateResourceWithID(name string, id string, args ...interface{}) (ResourceType, error) { - r, name, err := ctx.lookupResource(name) - if err != nil { - return nil, err - } - // We could have looked up an aliased resource. We need to correct the name - // for caching - name = r.Name - - argsMap, err := args2map(args) - if err != nil { - return nil, err - } - - if r.Factory == nil { - if len(args) > 0 { - return nil, errors.New("mock resources don't take any arguments. The resource '" + name + "' doesn't have a resource factory") - } - return ctx.createMockResource(name, r) - } - - // factory not only creates a resource, but may also provide an empty resource - // with the `Id` field set to look up an existing resource - res, err := r.Factory(ctx, argsMap) - if err != nil { - return nil, fmt.Errorf("failed to create resource '%s': %w", name, err) - } - if res == nil { - return nil, errors.New("resource factory produced a nil result for resource '" + name + "'") - } - - resResource := res.(ResourceType) - if id == "" { - id = resResource.MqlResource().Id - } else { - resResource.MqlResource().Id = id - } - - log.Trace().Str("name", name).Str("id", id).Msg("created resource") - - if ex, err := ctx.GetResource(name, id); err == nil { - resResource = ex - } else { - if err := resResource.Validate(); err != nil { - return nil, errors.New("failed to validate resource '" + name + "': " + err.Error()) - } - ctx.Set(name, id, res) - } - - return resResource, nil -} - -// CreateResource creates a new resource instance taking its name + args -func (ctx *Runtime) CreateResource(name string, args ...interface{}) (ResourceType, error) { - return ctx.CreateResourceWithID(name, "", args...) -} - -// CreateResourceWithAssetContext will create the Resource. If the asset "a" and provider "p" match -// what "ctx" already holds, we do not need to create a new Runtime and can attach it directly to the -// new resource. Otherwise, a new runtime is created, where the asset and provider are changed. -// We probably also need to do something about ctx.UpstreamConfig -func (ctx *Runtime) CreateResourceWithAssetContext(name string, a *inventory.Asset, p providers.Instance, args ...interface{}) (ResourceType, error) { - if p == nil { - p = ctx.Motor.Provider - } - if a == nil { - a = ctx.Motor.GetAsset() - } - nextCtx := ctx - // TODO: - // If we create a new motor, but p is shared, bad things may happen - // when closing the motor - m, err := motor.New(p) - if err != nil { - return nil, err - } - - m.SetAsset(a) - newCtx := ctx.cloneWithMotor(m) - nextCtx.children = append(nextCtx.children, newCtx) - nextCtx = newCtx - return nextCtx.CreateResourceWithID(name, "", args...) -} - -// GetRawResource resource instance by name and id -func (ctx *Runtime) getRawResource(name string, id string) (interface{}, bool) { - res, ok := ctx.cache.Load(name + "\x00" + id) - if !ok { - return nil, ok - } - return res.Data, ok -} - -// GetResource resource instance by name and id -func (ctx *Runtime) GetResource(name string, id string) (ResourceType, error) { - c, ok := ctx.getRawResource(name, id) - if !ok { - return nil, errors.New("cannot find cached resource " + name + " ID: " + id) - } - res, ok := c.(ResourceType) - if !ok { - return nil, errors.New("cached resource is not of ResourceType for " + name + " ID: " + id) - } - return res, nil -} - -// Set a resource by name and ID. Must be a valid Resource. -func (ctx *Runtime) Set(name string, id string, resource interface{}) { - ctx.cache.Store(name+"\x00"+id, &CacheEntry{ - Data: resource, - Valid: true, - }) -} - -// watch+update => observe it and callback results -// watch+compute => observe it and compute this field when the observed thing changes -// register => build more watch+compute relationships if needed -// trigger => force a field to send a result - -// WatchAndUpdate a resource field and call the function if it changes with its current value -func (ctx *Runtime) WatchAndUpdate(r ResourceType, field string, watcherUID string, callback func(res interface{}, err error)) error { - resource := r.MqlResource() - // log.Debug(). - // Str("src", resource.Name+"\x00"+resource.Id+"\x00"+field). - // Str("watcher", watcherUID). - // Msg("w+u> watch and update") - - // FIXME: calling resource.Fields instead of vv breaks everything!! Make it impossible to do so maybe? - fieldObj, err := ctx.Registry.Fields(resource.Name) - if err != nil { - return errors.New("tried to register field " + field + " in resource " + resource.UID() + ": " + err.Error()) - } - if fieldObj == nil { - return errors.New("field object " + field + " in resource " + resource.UID() + " is nil") - } - fieldUID := resource.FieldUID(field) - - processResult := func() { - log.Trace(). - Str("src", resource.Name+"\x00"+resource.Id+"\x00"+field). - Str("watcher", watcherUID). - Msg("w+u> process field result") - - data, ok := resource.Cache.Load(field) - if !ok { - callback(nil, errors.New("couldn't retrieve value of field \""+field+"\" in resource \""+resource.UID()+"\"")) - return - } - - callback(data.Data, data.Error) - } - - isInitial, exists, err := ctx.Observers.Watch(fieldUID, watcherUID, processResult) - if err != nil { - return err - } - if exists { - return nil - } - - // TODO: this is very special handling for when we create a copy of a list - // resource. in those cases its content (list) has already been filled, - // but without this block here it will try to compute the entire list from - // the ground up. It's more of a workaround right now and needs a better - // solution (eg an indicator for the copied resource?) - if field == "list" && isInitial { - data, ok := resource.Cache.Load(field) - if ok { - callback(data.Data, data.Error) - } - } - - // if the field wasn't registered in the chain of watchers yet, - // pull all its dependencies in - if isInitial { - if err = r.Register(field); err != nil { - return err - } - - err = r.MqlCompute(field) - // normal case most often: we called compute but it depends on something - // that is not ready - if _, ok := err.(NotReadyError); ok { - return nil - } - - // final case: it is computed and ready to go - log.Trace().Msg("w+u> initial process result") - processResult() - return nil - } - - data, ok := resource.Cache.Load(field) - if ok { - callback(data.Data, data.Error) - } - - return nil -} - -// Unregister will remove all watcher UIDs -func (ctx *Runtime) Unregister(watcherUID string) error { - log.Trace().Str("watchers", watcherUID).Msg("w+u> unregister") - return ctx.Observers.UnwatchAll(watcherUID) -} - -// WatchAndCompute watches a field in a resource and computes -// another resource + field once once this resource and field has changed -func (ctx *Runtime) WatchAndCompute(src ResourceType, sfield string, dst ResourceType, dfield string) error { - resource := dst.MqlResource() - fid := resource.FieldUID(dfield) - sid := src.MqlResource().FieldUID(sfield) - - isInitial, exists, err := ctx.Observers.Watch(sid, fid, func() { - // once the source field changes, we recalculate the destination field - ierr := dst.MqlCompute(dfield) - // if the field isnt ready, finish this execution - if _, ok := ierr.(NotReadyError); ok { - return - } - - // then we let all the dependent fields know that we just updated this resource field - ierr = ctx.Trigger(dst, dfield) - if ierr != nil { - log.Error().Str("resource+field-uid", fid).Msg("w+c> Failed to trigger resource field: " + ierr.Error()) - return - } - }) - log.Trace(). - Str("src", sid). - Str("dst", fid). - Bool("initial", isInitial). - Bool("exists", exists). - Err(err). - Msg("w+c> watch and compute") - - if err != nil { - return err - } - if exists { - return nil - } - - // if the field wasn't registered in the chain of watchers yet, - // pull all its dependencies in - if isInitial { - if err = src.Register(sfield); err != nil { - log.Error().Err(err).Msg("w+c> initial register failed") - return err - } - - err = src.MqlCompute(sfield) - if err != nil { - if _, ok := err.(NotReadyError); !ok { - log.Trace().Err(err).Msg("w+c> initial compute failed") - return err - } - } - } - - return nil -} - -// Trigger a resource-field is a way to request it to calculate its -// value and call the callback. It may use cached values at this point -func (ctx *Runtime) Trigger(r ResourceType, field string) error { - resource := r.MqlResource() - if field == "" { - return errors.New("cannot trigger a resource without specifying a field") - } - - log.Trace(). - Str("resource", resource.Name+":"+resource.Id). - Str("field", field). - Msg("trigger> trigger resource") - - res, ok := resource.Cache.LoadOrStore(field, &CacheEntry{}) - // data in cache means we can go ahead, it's nicely connected already - // if not it means that the underlying method was never called to compute its value - // we set the cache to an invalid value to make sure no one else triggers it - // then we ensure all dependencies send us their results - if ok { - entry := res.(*CacheEntry) - // if it's valid call whatever is listening to this field - if entry.Valid || entry.Error != nil { - return ctx.Observers.Trigger(resource.FieldUID(field)) - } - // if it's not we won't call listening fields yet, because things aren't ready - return NotReadyError{} - } - - return r.MqlCompute(field) -} - -func (r *Runtime) Close() { - for _, c := range r.children { - c.Close() - } - r.Motor.Close() -} diff --git a/_resources/runtime_test.go b/_resources/runtime_test.go deleted file mode 100644 index 9239b6d5d4..0000000000 --- a/_resources/runtime_test.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package resources - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestArg2Map(t *testing.T) { - args := []interface{}{"project", "mondoo", "zone", "us-central1-a"} - - argsmap, err := args2map(args) - assert.Nil(t, err, "should be able to convert args to map") - - assert.Equal(t, "mondoo", (*argsmap)["project"], "extracted project arg") - assert.Equal(t, "us-central1-a", (*argsmap)["zone"], "extracted zone arg") -} diff --git a/_resources/schema.go b/_resources/schema.go deleted file mode 100644 index f285cc9b61..0000000000 --- a/_resources/schema.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package resources - -func (s *Schema) Add(other *Schema) { - if other == nil { - return - } - - for k, v := range other.Resources { - s.Resources[k] = v - } -} - -func (s *Schema) Lookup(name string) *ResourceInfo { - return s.Resources[name] -} - -func (s *Schema) AllResources() map[string]*ResourceInfo { - return s.Resources -}