From 9d85d4befdb0f72d38f122200a5a5716590cde2c Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Wed, 11 Dec 2024 14:15:05 +0100 Subject: [PATCH 01/10] Refactor binding tests --- cmd/broker/binding_test.go | 5 - internal/broker/bind_create_test.go | 197 ++++++++++++++- internal/broker/bind_envtest_test.go | 361 +++++++++++++++++++++++++++ 3 files changed, 556 insertions(+), 7 deletions(-) create mode 100644 internal/broker/bind_envtest_test.go diff --git a/cmd/broker/binding_test.go b/cmd/broker/binding_test.go index f74f49642e..565b9ab16f 100644 --- a/cmd/broker/binding_test.go +++ b/cmd/broker/binding_test.go @@ -56,11 +56,6 @@ func TestBinding(t *testing.T) { suite.Log(string(b)) suite.Log(resp.Status) - respRuntimes := suite.CallAPI(http.MethodGet, "/runtimes?bindings=true", "") - b, _ = io.ReadAll(respRuntimes.Body) - suite.Log(string(b)) - suite.Log(resp.Status) - t.Run("should return 400 when expiration seconds parameter is string instead of int", func(t *testing.T) { resp = suite.CallAPI(http.MethodPut, fmt.Sprintf("oauth/v2/service_instances/%s/service_bindings/%s", iid, bid), `{ diff --git a/internal/broker/bind_create_test.go b/internal/broker/bind_create_test.go index 98c78e766d..2132a58c61 100644 --- a/internal/broker/bind_create_test.go +++ b/internal/broker/bind_create_test.go @@ -4,8 +4,13 @@ import ( "context" "encoding/json" "fmt" + "github.com/kyma-project/kyma-environment-broker/internal" + "github.com/kyma-project/kyma-environment-broker/internal/kubeconfig" + "github.com/pivotal-cf/brokerapi/v8/domain/apiresponses" "log/slog" + "net/http" "os" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" "time" @@ -24,9 +29,23 @@ import ( const ( instanceID1 = "1" - maxBindingsCount = 10 + maxBindingsCount = 1000 ) +func fixBindingConfig() BindingConfig { + return BindingConfig{ + Enabled: true, + BindablePlans: EnablePlans{ + fixture.PlanName, + }, + ExpirationSeconds: 600, + MaxExpirationSeconds: maxExpirationSeconds, + MinExpirationSeconds: minExpirationSeconds, + MaxBindingsCount: maxBindingsCount, + } + +} + type provider struct { } @@ -65,7 +84,8 @@ func TestCreateBindingEndpoint(t *testing.T) { BindablePlans: EnablePlans{ fixture.PlanName, }, - MaxBindingsCount: maxBindingsCount, + MaxBindingsCount: maxBindingsCount, + MinExpirationSeconds: minExpirationSeconds, } // event publisher @@ -97,6 +117,154 @@ func TestCreateBindingEndpoint(t *testing.T) { require.NotNil(t, binding.ExpiresAt) require.Empty(t, binding.Kubeconfig) }) + + t.Run("should return error when expiration_seconds is greater than maxExpirationSeconds", func(t *testing.T) { + // when + _, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-001", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{"expirationSeconds": 10}`), + }, false) + require.Error(t, err) + + // then + assert.IsType(t, err, &apiresponses.FailureResponse{}, "Get request returned error of unexpected type") + apierr := err.(*apiresponses.FailureResponse) + assert.Equal(t, http.StatusBadRequest, apierr.ValidatedStatusCode(nil), "Get request status code not matching") + }) + + t.Run("should return error when expiration_seconds is less than minExpirationSeconds", func(t *testing.T) { + // when + _, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-002", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{"expirationSeconds": 100000}`), + }, false) + require.Error(t, err) + + // then + assert.IsType(t, err, &apiresponses.FailureResponse{}, "Get request returned error of unexpected type") + apierr := err.(*apiresponses.FailureResponse) + assert.Equal(t, http.StatusBadRequest, apierr.ValidatedStatusCode(nil), "Get request status code not matching") + }) +} + +func TestCreateBindingExceedsAllowedNumberOfNonExpiredBindings(t *testing.T) { + // given + cfg := fixBindingConfig() + cfg.MaxBindingsCount = 1 + bindEndpoint, db := prepareBindingEndpoint(t, cfg) + err := db.Bindings().Insert(&internal.Binding{ + ID: "existing-binding", + InstanceID: instanceID1, + CreatedAt: time.Now().Add(-2 * time.Hour), + UpdatedAt: time.Time{}, + ExpiresAt: time.Now().Add(-1 * time.Hour), + Kubeconfig: "abcd", + ExpirationSeconds: 600, + CreatedBy: "", + }) + require.NoError(t, err) + + // when creating first binding + _, err = bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-001", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{}`), + }, false) + + require.NoError(t, err) + + // when creating second binding - expect error + _, err = bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-002", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{}`), + }, false) + + assert.IsType(t, err, &apiresponses.FailureResponse{}, "Get request returned error of unexpected type") + apierr := err.(*apiresponses.FailureResponse) + assert.Equal(t, http.StatusBadRequest, apierr.ValidatedStatusCode(nil), "Get request status code not matching") + +} + +func TestCreateBindng(t *testing.T) { + bindEndpoint, _ := prepareBindingEndpoint(t, fixBindingConfig()) + + t.Run("should return error when expiration_seconds is less than minExpirationSeconds", func(t *testing.T) { + _, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-001", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{"expiration_seconds": 10}`), + }, false) + + // then + assert.IsType(t, err, &apiresponses.FailureResponse{}, "Get request returned error of unexpected type") + apierr := err.(*apiresponses.FailureResponse) + assert.Equal(t, http.StatusBadRequest, apierr.ValidatedStatusCode(nil), "Get request status code not matching") + }) + + t.Run("should return error when expiration_seconds is greater than maxExpirationSeconds", func(t *testing.T) { + _, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-001", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{"expiration_seconds": 100000}`), + }, false) + + // then + assert.IsType(t, err, &apiresponses.FailureResponse{}, "Get request returned error of unexpected type") + apierr := err.(*apiresponses.FailureResponse) + assert.Equal(t, http.StatusBadRequest, apierr.ValidatedStatusCode(nil), "Get request status code not matching") + }) + + t.Run("should create binding", func(t *testing.T) { + binding, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-002", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{}`), + }, false) + + // then + require.NoError(t, err) + assert.NotEmptyf(t, binding.Credentials.(Credentials).Kubeconfig, "kubeconfig is empty") + }) + + t.Run("should create binding", func(t *testing.T) { + binding, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-003", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{}`), + }, false) + + // then + require.NoError(t, err) + assert.NotEmptyf(t, binding.Credentials.(Credentials).Kubeconfig, "kubeconfig is empty") + }) + + t.Run("should report a conflict", func(t *testing.T) { + _, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-004", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{"expiration_seconds": 1000}`), + }, false) + + // then + require.NoError(t, err) + + // when + _, err = bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-004", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{"expiration_seconds": 2000}`), + }, false) + // then + require.Error(t, err) + + assert.IsType(t, err, &apiresponses.FailureResponse{}, "Get request returned error of unexpected type") + apierr := err.(*apiresponses.FailureResponse) + assert.Equal(t, http.StatusConflict, apierr.ValidatedStatusCode(nil), "Get request status code not matching") + }) + } func TestCreatedBy(t *testing.T) { @@ -400,3 +568,28 @@ func TestCreateSecondBindingWithTheSameIdAndParamsNotExplicitlyDefined(t *testin assert.NoError(t, err) assert.Equal(t, binding.ExpiresAt.Format(expiresAtLayout), resp.Metadata.ExpiresAt) } + +func prepareBindingEndpoint(t *testing.T, cfg BindingConfig) (*BindEndpoint, storage.BrokerStorage) { + sch := internal.NewSchemeForTests(t) + fakeK8sSKRClient := fake.NewClientBuilder().WithScheme(sch).Build() + k8sClientProvider := kubeconfig.NewFakeK8sClientProvider(fakeK8sSKRClient) + db := storage.NewMemoryStorage() + log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelDebug, + })) + //// binding configuration + + logs := logrus.New() + logs.SetLevel(logrus.DebugLevel) + logs.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: time.RFC3339Nano, + }) + err := db.Instances().Insert(fixture.FixInstance(instanceID1)) + require.NoError(t, err) + + operation := fixture.FixOperation("operation-id", instanceID1, "provision") + err = db.Operations().InsertOperation(operation) + require.NoError(t, err) + + return NewBind(cfg, db, logs, k8sClientProvider, k8sClientProvider, event.NewPubSub(log)), db +} diff --git a/internal/broker/bind_envtest_test.go b/internal/broker/bind_envtest_test.go new file mode 100644 index 0000000000..1afdf6b9ce --- /dev/null +++ b/internal/broker/bind_envtest_test.go @@ -0,0 +1,361 @@ +package broker + +import ( + "context" + "encoding/json" + "fmt" + "github.com/golang-jwt/jwt/v4" + "github.com/kyma-project/kyma-environment-broker/internal" + brokerBindings "github.com/kyma-project/kyma-environment-broker/internal/broker/bindings" + "github.com/kyma-project/kyma-environment-broker/internal/event" + "github.com/kyma-project/kyma-environment-broker/internal/fixture" + "github.com/kyma-project/kyma-environment-broker/internal/kubeconfig" + "github.com/kyma-project/kyma-environment-broker/internal/storage" + "github.com/pivotal-cf/brokerapi/v8/domain" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "log/slog" + "os" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "testing" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + expirationSeconds = 600 + maxExpirationSeconds = 7200 + minExpirationSeconds = 600 +) + +func TestCreateBinding(t *testing.T) { + // given + + //// schema + sch := runtime.NewScheme() + err := corev1.AddToScheme(sch) + assert.NoError(t, err) + + logs := logrus.New() + logs.SetLevel(logrus.DebugLevel) + logs.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: time.RFC3339Nano, + }) + // prepare envtest to provide valid kubeconfig + envFirst, configFirst, clientFirst := createEnvTest(t) + defer func(env *envtest.Environment) { + err := env.Stop() + assert.NoError(t, err) + }(&envFirst) + kbcfgFirst := createKubeconfigFileForRestConfig(*configFirst) + + kcpClient := fake.NewClientBuilder(). + WithScheme(sch). + WithRuntimeObjects([]runtime.Object{ + &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: "kubeconfig-runtime-1", + Namespace: "kcp-system", + }, + Data: map[string][]byte{ + "config": kbcfgFirst, + }, + }, + }...). + Build() + + fmt.Println(string(kbcfgFirst)) + + //// secret check in assertions + err = clientFirst.Create(context.Background(), &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: "secret-to-check-first", + Namespace: "default", + }, + }) + require.NoError(t, err) + + bindingCfg := BindingConfig{ + Enabled: true, + BindablePlans: EnablePlans{ + fixture.PlanName, + }, + ExpirationSeconds: expirationSeconds, + MaxExpirationSeconds: maxExpirationSeconds, + MinExpirationSeconds: minExpirationSeconds, + MaxBindingsCount: maxBindingsCount, + } + db := storage.NewMemoryStorage() + + err = db.Instances().Insert(fixture.FixInstance(instanceID1)) + require.NoError(t, err) + operation := fixture.FixOperation("operation-001", instanceID1, internal.OperationTypeProvision) + err = db.Operations().InsertOperation(operation) + require.NoError(t, err) + + skrK8sClientProvider := kubeconfig.NewK8sClientFromSecretProvider(kcpClient) + log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelDebug, + })) + publisher := event.NewPubSub(log) + svc := NewBind(bindingCfg, db, logs, skrK8sClientProvider, skrK8sClientProvider, publisher) + unbindSvc := NewUnbind(logs, db, brokerBindings.NewServiceAccountBindingsManager(skrK8sClientProvider, skrK8sClientProvider), publisher) + + t.Run("should create a new service binding without error", func(t *testing.T) { + // When + response, err := svc.Bind(context.Background(), instanceID1, "binding-id", domain.BindDetails{ + AppGUID: "", + PlanID: "", + ServiceID: "", + BindResource: nil, + RawContext: json.RawMessage(`{}`), + RawParameters: json.RawMessage(`{"expirationSeconds": 660}`), + }, false) + + // then + require.NoError(t, err) + + assertClusterAccess(t, "secret-to-check-first", response.Credentials) + assertRolesExistence(t, brokerBindings.BindingName("binding-id"), response.Credentials) + assertTokenDuration(t, response.Credentials, 11*time.Minute) + }) + + t.Run("should create two bindings and unbind one of them", func(t *testing.T) { + // when + response1, err1 := svc.Bind(context.Background(), instanceID1, "binding-id1", domain.BindDetails{ + AppGUID: "", + PlanID: "", + ServiceID: "", + BindResource: nil, + RawContext: json.RawMessage(`{}`), + RawParameters: json.RawMessage(`{"expirationSeconds": 660}`), + }, false) + response2, err2 := svc.Bind(context.Background(), instanceID1, "binding-id2", domain.BindDetails{ + AppGUID: "", + PlanID: "", + ServiceID: "", + BindResource: nil, + RawContext: json.RawMessage(`{}`), + RawParameters: json.RawMessage(`{"expirationSeconds": 600}`), + }, false) + + // then + require.NoError(t, err1) + require.NoError(t, err2) + + assertClusterAccess(t, "secret-to-check-first", response1.Credentials) + assertClusterAccess(t, "secret-to-check-first", response2.Credentials) + + // when unbinding occurs for first binding, second should still work + _, err := unbindSvc.Unbind(context.Background(), instanceID1, "binding-id1", domain.UnbindDetails{}, false) + require.NoError(t, err) + + // then + assertClusterNoAccess(t, "secret-to-check-first", response1.Credentials) + assertClusterAccess(t, "secret-to-check-first", response2.Credentials) + }) +} + +type kubeconfigStruct struct { + Users []kubeconfigUser `yaml:"users"` +} + +type kubeconfigUser struct { + Name string `yaml:"name"` + User struct { + Token string `yaml:"token"` + } `yaml:"user"` +} + +func assertTokenDuration(t *testing.T, creds interface{}, expectedDuration time.Duration) { + + credentials, ok := creds.(Credentials) + require.True(t, ok) + config := credentials.Kubeconfig + var kubeconfigObj kubeconfigStruct + + err := yaml.Unmarshal([]byte(config), &kubeconfigObj) + require.NoError(t, err) + + var tokenDuration time.Duration + for _, user := range kubeconfigObj.Users { + if user.Name == "context" { + token, _, err := new(jwt.Parser).ParseUnverified(user.User.Token, jwt.MapClaims{}) + require.NoError(t, err) + + if claims, ok := token.Claims.(jwt.MapClaims); ok { + iat := int64(claims["iat"].(float64)) + exp := int64(claims["exp"].(float64)) + + issuedAt := time.Unix(iat, 0) + expiresAt := time.Unix(exp, 0) + + tokenDuration = expiresAt.Sub(issuedAt) + fmt.Printf("%v %v\n", expiresAt, issuedAt) + } else { + assert.Fail(t, "invalid token claims") + break + } + } + } + + assert.Equal(t, expectedDuration, tokenDuration) +} + +func createEnvTest(t *testing.T) (envtest.Environment, *rest.Config, client.Client) { + + fmt.Println("setup envtest") + + pid := internal.SetupEnvtest(t) + defer func() { + internal.CleanupEnvtestBinaries(pid) + }() + + fmt.Println("start envtest") + + env := envtest.Environment{ + ControlPlaneStartTimeout: 40 * time.Second, + } + var errEnvTest error + var config *rest.Config + err := wait.PollUntilContextTimeout(context.Background(), 500*time.Millisecond, 5*time.Second, true, func(context.Context) (done bool, err error) { + config, errEnvTest = env.Start() + if err != nil { + t.Logf("envtest could not start, retrying: %s", errEnvTest.Error()) + return false, nil + } + t.Logf("envtest started") + return true, nil + }) + require.NoError(t, err) + require.NoError(t, errEnvTest) + + skrClient, err := initClient(config) + require.NoError(t, err) + + err = skrClient.Create(context.Background(), &corev1.Namespace{ + ObjectMeta: v1.ObjectMeta{ + Name: brokerBindings.BindingNamespace, + }, + }) + require.NoError(t, err) + return env, config, skrClient +} + +func createKubeconfigFileForRestConfig(restConfig rest.Config) []byte { + const ( + userName = "user" + clusterName = "cluster" + contextName = "context" + ) + + clusters := make(map[string]*clientcmdapi.Cluster) + clusters[clusterName] = &clientcmdapi.Cluster{ + Server: restConfig.Host, + CertificateAuthorityData: restConfig.CAData, + } + contexts := make(map[string]*clientcmdapi.Context) + contexts[contextName] = &clientcmdapi.Context{ + Cluster: clusterName, + AuthInfo: userName, + } + authinfos := make(map[string]*clientcmdapi.AuthInfo) + authinfos[userName] = &clientcmdapi.AuthInfo{ + ClientCertificateData: restConfig.CertData, + ClientKeyData: restConfig.KeyData, + } + clientConfig := clientcmdapi.Config{ + Kind: "Config", + APIVersion: "v1", + Clusters: clusters, + Contexts: contexts, + CurrentContext: contextName, + AuthInfos: authinfos, + } + kubeconfig, _ := clientcmd.Write(clientConfig) + return kubeconfig +} + +func initClient(cfg *rest.Config) (client.Client, error) { + mapper, err := apiutil.NewDiscoveryRESTMapper(cfg) + if err != nil { + err = wait.Poll(time.Second, time.Minute, func() (bool, error) { + mapper, err = apiutil.NewDiscoveryRESTMapper(cfg) + if err != nil { + return false, nil + } + return true, nil + }) + if err != nil { + return nil, fmt.Errorf("while waiting for client mapper: %w", err) + } + } + cli, err := client.New(cfg, client.Options{Mapper: mapper}) + if err != nil { + return nil, fmt.Errorf("while creating a client: %w", err) + } + return cli, nil +} + +func assertClusterAccess(t *testing.T, controlSecretName string, creds interface{}) { + credentials, ok := creds.(Credentials) + require.True(t, ok) + kubeconfig := credentials.Kubeconfig + + newClient := kubeconfigClient(t, kubeconfig) + + _, err := newClient.CoreV1().Secrets("default").Get(context.Background(), controlSecretName, v1.GetOptions{}) + assert.NoError(t, err) +} + +func assertClusterNoAccess(t *testing.T, controlSecretName string, creds interface{}) { + credentials, ok := creds.(Credentials) + require.True(t, ok) + kubeconfig := credentials.Kubeconfig + + newClient := kubeconfigClient(t, kubeconfig) + + _, err := newClient.CoreV1().Secrets("default").Get(context.Background(), controlSecretName, v1.GetOptions{}) + assert.True(t, apierrors.IsForbidden(err)) +} + +func kubeconfigClient(t *testing.T, kubeconfig string) *kubernetes.Clientset { + config, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeconfig)) + assert.NoError(t, err) + + clientset, err := kubernetes.NewForConfig(config) + assert.NoError(t, err) + + return clientset +} + +func assertRolesExistence(t *testing.T, bindingID string, creds interface{}) { + credentials, ok := creds.(Credentials) + require.True(t, ok) + kubeconfig := credentials.Kubeconfig + + newClient := kubeconfigClient(t, kubeconfig) + + _, err := newClient.CoreV1().ServiceAccounts(brokerBindings.BindingNamespace).Get(context.Background(), bindingID, v1.GetOptions{}) + assert.NoError(t, err) + _, err = newClient.RbacV1().ClusterRoles().Get(context.Background(), bindingID, v1.GetOptions{}) + assert.NoError(t, err) + _, err = newClient.RbacV1().ClusterRoleBindings().Get(context.Background(), bindingID, v1.GetOptions{}) + assert.NoError(t, err) + _, err = newClient.RbacV1().ClusterRoleBindings().Get(context.Background(), bindingID, v1.GetOptions{}) + assert.NoError(t, err) +} From 7350bdbd06da7a4e79a940e806957379e5c8e866 Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Wed, 11 Dec 2024 14:15:33 +0100 Subject: [PATCH 02/10] refactor bidning tests - remove old env tests --- cmd/broker/bindings_envtest_test.go | 805 ---------------------------- 1 file changed, 805 deletions(-) delete mode 100644 cmd/broker/bindings_envtest_test.go diff --git a/cmd/broker/bindings_envtest_test.go b/cmd/broker/bindings_envtest_test.go deleted file mode 100644 index 301605857b..0000000000 --- a/cmd/broker/bindings_envtest_test.go +++ /dev/null @@ -1,805 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "os" - "testing" - "time" - - "github.com/kyma-project/kyma-environment-broker/internal/event" - - "github.com/golang-jwt/jwt/v4" - "github.com/google/uuid" - brokerBindings "github.com/kyma-project/kyma-environment-broker/internal/broker/bindings" - "gopkg.in/yaml.v2" - rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - - "code.cloudfoundry.org/lager" - "github.com/gorilla/mux" - "github.com/kyma-project/kyma-environment-broker/internal" - "github.com/kyma-project/kyma-environment-broker/internal/broker" - "github.com/kyma-project/kyma-environment-broker/internal/fixture" - "github.com/kyma-project/kyma-environment-broker/internal/kubeconfig" - "github.com/kyma-project/kyma-environment-broker/internal/storage" - "github.com/pivotal-cf/brokerapi/v8/domain" - "github.com/pivotal-cf/brokerapi/v8/handlers" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/envtest" -) - -type Kubeconfig struct { - Users []User `yaml:"users"` -} - -type User struct { - Name string `yaml:"name"` - User struct { - Token string `yaml:"token"` - } `yaml:"user"` -} - -const ( - instanceID1 = "1" - instanceID2 = "2" - instanceID3 = "max-bindings" - expirationSeconds = 10000 - maxExpirationSeconds = 7200 - minExpirationSeconds = 600 - bindingsPath = "v2/service_instances/%s/service_bindings/%s" - deleteParams = "?accepts_incomplete=false&service_id=%s&plan_id=%s" - maxBindingsCount = 10 -) - -var httpServer *httptest.Server - -func TestCreateBindingEndpoint(t *testing.T) { - t.Log("test create binding endpoint") - - // Given - //// logger - log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ - Level: slog.LevelDebug, - })) - - brokerLogger := lager.NewLogger("test") - brokerLogger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG)) - - //// schema - sch := runtime.NewScheme() - err := corev1.AddToScheme(sch) - assert.NoError(t, err) - - // prepare envtest to provide valid kubeconfig - envFirst, configFirst, clientFirst := createEnvTest(t) - - defer func(env *envtest.Environment) { - err := env.Stop() - assert.NoError(t, err) - }(&envFirst) - kbcfgFirst := createKubeconfigFileForRestConfig(*configFirst) - - //// secret check in assertions - err = clientFirst.Create(context.Background(), &corev1.Secret{ - ObjectMeta: v1.ObjectMeta{ - Name: "secret-to-check-first", - Namespace: "default", - }, - }) - require.NoError(t, err) - - // prepare envtest to provide valid kubeconfig for the second environment - envSecond, configSecond, clientSecond := createEnvTest(t) - - defer func(env *envtest.Environment) { - err := env.Stop() - assert.NoError(t, err) - }(&envSecond) - kbcfgSecond := createKubeconfigFileForRestConfig(*configSecond) - - //// secret check in assertions - err = clientSecond.Create(context.Background(), &corev1.Secret{ - ObjectMeta: v1.ObjectMeta{ - Name: "secret-to-check-second", - Namespace: "default", - }, - }) - - //// create fake kubernetes client - kcp - kcpClient := fake.NewClientBuilder(). - WithScheme(sch). - WithRuntimeObjects([]runtime.Object{ - &corev1.Secret{ - ObjectMeta: v1.ObjectMeta{ - Name: "kubeconfig-runtime-1", - Namespace: "kcp-system", - }, - Data: map[string][]byte{ - "config": kbcfgFirst, - }, - }, - &corev1.Secret{ - ObjectMeta: v1.ObjectMeta{ - Name: "kubeconfig-runtime-2", - Namespace: "kcp-system", - }, - Data: map[string][]byte{ - "config": kbcfgSecond, - }, - }, - &corev1.Secret{ - ObjectMeta: v1.ObjectMeta{ - Name: "kubeconfig-runtime-max-bindings", - Namespace: "kcp-system", - }, - Data: map[string][]byte{ - "config": kbcfgSecond, - }, - }, - }...). - Build() - - //// database - storageCleanup, db, err := GetStorageForE2ETests() - t.Cleanup(func() { - if storageCleanup != nil { - err := storageCleanup() - assert.NoError(t, err) - } - }) - assert.NoError(t, err) - - err = db.Instances().Insert(fixture.FixInstance(instanceID1)) - require.NoError(t, err) - operation := fixture.FixOperation("operation-001", instanceID1, internal.OperationTypeProvision) - err = db.Operations().InsertOperation(operation) - require.NoError(t, err) - - err = db.Instances().Insert(fixture.FixInstance(instanceID2)) - require.NoError(t, err) - operation = fixture.FixOperation("operation-002", instanceID2, internal.OperationTypeProvision) - err = db.Operations().InsertOperation(operation) - require.NoError(t, err) - - err = db.Instances().Insert(fixture.FixInstance(instanceID3)) - require.NoError(t, err) - operation = fixture.FixOperation("operation-003", instanceID3, internal.OperationTypeProvision) - err = db.Operations().InsertOperation(operation) - require.NoError(t, err) - - skrK8sClientProvider := kubeconfig.NewK8sClientFromSecretProvider(kcpClient) - - //// binding configuration - bindingCfg := &broker.BindingConfig{ - Enabled: true, - BindablePlans: broker.EnablePlans{ - fixture.PlanName, - }, - ExpirationSeconds: expirationSeconds, - MaxExpirationSeconds: maxExpirationSeconds, - MinExpirationSeconds: minExpirationSeconds, - MaxBindingsCount: maxBindingsCount, - } - - publisher := event.NewPubSub(log) - - //// api handler - bindEndpoint := broker.NewBind(*bindingCfg, db, log, skrK8sClientProvider, skrK8sClientProvider, publisher) - getBindingEndpoint := broker.NewGetBinding(log, db) - unbindEndpoint := broker.NewUnbind(log, db, brokerBindings.NewServiceAccountBindingsManager(skrK8sClientProvider, skrK8sClientProvider), publisher) - apiHandler := handlers.NewApiHandler(broker.KymaEnvironmentBroker{ - ServicesEndpoint: nil, - ProvisionEndpoint: nil, - DeprovisionEndpoint: nil, - UpdateEndpoint: nil, - GetInstanceEndpoint: nil, - LastOperationEndpoint: nil, - BindEndpoint: bindEndpoint, - UnbindEndpoint: unbindEndpoint, - GetBindingEndpoint: getBindingEndpoint, - LastBindingOperationEndpoint: nil, - }, brokerLogger) - - //// attach bindings api - router := mux.NewRouter() - router.HandleFunc("/v2/service_instances/{instance_id}/service_bindings/{binding_id}", apiHandler.Bind).Methods(http.MethodPut) - router.HandleFunc("/v2/service_instances/{instance_id}/service_bindings/{binding_id}", apiHandler.GetBinding).Methods(http.MethodGet) - router.HandleFunc("/v2/service_instances/{instance_id}/service_bindings/{binding_id}", apiHandler.Unbind).Methods(http.MethodDelete) - httpServer = httptest.NewServer(router) - defer httpServer.Close() - - t.Run("should create a new service binding without error", func(t *testing.T) { - // When - response := createBinding(instanceID1, "binding-id", t) - defer response.Body.Close() - - binding := unmarshal(t, response) - require.Equal(t, http.StatusCreated, response.StatusCode) - - duration, err := getTokenDurationFromBinding(t, binding) - require.NoError(t, err) - assert.Equal(t, expirationSeconds*time.Second, duration) - - assert.NotEmpty(t, binding.Metadata.ExpiresAt) - - //// verify connectivity using kubeconfig from the generated binding - assertClusterAccess(t, "secret-to-check-first", binding) - assertRolesExistence(t, brokerBindings.BindingName("binding-id"), binding) - }) - - t.Run("should create a new service binding with custom token expiration time", func(t *testing.T) { - const customExpirationSeconds = 900 - - // When - response := createBindingWithCustomExpiration(instanceID1, "binding-id2", customExpirationSeconds, t) - - defer response.Body.Close() - - binding := unmarshal(t, response) - require.Equal(t, http.StatusCreated, response.StatusCode) - - duration, err := getTokenDurationFromBinding(t, binding) - require.NoError(t, err) - assert.Equal(t, customExpirationSeconds*time.Second, duration) - }) - - t.Run("should return error when expiration_seconds is greater than maxExpirationSeconds", func(t *testing.T) { - const customExpirationSeconds = 7201 - - // When - response := createBindingWithCustomExpiration(instanceID1, "binding-id3", customExpirationSeconds, t) - - defer response.Body.Close() - require.Equal(t, http.StatusBadRequest, response.StatusCode) - }) - - t.Run("should return error when expiration_seconds is less than minExpirationSeconds", func(t *testing.T) { - const customExpirationSeconds = 60 - - // When - response := createBindingWithCustomExpiration(instanceID1, "binding-id4", customExpirationSeconds, t) - - defer response.Body.Close() - require.Equal(t, http.StatusBadRequest, response.StatusCode) - }) - - t.Run("should return 404 for not existing binding", func(t *testing.T) { - // when - response := getBinding(instanceID1, uuid.New().String(), t) - defer response.Body.Close() - - // then - require.Equal(t, http.StatusNotFound, response.StatusCode) - }) - - t.Run("should return created kubeconfig", func(t *testing.T) { - // given - bindingID := uuid.New().String() - - // when - response := createBinding(instanceID1, bindingID, t) - defer response.Body.Close() - - require.Equal(t, http.StatusCreated, response.StatusCode) - - // then - binding := getBindingUnmarshalled(instanceID1, bindingID, t) - - duration, err := getTokenDurationFromBinding(t, binding) - require.NoError(t, err) - assert.Equal(t, expirationSeconds*time.Second, duration) - - //// verify connectivity using kubeconfig from the generated binding - assertClusterAccess(t, "secret-to-check-first", binding) - }) - - t.Run("should invalidate binding when cluster role binding is removed", func(t *testing.T) { - // given - bindingID := uuid.New().String() - - response := createBinding(instanceID1, bindingID, t) - defer response.Body.Close() - - require.Equal(t, http.StatusCreated, response.StatusCode) - - binding := getBindingUnmarshalled(instanceID1, bindingID, t) - - duration, err := getTokenDurationFromBinding(t, binding) - require.NoError(t, err) - assert.Equal(t, expirationSeconds*time.Second, duration) - - assertClusterAccess(t, "secret-to-check-first", binding) - - // when - err = clientFirst.Delete(context.Background(), &rbacv1.ClusterRoleBinding{ - ObjectMeta: v1.ObjectMeta{ - Name: brokerBindings.BindingName(bindingID), - Namespace: brokerBindings.BindingNamespace, - }, - }) - assert.NoError(t, err) - - // then - assertClusterNoAccess(t, "secret-to-check-first", binding) - }) - - t.Run("should return created bindings when multiple bindings created", func(t *testing.T) { - // given - firstInstanceFirstBindingID, firstInstancefirstBinding := createBindingWithRandomBindingID(instanceID1, httpServer, t) - firstInstanceFirstBindingDB, err := db.Bindings().Get(instanceID1, firstInstanceFirstBindingID) - assert.NoError(t, err) - - secondInstanceBindingID, secondInstanceFirstBinding := createBindingWithRandomBindingID(instanceID2, httpServer, t) - secondInstanceFirstBindingDB, err := db.Bindings().Get(instanceID2, secondInstanceBindingID) - assert.NoError(t, err) - - firstInstanceSecondBindingID, firstInstanceSecondBinding := createBindingWithRandomBindingID(instanceID1, httpServer, t) - firstInstanceSecondBindingDB, err := db.Bindings().Get(instanceID1, firstInstanceSecondBindingID) - assert.NoError(t, err) - - // when - first binding to the first instance - - response := getBinding(instanceID1, firstInstanceFirstBindingID, t) - defer response.Body.Close() - - // then - assert.Equal(t, http.StatusOK, response.StatusCode) - binding := unmarshal(t, response) - assertKubeconfigsEqual(t, firstInstancefirstBinding, binding) - assert.Equal(t, firstInstanceFirstBindingDB.Kubeconfig, binding.Credentials.(map[string]interface{})["kubeconfig"]) - assertClusterAccess(t, "secret-to-check-first", binding) - - // when - binding to the second instance - response = getBinding(instanceID2, secondInstanceBindingID, t) - defer response.Body.Close() - - // then - assert.Equal(t, http.StatusOK, response.StatusCode) - binding = unmarshal(t, response) - assertKubeconfigsEqual(t, secondInstanceFirstBinding, binding) - assert.Equal(t, secondInstanceFirstBindingDB.Kubeconfig, binding.Credentials.(map[string]interface{})["kubeconfig"]) - assertClusterAccess(t, "secret-to-check-second", binding) - - // when - second binding to the first instance - response = getBinding(instanceID1, firstInstanceSecondBindingID, t) - defer response.Body.Close() - - // then - assert.Equal(t, http.StatusOK, response.StatusCode) - binding = unmarshal(t, response) - assertKubeconfigsEqual(t, firstInstanceSecondBinding, binding) - assert.Equal(t, firstInstanceSecondBindingDB.Kubeconfig, binding.Credentials.(map[string]interface{})["kubeconfig"]) - assertClusterAccess(t, "secret-to-check-first", binding) - }) - - t.Run("should delete created binding", func(t *testing.T) { - // given - bindingID, binding := createBindingWithRandomBindingID(instanceID1, httpServer, t) - bindingFromDB, err := db.Bindings().Get(instanceID1, bindingID) - assert.NoError(t, err) - assert.Equal(t, binding.Credentials.(map[string]interface{})["kubeconfig"], bindingFromDB.Kubeconfig) - - // when - path := fmt.Sprintf(bindingsPath+deleteParams, instanceID1, bindingID, "123", fixture.PlanId) - response := CallAPI(httpServer, http.MethodDelete, path, "", t) - defer response.Body.Close() - - // then - assert.Equal(t, http.StatusOK, response.StatusCode) - bindingFromDB, err = db.Bindings().Get(instanceID1, bindingID) - assert.Error(t, err) - assert.Nil(t, bindingFromDB) - }) - - t.Run("should delete created binding and fail after the second call", func(t *testing.T) { - // given - bindingID, binding := createBindingWithRandomBindingID(instanceID1, httpServer, t) - bindingFromDB, err := db.Bindings().Get(instanceID1, bindingID) - assert.NoError(t, err) - assert.Equal(t, binding.Credentials.(map[string]interface{})["kubeconfig"], bindingFromDB.Kubeconfig) - - // when - path := fmt.Sprintf(bindingsPath+deleteParams, instanceID1, bindingID, "123", fixture.PlanId) - response := CallAPI(httpServer, http.MethodDelete, path, "", t) - defer response.Body.Close() - - // then - assert.Equal(t, http.StatusOK, response.StatusCode) - bindingFromDB, err = db.Bindings().Get(instanceID1, bindingID) - assert.Error(t, err) - assert.Nil(t, bindingFromDB) - - // when - path = fmt.Sprintf(bindingsPath+deleteParams, instanceID1, bindingID, "123", fixture.PlanId) - response = CallAPI(httpServer, http.MethodDelete, path, "", t) - defer response.Body.Close() - - // then - assert.Equal(t, http.StatusGone, response.StatusCode) - bindingFromDB, err = db.Bindings().Get(instanceID1, bindingID) - assert.Error(t, err) - assert.Nil(t, bindingFromDB) - }) - - t.Run("should selectively delete created binding and its service account resources", func(t *testing.T) { - // given - instanceFirst := "1" - - //// first instance first binding - createdBindingIDInstanceFirstFirst, createdBindingInstanceFirstFirst := createBindingWithRandomBindingID(instanceFirst, httpServer, t) - - assertExistsAndKubeconfigCreated(t, createdBindingInstanceFirstFirst, createdBindingIDInstanceFirstFirst, instanceFirst, httpServer, db) - - assertResourcesExistence(t, clientFirst, createdBindingIDInstanceFirstFirst) - - //// first instance second binding - createdBindingIDInstanceFirstSecond, createdBindingInstanceFirstSecond := createBindingWithRandomBindingID(instanceFirst, httpServer, t) - - assertExistsAndKubeconfigCreated(t, createdBindingInstanceFirstSecond, createdBindingIDInstanceFirstSecond, instanceFirst, httpServer, db) - - assertResourcesExistence(t, clientFirst, createdBindingIDInstanceFirstSecond) - - //// second instance first binding - instanceSecond := "2" - createdBindingIDInstanceSecondFirst, createdBindingInstanceSecondFirst := createBindingWithRandomBindingID(instanceSecond, httpServer, t) - - assertExistsAndKubeconfigCreated(t, createdBindingInstanceSecondFirst, createdBindingIDInstanceSecondFirst, instanceSecond, httpServer, db) - - assertResourcesExistence(t, clientSecond, createdBindingIDInstanceSecondFirst) - - //// second instance second binding - createdBindingIDInstanceSecondSecond, createdBindingInstanceSecondSecond := createBindingWithRandomBindingID(instanceSecond, httpServer, t) - - assertExistsAndKubeconfigCreated(t, createdBindingInstanceSecondSecond, createdBindingIDInstanceSecondSecond, instanceSecond, httpServer, db) - - assertResourcesExistence(t, clientSecond, createdBindingIDInstanceSecondSecond) - - // when - path := fmt.Sprintf(bindingsPath+deleteParams, instanceFirst, createdBindingIDInstanceFirstFirst, "123", fixture.PlanId) - - response := CallAPI(httpServer, http.MethodDelete, path, "", t) - defer response.Body.Close() - - // then - assert.Equal(t, http.StatusOK, response.StatusCode) - - assertServiceAccountsNotExists(t, clientFirst, createdBindingIDInstanceFirstFirst) - - assertExistsAndKubeconfigCreated(t, createdBindingInstanceFirstSecond, createdBindingIDInstanceFirstSecond, instanceFirst, httpServer, db) - - assertResourcesExistence(t, clientFirst, createdBindingIDInstanceFirstSecond) - - assertExistsAndKubeconfigCreated(t, createdBindingInstanceSecondFirst, createdBindingIDInstanceSecondFirst, instanceSecond, httpServer, db) - - assertResourcesExistence(t, clientSecond, createdBindingIDInstanceSecondFirst) - - assertExistsAndKubeconfigCreated(t, createdBindingInstanceSecondSecond, createdBindingIDInstanceSecondSecond, instanceSecond, httpServer, db) - - assertResourcesExistence(t, clientSecond, createdBindingIDInstanceSecondSecond) - - removedBinding, err := db.Bindings().Get(instanceFirst, createdBindingIDInstanceFirstFirst) - assert.Error(t, err) - assert.Nil(t, removedBinding) - - }) - - t.Run("should return error when attempting to add a new binding when the maximum number of non expired bindings has already been reached", func(t *testing.T) { - // given - create max number of bindings - var response *http.Response - - for i := 0; i < maxBindingsCount; i++ { - response = createBinding(instanceID3, uuid.New().String(), t) - defer response.Body.Close() - require.Equal(t, http.StatusCreated, response.StatusCode) - } - - // when - create one more binding - response = createBinding(instanceID3, uuid.New().String(), t) - defer response.Body.Close() - - //then - require.Equal(t, http.StatusBadRequest, response.StatusCode) - }) -} - -func assertResourcesExistence(t *testing.T, k8sClient client.Client, bindingID string) { - name := brokerBindings.BindingName(bindingID) - - serviceAccount := corev1.ServiceAccount{} - err := k8sClient.Get(context.Background(), client.ObjectKey{Name: name, Namespace: brokerBindings.BindingNamespace}, &serviceAccount) - assert.NoError(t, err) - assert.NotNil(t, serviceAccount) - - clusterRole := rbacv1.ClusterRole{} - err = k8sClient.Get(context.Background(), client.ObjectKey{Name: name, Namespace: brokerBindings.BindingNamespace}, &clusterRole) - assert.NoError(t, err) - assert.NotNil(t, clusterRole) - - clusterRoleBinding := rbacv1.ClusterRoleBinding{} - err = k8sClient.Get(context.Background(), client.ObjectKey{Name: name, Namespace: brokerBindings.BindingNamespace}, &clusterRoleBinding) - assert.NoError(t, err) - assert.NotNil(t, clusterRoleBinding) -} - -func assertServiceAccountsNotExists(t *testing.T, k8sClient client.Client, bindingID string) { - name := brokerBindings.BindingName(bindingID) - - serviceAccount := corev1.ServiceAccount{} - err := k8sClient.Get(context.Background(), client.ObjectKey{Name: name, Namespace: brokerBindings.BindingNamespace}, &serviceAccount) - assert.True(t, apierrors.IsNotFound(err)) - - clusterRole := rbacv1.ClusterRole{} - err = k8sClient.Get(context.Background(), client.ObjectKey{Name: name, Namespace: brokerBindings.BindingNamespace}, &clusterRole) - assert.True(t, apierrors.IsNotFound(err)) - - clusterRoleBinding := rbacv1.ClusterRoleBinding{} - err = k8sClient.Get(context.Background(), client.ObjectKey{Name: name, Namespace: brokerBindings.BindingNamespace}, &clusterRoleBinding) - assert.True(t, apierrors.IsNotFound(err)) -} - -func assertExistsAndKubeconfigCreated(t *testing.T, actual domain.Binding, bindingID, instanceID string, httpServer *httptest.Server, db storage.BrokerStorage) { - expected, err := db.Bindings().Get(instanceID, bindingID) - require.NoError(t, err) - require.Equal(t, actual.Credentials.(map[string]interface{})["kubeconfig"], expected.Kubeconfig) -} - -func assertClusterAccess(t *testing.T, controlSecretName string, binding domain.Binding) { - - credentials, ok := binding.Credentials.(map[string]interface{}) - require.True(t, ok) - kubeconfig := credentials["kubeconfig"].(string) - - newClient := kubeconfigClient(t, kubeconfig) - - _, err := newClient.CoreV1().Secrets("default").Get(context.Background(), controlSecretName, v1.GetOptions{}) - assert.NoError(t, err) -} - -func assertClusterNoAccess(t *testing.T, controlSecretName string, binding domain.Binding) { - - credentials, ok := binding.Credentials.(map[string]interface{}) - require.True(t, ok) - kubeconfig := credentials["kubeconfig"].(string) - - newClient := kubeconfigClient(t, kubeconfig) - - _, err := newClient.CoreV1().Secrets("default").Get(context.Background(), controlSecretName, v1.GetOptions{}) - assert.True(t, apierrors.IsForbidden(err)) -} - -func assertRolesExistence(t *testing.T, bindingID string, binding domain.Binding) { - - credentials, ok := binding.Credentials.(map[string]interface{}) - require.True(t, ok) - kubeconfig := credentials["kubeconfig"].(string) - - newClient := kubeconfigClient(t, kubeconfig) - - _, err := newClient.CoreV1().ServiceAccounts(brokerBindings.BindingNamespace).Get(context.Background(), bindingID, v1.GetOptions{}) - assert.NoError(t, err) - _, err = newClient.RbacV1().ClusterRoles().Get(context.Background(), bindingID, v1.GetOptions{}) - assert.NoError(t, err) - _, err = newClient.RbacV1().ClusterRoleBindings().Get(context.Background(), bindingID, v1.GetOptions{}) - assert.NoError(t, err) - _, err = newClient.RbacV1().ClusterRoleBindings().Get(context.Background(), bindingID, v1.GetOptions{}) - assert.NoError(t, err) -} - -func assertKubeconfigsEqual(t *testing.T, expected, actual domain.Binding) { - assert.Equal(t, expected.Credentials.(map[string]interface{})["kubeconfig"], actual.Credentials.(map[string]interface{})["kubeconfig"]) -} - -func createBindingWithRandomBindingID(instanceID string, httpServer *httptest.Server, t *testing.T) (string, domain.Binding) { - bindingID := uuid.New().String() - - response := createBinding(instanceID, bindingID, t) - defer response.Body.Close() - require.Equal(t, http.StatusCreated, response.StatusCode) - - createdBinding := unmarshal(t, response) - - return bindingID, createdBinding -} - -func createKubeconfigFileForRestConfig(restConfig rest.Config) []byte { - const ( - userName = "user" - clusterName = "cluster" - contextName = "context" - ) - - clusters := make(map[string]*clientcmdapi.Cluster) - clusters[clusterName] = &clientcmdapi.Cluster{ - Server: restConfig.Host, - CertificateAuthorityData: restConfig.CAData, - } - contexts := make(map[string]*clientcmdapi.Context) - contexts[contextName] = &clientcmdapi.Context{ - Cluster: clusterName, - AuthInfo: userName, - } - authinfos := make(map[string]*clientcmdapi.AuthInfo) - authinfos[userName] = &clientcmdapi.AuthInfo{ - ClientCertificateData: restConfig.CertData, - ClientKeyData: restConfig.KeyData, - } - clientConfig := clientcmdapi.Config{ - Kind: "Config", - APIVersion: "v1", - Clusters: clusters, - Contexts: contexts, - CurrentContext: contextName, - AuthInfos: authinfos, - } - kubeconfig, _ := clientcmd.Write(clientConfig) - return kubeconfig -} - -func CallAPI(httpServer *httptest.Server, method string, path string, body string, t *testing.T) *http.Response { - cli := httpServer.Client() - req, err := http.NewRequest(method, fmt.Sprintf("%s/%s", httpServer.URL, path), bytes.NewBuffer([]byte(body))) - req.Header.Set("X-Broker-API-Version", "2.14") - - require.NoError(t, err) - - resp, err := cli.Do(req) - require.NoError(t, err) - return resp -} - -func createBindingWithCustomExpiration(instanceID string, bindingID string, customExpirationSeconds int, t *testing.T) *http.Response { - path := getPath(instanceID, bindingID) - return CallAPI(httpServer, http.MethodPut, - path, fmt.Sprintf(` - { - "service_id": "123", - "plan_id": "%s", - "parameters": { - "expiration_seconds": %v - } - }`, fixture.PlanId, customExpirationSeconds), t) -} - -func createBinding(instanceID string, bindingID string, t *testing.T) *http.Response { - path := getPath(instanceID, bindingID) - return CallAPI(httpServer, http.MethodPut, - path, fmt.Sprintf(` - { - "service_id": "123", - "plan_id": "%s", - "context": { - "email": "john.smith@email.com", - "origin": "origin" - } - }`, fixture.PlanId), t) -} - -func getBinding(instanceID string, bindingID string, t *testing.T) *http.Response { - return CallAPI(httpServer, http.MethodGet, getPath(instanceID, bindingID), "", t) -} - -func getBindingUnmarshalled(instanceID string, bindingID string, t *testing.T) domain.Binding { - response := getBinding(instanceID, bindingID, t) - require.Equal(t, http.StatusOK, response.StatusCode) - - return unmarshal(t, response) -} - -func getPath(instanceID, bindingID string) string { - return fmt.Sprintf(bindingsPath, instanceID, bindingID) -} - -func kubeconfigClient(t *testing.T, kubeconfig string) *kubernetes.Clientset { - config, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeconfig)) - assert.NoError(t, err) - - clientset, err := kubernetes.NewForConfig(config) - assert.NoError(t, err) - - return clientset -} - -func unmarshal(t *testing.T, response *http.Response) domain.Binding { - content, err := io.ReadAll(response.Body) - require.NoError(t, err) - - t.Logf("response content is: %v", string(content)) - - assert.Contains(t, string(content), "credentials") - - var binding domain.Binding - err = json.Unmarshal(content, &binding) - require.NoError(t, err) - - t.Logf("binding: %v", binding.Credentials) - - return binding -} - -func getTokenDurationFromBinding(t *testing.T, binding domain.Binding) (time.Duration, error) { - credentials, ok := binding.Credentials.(map[string]interface{}) - require.True(t, ok) - kubeconfig := credentials["kubeconfig"].(string) - - return getTokenDuration(t, kubeconfig) -} - -func getTokenDuration(t *testing.T, config string) (time.Duration, error) { - var kubeconfig Kubeconfig - - err := yaml.Unmarshal([]byte(config), &kubeconfig) - require.NoError(t, err) - - for _, user := range kubeconfig.Users { - if user.Name == "context" { - token, _, err := new(jwt.Parser).ParseUnverified(user.User.Token, jwt.MapClaims{}) - require.NoError(t, err) - - if claims, ok := token.Claims.(jwt.MapClaims); ok { - iat := int64(claims["iat"].(float64)) - exp := int64(claims["exp"].(float64)) - - issuedAt := time.Unix(iat, 0) - expiresAt := time.Unix(exp, 0) - - return expiresAt.Sub(issuedAt), nil - } else { - return 0, fmt.Errorf("invalid token claims") - } - } - } - return 0, fmt.Errorf("user with name 'context' not found") -} - -func createEnvTest(t *testing.T) (envtest.Environment, *rest.Config, client.Client) { - pid := internal.SetupEnvtest(t) - defer func() { - internal.CleanupEnvtestBinaries(pid) - }() - - env := envtest.Environment{ - ControlPlaneStartTimeout: 40 * time.Second, - } - var errEnvTest error - var config *rest.Config - err := wait.PollUntilContextTimeout(context.Background(), 500*time.Millisecond, 5*time.Second, false, func(ctx context.Context) (done bool, err error) { - config, errEnvTest = env.Start() - if err != nil { - t.Logf("envtest could not start, retrying: %s", errEnvTest.Error()) - return false, nil - } - t.Logf("envtest started") - return true, nil - }) - require.NoError(t, err) - require.NoError(t, errEnvTest) - - skrClient, err := initClient(config) - require.NoError(t, err) - - err = skrClient.Create(context.Background(), &corev1.Namespace{ - ObjectMeta: v1.ObjectMeta{ - Name: brokerBindings.BindingNamespace, - }, - }) - require.NoError(t, err) - return env, config, skrClient -} From 2ff23ceb2d864d97ed9db460c516a0569d11f02f Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Wed, 11 Dec 2024 15:13:57 +0100 Subject: [PATCH 03/10] wip --- internal/broker/bind_create_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/broker/bind_create_test.go b/internal/broker/bind_create_test.go index 2132a58c61..613534e4ac 100644 --- a/internal/broker/bind_create_test.go +++ b/internal/broker/bind_create_test.go @@ -29,7 +29,7 @@ import ( const ( instanceID1 = "1" - maxBindingsCount = 1000 + maxBindingsCount = 10 ) func fixBindingConfig() BindingConfig { From fd9db952533e6d4a9c1e3dd3c925a08d8a65b266 Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Wed, 11 Dec 2024 15:14:37 +0100 Subject: [PATCH 04/10] wip --- internal/broker/bind_create_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/internal/broker/bind_create_test.go b/internal/broker/bind_create_test.go index 613534e4ac..6e31e11fe0 100644 --- a/internal/broker/bind_create_test.go +++ b/internal/broker/bind_create_test.go @@ -229,18 +229,6 @@ func TestCreateBindng(t *testing.T) { assert.NotEmptyf(t, binding.Credentials.(Credentials).Kubeconfig, "kubeconfig is empty") }) - t.Run("should create binding", func(t *testing.T) { - binding, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-003", domain.BindDetails{ - ServiceID: "123", - PlanID: fixture.PlanId, - RawParameters: json.RawMessage(`{}`), - }, false) - - // then - require.NoError(t, err) - assert.NotEmptyf(t, binding.Credentials.(Credentials).Kubeconfig, "kubeconfig is empty") - }) - t.Run("should report a conflict", func(t *testing.T) { _, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-004", domain.BindDetails{ ServiceID: "123", From f9f71d3ca3711ded6a3f8d0507aa52be309ae3fa Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Wed, 11 Dec 2024 15:20:40 +0100 Subject: [PATCH 05/10] wip --- internal/broker/bind_create_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/broker/bind_create_test.go b/internal/broker/bind_create_test.go index 6e31e11fe0..6b8c20614b 100644 --- a/internal/broker/bind_create_test.go +++ b/internal/broker/bind_create_test.go @@ -4,16 +4,17 @@ import ( "context" "encoding/json" "fmt" - "github.com/kyma-project/kyma-environment-broker/internal" - "github.com/kyma-project/kyma-environment-broker/internal/kubeconfig" - "github.com/pivotal-cf/brokerapi/v8/domain/apiresponses" "log/slog" "net/http" "os" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" "time" + "github.com/kyma-project/kyma-environment-broker/internal" + "github.com/kyma-project/kyma-environment-broker/internal/kubeconfig" + "github.com/pivotal-cf/brokerapi/v8/domain/apiresponses" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/kyma-project/kyma-environment-broker/internal/event" "github.com/google/uuid" From 478307caec6729be63bcbc608d0c443be31ef424 Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Thu, 12 Dec 2024 08:18:49 +0100 Subject: [PATCH 06/10] wip --- internal/broker/bind_create_test.go | 1 - internal/broker/bind_envtest_test.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/internal/broker/bind_create_test.go b/internal/broker/bind_create_test.go index 6b8c20614b..d1f9a50e04 100644 --- a/internal/broker/bind_create_test.go +++ b/internal/broker/bind_create_test.go @@ -566,7 +566,6 @@ func prepareBindingEndpoint(t *testing.T, cfg BindingConfig) (*BindEndpoint, sto log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ Level: slog.LevelDebug, })) - //// binding configuration logs := logrus.New() logs.SetLevel(logrus.DebugLevel) diff --git a/internal/broker/bind_envtest_test.go b/internal/broker/bind_envtest_test.go index 1afdf6b9ce..1ea3ba3f01 100644 --- a/internal/broker/bind_envtest_test.go +++ b/internal/broker/bind_envtest_test.go @@ -78,8 +78,6 @@ func TestCreateBinding(t *testing.T) { }...). Build() - fmt.Println(string(kbcfgFirst)) - //// secret check in assertions err = clientFirst.Create(context.Background(), &corev1.Secret{ ObjectMeta: v1.ObjectMeta{ From 34f9b32974391cbf0630310a385819979329faa1 Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Thu, 12 Dec 2024 08:21:14 +0100 Subject: [PATCH 07/10] wip --- internal/broker/bind_create_test.go | 7 +------ internal/broker/bind_envtest_test.go | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/internal/broker/bind_create_test.go b/internal/broker/bind_create_test.go index d1f9a50e04..6d6f295136 100644 --- a/internal/broker/bind_create_test.go +++ b/internal/broker/bind_create_test.go @@ -567,11 +567,6 @@ func prepareBindingEndpoint(t *testing.T, cfg BindingConfig) (*BindEndpoint, sto Level: slog.LevelDebug, })) - logs := logrus.New() - logs.SetLevel(logrus.DebugLevel) - logs.SetFormatter(&logrus.JSONFormatter{ - TimestampFormat: time.RFC3339Nano, - }) err := db.Instances().Insert(fixture.FixInstance(instanceID1)) require.NoError(t, err) @@ -579,5 +574,5 @@ func prepareBindingEndpoint(t *testing.T, cfg BindingConfig) (*BindEndpoint, sto err = db.Operations().InsertOperation(operation) require.NoError(t, err) - return NewBind(cfg, db, logs, k8sClientProvider, k8sClientProvider, event.NewPubSub(log)), db + return NewBind(cfg, db, log, k8sClientProvider, k8sClientProvider, event.NewPubSub(log)), db } diff --git a/internal/broker/bind_envtest_test.go b/internal/broker/bind_envtest_test.go index 1ea3ba3f01..fd040f1dce 100644 --- a/internal/broker/bind_envtest_test.go +++ b/internal/broker/bind_envtest_test.go @@ -110,8 +110,8 @@ func TestCreateBinding(t *testing.T) { Level: slog.LevelDebug, })) publisher := event.NewPubSub(log) - svc := NewBind(bindingCfg, db, logs, skrK8sClientProvider, skrK8sClientProvider, publisher) - unbindSvc := NewUnbind(logs, db, brokerBindings.NewServiceAccountBindingsManager(skrK8sClientProvider, skrK8sClientProvider), publisher) + svc := NewBind(bindingCfg, db, log, skrK8sClientProvider, skrK8sClientProvider, publisher) + unbindSvc := NewUnbind(log, db, brokerBindings.NewServiceAccountBindingsManager(skrK8sClientProvider, skrK8sClientProvider), publisher) t.Run("should create a new service binding without error", func(t *testing.T) { // When From cbbf4ea95a7ea5c75c8e087c1a9f5aa7a6919b49 Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Thu, 12 Dec 2024 08:39:34 +0100 Subject: [PATCH 08/10] wip --- internal/broker/bind_envtest_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/broker/bind_envtest_test.go b/internal/broker/bind_envtest_test.go index fd040f1dce..17e27727fa 100644 --- a/internal/broker/bind_envtest_test.go +++ b/internal/broker/bind_envtest_test.go @@ -121,7 +121,7 @@ func TestCreateBinding(t *testing.T) { ServiceID: "", BindResource: nil, RawContext: json.RawMessage(`{}`), - RawParameters: json.RawMessage(`{"expirationSeconds": 660}`), + RawParameters: json.RawMessage(`{"expiration_seconds": 660}`), }, false) // then From d56538b878020e9ef7f41eae7fa4dfcc17c7c4a9 Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Thu, 12 Dec 2024 14:46:59 +0100 Subject: [PATCH 09/10] wip --- internal/broker/bind_create_test.go | 67 +++++++++++++++-------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/internal/broker/bind_create_test.go b/internal/broker/bind_create_test.go index 6d6f295136..e1bc5dad2c 100644 --- a/internal/broker/bind_create_test.go +++ b/internal/broker/bind_create_test.go @@ -58,7 +58,7 @@ func (p *provider) KubeconfigForRuntimeID(runtimeID string) ([]byte, error) { return []byte{}, nil } -func TestCreateBindingEndpoint(t *testing.T) { +func TestCreateBindingEndpoint_dbInsertionInCaseOfError(t *testing.T) { t.Log("test create binding endpoint") // Given @@ -118,36 +118,6 @@ func TestCreateBindingEndpoint(t *testing.T) { require.NotNil(t, binding.ExpiresAt) require.Empty(t, binding.Kubeconfig) }) - - t.Run("should return error when expiration_seconds is greater than maxExpirationSeconds", func(t *testing.T) { - // when - _, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-001", domain.BindDetails{ - ServiceID: "123", - PlanID: fixture.PlanId, - RawParameters: json.RawMessage(`{"expirationSeconds": 10}`), - }, false) - require.Error(t, err) - - // then - assert.IsType(t, err, &apiresponses.FailureResponse{}, "Get request returned error of unexpected type") - apierr := err.(*apiresponses.FailureResponse) - assert.Equal(t, http.StatusBadRequest, apierr.ValidatedStatusCode(nil), "Get request status code not matching") - }) - - t.Run("should return error when expiration_seconds is less than minExpirationSeconds", func(t *testing.T) { - // when - _, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-002", domain.BindDetails{ - ServiceID: "123", - PlanID: fixture.PlanId, - RawParameters: json.RawMessage(`{"expirationSeconds": 100000}`), - }, false) - require.Error(t, err) - - // then - assert.IsType(t, err, &apiresponses.FailureResponse{}, "Get request returned error of unexpected type") - apierr := err.(*apiresponses.FailureResponse) - assert.Equal(t, http.StatusBadRequest, apierr.ValidatedStatusCode(nil), "Get request status code not matching") - }) } func TestCreateBindingExceedsAllowedNumberOfNonExpiredBindings(t *testing.T) { @@ -189,7 +159,7 @@ func TestCreateBindingExceedsAllowedNumberOfNonExpiredBindings(t *testing.T) { } -func TestCreateBindng(t *testing.T) { +func TestCreateBindingEndpoint(t *testing.T) { bindEndpoint, _ := prepareBindingEndpoint(t, fixBindingConfig()) t.Run("should return error when expiration_seconds is less than minExpirationSeconds", func(t *testing.T) { @@ -230,6 +200,18 @@ func TestCreateBindng(t *testing.T) { assert.NotEmptyf(t, binding.Credentials.(Credentials).Kubeconfig, "kubeconfig is empty") }) + t.Run("should create binding with expiration seconds provided", func(t *testing.T) { + binding, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-003", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{"expiration_seconds": 1000}`), + }, false) + + // then + require.NoError(t, err) + assert.NotEmptyf(t, binding.Credentials.(Credentials).Kubeconfig, "kubeconfig is empty") + }) + t.Run("should report a conflict", func(t *testing.T) { _, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-004", domain.BindDetails{ ServiceID: "123", @@ -254,6 +236,27 @@ func TestCreateBindng(t *testing.T) { assert.Equal(t, http.StatusConflict, apierr.ValidatedStatusCode(nil), "Get request status code not matching") }) + t.Run("should not report a conflict if parameters are the same", func(t *testing.T) { + _, err := bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-005", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{"expiration_seconds": 1000}`), + }, false) + + // then + require.NoError(t, err) + + // when + _, err = bindEndpoint.Bind(context.Background(), instanceID1, "binding-id-005", domain.BindDetails{ + ServiceID: "123", + PlanID: fixture.PlanId, + RawParameters: json.RawMessage(`{"expiration_seconds": 1000}`), + }, false) + + // then + require.NoError(t, err) + }) + } func TestCreatedBy(t *testing.T) { From 9f93c3257287fbf0510ab543753f3f99bc6e9201 Mon Sep 17 00:00:00 2001 From: Miskiewicz Date: Thu, 12 Dec 2024 15:36:56 +0100 Subject: [PATCH 10/10] imports --- go.mod | 2 +- internal/broker/bind_envtest_test.go | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3fbe9a4606..32bd11f38b 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.20.5 github.com/sebdah/goldie/v2 v2.5.5 + github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.10.0 github.com/vrischmann/envconfig v1.3.0 golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f @@ -103,7 +104,6 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sergi/go-diff v1.3.1 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect github.com/sosodev/duration v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect diff --git a/internal/broker/bind_envtest_test.go b/internal/broker/bind_envtest_test.go index 17e27727fa..a15d84bc8a 100644 --- a/internal/broker/bind_envtest_test.go +++ b/internal/broker/bind_envtest_test.go @@ -4,6 +4,11 @@ import ( "context" "encoding/json" "fmt" + "log/slog" + "os" + "testing" + "time" + "github.com/golang-jwt/jwt/v4" "github.com/kyma-project/kyma-environment-broker/internal" brokerBindings "github.com/kyma-project/kyma-environment-broker/internal/broker/bindings" @@ -24,14 +29,10 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "log/slog" - "os" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/envtest" - "testing" - "time" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" )