From 4efa6583786cd50bcedf279f66a34a3c9cd2db00 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Wed, 29 May 2024 11:01:19 +0200 Subject: [PATCH] refactor!: have knuu as an object (#356) * chore: refactor knuu to an object * fix: clone method * chore: comments added to old file * chore: add deprecated annotation to the old code * chore: refactor knuu helper funcs for minio * chore: added a sample unittest * fix: increase timeout limit * chore: refactor proxy to fit into the new knuu pkg * chore: rename for more clarity --- e2e/basic/build_from_git_test.go | 33 +- go.mod | 1 + go.sum | 2 + pkg/{knuu => instance}/bittwister.go | 2 +- .../destroy.go} | 12 +- pkg/instance/errors.go | 228 +++ pkg/instance/executor.go | 54 + .../instance_helper.go => instance/helper.go} | 92 +- pkg/instance/instance.go | 1363 ++++++++++++++++ .../instance_otel.go => instance/otel.go} | 11 +- .../instance_pool.go => instance/pool.go} | 33 +- .../instance_state.go => instance/state.go} | 2 +- .../instance_type.go => instance/type.go} | 2 +- pkg/k8s/k8s.go | 11 +- pkg/k8s/types.go | 79 + pkg/knuu/errors.go | 2 + pkg/knuu/executor.go | 54 - pkg/knuu/instance.go | 1416 +---------------- pkg/knuu/instance_old.go | 444 ++++++ pkg/knuu/knuu.go | 312 ++-- pkg/knuu/knuu_old.go | 145 ++ pkg/knuu/knuu_test.go | 148 ++ pkg/knuu/minio.go | 24 +- pkg/knuu/preloader_old.go | 42 + pkg/preloader/errors.go | 38 + pkg/{knuu => preloader}/preloader.go | 63 +- pkg/system/dependencies.go | 19 + pkg/traefik/traefik.go | 52 +- 28 files changed, 2900 insertions(+), 1784 deletions(-) rename pkg/{knuu => instance}/bittwister.go (99%) rename pkg/{knuu/instance_destroy.go => instance/destroy.go} (81%) create mode 100644 pkg/instance/errors.go create mode 100644 pkg/instance/executor.go rename pkg/{knuu/instance_helper.go => instance/helper.go} (86%) create mode 100644 pkg/instance/instance.go rename pkg/{knuu/instance_otel.go => instance/otel.go} (96%) rename pkg/{knuu/instance_pool.go => instance/pool.go} (72%) rename pkg/{knuu/instance_state.go => instance/state.go} (97%) rename pkg/{knuu/instance_type.go => instance/type.go} (96%) create mode 100644 pkg/k8s/types.go delete mode 100644 pkg/knuu/executor.go create mode 100644 pkg/knuu/instance_old.go create mode 100644 pkg/knuu/knuu_old.go create mode 100644 pkg/knuu/knuu_test.go create mode 100644 pkg/knuu/preloader_old.go create mode 100644 pkg/preloader/errors.go rename pkg/{knuu => preloader}/preloader.go (59%) create mode 100644 pkg/system/dependencies.go diff --git a/e2e/basic/build_from_git_test.go b/e2e/basic/build_from_git_test.go index bf3f7a2..2ab6d0f 100644 --- a/e2e/basic/build_from_git_test.go +++ b/e2e/basic/build_from_git_test.go @@ -2,13 +2,13 @@ package basic import ( "context" - "os" "testing" "time" "github.com/stretchr/testify/require" "github.com/celestiaorg/knuu/pkg/builder" + "github.com/celestiaorg/knuu/pkg/instance" "github.com/celestiaorg/knuu/pkg/knuu" ) @@ -18,20 +18,18 @@ func TestBuildFromGit(t *testing.T) { t.Parallel() // Setup - // This code is a bit dirty due to the current limitations of knuu - // After refactoring knuu, this test must be either removed or updated - require.NoError(t, os.Setenv("KNUU_BUILDER", "kubernetes"), "Error setting KNUU_BUILDER Env") - require.NoError(t, knuu.CleanUp(), "Error cleaning up knuu") - require.NoError(t, knuu.Initialize(), "Error initializing knuu") - - instance, err := knuu.NewInstance("my-instance") - require.NoError(t, err, "Error creating instance") - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Minute) defer cancel() + // The default image builder is kaniko here + kn, err := knuu.New(ctx) + require.NoError(t, err, "Error creating knuu") + + sampleInstance, err := kn.NewInstance("git-builder") + require.NoError(t, err, "Error creating instance") + // This is a blocking call which builds the image from git repo - err = instance.SetGitRepo(ctx, builder.GitContext{ + err = sampleInstance.SetGitRepo(ctx, builder.GitContext{ Repo: "https://github.com/celestiaorg/celestia-app.git", Branch: "main", // Commit: "5ce94f4f010e366df301d25cd5d797c3147ff884", @@ -40,21 +38,20 @@ func TestBuildFromGit(t *testing.T) { }) require.NoError(t, err, "Error setting git repo") - require.NoError(t, instance.SetCommand("sleep", "infinity"), "Error setting command") + require.NoError(t, sampleInstance.SetCommand("sleep", "infinity"), "Error setting command") - err = instance.AddFileBytes([]byte("Hello, world!"), "/home/hello.txt", "root:root") + err = sampleInstance.AddFileBytes([]byte("Hello, world!"), "/home/hello.txt", "root:root") require.NoError(t, err, "Error adding file") - require.NoError(t, instance.Commit(), "Error committing instance") + require.NoError(t, sampleInstance.Commit(), "Error committing instance") t.Cleanup(func() { - require.NoError(t, knuu.BatchDestroy(instance)) + require.NoError(t, instance.BatchDestroy(ctx, sampleInstance)) }) - // Test logic - require.NoError(t, instance.Start(), "Error starting instance") + require.NoError(t, sampleInstance.Start(ctx), "Error starting instance") - data, err := instance.GetFileBytes("/home/hello.txt") + data, err := sampleInstance.GetFileBytes(ctx, "/home/hello.txt") require.NoError(t, err, "Error getting file bytes") require.Equal(t, []byte("Hello, world!"), data, "File bytes do not match") diff --git a/go.mod b/go.mod index 918d415..ee4b5b6 100644 --- a/go.mod +++ b/go.mod @@ -66,6 +66,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rs/xid v1.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.2 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.26.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect diff --git a/go.sum b/go.sum index 5445225..0428618 100644 --- a/go.sum +++ b/go.sum @@ -167,6 +167,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/pkg/knuu/bittwister.go b/pkg/instance/bittwister.go similarity index 99% rename from pkg/knuu/bittwister.go rename to pkg/instance/bittwister.go index bb79ea7..d45bc55 100644 --- a/pkg/knuu/bittwister.go +++ b/pkg/instance/bittwister.go @@ -1,4 +1,4 @@ -package knuu +package instance import ( "context" diff --git a/pkg/knuu/instance_destroy.go b/pkg/instance/destroy.go similarity index 81% rename from pkg/knuu/instance_destroy.go rename to pkg/instance/destroy.go index 63e7bd0..8157710 100644 --- a/pkg/knuu/instance_destroy.go +++ b/pkg/instance/destroy.go @@ -1,4 +1,4 @@ -package knuu +package instance import ( "context" @@ -9,7 +9,7 @@ import ( // Destroy destroys the instance // This function can only be called in the state 'Started' or 'Destroyed' -func (i *Instance) Destroy() error { +func (i *Instance) Destroy(ctx context.Context) error { if i.state == Destroyed { return nil } @@ -18,10 +18,6 @@ func (i *Instance) Destroy() error { return ErrDestroyingNotAllowed.WithParams(i.state.String()) } - // TODO: receive context from the user in the breaking refactor - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - if err := i.destroyPod(ctx); err != nil { return ErrDestroyingPod.WithParams(i.k8sName).Wrap(err) } @@ -45,7 +41,7 @@ func (i *Instance) Destroy() error { } // BatchDestroy destroys a list of instances. -func BatchDestroy(instances ...*Instance) error { +func BatchDestroy(ctx context.Context, instances ...*Instance) error { if os.Getenv("KNUU_SKIP_CLEANUP") == "true" { logrus.Info("Skipping cleanup") return nil @@ -55,7 +51,7 @@ func BatchDestroy(instances ...*Instance) error { if instance == nil { continue } - if err := instance.Destroy(); err != nil { + if err := instance.Destroy(ctx); err != nil { return err } } diff --git a/pkg/instance/errors.go b/pkg/instance/errors.go new file mode 100644 index 0000000..a8449b9 --- /dev/null +++ b/pkg/instance/errors.go @@ -0,0 +1,228 @@ +package instance + +import ( + "fmt" +) + +type Error struct { + Code string + Message string + Err error + Params []interface{} +} + +func (e *Error) Error() string { + if e.Err == e { + return e.Message + } + + msg := fmt.Sprintf(e.Message, e.Params...) + if e.Err != nil { + return fmt.Sprintf("%s: %v", msg, e.Err) + } + return msg +} + +func (e *Error) Wrap(err error) error { + e.Err = err + return e +} + +func (e *Error) WithParams(params ...interface{}) *Error { + e.Params = params + return e +} + +var ( + ErrBitTwisterFailedToStart = &Error{Code: "BitTwisterFailedToStart", Message: "BitTwister failed to start"} + ErrCreatingInstance = &Error{Code: "CreatingInstance", Message: "error creating instance"} + ErrSettingImage = &Error{Code: "SettingImage", Message: "error setting image"} + ErrCommittingInstance = &Error{Code: "CommittingInstance", Message: "error committing instance"} + ErrSettingArgs = &Error{Code: "SettingArgs", Message: "error setting args"} + ErrSettingMemory = &Error{Code: "SettingMemory", Message: "error setting memory"} + ErrSettingCPU = &Error{Code: "SettingCPU", Message: "error setting cpu"} + ErrStartingInstance = &Error{Code: "StartingInstance", Message: "error starting instance"} + ErrWaitingInstanceIsRunning = &Error{Code: "WaitingInstanceIsRunning", Message: "error waiting for instance to be running"} + ErrPortNumberOutOfRange = &Error{Code: "PortNumberOutOfRange", Message: "port number '%d' is out of range"} + ErrDeployingService = &Error{Code: "DeployingService", Message: "error deploying service '%s'"} + ErrGettingService = &Error{Code: "GettingService", Message: "error getting service '%s'"} + ErrPatchingService = &Error{Code: "PatchingService", Message: "error patching service '%s'"} + ErrFailedToCreateServiceAccount = &Error{Code: "FailedToCreateServiceAccount", Message: "failed to create service account"} + ErrFailedToCreateRole = &Error{Code: "FailedToCreateRole", Message: "failed to create role"} + ErrFailedToCreateRoleBinding = &Error{Code: "FailedToCreateRoleBinding", Message: "failed to create role binding"} + ErrFailedToDeployPod = &Error{Code: "FailedToDeployPod", Message: "failed to deploy pod"} + ErrFailedToDeletePod = &Error{Code: "FailedToDeletePod", Message: "failed to delete pod"} + ErrFailedToDeleteServiceAccount = &Error{Code: "FailedToDeleteServiceAccount", Message: "failed to delete service account"} + ErrFailedToDeleteRole = &Error{Code: "FailedToDeleteRole", Message: "failed to delete role"} + ErrFailedToDeleteRoleBinding = &Error{Code: "FailedToDeleteRoleBinding", Message: "failed to delete role binding"} + ErrDeployingServiceForInstance = &Error{Code: "DeployingServiceForInstance", Message: "error deploying service for instance '%s'"} + ErrPatchingServiceForInstance = &Error{Code: "PatchingServiceForInstance", Message: "error patching service for instance '%s'"} + ErrFailedToOpenFile = &Error{Code: "FailedToOpenFile", Message: "failed to open file"} + ErrFailedToReadFile = &Error{Code: "FailedToReadFile", Message: "failed to read file"} + ErrFailedToCreateConfigMap = &Error{Code: "FailedToCreateConfigMap", Message: "failed to create configmap"} + ErrFailedToDeleteConfigMap = &Error{Code: "FailedToDeleteConfigMap", Message: "failed to delete configmap"} + ErrFailedToDeployOrPatchService = &Error{Code: "FailedToDeployOrPatchService", Message: "failed to deploy or patch service"} + ErrDeployingServiceForSidecar = &Error{Code: "DeployingServiceForSidecar", Message: "error deploying service for sidecar '%s' of instance '%s', a sidecar cannot have a service"} + ErrPatchingServiceForSidecar = &Error{Code: "PatchingServiceForSidecar", Message: "error patching service for sidecar '%s' of instance '%s', a sidecar cannot have a service"} + ErrDeployingVolumeForInstance = &Error{Code: "DeployingVolumeForInstance", Message: "error deploying volume for instance '%s'"} + ErrDeployingFilesForInstance = &Error{Code: "DeployingFilesForInstance", Message: "error deploying files for instance '%s'"} + ErrDestroyingVolumeForInstance = &Error{Code: "DestroyingVolumeForInstance", Message: "error destroying volume for instance '%s'"} + ErrDestroyingFilesForInstance = &Error{Code: "DestroyingFilesForInstance", Message: "error destroying files for instance '%s'"} + ErrDestroyingServiceForInstance = &Error{Code: "DestroyingServiceForInstance", Message: "error destroying service for instance '%s'"} + ErrCheckingNetworkStatusForInstance = &Error{Code: "CheckingNetworkStatusForInstance", Message: "error checking network status for instance '%s'"} + ErrEnablingNetworkForInstance = &Error{Code: "EnablingNetworkForInstance", Message: "error enabling network for instance '%s'"} + ErrGeneratingUUID = &Error{Code: "GeneratingUUID", Message: "error generating UUID"} + ErrGettingFreePort = &Error{Code: "GettingFreePort", Message: "error getting free port"} + ErrSrcMustBeSet = &Error{Code: "SrcMustBeSet", Message: "src must be set"} + ErrDestMustBeSet = &Error{Code: "DestMustBeSet", Message: "dest must be set"} + ErrChownMustBeSet = &Error{Code: "ChownMustBeSet", Message: "chown must be set"} + ErrChownMustBeInFormatUserGroup = &Error{Code: "ChownMustBeInFormatUserGroup", Message: "chown must be in format 'user:group'"} + ErrAddingFileToInstance = &Error{Code: "AddingFileToInstance", Message: "error adding file '%s' to instance '%s'"} + ErrReplacingPod = &Error{Code: "ReplacingPod", Message: "error replacing pod"} + ErrApplyingFunctionToInstance = &Error{Code: "ApplyingFunctionToInstance", Message: "error applying function to instance '%s'"} + ErrSettingNotAllowed = &Error{Code: "SettingNotAllowed", Message: "setting %s is only allowed in state 'Preparing' or 'Committed'. Current state is '%s'"} + ErrCreatingOtelCollectorInstance = &Error{Code: "CreatingOtelCollectorInstance", Message: "error creating otel collector instance '%s'"} + ErrSettingBitTwisterImage = &Error{Code: "SettingBitTwisterImage", Message: "error setting image for bit-twister instance"} + ErrAddingBitTwisterPort = &Error{Code: "AddingBitTwisterPort", Message: "error adding BitTwister port"} + ErrGettingInstanceIP = &Error{Code: "GettingInstanceIP", Message: "error getting IP of instance '%s'"} + ErrCommittingBitTwisterInstance = &Error{Code: "CommittingBitTwisterInstance", Message: "error committing bit-twister instance"} + ErrSettingBitTwisterEnv = &Error{Code: "SettingBitTwisterEnv", Message: "error setting environment variable for bit-twister instance"} + ErrCreatingBitTwisterInstance = &Error{Code: "CreatingBitTwisterInstance", Message: "error creating bit-twister instance '%s'"} + ErrSettingBitTwisterPrivileged = &Error{Code: "SettingBitTwisterPrivileged", Message: "error setting privileged for bit-twister instance '%s'"} + ErrAddingBitTwisterCapability = &Error{Code: "AddingBitTwisterCapability", Message: "error adding capability for bit-twister instance '%s'"} + ErrAddingBitTwisterSidecar = &Error{Code: "AddingBitTwisterSidecar", Message: "error adding bit-twister sidecar to instance '%s'"} + ErrCreatingOtelAgentInstance = &Error{Code: "CreatingOtelAgentInstance", Message: "error creating otel-agent instance"} + ErrSettingOtelAgentImage = &Error{Code: "SettingOtelAgentImage", Message: "error setting image for otel-agent instance"} + ErrAddingOtelAgentPort = &Error{Code: "AddingOtelAgentPort", Message: "error adding port for otel-agent instance"} + ErrSettingOtelAgentCPU = &Error{Code: "SettingOtelAgentCPU", Message: "error setting CPU for otel-agent instance"} + ErrSettingOtelAgentMemory = &Error{Code: "SettingOtelAgentMemory", Message: "error setting memory for otel-agent instance"} + ErrCommittingOtelAgentInstance = &Error{Code: "CommittingOtelAgentInstance", Message: "error committing otel-agent instance"} + ErrMarshalingYAML = &Error{Code: "MarshalingYAML", Message: "error marshaling YAML"} + ErrAddingOtelAgentConfigFile = &Error{Code: "AddingOtelAgentConfigFile", Message: "error adding otel-agent config file"} + ErrSettingOtelAgentCommand = &Error{Code: "SettingOtelAgentCommand", Message: "error setting command for otel-agent instance"} + ErrCreatingPoolNotAllowed = &Error{Code: "CreatingPoolNotAllowed", Message: "creating a pool is only allowed in state 'Committed' or 'Destroyed'. Current state is '%s'"} + ErrGeneratingK8sName = &Error{Code: "GeneratingK8sName", Message: "error generating k8s name for instance '%s'"} + ErrEnablingBitTwister = &Error{Code: "EnablingBitTwister", Message: "enabling BitTwister is not allowed in state 'Started'"} + ErrSettingImageNotAllowed = &Error{Code: "SettingImageNotAllowed", Message: "setting image is only allowed in state 'None' and 'Started'. Current state is '%s'"} + ErrCreatingBuilder = &Error{Code: "CreatingBuilder", Message: "error creating builder"} + ErrSettingImageNotAllowedForSidecarsStarted = &Error{Code: "SettingImageNotAllowedForSidecarsStarted", Message: "setting image is not allowed for sidecars when in state 'Started'"} + ErrSettingGitRepo = &Error{Code: "SettingGitRepo", Message: "setting git repo is only allowed in state 'None'. Current state is '%s'"} + ErrGettingBuildContext = &Error{Code: "GettingBuildContext", Message: "error getting build context"} + ErrGettingImageName = &Error{Code: "GettingImageName", Message: "error getting image name"} + ErrSettingImageNotAllowedForSidecars = &Error{Code: "SettingImageNotAllowedForSidecars", Message: "setting image is not allowed for sidecars"} + ErrSettingCommand = &Error{Code: "SettingCommand", Message: "setting command is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrSettingArgsNotAllowed = &Error{Code: "SettingArgsNotAllowed", Message: "setting args is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrAddingPortNotAllowed = &Error{Code: "AddingPortNotAllowed", Message: "adding port is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrPortAlreadyRegistered = &Error{Code: "PortAlreadyRegistered", Message: "TCP port '%d' is already in registered"} + ErrRandomPortForwardingNotAllowed = &Error{Code: "RandomPortForwardingNotAllowed", Message: "random port forwarding is only allowed in state 'Started'. Current state is '%s"} + ErrPortNotRegistered = &Error{Code: "PortNotRegistered", Message: "TCP port '%d' is not registered"} + ErrGettingPodFromReplicaSet = &Error{Code: "GettingPodFromReplicaSet", Message: "error getting pod from replicaset '%s'"} + ErrForwardingPort = &Error{Code: "ForwardingPort", Message: "error forwarding port after %d retries"} + ErrUDPPortAlreadyRegistered = &Error{Code: "UDPPortAlreadyRegistered", Message: "UDP port '%d' is already in registered"} + ErrExecutingCommandNotAllowed = &Error{Code: "ExecutingCommandNotAllowed", Message: "executing command is only allowed in state 'Preparing' or 'Started'. Current state is '%s"} + ErrExecutingCommandInInstance = &Error{Code: "ExecutingCommandInInstance", Message: "error executing command '%s' in instance '%s'"} + ErrExecutingCommandInSidecar = &Error{Code: "ExecutingCommandInSidecar", Message: "error executing command '%s' in sidecar '%s' of instance '%s'"} + ErrAddingFileNotAllowed = &Error{Code: "AddingFileNotAllowed", Message: "adding file is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrSrcDoesNotExist = &Error{Code: "SrcDoesNotExist", Message: "src '%s' does not exist"} + ErrCreatingDirectory = &Error{Code: "CreatingDirectory", Message: "error creating directory"} + ErrFailedToCreateDestFile = &Error{Code: "FailedToCreateDestFile", Message: "failed to create destination file '%s'"} + ErrFailedToOpenSrcFile = &Error{Code: "FailedToOpenSrcFile", Message: "failed to open source file '%s'"} + ErrFailedToCopyFile = &Error{Code: "FailedToCopyFile", Message: "failed to copy from source '%s' to destination '%s'"} + ErrSrcDoesNotExistOrIsDirectory = &Error{Code: "SrcDoesNotExistOrIsDirectory", Message: "src '%s' does not exist or is a directory"} + ErrInvalidFormat = &Error{Code: "InvalidFormat", Message: "invalid format"} + ErrFailedToConvertToInt64 = &Error{Code: "FailedToConvertToInt64", Message: "failed to convert to int64"} + ErrAllFilesMustHaveSameGroup = &Error{Code: "AllFilesMustHaveSameGroup", Message: "all files must have the same group"} + ErrAddingFolderNotAllowed = &Error{Code: "AddingFolderNotAllowed", Message: "adding folder is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrSrcDoesNotExistOrIsNotDirectory = &Error{Code: "SrcDoesNotExistOrIsNotDirectory", Message: "src '%s' does not exist or is not a directory"} + ErrCopyingFolderToInstance = &Error{Code: "CopyingFolderToInstance", Message: "error copying folder '%s' to instance '%s"} + ErrSettingUserNotAllowed = &Error{Code: "SettingUserNotAllowed", Message: "setting user is only allowed in state 'Preparing'. Current state is '%s"} + ErrSettingUser = &Error{Code: "SettingUser", Message: "error setting user '%s' for instance '%s"} + ErrCommittingNotAllowed = &Error{Code: "CommittingNotAllowed", Message: "committing is only allowed in state 'Preparing'. Current state is '%s"} + ErrGettingImageRegistry = &Error{Code: "GettingImageRegistry", Message: "error getting image registry"} + ErrGeneratingImageHash = &Error{Code: "GeneratingImageHash", Message: "error generating image hash"} + ErrPushingImage = &Error{Code: "PushingImage", Message: "error pushing image for instance '%s'"} + ErrAddingVolumeNotAllowed = &Error{Code: "AddingVolumeNotAllowed", Message: "adding volume is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrSettingMemoryNotAllowed = &Error{Code: "SettingMemoryNotAllowed", Message: "setting memory is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrSettingCPUNotAllowed = &Error{Code: "SettingCPUNotAllowed", Message: "setting cpu is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrSettingEnvNotAllowed = &Error{Code: "SettingEnvNotAllowed", Message: "setting environment variable is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrGettingServiceForInstance = &Error{Code: "GettingServiceForInstance", Message: "error retrieving deployed service for instance '%s'"} + ErrGettingServiceIP = &Error{Code: "GettingServiceIP", Message: "IP address is not available for service '%s'"} + ErrGettingFileNotAllowed = &Error{Code: "GettingFileNotAllowed", Message: "getting file is only allowed in state 'Started', 'Preparing' or 'Committed'. Current state is '%s"} + ErrGettingFile = &Error{Code: "GettingFile", Message: "error getting file '%s' from instance '%s"} + ErrReadingFile = &Error{Code: "ReadingFile", Message: "error reading file '%s' from running instance '%s"} + ErrReadingFileNotAllowed = &Error{Code: "ReadingFileNotAllowed", Message: "reading file is only allowed in state 'Started'. Current state is '%s"} + ErrReadingFileFromInstance = &Error{Code: "ReadingFileFromInstance", Message: "error reading file '%s' from running instance '%s"} + ErrAddingPolicyRuleNotAllowed = &Error{Code: "AddingPolicyRuleNotAllowed", Message: "adding policy rule is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrSettingProbeNotAllowed = &Error{Code: "SettingProbeNotAllowed", Message: "setting probe is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrAddingSidecarNotAllowed = &Error{Code: "AddingSidecarNotAllowed", Message: "adding sidecar is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrSidecarIsNil = &Error{Code: "SidecarIsNil", Message: "sidecar is nil"} + ErrSidecarCannotBeSameInstance = &Error{Code: "SidecarCannotBeSameInstance", Message: "sidecar cannot be the same instance"} + ErrSidecarNotCommitted = &Error{Code: "SidecarNotCommitted", Message: "sidecar '%s' is not in state 'Committed'"} + ErrSidecarCannotHaveSidecar = &Error{Code: "SidecarCannotHaveSidecar", Message: "sidecar '%s' cannot have a sidecar"} + ErrSidecarAlreadySidecar = &Error{Code: "SidecarAlreadySidecar", Message: "sidecar '%s' is already a sidecar"} + ErrSettingPrivilegedNotAllowed = &Error{Code: "SettingPrivilegedNotAllowed", Message: "setting privileged is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrAddingCapabilityNotAllowed = &Error{Code: "AddingCapabilityNotAllowed", Message: "adding capability is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrAddingCapabilitiesNotAllowed = &Error{Code: "AddingCapabilitiesNotAllowed", Message: "adding capabilities is only allowed in state 'Preparing' or 'Committed'. Current state is '%s"} + ErrStartingNotAllowed = &Error{Code: "StartingNotAllowed", Message: "starting is only allowed in state 'Committed' or 'Stopped'. Current state of sidecar '%s' is '%s'"} + ErrStartingNotAllowedForSidecar = &Error{Code: "StartingNotAllowedForSidecar", Message: "starting is only allowed in state 'Committed' or 'Stopped'. Current state of sidecar '%s' is '%s"} + ErrStartingSidecarNotAllowed = &Error{Code: "StartingSidecarNotAllowed", Message: "starting a sidecar is not allowed"} + ErrAddingOtelCollectorSidecar = &Error{Code: "AddingOtelCollectorSidecar", Message: "error adding OpenTelemetry collector sidecar for instance '%s'"} + ErrAddingNetworkSidecar = &Error{Code: "AddingNetworkSidecar", Message: "error adding network sidecar for instance '%s'"} + ErrDeployingResourcesForInstance = &Error{Code: "DeployingResourcesForInstance", Message: "error deploying resources for instance '%s'"} + ErrDeployingResourcesForSidecars = &Error{Code: "DeployingResourcesForSidecars", Message: "error deploying resources for sidecars of instance '%s'"} + ErrDeployingPodForInstance = &Error{Code: "DeployingPodForInstance", Message: "error deploying pod for instance '%s'"} + ErrWaitingForInstanceRunning = &Error{Code: "WaitingForInstanceRunning", Message: "error waiting for instance '%s' to be running"} + ErrCheckingIfInstanceRunningNotAllowed = &Error{Code: "CheckingIfInstanceRunningNotAllowed", Message: "checking if instance is running is only allowed in state 'Started'. Current state is '%s"} + ErrWaitingForInstanceNotAllowed = &Error{Code: "WaitingForInstanceNotAllowed", Message: "waiting for instance is only allowed in state 'Started'. Current state is '%s"} + ErrWaitingForInstanceTimeout = &Error{Code: "WaitingForInstanceTimeout", Message: "timeout while waiting for instance '%s' to be running"} + ErrCheckingIfInstanceRunning = &Error{Code: "CheckingIfInstanceRunning", Message: "error checking if instance '%s' is running"} + ErrDisablingNetworkNotAllowed = &Error{Code: "DisablingNetworkNotAllowed", Message: "disabling network is only allowed in state 'Started'. Current state is '%s"} + ErrDisablingNetwork = &Error{Code: "DisablingNetwork", Message: "error disabling network for instance '%s'"} + ErrSettingBandwidthLimitNotAllowed = &Error{Code: "SettingBandwidthLimitNotAllowed", Message: "setting bandwidth limit is only allowed in state 'Started'. Current state is '%s"} + ErrSettingBandwidthLimitNotAllowedBitTwister = &Error{Code: "SettingBandwidthLimitNotAllowedBitTwister", Message: "setting bandwidth limit is only allowed if BitTwister is enabled"} + ErrStoppingBandwidthLimit = &Error{Code: "StoppingBandwidthLimit", Message: "error stopping bandwidth limit for instance '%s'"} + ErrSettingBandwidthLimit = &Error{Code: "SettingBandwidthLimit", Message: "error setting bandwidth limit for instance '%s'"} + ErrSettingLatencyJitterNotAllowed = &Error{Code: "SettingLatencyJitterNotAllowed", Message: "setting latency/jitter is only allowed in state 'Started'. Current state is '%s"} + ErrSettingLatencyJitterNotAllowedBitTwister = &Error{Code: "SettingLatencyJitterNotAllowedBitTwister", Message: "setting latency/jitter is only allowed if BitTwister is enabled"} + ErrStoppingLatencyJitter = &Error{Code: "StoppingLatencyJitter", Message: "error stopping latency/jitter for instance '%s'"} + ErrSettingLatencyJitter = &Error{Code: "SettingLatencyJitter", Message: "error setting latency/jitter for instance '%s'"} + ErrSettingPacketLossNotAllowed = &Error{Code: "SettingPacketLossNotAllowed", Message: "setting packetloss is only allowed in state 'Started'. Current state is '%s"} + ErrSettingPacketLossNotAllowedBitTwister = &Error{Code: "SettingPacketLossNotAllowedBitTwister", Message: "setting packetloss is only allowed if BitTwister is enabled"} + ErrStoppingPacketLoss = &Error{Code: "StoppingPacketLoss", Message: "error stopping packetloss for instance '%s'"} + ErrSettingPacketLoss = &Error{Code: "SettingPacketLoss", Message: "error setting packetloss for instance '%s'"} + ErrEnablingNetworkNotAllowed = &Error{Code: "EnablingNetworkNotAllowed", Message: "enabling network is only allowed in state 'Started'. Current state is '%s"} + ErrEnablingNetwork = &Error{Code: "EnablingNetwork", Message: "error enabling network for instance '%s'"} + ErrCheckingIfNetworkDisabledNotAllowed = &Error{Code: "CheckingIfNetworkDisabledNotAllowed", Message: "checking if network is disabled is only allowed in state 'Started'. Current state is '%s"} + ErrWaitingForInstanceStoppedNotAllowed = &Error{Code: "WaitingForInstanceStoppedNotAllowed", Message: "waiting for instance is only allowed in state 'Stopped'. Current state is '%s"} + ErrCheckingIfInstanceStopped = &Error{Code: "CheckingIfInstanceStopped", Message: "error checking if instance '%s' is running"} + ErrStoppingNotAllowed = &Error{Code: "StoppingNotAllowed", Message: "stopping is only allowed in state 'Started'. Current state is '%s"} + ErrDestroyingNotAllowed = &Error{Code: "DestroyingNotAllowed", Message: "destroying is only allowed in state 'Started' or 'Destroyed'. Current state is '%s"} + ErrDestroyingPod = &Error{Code: "DestroyingPod", Message: "error destroying pod for instance '%s'"} + ErrDestroyingResourcesForInstance = &Error{Code: "DestroyingResourcesForInstance", Message: "error destroying resources for instance '%s'"} + ErrDestroyingResourcesForSidecars = &Error{Code: "DestroyingResourcesForSidecars", Message: "error destroying resources for sidecars of instance '%s'"} + ErrCloningNotAllowed = &Error{Code: "CloningNotAllowed", Message: "cloning is only allowed in state 'Committed'. Current state is '%s"} + ErrCloningNotAllowedForSidecar = &Error{Code: "CloningNotAllowedForSidecar", Message: "cloning is only allowed in state 'Committed'. Current state is '%s"} + ErrGeneratingK8sNameForSidecar = &Error{Code: "GeneratingK8sNameForSidecar", Message: "error generating k8s name for instance '%s'"} + ErrCannotInitializeKnuuWithEmptyScope = &Error{Code: "Cannot Initialize Knuu With Empty Scope", Message: "cannot initialize knuu with empty scope"} + ErrCannotInitializeK8s = &Error{Code: "Cannot Initialize K8s", Message: "cannot initialize k8s"} + ErrCreatingNamespace = &Error{Code: "CreatingNamespace", Message: "creating namespace %s"} + ErrCannotParseTimeout = &Error{Code: "Cannot Parse Timeout", Message: "cannot parse timeout"} + ErrCannotHandleTimeout = &Error{Code: "Cannot Handle Timeout", Message: "cannot handle timeout"} + ErrInvalidKnuuBuilder = &Error{Code: "Invalid Knuu Builder", Message: "invalid KNUU_BUILDER, available [kubernetes, docker], value used: %s"} + ErrCannotCreateInstance = &Error{Code: "Cannot Create Instance", Message: "cannot create instance"} + ErrCannotSetImage = &Error{Code: "Cannot Set Image", Message: "cannot set image"} + ErrCannotCommitInstance = &Error{Code: "Cannot Commit Instance", Message: "cannot commit instance"} + ErrCannotSetCommand = &Error{Code: "Cannot Set Command", Message: "cannot set command"} + ErrCannotAddPolicyRule = &Error{Code: "Cannot Add Policy Rule", Message: "cannot add policy rule"} + ErrCannotStartInstance = &Error{Code: "Cannot Start Instance", Message: "cannot start instance"} + ErrMinioNotInitialized = &Error{Code: "MinioNotInitialized", Message: "minio not initialized"} + ErrGeneratingK8sNameForPreloader = &Error{Code: "GeneratingK8sNameForPreloader", Message: "error generating k8s name for preloader"} + ErrCannotLoadEnv = &Error{Code: "Cannot Load Env", Message: "cannot load env"} + ErrMaximumVolumesExceeded = &Error{Code: "MaximumVolumesExceeded", Message: "maximum volumes exceeded for instance '%s'"} + ErrCustomResourceDefinitionDoesNotExist = &Error{Code: "CustomResourceDefinitionDoesNotExist", Message: "custom resource definition %s does not exist"} + ErrFileIsNotSubFolderOfVolumes = &Error{Code: "FileIsNotSubFolderOfVolumes", Message: "the file '%s' is not a sub folder of any added volume"} + ErrCannotInitializeKnuu = &Error{Code: "Cannot Initialize Knuu", Message: "cannot initialize knuu"} + ErrAddingToProxy = &Error{Code: "AddingToProxy", Message: "error adding '%s' to traefik proxy for service '%s'"} + ErrGettingProxyURL = &Error{Code: "GettingProxyURL", Message: "error getting proxy URL for service '%s'"} + ErrProxyNotInitialized = &Error{Code: "ProxyNotInitialized", Message: "proxy not initialized"} +) diff --git a/pkg/instance/executor.go b/pkg/instance/executor.go new file mode 100644 index 0000000..3618a3c --- /dev/null +++ b/pkg/instance/executor.go @@ -0,0 +1,54 @@ +package instance + +import ( + "context" + + "github.com/celestiaorg/knuu/pkg/system" +) + +const ( + executorDefaultImage = "docker.io/nicolaka/netshoot:latest" + executorName = "executor" + sleepCommand = "sleep" + infinityArg = "infinity" + memoryLimit = "100M" + cpuLimit = "100m" +) + +type Executor struct { + *Instance +} + +func NewExecutor(ctx context.Context, sysDeps system.SystemDependencies) (*Executor, error) { + i, err := New(executorName, sysDeps) + if err != nil { + return nil, ErrCreatingInstance.Wrap(err) + } + + if err := i.SetImage(ctx, executorDefaultImage); err != nil { + return nil, ErrSettingImage.Wrap(err) + } + + if err := i.Commit(); err != nil { + return nil, ErrCommittingInstance.Wrap(err) + } + + if err := i.SetArgs(sleepCommand, infinityArg); err != nil { + return nil, ErrSettingArgs.Wrap(err) + } + + if err := i.SetMemory(memoryLimit, memoryLimit); err != nil { + return nil, ErrSettingMemory.Wrap(err) + } + + if err := i.SetCPU(cpuLimit); err != nil { + return nil, ErrSettingCPU.Wrap(err) + } + i.instanceType = ExecutorInstance + + if err := i.Start(ctx); err != nil { + return nil, ErrStartingInstance.Wrap(err) + } + + return &Executor{Instance: i}, nil +} diff --git a/pkg/knuu/instance_helper.go b/pkg/instance/helper.go similarity index 86% rename from pkg/knuu/instance_helper.go rename to pkg/instance/helper.go index 9256d00..14fc499 100644 --- a/pkg/knuu/instance_helper.go +++ b/pkg/instance/helper.go @@ -1,4 +1,4 @@ -package knuu +package instance import ( "context" @@ -8,7 +8,6 @@ import ( "os" "path/filepath" "strings" - "time" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -67,8 +66,8 @@ func (i *Instance) getLabels() map[string]string { return map[string]string{ "app": i.k8sName, "k8s.kubernetes.io/managed-by": "knuu", - "knuu.sh/scope": testScope, - "knuu.sh/test-started": startTime, + "knuu.sh/scope": i.TestScope, + "knuu.sh/test-started": i.StartTime, "knuu.sh/name": i.name, "knuu.sh/k8s-name": i.k8sName, "knuu.sh/type": i.instanceType.String(), @@ -91,7 +90,7 @@ func (i *Instance) deployService(ctx context.Context, portsTCP, portsUDP []int) labels := i.getLabels() labelSelectors := labels - service, err := k8sClient.CreateService(ctx, serviceName, labels, labelSelectors, portsTCP, portsUDP) + service, err := i.K8sCli.CreateService(ctx, serviceName, labels, labelSelectors, portsTCP, portsUDP) if err != nil { return ErrDeployingService.WithParams(i.k8sName).Wrap(err) } @@ -111,7 +110,7 @@ func (i *Instance) patchService(ctx context.Context, portsTCP, portsUDP []int) e labels := i.getLabels() labelSelectors := labels - service, err := k8sClient.PatchService(ctx, serviceName, labels, labelSelectors, portsTCP, portsUDP) + service, err := i.K8sCli.PatchService(ctx, serviceName, labels, labelSelectors, portsTCP, portsUDP) if err != nil { return ErrPatchingService.WithParams(serviceName).Wrap(err) } @@ -122,7 +121,7 @@ func (i *Instance) patchService(ctx context.Context, portsTCP, portsUDP []int) e // destroyService destroys the service for the instance func (i *Instance) destroyService(ctx context.Context) error { - return k8sClient.DeleteService(ctx, i.k8sName) + return i.K8sCli.DeleteService(ctx, i.k8sName) } // deployPod deploys the pod for the instance @@ -131,16 +130,16 @@ func (i *Instance) deployPod(ctx context.Context) error { labels := i.getLabels() // create a service account for the pod - if err := k8sClient.CreateServiceAccount(ctx, i.k8sName, labels); err != nil { + if err := i.K8sCli.CreateServiceAccount(ctx, i.k8sName, labels); err != nil { return ErrFailedToCreateServiceAccount.Wrap(err) } // create a role and role binding for the pod if there are policy rules if len(i.policyRules) > 0 { - if err := k8sClient.CreateRole(ctx, i.k8sName, labels, i.policyRules); err != nil { + if err := i.K8sCli.CreateRole(ctx, i.k8sName, labels, i.policyRules); err != nil { return ErrFailedToCreateRole.Wrap(err) } - if err := k8sClient.CreateRoleBinding(ctx, i.k8sName, labels, i.k8sName, i.k8sName); err != nil { + if err := i.K8sCli.CreateRoleBinding(ctx, i.k8sName, labels, i.k8sName, i.k8sName); err != nil { return ErrFailedToCreateRoleBinding.Wrap(err) } } @@ -148,7 +147,7 @@ func (i *Instance) deployPod(ctx context.Context) error { replicaSetSetConfig := i.prepareReplicaSetConfig() // Deploy the statefulSet - replicaSet, err := k8sClient.CreateReplicaSet(ctx, replicaSetSetConfig, true) + replicaSet, err := i.K8sCli.CreateReplicaSet(ctx, replicaSetSetConfig, true) if err != nil { return ErrFailedToDeployPod.Wrap(err) } @@ -167,21 +166,21 @@ func (i *Instance) deployPod(ctx context.Context) error { // Skips if the pod is already destroyed func (i *Instance) destroyPod(ctx context.Context) error { grace := int64(0) - err := k8sClient.DeleteReplicaSetWithGracePeriod(ctx, i.k8sName, &grace) + err := i.K8sCli.DeleteReplicaSetWithGracePeriod(ctx, i.k8sName, &grace) if err != nil { return ErrFailedToDeletePod.Wrap(err) } // Delete the service account for the pod - if err := k8sClient.DeleteServiceAccount(ctx, i.k8sName); err != nil { + if err := i.K8sCli.DeleteServiceAccount(ctx, i.k8sName); err != nil { return ErrFailedToDeleteServiceAccount.Wrap(err) } // Delete the role and role binding for the pod if there are policy rules if len(i.policyRules) > 0 { - if err := k8sClient.DeleteRole(ctx, i.k8sName); err != nil { + if err := i.K8sCli.DeleteRole(ctx, i.k8sName); err != nil { return ErrFailedToDeleteRole.Wrap(err) } - if err := k8sClient.DeleteRoleBinding(ctx, i.k8sName); err != nil { + if err := i.K8sCli.DeleteRoleBinding(ctx, i.k8sName); err != nil { return ErrFailedToDeleteRoleBinding.Wrap(err) } } @@ -193,7 +192,7 @@ func (i *Instance) destroyPod(ctx context.Context) error { func (i *Instance) deployOrPatchService(ctx context.Context, portsTCP, portsUDP []int) error { if len(portsTCP) != 0 || len(portsUDP) != 0 { logrus.Debugf("Ports not empty, deploying service for instance '%s'", i.k8sName) - svc, _ := k8sClient.GetService(ctx, i.k8sName) + svc, _ := i.K8sCli.GetService(ctx, i.k8sName) if svc == nil { err := i.deployService(ctx, portsTCP, portsUDP) if err != nil { @@ -215,7 +214,7 @@ func (i *Instance) deployVolume(ctx context.Context) error { for _, volume := range i.volumes { size.Add(resource.MustParse(volume.Size)) } - k8sClient.CreatePersistentVolumeClaim(ctx, i.k8sName, i.getLabels(), size) + i.K8sCli.CreatePersistentVolumeClaim(ctx, i.k8sName, i.getLabels(), size) logrus.Debugf("Deployed persistent volume '%s'", i.k8sName) return nil @@ -223,7 +222,7 @@ func (i *Instance) deployVolume(ctx context.Context) error { // destroyVolume destroys the volume for the instance func (i *Instance) destroyVolume(ctx context.Context) error { - k8sClient.DeletePersistentVolumeClaim(ctx, i.k8sName) + i.K8sCli.DeletePersistentVolumeClaim(ctx, i.k8sName) logrus.Debugf("Destroyed persistent volume '%s'", i.k8sName) return nil @@ -256,7 +255,7 @@ func (i *Instance) deployFiles(ctx context.Context) error { } // create configmap - if _, err := k8sClient.CreateConfigMap(ctx, i.k8sName, i.getLabels(), data); err != nil { + if _, err := i.K8sCli.CreateConfigMap(ctx, i.k8sName, i.getLabels(), data); err != nil { return ErrFailedToCreateConfigMap.Wrap(err) } @@ -267,7 +266,7 @@ func (i *Instance) deployFiles(ctx context.Context) error { // destroyFiles destroys the files for the instance func (i *Instance) destroyFiles(ctx context.Context) error { - if err := k8sClient.DeleteConfigMap(ctx, i.k8sName); err != nil { + if err := i.K8sCli.DeleteConfigMap(ctx, i.k8sName); err != nil { return ErrFailedToDeleteConfigMap.Wrap(err) } @@ -330,13 +329,13 @@ func (i *Instance) destroyResources(ctx context.Context) error { // disable network only for non-sidecar instances if !i.isSidecar { // enable network when network is disabled - disableNetwork, err := i.NetworkIsDisabled() + disableNetwork, err := i.NetworkIsDisabled(ctx) if err != nil { logrus.Debugf("error checking network status for instance") return ErrCheckingNetworkStatusForInstance.WithParams(i.k8sName).Wrap(err) } if disableNetwork { - err := i.EnableNetwork() + err := i.EnableNetwork(ctx) if err != nil { logrus.Debugf("error enabling network for instance") return ErrEnablingNetworkForInstance.WithParams(i.k8sName).Wrap(err) @@ -388,17 +387,10 @@ func (i *Instance) cloneWithSuffix(suffix string) *Instance { obsyConfig: i.obsyConfig, securityContext: &clonedSecurityContext, BitTwister: &clonedBitTwister, + SystemDependencies: i.SystemDependencies, } } -func generateK8sName(name string) (string, error) { - uuid, err := uuid.NewRandom() - if err != nil { - return "", ErrGeneratingUUID.Wrap(err) - } - return fmt.Sprintf("%s-%s", name, uuid.String()[:8]), nil -} - // getFreePort returns a free port func getFreePortTCP() (int, error) { // Get a random port @@ -515,7 +507,7 @@ func (i *Instance) prepareReplicaSetConfig() k8s.ReplicaSetConfig { } // Generate the pod configuration podConfig := k8s.PodConfig{ - Namespace: k8sClient.Namespace(), + Namespace: i.K8sCli.Namespace(), Name: i.k8sName, Labels: i.getLabels(), ServiceAccountName: i.k8sName, @@ -525,7 +517,7 @@ func (i *Instance) prepareReplicaSetConfig() k8s.ReplicaSetConfig { } // Generate the ReplicaSet configuration statefulSetConfig := k8s.ReplicaSetConfig{ - Namespace: k8sClient.Namespace(), + Namespace: i.K8sCli.Namespace(), Name: i.k8sName, Labels: i.getLabels(), Replicas: 1, @@ -542,11 +534,11 @@ func (i *Instance) setImageWithGracePeriod(ctx context.Context, imageName string replicaSetConfig := i.prepareReplicaSetConfig() // Replace the pod with a new one, using the given image - _, err := k8sClient.ReplaceReplicaSetWithGracePeriod(ctx, replicaSetConfig, gracePeriod) + _, err := i.K8sCli.ReplaceReplicaSetWithGracePeriod(ctx, replicaSetConfig, gracePeriod) if err != nil { return ErrReplacingPod.Wrap(err) } - if err := i.WaitInstanceIsRunning(); err != nil { + if err := i.WaitInstanceIsRunning(ctx); err != nil { return ErrWaitingInstanceIsRunning.Wrap(err) } @@ -576,7 +568,11 @@ func setStateForSidecars(sidecars []*Instance, state InstanceState) { // isObservabilityEnabled returns true if observability is enabled func (i *Instance) isObservabilityEnabled() bool { - return i.obsyConfig.otlpPort != 0 || i.obsyConfig.prometheusEndpointPort != 0 || i.obsyConfig.jaegerGrpcPort != 0 || i.obsyConfig.jaegerThriftCompactPort != 0 || i.obsyConfig.jaegerThriftHttpPort != 0 + return i.obsyConfig.otlpPort != 0 || + i.obsyConfig.prometheusEndpointPort != 0 || + i.obsyConfig.jaegerGrpcPort != 0 || + i.obsyConfig.jaegerThriftCompactPort != 0 || + i.obsyConfig.jaegerThriftHttpPort != 0 } func (i *Instance) validateStateForObsy(endpoint string) error { @@ -586,8 +582,8 @@ func (i *Instance) validateStateForObsy(endpoint string) error { return nil } -func (i *Instance) addOtelCollectorSidecar() error { - otelSidecar, err := i.createOtelCollectorInstance() +func (i *Instance) addOtelCollectorSidecar(ctx context.Context) error { + otelSidecar, err := i.createOtelCollectorInstance(ctx) if err != nil { return ErrCreatingOtelCollectorInstance.WithParams(i.k8sName).Wrap(err) } @@ -597,13 +593,13 @@ func (i *Instance) addOtelCollectorSidecar() error { return nil } -func (i *Instance) createBitTwisterInstance() (*Instance, error) { - bt, err := NewInstance("bit-twister") +func (i *Instance) createBitTwisterInstance(ctx context.Context) (*Instance, error) { + bt, err := New("bit-twister", i.SystemDependencies) if err != nil { return nil, ErrCreatingBitTwisterInstance.Wrap(err) } - if err := bt.SetImage(i.BitTwister.Image()); err != nil { + if err := bt.SetImage(ctx, i.BitTwister.Image()); err != nil { return nil, ErrSettingBitTwisterImage.Wrap(err) } @@ -611,21 +607,11 @@ func (i *Instance) createBitTwisterInstance() (*Instance, error) { if err := bt.AddPortTCP(i.BitTwister.Port()); err != nil { return nil, ErrAddingBitTwisterPort.Wrap(err) } - - // TODO: remove this when pkg refactor - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - serviceName := i.k8sName // the main instance name - err = traefikClient.AddHost(ctx, serviceName, bt.k8sName, i.BitTwister.Port()) + btURL, err := i.AddHost(ctx, i.BitTwister.Port()) if err != nil { return nil, ErrAddingToProxy.WithParams(bt.k8sName, serviceName).Wrap(err) } - - btURL, err := traefikClient.URL(ctx, bt.k8sName) - if err != nil { - return nil, ErrGettingBitTwisterPath.Wrap(err) - } logrus.Debugf("BitTwister URL: %s", btURL) i.BitTwister.SetNewClientByURL(btURL) @@ -641,8 +627,8 @@ func (i *Instance) createBitTwisterInstance() (*Instance, error) { return bt, nil } -func (i *Instance) addBitTwisterSidecar() error { - networkConfigSidecar, err := i.createBitTwisterInstance() +func (i *Instance) addBitTwisterSidecar(ctx context.Context) error { + networkConfigSidecar, err := i.createBitTwisterInstance(ctx) if err != nil { return ErrCreatingBitTwisterInstance.WithParams(i.k8sName).Wrap(err) } diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go new file mode 100644 index 0000000..36b054b --- /dev/null +++ b/pkg/instance/instance.go @@ -0,0 +1,1363 @@ +package instance + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + appv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/sirupsen/logrus" + + "github.com/celestiaorg/bittwister/sdk" + + "github.com/celestiaorg/knuu/pkg/builder" + "github.com/celestiaorg/knuu/pkg/container" + "github.com/celestiaorg/knuu/pkg/k8s" + "github.com/celestiaorg/knuu/pkg/names" + "github.com/celestiaorg/knuu/pkg/system" +) + +// We need to retry here because the port forwarding might fail as getFreePortTCP() might not free the port fast enough +const ( + maxRetries = 5 + retryInterval = 5 * time.Second +) + +// ObsyConfig represents the configuration for the obsy sidecar +type ObsyConfig struct { + // otelCollectorVersion is the version of the otel collector to use + otelCollectorVersion string + + // prometheusEndpointPort is the port on which the prometheus server will be exposed + prometheusEndpointPort int + // prometheusEndpointJobName is the name of the prometheus job + prometheusEndpointJobName string + // prometheusEndpointScrapeInterval is the scrape interval for the prometheus job + prometheusEndpointScrapeInterval string + + // jaegerGrpcPort is the port on which the jaeger grpc server is exposed + jaegerGrpcPort int + // jaegerThriftCompactPort is the port on which the jaeger thrift compact server is exposed + jaegerThriftCompactPort int + // jaegerThriftHttpPort is the port on which the jaeger thrift http server is exposed + jaegerThriftHttpPort int + // jaegerEndpoint is the endpoint of the jaeger collector where spans will be sent to + jaegerEndpoint string + + // otlpPort is the port on which the otlp server is exposed + otlpPort int + // otlpEndpoint is the endpoint of the otlp collector where spans will be sent to + otlpEndpoint string + // otlpUsername is the username to use for the otlp collector + otlpUsername string + // otlpPassword is the password to use for the otlp collector + otlpPassword string + + // prometheusExporterEndpoint is the endpoint of the prometheus exporter + prometheusExporterEndpoint string + + // prometheusRemoteWriteExporterEndpoint is the endpoint of the prometheus remote write + prometheusRemoteWriteExporterEndpoint string +} + +// SecurityContext represents the security settings for a container +type SecurityContext struct { + // Privileged indicates whether the container should be run in privileged mode + privileged bool + + // CapabilitiesAdd is the list of capabilities to add to the container + capabilitiesAdd []string +} + +// Instance represents a instance +type Instance struct { + system.SystemDependencies + name string + imageName string + k8sName string + state InstanceState + instanceType InstanceType + kubernetesService *v1.Service + builderFactory *container.BuilderFactory + kubernetesReplicaSet *appv1.ReplicaSet + portsTCP []int + portsUDP []int + command []string + args []string + env map[string]string + volumes []*k8s.Volume + memoryRequest string + memoryLimit string + cpuRequest string + policyRules []rbacv1.PolicyRule + livenessProbe *v1.Probe + readinessProbe *v1.Probe + startupProbe *v1.Probe + files []*k8s.File + isSidecar bool + parentInstance *Instance + sidecars []*Instance + fsGroup int64 + obsyConfig *ObsyConfig + securityContext *SecurityContext + BitTwister *btConfig +} + +func New(name string, sysDeps system.SystemDependencies) (*Instance, error) { + k8sName, err := names.NewRandomK8(name) + if err != nil { + return nil, ErrGeneratingK8sName.WithParams(name).Wrap(err) + } + + obsyConfig := &ObsyConfig{ + otelCollectorVersion: "0.83.0", + otlpPort: 0, + prometheusEndpointPort: 0, + prometheusEndpointJobName: "", + prometheusEndpointScrapeInterval: "", + jaegerGrpcPort: 0, + jaegerThriftCompactPort: 0, + jaegerThriftHttpPort: 0, + otlpEndpoint: "", + otlpUsername: "", + otlpPassword: "", + jaegerEndpoint: "", + prometheusExporterEndpoint: "", + prometheusRemoteWriteExporterEndpoint: "", + } + securityContext := &SecurityContext{ + privileged: false, + capabilitiesAdd: make([]string, 0), + } + + // Create the instance + return &Instance{ + name: name, + k8sName: k8sName, + imageName: "", + state: None, + instanceType: BasicInstance, + portsTCP: make([]int, 0), + portsUDP: make([]int, 0), + command: make([]string, 0), + args: make([]string, 0), + env: make(map[string]string), + volumes: make([]*k8s.Volume, 0), + memoryRequest: "", + memoryLimit: "", + cpuRequest: "", + policyRules: make([]rbacv1.PolicyRule, 0), + livenessProbe: nil, + readinessProbe: nil, + startupProbe: nil, + files: make([]*k8s.File, 0), + isSidecar: false, + parentInstance: nil, + sidecars: make([]*Instance, 0), + obsyConfig: obsyConfig, + securityContext: securityContext, + BitTwister: getBitTwisterDefaultConfig(), + SystemDependencies: sysDeps, + }, nil +} + +func (i *Instance) EnableBitTwister() error { + if i.IsInState(Started) { + return ErrEnablingBitTwister + } + i.BitTwister.enable() + return nil +} + +func (i *Instance) DisableBitTwister() error { + i.BitTwister.disable() + return nil +} + +// Name returns the name of the instance +func (i *Instance) Name() string { + return i.name +} + +func (i *Instance) SetInstanceType(instanceType InstanceType) { + i.instanceType = instanceType +} + +// SetImage sets the image of the instance. +// When calling in state 'Started', make sure to call AddVolume() before. +// It is only allowed in the 'None' and 'Started' states. +func (i *Instance) SetImage(ctx context.Context, image string) error { + if !i.IsInState(None, Started) { + return ErrSettingImageNotAllowed.WithParams(i.state.String()) + } + + if i.state == None { + // Use the builder to build a new image + factory, err := container.NewBuilderFactory(image, i.getBuildDir(), i.ImageBuilder) + if err != nil { + return ErrCreatingBuilder.Wrap(err) + } + i.builderFactory = factory + i.state = Preparing + + return nil + } + + if i.isSidecar { + return ErrSettingImageNotAllowedForSidecarsStarted + } + return i.setImageWithGracePeriod(ctx, image, nil) +} + +// SetGitRepo builds the image from the given git repo, pushes it +// to the registry under the given name and sets the image of the instance. +func (i *Instance) SetGitRepo(ctx context.Context, gitContext builder.GitContext) error { + if !i.IsInState(None) { + return ErrSettingGitRepo.WithParams(i.state.String()) + } + + bCtx, err := gitContext.BuildContext() + if err != nil { + return ErrGettingBuildContext.Wrap(err) + } + imageName, err := builder.DefaultImageName(bCtx) + if err != nil { + return ErrGettingImageName.Wrap(err) + } + + factory, err := container.NewBuilderFactory(imageName, i.getBuildDir(), i.ImageBuilder) + if err != nil { + return ErrCreatingBuilder.Wrap(err) + } + i.builderFactory = factory + i.state = Preparing + + return i.builderFactory.BuildImageFromGitRepo(ctx, gitContext, imageName) +} + +// SetImageInstant sets the image of the instance without a grace period. +// Instant means that the pod is replaced without a grace period of 1 second. +// It is only allowed in the 'Running' state. +func (i *Instance) SetImageInstant(ctx context.Context, image string) error { + if !i.IsInState(Started) { + return ErrSettingImageNotAllowedForSidecarsStarted.WithParams(i.state.String()) + } + + if i.isSidecar { + return ErrSettingImageNotAllowedForSidecars + } + + gracePeriod := int64(0) + return i.setImageWithGracePeriod(ctx, image, &gracePeriod) +} + +// SetCommand sets the command to run in the instance +// This function can only be called when the instance is in state 'Preparing' or 'Committed' +func (i *Instance) SetCommand(command ...string) error { + if !i.IsInState(Preparing, Committed) { + return ErrSettingCommand.WithParams(i.state.String()) + } + i.command = command + return nil +} + +// SetArgs sets the arguments passed to the instance +// This function can only be called in the states 'Preparing' or 'Committed' +func (i *Instance) SetArgs(args ...string) error { + if !i.IsInState(Preparing, Committed) { + return ErrSettingArgsNotAllowed.WithParams(i.state.String()) + } + i.args = args + return nil +} + +// AddPortTCP adds a TCP port to the instance +// This function can be called in the states 'Preparing' and 'Committed' +func (i *Instance) AddPortTCP(port int) error { + if !i.IsInState(Preparing, Committed) { + return ErrAddingPortNotAllowed.WithParams(i.state.String()) + } + err := validatePort(port) + if err != nil { + return err + } + if i.isTCPPortRegistered(port) { + return ErrPortAlreadyRegistered.WithParams(port) + } + i.portsTCP = append(i.portsTCP, port) + logrus.Debugf("Added TCP port '%d' to instance '%s'", port, i.name) + return nil +} + +// PortForwardTCP forwards the given port to a random port on the host +// This function can only be called in the state 'Started' +func (i *Instance) PortForwardTCP(ctx context.Context, port int) (int, error) { + if !i.IsInState(Started) { + return -1, ErrRandomPortForwardingNotAllowed.WithParams(i.state.String()) + } + err := validatePort(port) + if err != nil { + return 0, err + } + if !i.isTCPPortRegistered(port) { + return -1, ErrPortNotRegistered.WithParams(port) + } + // Get a random port on the host + localPort, err := getFreePortTCP() + if err != nil { + return -1, ErrGettingFreePort.WithParams(port) + } + + // Forward the port + pod, err := i.K8sCli.GetFirstPodFromReplicaSet(ctx, i.k8sName) + if err != nil { + return -1, ErrGettingPodFromReplicaSet.WithParams(i.k8sName).Wrap(err) + } + + for attempt := 1; attempt <= maxRetries; attempt++ { + err := i.K8sCli.PortForwardPod(ctx, pod.Name, localPort, port) + if err == nil { + break + } + if attempt == maxRetries { + return -1, ErrForwardingPort.WithParams(maxRetries) + } + logrus.Debugf("Forwarding port %d failed, cause: %v, retrying after %v (retry %d/%d)", port, err, retryInterval, attempt, maxRetries) + time.Sleep(retryInterval) + } + return localPort, nil +} + +// AddPortUDP adds a UDP port to the instance +// This function can be called in the states 'Preparing' and 'Committed' +func (i *Instance) AddPortUDP(port int) error { + if !i.IsInState(Preparing, Committed) { + return ErrAddingPortNotAllowed.WithParams(i.state.String()) + } + err := validatePort(port) + if err != nil { + return err + } + if i.isUDPPortRegistered(port) { + return ErrUDPPortAlreadyRegistered.WithParams(port) + } + i.portsUDP = append(i.portsUDP, port) + logrus.Debugf("Added UDP port '%d' to instance '%s'", port, i.k8sName) + return nil +} + +// ExecuteCommand executes the given command in the instance +// This function can only be called in the states 'Preparing' and 'Started' +// The context can be used to cancel the command and it is only possible in start state +func (i *Instance) ExecuteCommand(ctx context.Context, command ...string) (string, error) { + if !i.IsInState(Preparing, Started) { + return "", ErrExecutingCommandNotAllowed.WithParams(i.state.String()) + } + + if i.IsInState(Preparing) { + output, err := i.builderFactory.ExecuteCmdInBuilder(command) + if err != nil { + return "", ErrExecutingCommandInInstance.WithParams(command, i.name).Wrap(err) + } + return output, nil + } + + var ( + instanceName string + eErr *Error + containerName = i.k8sName + ) + + if i.isSidecar { + instanceName = i.parentInstance.k8sName + eErr = ErrExecutingCommandInSidecar.WithParams(command, i.k8sName, i.parentInstance.k8sName) + } else { + instanceName = i.k8sName + eErr = ErrExecutingCommandInInstance.WithParams(command, i.k8sName) + } + + pod, err := i.K8sCli.GetFirstPodFromReplicaSet(ctx, instanceName) + if err != nil { + return "", ErrGettingPodFromReplicaSet.WithParams(i.k8sName).Wrap(err) + } + + commandWithShell := []string{"/bin/sh", "-c", strings.Join(command, " ")} + output, err := i.K8sCli.RunCommandInPod(ctx, pod.Name, containerName, commandWithShell) + if err != nil { + return "", eErr.Wrap(err) + } + return output, nil +} + +// checkStateForAddingFile checks if the current state allows adding a file +func (i *Instance) checkStateForAddingFile() error { + if !i.IsInState(Preparing, Committed) { + return ErrAddingFileNotAllowed.WithParams(i.state.String()) + } + return nil +} + +// AddFile adds a file to the instance +// This function can only be called in the state 'Preparing' +func (i *Instance) AddFile(src string, dest string, chown string) error { + if err := i.checkStateForAddingFile(); err != nil { + return err + } + + err := i.validateFileArgs(src, dest, chown) + if err != nil { + return err + } + + // check if src exists (either as file or as folder) + if _, err := os.Stat(src); os.IsNotExist(err) { + return ErrSrcDoesNotExist.WithParams(src).Wrap(err) + } + + // copy file to build dir + dstPath := filepath.Join(i.getBuildDir(), dest) + + // make sure dir exists + err = os.MkdirAll(filepath.Dir(dstPath), os.ModePerm) + if err != nil { + return ErrCreatingDirectory.Wrap(err) + } + // Create destination file making sure the path is writeable. + dst, err := os.Create(dstPath) + if err != nil { + return ErrFailedToCreateDestFile.WithParams(dstPath).Wrap(err) + } + defer dst.Close() + + // Open source file for reading. + srcFile, err := os.Open(src) + if err != nil { + return ErrFailedToOpenSrcFile.WithParams(src).Wrap(err) + } + defer srcFile.Close() + + // Copy the contents from source file to destination file + _, err = io.Copy(dst, srcFile) + if err != nil { + return ErrFailedToCopyFile.WithParams(src, dstPath).Wrap(err) + } + + switch i.state { + case Preparing: + err := i.addFileToBuilder(src, dest, chown) + if err != nil { + return err + } + case Committed: + // check if the dest is a sub folder of added volumes and print a warning if not + if !i.isSubFolderOfVolumes(dest) { + return ErrFileIsNotSubFolderOfVolumes.WithParams(dest) + } + + // only allow files, not folders + srcInfo, err := os.Stat(src) + if os.IsNotExist(err) || srcInfo.IsDir() { + return ErrSrcDoesNotExistOrIsDirectory.WithParams(src).Wrap(err) + } + file := i.K8sCli.NewFile(dstPath, dest) + + // the user provided a chown string (e.g. "10001:10001") and we only need the group (second part) + parts := strings.Split(chown, ":") + if len(parts) != 2 { + return ErrInvalidFormat + } + + // second part of array, base of number is 10, and we want a 64-bit integer + group, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return ErrFailedToConvertToInt64.Wrap(err) + } + + if i.fsGroup != 0 && i.fsGroup != group { + return ErrAllFilesMustHaveSameGroup + } else { + i.fsGroup = group + } + + i.files = append(i.files, file) + } + + logrus.Debugf("Added file '%s' to instance '%s'", dest, i.name) + return nil +} + +// AddFolder adds a folder to the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) AddFolder(src string, dest string, chown string) error { + if !i.IsInState(Preparing, Committed) { + return ErrAddingFolderNotAllowed.WithParams(i.state.String()) + } + + i.validateFileArgs(src, dest, chown) + + // check if src exists (should be a folder) + srcInfo, err := os.Stat(src) + if os.IsNotExist(err) || !srcInfo.IsDir() { + return ErrSrcDoesNotExistOrIsNotDirectory.WithParams(src).Wrap(err) + } + + // iterate over the files/directories in the src + err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // create the destination path + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + dstPath := filepath.Join(i.getBuildDir(), dest, relPath) + + if info.IsDir() { + // create directory at destination path + return os.MkdirAll(dstPath, os.ModePerm) + } + // copy file to destination path + return i.AddFile(path, filepath.Join(dest, relPath), chown) + }) + + if err != nil { + return ErrCopyingFolderToInstance.WithParams(src, i.name).Wrap(err) + } + + logrus.Debugf("Added folder '%s' to instance '%s'", dest, i.name) + return nil +} + +// AddFileBytes adds a file with the given content to the instance +// This function can only be called in the state 'Preparing' +func (i *Instance) AddFileBytes(bytes []byte, dest string, chown string) error { + if err := i.checkStateForAddingFile(); err != nil { + return err + } + + // create a temporary file + tmpfile, err := os.CreateTemp("", "temp") + if err != nil { + return err + } + defer os.Remove(tmpfile.Name()) // clean up + + // write bytes to the temporary file + if _, err := tmpfile.Write(bytes); err != nil { + return err + } + if err := tmpfile.Close(); err != nil { + return err + } + + // use AddFile to copy the temp file to the destination + return i.AddFile(tmpfile.Name(), dest, chown) +} + +// SetUser sets the user for the instance +// This function can only be called in the state 'Preparing' +func (i *Instance) SetUser(user string) error { + if !i.IsInState(Preparing) { + return ErrSettingUserNotAllowed.WithParams(i.state.String()) + } + err := i.builderFactory.SetUser(user) + if err != nil { + return ErrSettingUser.WithParams(user, i.name).Wrap(err) + } + logrus.Debugf("Set user '%s' for instance '%s'", user, i.name) + return nil +} + +// imageCache maps image hash values to image names +var imageCache = make(map[string]string) + +// checkImageHashInCache checks if the given image hash exists in the cache. +func checkImageHashInCache(imageHash string) (imageName string, exists bool) { + imageName, exists = imageCache[imageHash] + return imageName, exists +} + +// updateImageCacheWithHash adds or updates the image cache with the given hash and image name. +func updateImageCacheWithHash(imageHash, imageName string) { + imageCache[imageHash] = imageName // Update the cache with the new hash and image name +} + +// Commit commits the instance +// This function can only be called in the state 'Preparing' +func (i *Instance) Commit() error { + if !i.IsInState(Preparing) { + return ErrCommittingNotAllowed.WithParams(i.state.String()) + } + if i.builderFactory.Changed() { + // TODO: To speed up the process, the image name could be dependent on the hash of the image + imageName, err := i.getImageRegistry() + if err != nil { + return ErrGettingImageRegistry.Wrap(err) + } + + // Generate a hash for the current image + imageHash, err := i.builderFactory.GenerateImageHash() + if err != nil { + return ErrGeneratingImageHash.Wrap(err) + } + + // Check if the generated image hash already exists in the cache, otherwise, we build it. + cachedImageName, exists := checkImageHashInCache(imageHash) + if exists { + i.imageName = cachedImageName + logrus.Debugf("Using cached image for instance '%s'", i.name) + } else { + logrus.Debugf("Cannot use any cached image for instance '%s'", i.name) + err = i.builderFactory.PushBuilderImage(imageName) + if err != nil { + return ErrPushingImage.WithParams(i.name).Wrap(err) + } + updateImageCacheWithHash(imageHash, imageName) + i.imageName = imageName + logrus.Debugf("Pushed new image for instance '%s'", i.name) + } + } else { + i.imageName = i.builderFactory.ImageNameFrom() + logrus.Debugf("No need to build and push image for instance '%s'", i.name) + } + i.state = Committed + logrus.Debugf("Set state of instance '%s' to '%s'", i.name, i.state.String()) + + return nil +} + +// AddVolume adds a volume to the instance +// The owner of the volume is set to 0, if you want to set a custom owner use AddVolumeWithOwner +// This function can only be called in the states 'Preparing' and 'Committed' +func (i *Instance) AddVolume(path, size string) error { + // temporary feat, we will remove it once we can add multiple volumes + if len(i.volumes) > 0 { + logrus.Debugf("Maximum volumes exceeded for instance '%s', volumes: %d", i.name, len(i.volumes)) + return ErrMaximumVolumesExceeded.WithParams(i.name) + } + i.AddVolumeWithOwner(path, size, 0) + return nil +} + +// AddVolumeWithOwner adds a volume to the instance with the given owner +// This function can only be called in the states 'Preparing' and 'Committed' +func (i *Instance) AddVolumeWithOwner(path, size string, owner int64) error { + if !i.IsInState(Preparing, Committed) { + return ErrAddingVolumeNotAllowed.WithParams(i.state.String()) + } + // temporary feat, we will remove it once we can add multiple volumes + if len(i.volumes) > 0 { + logrus.Debugf("Maximum volumes exceeded for instance '%s', volumes: %d", i.name, len(i.volumes)) + return ErrMaximumVolumesExceeded.WithParams(i.name) + } + volume := i.K8sCli.NewVolume(path, size, owner) + i.volumes = append(i.volumes, volume) + logrus.Debugf("Added volume '%s' with size '%s' and owner '%d' to instance '%s'", path, size, owner, i.name) + return nil +} + +// SetMemory sets the memory of the instance +// This function can only be called in the states 'Preparing' and 'Committed' +func (i *Instance) SetMemory(request, limit string) error { + if !i.IsInState(Preparing, Committed) { + return ErrSettingMemoryNotAllowed.WithParams(i.state.String()) + } + i.memoryRequest = request + i.memoryLimit = limit + logrus.Debugf("Set memory to '%s' and limit to '%s' in instance '%s'", request, limit, i.name) + return nil +} + +// SetCPU sets the CPU of the instance +// This function can only be called in the states 'Preparing' and 'Committed' +func (i *Instance) SetCPU(request string) error { + if !i.IsInState(Preparing, Committed) { + return ErrSettingCPUNotAllowed.WithParams(i.state.String()) + } + i.cpuRequest = request + logrus.Debugf("Set cpu to '%s' in instance '%s'", request, i.name) + return nil +} + +// SetEnvironmentVariable sets the given environment variable in the instance +// This function can only be called in the states 'Preparing' and 'Committed' +func (i *Instance) SetEnvironmentVariable(key, value string) error { + if !i.IsInState(Preparing, Committed) { + return ErrSettingEnvNotAllowed.WithParams(i.state.String()) + } + if i.state == Preparing { + err := i.builderFactory.SetEnvVar(key, value) + if err != nil { + return err + } + } else if i.state == Committed { + i.env[key] = value + } + logrus.Debugf("Set environment variable '%s' to '%s' in instance '%s'", key, value, i.name) + return nil +} + +// GetIP returns the IP of the instance +// This function can only be called in the states 'Preparing' and 'Started' +func (i *Instance) GetIP(ctx context.Context) (string, error) { + // Check if i.kubernetesService already has the IP + if i.kubernetesService != nil && i.kubernetesService.Spec.ClusterIP != "" { + return i.kubernetesService.Spec.ClusterIP, nil + } + // If not, proceed with the existing logic to deploy the service and get the IP + svc, err := i.K8sCli.GetService(ctx, i.k8sName) + if err != nil || svc == nil { + // Service does not exist, so we need to deploy it + err := i.deployService(ctx, i.portsTCP, i.portsUDP) + if err != nil { + return "", ErrDeployingServiceForInstance.WithParams(i.k8sName).Wrap(err) + } + svc, err = i.K8sCli.GetService(ctx, i.k8sName) + if err != nil { + return "", ErrGettingServiceForInstance.WithParams(i.k8sName).Wrap(err) + } + } + + ip := svc.Spec.ClusterIP + if ip == "" { + return "", ErrGettingServiceIP.WithParams(i.k8sName) + } + + // Update i.kubernetesService for future reference + i.kubernetesService = svc + + return ip, nil +} + +// GetFileBytes returns the content of the given file +// This function can only be called in the states 'Preparing' and 'Committed' +func (i *Instance) GetFileBytes(ctx context.Context, file string) ([]byte, error) { + if !i.IsInState(Preparing, Committed, Started) { + return nil, ErrGettingFileNotAllowed.WithParams(i.state.String()) + } + + if i.state != Started { + bytes, err := i.builderFactory.ReadFileFromBuilder(file) + if err != nil { + return nil, ErrGettingFile.WithParams(file, i.name).Wrap(err) + } + return bytes, nil + } + + rc, err := i.ReadFileFromRunningInstance(ctx, file) + if err != nil { + return nil, ErrReadingFile.WithParams(file, i.name).Wrap(err) + } + + defer rc.Close() + return io.ReadAll(rc) +} + +func (i *Instance) ReadFileFromRunningInstance(ctx context.Context, filePath string) (io.ReadCloser, error) { + if !i.IsInState(Started) { + return nil, ErrReadingFileNotAllowed.WithParams(i.state.String()) + } + + // Not the best solution, we need to find a better one. + // Tested with a 110MB+ file and it worked. + fileContent, err := i.ExecuteCommand(ctx, "cat", filePath) + if err != nil { + return nil, ErrReadingFileFromInstance.WithParams(filePath, i.name).Wrap(err) + } + return io.NopCloser(strings.NewReader(fileContent)), nil +} + +// AddPolicyRule adds a policy rule to the instance +// This function can only be called in the states 'Preparing' and 'Committed' +func (i *Instance) AddPolicyRule(rule rbacv1.PolicyRule) error { + if !i.IsInState(Preparing, Committed) { + return ErrAddingPolicyRuleNotAllowed.WithParams(i.state.String()) + } + i.policyRules = append(i.policyRules, rule) + return nil +} + +// checkStateForProbe checks if the current state is allowed for setting a probe +func (i *Instance) checkStateForProbe() error { + if !i.IsInState(Preparing, Committed) { + return ErrSettingProbeNotAllowed.WithParams(i.state.String()) + } + return nil +} + +// SetLivenessProbe sets the liveness probe of the instance +// A live probe is a probe that is used to determine if the instance is still alive, and should be restarted if not +// See usage documentation: https://pkg.go.dev/i.K8sCli.io/api/core/v1@v0.27.3#Probe +// This function can only be called in the states 'Preparing' and 'Committed' +func (i *Instance) SetLivenessProbe(livenessProbe *v1.Probe) error { + if err := i.checkStateForProbe(); err != nil { + return err + } + i.livenessProbe = livenessProbe + logrus.Debugf("Set liveness probe to '%s' in instance '%s'", livenessProbe, i.name) + return nil +} + +// SetReadinessProbe sets the readiness probe of the instance +// A readiness probe is a probe that is used to determine if the instance is ready to receive traffic +// See usage documentation: https://pkg.go.dev/i.K8sCli.io/api/core/v1@v0.27.3#Probe +// This function can only be called in the states 'Preparing' and 'Committed' +func (i *Instance) SetReadinessProbe(readinessProbe *v1.Probe) error { + if err := i.checkStateForProbe(); err != nil { + return err + } + i.readinessProbe = readinessProbe + logrus.Debugf("Set readiness probe to '%s' in instance '%s'", readinessProbe, i.name) + return nil +} + +// SetStartupProbe sets the startup probe of the instance +// A startup probe is a probe that is used to determine if the instance is ready to receive traffic after a startup +// See usage documentation: https://pkg.go.dev/i.K8sCli.io/api/core/v1@v0.27.3#Probe +// This function can only be called in the states 'Preparing' and 'Committed' +func (i *Instance) SetStartupProbe(startupProbe *v1.Probe) error { + if err := i.checkStateForProbe(); err != nil { + return err + } + i.startupProbe = startupProbe + logrus.Debugf("Set startup probe to '%s' in instance '%s'", startupProbe, i.name) + return nil +} + +// AddSidecar adds a sidecar to the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) AddSidecar(sidecar *Instance) error { + + if !i.IsInState(Preparing, Committed) { + return ErrAddingSidecarNotAllowed.WithParams(i.state.String()) + } + if sidecar == nil { + return ErrSidecarIsNil + } + if sidecar == i { + return ErrSidecarCannotBeSameInstance + } + if sidecar.state != Committed { + return ErrSidecarNotCommitted.WithParams(sidecar.name) + } + if i.isSidecar { + return ErrSidecarCannotHaveSidecar.WithParams(i.name) + } + if sidecar.isSidecar { + return ErrSidecarAlreadySidecar.WithParams(sidecar.name) + } + + i.sidecars = append(i.sidecars, sidecar) + sidecar.isSidecar = true + sidecar.parentInstance = i + logrus.Debugf("Added sidecar '%s' to instance '%s'", sidecar.name, i.name) + return nil +} + +// SetOtelCollectorVersion sets the OpenTelemetry collector version for the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) SetOtelCollectorVersion(version string) error { + if err := i.validateStateForObsy("OpenTelemetry collector version"); err != nil { + return err + } + i.obsyConfig.otelCollectorVersion = version + logrus.Debugf("Set OpenTelemetry collector version '%s' for instance '%s'", version, i.name) + return nil +} + +// SetOtelEndpoint sets the OpenTelemetry endpoint for the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) SetOtelEndpoint(port int) error { + if err := i.validateStateForObsy("OpenTelemetry endpoint"); err != nil { + return err + } + i.obsyConfig.otlpPort = port + logrus.Debugf("Set OpenTelemetry endpoint '%d' for instance '%s'", port, i.name) + return nil +} + +// SetPrometheusEndpoint sets the Prometheus endpoint for the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) SetPrometheusEndpoint(port int, jobName, scapeInterval string) error { + if err := i.validateStateForObsy("Prometheus endpoint"); err != nil { + return err + } + i.obsyConfig.prometheusEndpointPort = port + i.obsyConfig.prometheusEndpointJobName = jobName + i.obsyConfig.prometheusEndpointScrapeInterval = scapeInterval + logrus.Debugf("Set Prometheus endpoint '%d' for instance '%s'", port, i.name) + return nil +} + +// SetJaegerEndpoint sets the Jaeger endpoint for the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) SetJaegerEndpoint(grpcPort, thriftCompactPort, thriftHttpPort int) error { + if err := i.validateStateForObsy("Jaeger endpoint"); err != nil { + return err + } + i.obsyConfig.jaegerGrpcPort = grpcPort + i.obsyConfig.jaegerThriftCompactPort = thriftCompactPort + i.obsyConfig.jaegerThriftHttpPort = thriftHttpPort + logrus.Debugf("Set Jaeger endpoints '%d', '%d' and '%d' for instance '%s'", grpcPort, thriftCompactPort, thriftHttpPort, i.name) + return nil +} + +// SetOtlpExporter sets the OTLP exporter for the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) SetOtlpExporter(endpoint, username, password string) error { + if err := i.validateStateForObsy("OTLP exporter"); err != nil { + return err + } + i.obsyConfig.otlpEndpoint = endpoint + i.obsyConfig.otlpUsername = username + i.obsyConfig.otlpPassword = password + logrus.Debugf("Set OTLP exporter '%s' for instance '%s'", endpoint, i.name) + return nil +} + +// SetJaegerExporter sets the Jaeger exporter for the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) SetJaegerExporter(endpoint string) error { + if err := i.validateStateForObsy("Jaeger exporter"); err != nil { + return err + } + i.obsyConfig.jaegerEndpoint = endpoint + logrus.Debugf("Set Jaeger exporter '%s' for instance '%s'", endpoint, i.name) + return nil +} + +// SetPrometheusExporter sets the Prometheus exporter for the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) SetPrometheusExporter(endpoint string) error { + if err := i.validateStateForObsy("Prometheus exporter"); err != nil { + return err + } + i.obsyConfig.prometheusExporterEndpoint = endpoint + logrus.Debugf("Set Prometheus exporter '%s' for instance '%s'", endpoint, i.name) + return nil +} + +// SetPrometheusRemoteWriteExporter sets the Prometheus remote write exporter for the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) SetPrometheusRemoteWriteExporter(endpoint string) error { + if err := i.validateStateForObsy("Prometheus remote write exporter"); err != nil { + return err + } + i.obsyConfig.prometheusRemoteWriteExporterEndpoint = endpoint + logrus.Debugf("Set Prometheus remote write exporter '%s' for instance '%s'", endpoint, i.name) + return nil +} + +// SetPrivileged sets the privileged status for the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) SetPrivileged(privileged bool) error { + if !i.IsInState(Preparing, Committed) { + return ErrSettingPrivilegedNotAllowed.WithParams(i.state.String()) + } + i.securityContext.privileged = privileged + logrus.Debugf("Set privileged to '%t' for instance '%s'", privileged, i.name) + return nil +} + +// AddCapability adds a capability to the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) AddCapability(capability string) error { + if !i.IsInState(Preparing, Committed) { + return ErrAddingCapabilityNotAllowed.WithParams(i.state.String()) + } + i.securityContext.capabilitiesAdd = append(i.securityContext.capabilitiesAdd, capability) + logrus.Debugf("Added capability '%s' to instance '%s'", capability, i.name) + return nil +} + +// AddCapabilities adds multiple capabilities to the instance +// This function can only be called in the state 'Preparing' or 'Committed' +func (i *Instance) AddCapabilities(capabilities []string) error { + if !i.IsInState(Preparing, Committed) { + return ErrAddingCapabilitiesNotAllowed.WithParams(i.state.String()) + } + for _, capability := range capabilities { + i.securityContext.capabilitiesAdd = append(i.securityContext.capabilitiesAdd, capability) + logrus.Debugf("Added capability '%s' to instance '%s'", capability, i.name) + } + return nil +} + +// StartAsync starts the instance without waiting for it to be ready +// This function can only be called in the state 'Committed' or 'Stopped' +// This function will replace StartWithoutWait +func (i *Instance) StartAsync(ctx context.Context) error { + if err := i.StartWithoutWait(ctx); err != nil { + return err + } + return nil +} + +// StartWithoutWait starts the instance without waiting for it to be ready +// This function can only be called in the state 'Committed' or 'Stopped' +func (i *Instance) StartWithoutWait(ctx context.Context) error { + if !i.IsInState(Committed, Stopped) { + return ErrStartingNotAllowed.WithParams(i.state.String()) + } + if err := applyFunctionToInstances(i.sidecars, func(sidecar Instance) error { + if !sidecar.IsInState(Committed, Stopped) { + return ErrStartingNotAllowedForSidecar.WithParams(sidecar.name, sidecar.state.String()) + } + return nil + }); err != nil { + return err + } + if i.isSidecar { + return ErrStartingSidecarNotAllowed + } + + if i.state == Committed { + // deploy otel collector if observability is enabled + if i.isObservabilityEnabled() { + if err := i.addOtelCollectorSidecar(ctx); err != nil { + return ErrAddingOtelCollectorSidecar.WithParams(i.k8sName).Wrap(err) + } + } + + if i.BitTwister.Enabled() { + if err := i.addBitTwisterSidecar(ctx); err != nil { + return ErrAddingNetworkSidecar.WithParams(i.k8sName).Wrap(err) + } + } + + if err := i.deployResources(ctx); err != nil { + return ErrDeployingResourcesForInstance.WithParams(i.k8sName).Wrap(err) + } + if err := applyFunctionToInstances(i.sidecars, func(sidecar Instance) error { + return sidecar.deployResources(ctx) + }); err != nil { + return ErrDeployingResourcesForSidecars.WithParams(i.k8sName).Wrap(err) + } + } + + err := i.deployPod(ctx) + if err != nil { + return ErrDeployingPodForInstance.WithParams(i.k8sName).Wrap(err) + } + i.state = Started + setStateForSidecars(i.sidecars, Started) + logrus.Debugf("Set state of instance '%s' to '%s'", i.k8sName, i.state.String()) + + return nil +} + +// Start starts the instance and waits for it to be ready +// This function can only be called in the state 'Committed' and 'Stopped' +func (i *Instance) Start(ctx context.Context) error { + if err := i.StartWithoutWait(ctx); err != nil { + return err + } + + err := i.WaitInstanceIsRunning(ctx) + if err != nil { + return ErrWaitingForInstanceRunning.WithParams(i.k8sName).Wrap(err) + } + + return nil +} + +// IsRunning returns true if the instance is running +// This function can only be called in the state 'Started' +func (i *Instance) IsRunning(ctx context.Context) (bool, error) { + if !i.IsInState(Started, Stopped) { + return false, ErrCheckingIfInstanceRunningNotAllowed.WithParams(i.state.String()) + } + + return i.K8sCli.IsReplicaSetRunning(ctx, i.k8sName) +} + +// WaitInstanceIsRunning waits until the instance is running +// This function can only be called in the state 'Started' +func (i *Instance) WaitInstanceIsRunning(ctx context.Context) error { + if !i.IsInState(Started) { + return ErrWaitingForInstanceNotAllowed.WithParams(i.state.String()) + } + timeout := time.After(1 * time.Minute) + tick := time.Tick(1 * time.Second) + + for { + select { + case <-timeout: + return ErrWaitingForInstanceTimeout.WithParams(i.k8sName) + case <-tick: + running, err := i.IsRunning(ctx) + if err != nil { + return ErrCheckingIfInstanceRunning.WithParams(i.k8sName).Wrap(err) + } + if running { + return nil + } + } + } +} + +// DisableNetwork disables the network of the instance +// This does not apply to executor instances +// This function can only be called in the state 'Started' +func (i *Instance) DisableNetwork(ctx context.Context) error { + if !i.IsInState(Started) { + return ErrDisablingNetworkNotAllowed.WithParams(i.state.String()) + } + executorSelectorMap := map[string]string{ + "knuu.sh/type": ExecutorInstance.String(), + } + + err := i.K8sCli.CreateNetworkPolicy(ctx, i.k8sName, i.getLabels(), executorSelectorMap, executorSelectorMap) + if err != nil { + return ErrDisablingNetwork.WithParams(i.k8sName).Wrap(err) + } + return nil +} + +// SetBandwidthLimit sets the bandwidth limit of the instance +// bandwidth limit in bps (e.g. 1000 for 1Kbps) +// Currently, only one of bandwidth, jitter, latency or packet loss can be set +// This function can only be called in the state 'Commited' +func (i *Instance) SetBandwidthLimit(limit int64) error { + if !i.IsInState(Started) { + return ErrSettingBandwidthLimitNotAllowed.WithParams(i.state.String()) + } + if !i.BitTwister.Enabled() { + return ErrSettingBandwidthLimitNotAllowedBitTwister + } + + // We first need to stop it, otherwise we get an error + if err := i.BitTwister.Client().BandwidthStop(); err != nil { + if !sdk.IsErrorServiceNotInitialized(err) && + !sdk.IsErrorServiceNotReady(err) && + !sdk.IsErrorServiceNotStarted(err) { + return ErrStoppingBandwidthLimit.WithParams(i.k8sName).Wrap(err) + } + } + + err := i.BitTwister.Client().BandwidthStart(sdk.BandwidthStartRequest{ + NetworkInterfaceName: i.BitTwister.NetworkInterface(), + Limit: limit, + }) + if err != nil { + return ErrSettingBandwidthLimit.WithParams(i.k8sName).Wrap(err) + } + + logrus.Debugf("Set bandwidth limit to '%d' in instance '%s'", limit, i.name) + return nil +} + +// SetLatency sets the latency of the instance +// latency in ms (e.g. 1000 for 1s) +// jitter in ms (e.g. 1000 for 1s) +// Currently, only one of bandwidth, jitter, latency or packet loss can be set +// This function can only be called in the state 'Commited' +func (i *Instance) SetLatencyAndJitter(latency, jitter int64) error { + if !i.IsInState(Started) { + return ErrSettingLatencyJitterNotAllowed.WithParams(i.state.String()) + } + if !i.BitTwister.Enabled() { + return ErrSettingLatencyJitterNotAllowedBitTwister + } + + // We first need to stop it, otherwise we get an error + if err := i.BitTwister.Client().LatencyStop(); err != nil { + if !sdk.IsErrorServiceNotInitialized(err) && + !sdk.IsErrorServiceNotReady(err) && + !sdk.IsErrorServiceNotStarted(err) { + return ErrStoppingLatencyJitter.WithParams(i.k8sName).Wrap(err) + } + } + + err := i.BitTwister.Client().LatencyStart(sdk.LatencyStartRequest{ + NetworkInterfaceName: i.BitTwister.NetworkInterface(), + Latency: latency, + Jitter: jitter, + }) + if err != nil { + return ErrSettingLatencyJitter.WithParams(i.k8sName).Wrap(err) + } + + logrus.Debugf("Set latency to '%d' and jitter to '%d' in instance '%s'", latency, jitter, i.name) + return nil +} + +// SetPacketLoss sets the packet loss of the instance +// packet loss in percent (e.g. 10 for 10%) +// Currently, only one of bandwidth, jitter, latency or packet loss can be set +// This function can only be called in the state 'Commited' +func (i *Instance) SetPacketLoss(packetLoss int32) error { + if !i.IsInState(Started) { + return ErrSettingPacketLossNotAllowed.WithParams(i.state.String()) + } + if !i.BitTwister.Enabled() { + return ErrSettingPacketLossNotAllowedBitTwister + } + + // We first need to stop it, otherwise we get an error + if err := i.BitTwister.Client().PacketlossStop(); err != nil { + if !sdk.IsErrorServiceNotInitialized(err) && + !sdk.IsErrorServiceNotReady(err) && + !sdk.IsErrorServiceNotStarted(err) { + return ErrStoppingPacketLoss.WithParams(i.k8sName).Wrap(err) + } + } + + err := i.BitTwister.Client().PacketlossStart(sdk.PacketLossStartRequest{ + NetworkInterfaceName: i.BitTwister.NetworkInterface(), + PacketLossRate: packetLoss, + }) + if err != nil { + return ErrSettingPacketLoss.WithParams(i.k8sName).Wrap(err) + } + + logrus.Debugf("Set packet loss to '%d' in instance '%s'", packetLoss, i.name) + return nil +} + +// EnableNetwork enables the network of the instance +// This function can only be called in the state 'Started' +func (i *Instance) EnableNetwork(ctx context.Context) error { + if !i.IsInState(Started) { + return ErrEnablingNetworkNotAllowed.WithParams(i.state.String()) + } + + err := i.K8sCli.DeleteNetworkPolicy(ctx, i.k8sName) + if err != nil { + return ErrEnablingNetwork.WithParams(i.k8sName).Wrap(err) + } + return nil +} + +// NetworkIsDisabled returns true if the network of the instance is disabled +// This function can only be called in the state 'Started' +func (i *Instance) NetworkIsDisabled(ctx context.Context) (bool, error) { + if !i.IsInState(Started) { + return false, ErrCheckingIfNetworkDisabledNotAllowed.WithParams(i.state.String()) + } + + return i.K8sCli.NetworkPolicyExists(ctx, i.k8sName), nil +} + +// WaitInstanceIsStopped waits until the instance is not running anymore +// This function can only be called in the state 'Stopped' +func (i *Instance) WaitInstanceIsStopped(ctx context.Context) error { + if !i.IsInState(Stopped) { + return ErrWaitingForInstanceStoppedNotAllowed.WithParams(i.state.String()) + } + for { + running, err := i.IsRunning(ctx) + if !running { + break + } + if err != nil { + return ErrCheckingIfInstanceStopped.WithParams(i.k8sName).Wrap(err) + } + } + + return nil +} + +// Stop stops the instance +// CAUTION: In order to keep data of the instance, you need to use AddVolume() before. +// This function can only be called in the state 'Started' +func (i *Instance) Stop(ctx context.Context) error { + if !i.IsInState(Started) { + return ErrStoppingNotAllowed.WithParams(i.state.String()) + + } + + if err := i.destroyPod(ctx); err != nil { + return ErrDestroyingPod.WithParams(i.k8sName).Wrap(err) + } + i.state = Stopped + setStateForSidecars(i.sidecars, Stopped) + logrus.Debugf("Set state of instance '%s' to '%s'", i.k8sName, i.state.String()) + + return nil +} + +// Clone creates a clone of the instance +// This function can only be called in the state 'Committed' +// When cloning an instance that is a sidecar, the clone will be not a sidecar +// When cloning an instance with sidecars, the sidecars will be cloned as well +func (i *Instance) Clone() (*Instance, error) { + if !i.IsInState(Committed) { + return nil, ErrCloningNotAllowed.WithParams(i.state.String()) + } + + newK8sName, err := names.NewRandomK8(i.name) + if err != nil { + return nil, ErrGeneratingK8sName.WithParams(i.name).Wrap(err) + } + // Create a new instance with the same attributes as the original instance + ins := i.cloneWithSuffix("") + ins.k8sName = newK8sName + return ins, nil +} + +// CloneWithName creates a clone of the instance with a given name +// This function can only be called in the state 'Committed' +// When cloning an instance that is a sidecar, the clone will be not a sidecar +// When cloning an instance with sidecars, the sidecars will be cloned as well +func (i *Instance) CloneWithName(name string) (*Instance, error) { + if !i.IsInState(Committed) { + return nil, ErrCloningNotAllowedForSidecar.WithParams(i.state.String()) + } + + newK8sName, err := names.NewRandomK8(name) + if err != nil { + return nil, ErrGeneratingK8sNameForSidecar.WithParams(name).Wrap(err) + } + // Create a new instance with the same attributes as the original instance + ins := i.cloneWithSuffix("") + ins.name = name + ins.k8sName = newK8sName + return ins, nil +} + +// CreateCustomResource creates a custom resource for the instance +// The names and namespace are set and overridden by knuu +func (i *Instance) CreateCustomResource(ctx context.Context, gvr *schema.GroupVersionResource, obj *map[string]interface{}) error { + crdExists, err := i.CustomResourceDefinitionExists(ctx, gvr) + if err != nil { + return err + } + if !crdExists { + return ErrCustomResourceDefinitionDoesNotExist.WithParams(gvr.Resource) + } + + return i.K8sCli.CreateCustomResource(ctx, i.k8sName, gvr, obj) +} + +// CustomResourceDefinitionExists checks if the custom resource definition exists +func (i *Instance) CustomResourceDefinitionExists(ctx context.Context, gvr *schema.GroupVersionResource) (bool, error) { + return i.K8sCli.CustomResourceDefinitionExists(ctx, gvr), nil +} + +func (i *Instance) AddHost(ctx context.Context, port int) (host string, err error) { + if i.Proxy == nil { + return "", ErrProxyNotInitialized + } + + prefix := fmt.Sprintf("%s-%d", i.k8sName, port) + if err := i.Proxy.AddHost(ctx, i.k8sName, prefix, port); err != nil { + return "", ErrAddingToProxy.WithParams(i.k8sName).Wrap(err) + } + host, err = i.Proxy.URL(ctx, prefix) + if err != nil { + return "", ErrGettingProxyURL.WithParams(i.k8sName).Wrap(err) + } + return host, nil +} diff --git a/pkg/knuu/instance_otel.go b/pkg/instance/otel.go similarity index 96% rename from pkg/knuu/instance_otel.go rename to pkg/instance/otel.go index 74fd7dd..41943fc 100644 --- a/pkg/knuu/instance_otel.go +++ b/pkg/instance/otel.go @@ -1,6 +1,7 @@ -package knuu +package instance import ( + "context" "fmt" "gopkg.in/yaml.v3" @@ -175,13 +176,13 @@ type Action struct { Action string `yaml:"action,omitempty"` } -func (i *Instance) createOtelCollectorInstance() (*Instance, error) { - otelAgent, err := NewInstance("otel-agent") +func (i *Instance) createOtelCollectorInstance(ctx context.Context) (*Instance, error) { + otelAgent, err := New("otel-agent", i.SystemDependencies) if err != nil { return nil, ErrCreatingOtelAgentInstance.Wrap(err) } - if err := otelAgent.SetImage(fmt.Sprintf("otel/opentelemetry-collector-contrib:%s", i.obsyConfig.otelCollectorVersion)); err != nil { + if err := otelAgent.SetImage(ctx, fmt.Sprintf("otel/opentelemetry-collector-contrib:%s", i.obsyConfig.otelCollectorVersion)); err != nil { return nil, ErrSettingOtelAgentImage.Wrap(err) } if err := otelAgent.AddPortTCP(8888); err != nil { @@ -433,7 +434,7 @@ func (i *Instance) createProcessors() Processors { Actions: []Action{ { Key: "namespace", - Value: k8sClient.Namespace(), + Value: i.K8sCli.Namespace(), Action: "insert", }, }, diff --git a/pkg/knuu/instance_pool.go b/pkg/instance/pool.go similarity index 72% rename from pkg/knuu/instance_pool.go rename to pkg/instance/pool.go index e6ae359..22fd43a 100644 --- a/pkg/knuu/instance_pool.go +++ b/pkg/instance/pool.go @@ -1,6 +1,7 @@ -package knuu +package instance import ( + "context" "fmt" "github.com/sirupsen/logrus" @@ -12,14 +13,9 @@ type InstancePool struct { amount int } -// Instances returns the instances in the instance pool -func (i *InstancePool) Instances() []*Instance { - return i.instances -} - -// CreatePool creates a pool of instances +// NewPool creates a pool of instances // This function can only be called in the state 'Committed' -func (i *Instance) CreatePool(amount int) (*InstancePool, error) { +func (i *Instance) NewPool(amount int) (*InstancePool, error) { if !i.IsInState(Committed) { return nil, ErrCreatingPoolNotAllowed.WithParams(i.state.String()) } @@ -37,10 +33,15 @@ func (i *Instance) CreatePool(amount int) (*InstancePool, error) { }, nil } +// Instances returns the instances in the instance pool +func (i *InstancePool) Instances() []*Instance { + return i.instances +} + // StartWithoutWait starts all instances in the instance pool without waiting for them to be running -func (i *InstancePool) StartWithoutWait() error { +func (i *InstancePool) StartWithoutWait(ctx context.Context) error { for _, instance := range i.instances { - err := instance.StartWithoutWait() + err := instance.StartWithoutWait(ctx) if err != nil { return err } @@ -49,9 +50,9 @@ func (i *InstancePool) StartWithoutWait() error { } // Start starts all instances in the instance pool -func (i *InstancePool) Start() error { +func (i *InstancePool) Start(ctx context.Context) error { for _, instance := range i.instances { - err := instance.Start() + err := instance.Start(ctx) if err != nil { return err } @@ -60,9 +61,9 @@ func (i *InstancePool) Start() error { } // Destroy destroys all instances in the instance pool -func (i *InstancePool) Destroy() error { +func (i *InstancePool) Destroy(ctx context.Context) error { for _, instance := range i.instances { - err := instance.Destroy() + err := instance.Destroy(ctx) if err != nil { return err } @@ -71,9 +72,9 @@ func (i *InstancePool) Destroy() error { } // WaitInstancePoolIsRunning waits until all instances in the instance pool are running -func (i *InstancePool) WaitInstancePoolIsRunning() error { +func (i *InstancePool) WaitInstancePoolIsRunning(ctx context.Context) error { for _, instance := range i.instances { - err := instance.WaitInstanceIsRunning() + err := instance.WaitInstanceIsRunning(ctx) if err != nil { return err } diff --git a/pkg/knuu/instance_state.go b/pkg/instance/state.go similarity index 97% rename from pkg/knuu/instance_state.go rename to pkg/instance/state.go index 15d5704..d54fb31 100644 --- a/pkg/knuu/instance_state.go +++ b/pkg/instance/state.go @@ -1,4 +1,4 @@ -package knuu +package instance // InstanceState represents the state of the instance type InstanceState int diff --git a/pkg/knuu/instance_type.go b/pkg/instance/type.go similarity index 96% rename from pkg/knuu/instance_type.go rename to pkg/instance/type.go index 110116f..f71ed3d 100644 --- a/pkg/knuu/instance_type.go +++ b/pkg/instance/type.go @@ -1,4 +1,4 @@ -package knuu +package instance // InstanceType represents the type of the instance type InstanceType int diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index b53f529..7810275 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -35,6 +35,8 @@ type Client struct { namespace string } +var _ KubeManager = &Client{} + func New(ctx context.Context, namespace string) (*Client, error) { config, err := getClusterConfig() if err != nil { @@ -87,9 +89,12 @@ func (c *Client) Namespace() string { // isClusterEnvironment checks if the program is running in a Kubernetes cluster. func isClusterEnvironment() bool { - _, errToken := os.Stat(tokenPath) - _, errCert := os.Stat(certPath) - return errToken == nil && errCert == nil + return fileExists(tokenPath) && fileExists(certPath) +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil } // getClusterConfig returns the appropriate Kubernetes cluster configuration. diff --git a/pkg/k8s/types.go b/pkg/k8s/types.go new file mode 100644 index 0000000..f33e179 --- /dev/null +++ b/pkg/k8s/types.go @@ -0,0 +1,79 @@ +package k8s + +import ( + "context" + + appv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +type KubeManager interface { + Clientset() *kubernetes.Clientset + CreateClusterRole(ctx context.Context, name string, labels map[string]string, policyRules []rbacv1.PolicyRule) error + CreateClusterRoleBinding(ctx context.Context, name string, labels map[string]string, clusterRole, serviceAccount string) error + CreateConfigMap(ctx context.Context, name string, labels, data map[string]string) (*v1.ConfigMap, error) + CreateCustomResource(ctx context.Context, name string, gvr *schema.GroupVersionResource, obj *map[string]interface{}) error + CreateDaemonSet(ctx context.Context, name string, labels map[string]string, initContainers []v1.Container, containers []v1.Container) (*appv1.DaemonSet, error) + CreateNamespace(ctx context.Context, name string) error + CreateNetworkPolicy(ctx context.Context, name string, selectorMap, ingressSelectorMap, egressSelectorMap map[string]string) error + CreatePersistentVolumeClaim(ctx context.Context, name string, labels map[string]string, size resource.Quantity) error + CreateReplicaSet(ctx context.Context, rsConfig ReplicaSetConfig, init bool) (*appv1.ReplicaSet, error) + CreateRole(ctx context.Context, name string, labels map[string]string, policyRules []rbacv1.PolicyRule) error + CreateRoleBinding(ctx context.Context, name string, labels map[string]string, role, serviceAccount string) error + CreateService(ctx context.Context, name string, labels, selectorMap map[string]string, portsTCP, portsUDP []int) (*v1.Service, error) + CreateServiceAccount(ctx context.Context, name string, labels map[string]string) error + CustomResourceDefinitionExists(ctx context.Context, gvr *schema.GroupVersionResource) bool + DaemonSetExists(ctx context.Context, name string) (bool, error) + DeleteConfigMap(ctx context.Context, name string) error + DeleteDaemonSet(ctx context.Context, name string) error + DeleteNamespace(ctx context.Context, name string) error + DeleteNetworkPolicy(ctx context.Context, name string) error + DeletePersistentVolumeClaim(ctx context.Context, name string) error + DeletePod(ctx context.Context, name string) error + DeletePodWithGracePeriod(ctx context.Context, name string, gracePeriodSeconds *int64) error + DeleteReplicaSet(ctx context.Context, name string) error + DeleteReplicaSetWithGracePeriod(ctx context.Context, name string, gracePeriodSeconds *int64) error + DeleteRole(ctx context.Context, name string) error + DeleteRoleBinding(ctx context.Context, name string) error + DeleteService(ctx context.Context, name string) error + DeleteServiceAccount(ctx context.Context, name string) error + DeployPod(ctx context.Context, podConfig PodConfig, init bool) (*corev1.Pod, error) + DynamicClient() dynamic.Interface + GetConfigMap(ctx context.Context, name string) (*corev1.ConfigMap, error) + GetDaemonSet(ctx context.Context, name string) (*appv1.DaemonSet, error) + GetFirstPodFromReplicaSet(ctx context.Context, name string) (*corev1.Pod, error) + GetNamespace(ctx context.Context, name string) (*corev1.Namespace, error) + GetNetworkPolicy(ctx context.Context, name string) (*netv1.NetworkPolicy, error) + GetService(ctx context.Context, name string) (*corev1.Service, error) + GetServiceEndpoint(ctx context.Context, name string) (string, error) + GetServiceIP(ctx context.Context, name string) (string, error) + IsPodRunning(ctx context.Context, name string) (bool, error) + IsReplicaSetRunning(ctx context.Context, name string) (bool, error) + Namespace() string + NamespaceExists(ctx context.Context, name string) bool + NetworkPolicyExists(ctx context.Context, name string) bool + NewFile(source, dest string) *File + NewVolume(path, size string, owner int64) *Volume + PatchService(ctx context.Context, name string, labels, selectorMap map[string]string, portsTCP, portsUDP []int) (*v1.Service, error) + PortForwardPod(ctx context.Context, podName string, localPort, remotePort int) error + ReplicaSetExists(ctx context.Context, name string) (bool, error) + ReplacePod(ctx context.Context, podConfig PodConfig) (*corev1.Pod, error) + ReplacePodWithGracePeriod(ctx context.Context, podConfig PodConfig, gracePeriod *int64) (*corev1.Pod, error) + ReplaceReplicaSet(ctx context.Context, ReplicaSetConfig ReplicaSetConfig) (*appv1.ReplicaSet, error) + ReplaceReplicaSetWithGracePeriod(ctx context.Context, ReplicaSetConfig ReplicaSetConfig, gracePeriod *int64) (*appv1.ReplicaSet, error) + RunCommandInPod(ctx context.Context, podName, containerName string, cmd []string) (string, error) + getPersistentVolumeClaim(ctx context.Context, name string) (*corev1.PersistentVolumeClaim, error) + getPod(ctx context.Context, name string) (*corev1.Pod, error) + getReplicaSet(ctx context.Context, name string) (*appv1.ReplicaSet, error) + ConfigMapExists(ctx context.Context, name string) (bool, error) + UpdateDaemonSet(ctx context.Context, name string, labels map[string]string, initContainers []v1.Container, containers []v1.Container) (*appv1.DaemonSet, error) + WaitForDeployment(ctx context.Context, name string) error + WaitForService(ctx context.Context, name string) error +} diff --git a/pkg/knuu/errors.go b/pkg/knuu/errors.go index a033233..9e84713 100644 --- a/pkg/knuu/errors.go +++ b/pkg/knuu/errors.go @@ -221,6 +221,7 @@ var ( ErrMaximumVolumesExceeded = &Error{Code: "MaximumVolumesExceeded", Message: "maximum volumes exceeded for instance '%s'"} ErrCustomResourceDefinitionDoesNotExist = &Error{Code: "CustomResourceDefinitionDoesNotExist", Message: "custom resource definition %s does not exist"} ErrFileIsNotSubFolderOfVolumes = &Error{Code: "FileIsNotSubFolderOfVolumes", Message: "the file '%s' is not a sub folder of any added volume"} + ErrCannotInitializeKnuu = &Error{Code: "Cannot Initialize Knuu", Message: "cannot initialize knuu"} ErrCannotDeployTraefik = &Error{Code: "Cannot Deploy Traefik", Message: "cannot deploy Traefik"} ErrGettingBitTwisterPath = &Error{Code: "GettingBitTwisterPath", Message: "error getting BitTwister path"} ErrFailedToAddHostToTraefik = &Error{Code: "FailedToAddHostToTraefik", Message: "failed to add host to traefik"} @@ -230,4 +231,5 @@ var ( ErrAddingToProxy = &Error{Code: "AddingToTraefikProxy", Message: "error adding '%s' to traefik proxy for service '%s'"} ErrCannotGetTraefikEndpoint = &Error{Code: "CannotGetTraefikEndpoint", Message: "cannot get traefik endpoint"} ErrGettingProxyURL = &Error{Code: "GettingProxyURL", Message: "error getting proxy URL for service '%s'"} + ErrTraefikAPINotAvailable = &Error{Code: "TraefikAPINotAvailable", Message: "traefik API is not available"} ) diff --git a/pkg/knuu/executor.go b/pkg/knuu/executor.go deleted file mode 100644 index eb44849..0000000 --- a/pkg/knuu/executor.go +++ /dev/null @@ -1,54 +0,0 @@ -package knuu - -type Executor struct { - *Instance -} - -func NewExecutor() (*Executor, error) { - instance, err := NewInstance("executor") - if err != nil { - return nil, ErrCreatingInstance.Wrap(err) - } - err = instance.SetImage("docker.io/nicolaka/netshoot:latest") - if err != nil { - return nil, ErrSettingImage.Wrap(err) - } - err = instance.Commit() - if err != nil { - return nil, ErrCommittingInstance.Wrap(err) - } - err = instance.SetArgs("sleep", "infinity") - if err != nil { - return nil, ErrSettingArgs.Wrap(err) - } - err = instance.SetMemory("100M", "100M") - if err != nil { - return nil, ErrSettingMemory.Wrap(err) - } - err = instance.SetCPU("100m") - if err != nil { - return nil, ErrSettingCPU.Wrap(err) - } - instance.instanceType = ExecutorInstance - err = instance.Start() - if err != nil { - return nil, ErrStartingInstance.Wrap(err) - } - err = instance.WaitInstanceIsRunning() - if err != nil { - return nil, ErrWaitingInstanceIsRunning.Wrap(err) - } - return &Executor{Instance: instance}, nil -} - -// func (e *Executor) ExecuteCommand(command ...string) (string, error) { -// return e.ExecuteCommand(command...) -// } - -// func (e *Executor) ExecuteCommandWithContext(ctx context.Context, command ...string) (string, error) { -// return e.ExecuteCommandWithContext(ctx, command...) -// } - -// func (e *Executor) Destroy() error { -// return e.Destroy() -// } diff --git a/pkg/knuu/instance.go b/pkg/knuu/instance.go index 5cc5278..7f8398a 100644 --- a/pkg/knuu/instance.go +++ b/pkg/knuu/instance.go @@ -1,1419 +1,21 @@ +// Package knuu provides the core functionality of knuu. package knuu import ( "context" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "time" - appv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - - "github.com/sirupsen/logrus" - - "github.com/celestiaorg/bittwister/sdk" - - "github.com/celestiaorg/knuu/pkg/builder" - "github.com/celestiaorg/knuu/pkg/container" - "github.com/celestiaorg/knuu/pkg/k8s" + "github.com/celestiaorg/knuu/pkg/instance" + "github.com/celestiaorg/knuu/pkg/preloader" ) -// ObsyConfig represents the configuration for the obsy sidecar -type ObsyConfig struct { - // otelCollectorVersion is the version of the otel collector to use - otelCollectorVersion string - - // prometheusEndpointPort is the port on which the prometheus server will be exposed - prometheusEndpointPort int - // prometheusEndpointJobName is the name of the prometheus job - prometheusEndpointJobName string - // prometheusEndpointScrapeInterval is the scrape interval for the prometheus job - prometheusEndpointScrapeInterval string - - // jaegerGrpcPort is the port on which the jaeger grpc server is exposed - jaegerGrpcPort int - // jaegerThriftCompactPort is the port on which the jaeger thrift compact server is exposed - jaegerThriftCompactPort int - // jaegerThriftHttpPort is the port on which the jaeger thrift http server is exposed - jaegerThriftHttpPort int - // jaegerEndpoint is the endpoint of the jaeger collector where spans will be sent to - jaegerEndpoint string - - // otlpPort is the port on which the otlp server is exposed - otlpPort int - // otlpEndpoint is the endpoint of the otlp collector where spans will be sent to - otlpEndpoint string - // otlpUsername is the username to use for the otlp collector - otlpUsername string - // otlpPassword is the password to use for the otlp collector - otlpPassword string - - // prometheusExporterEndpoint is the endpoint of the prometheus exporter - prometheusExporterEndpoint string - - // prometheusRemoteWriteExporterEndpoint is the endpoint of the prometheus remote write - prometheusRemoteWriteExporterEndpoint string -} - -// SecurityContext represents the security settings for a container -type SecurityContext struct { - // Privileged indicates whether the container should be run in privileged mode - privileged bool - - // CapabilitiesAdd is the list of capabilities to add to the container - capabilitiesAdd []string -} - -// Instance represents a instance -type Instance struct { - name string - imageName string - k8sName string - state InstanceState - instanceType InstanceType - kubernetesService *v1.Service - builderFactory *container.BuilderFactory - kubernetesReplicaSet *appv1.ReplicaSet - portsTCP []int - portsUDP []int - command []string - args []string - env map[string]string - volumes []*k8s.Volume - memoryRequest string - memoryLimit string - cpuRequest string - policyRules []rbacv1.PolicyRule - livenessProbe *v1.Probe - readinessProbe *v1.Probe - startupProbe *v1.Probe - files []*k8s.File - isSidecar bool - parentInstance *Instance - sidecars []*Instance - fsGroup int64 - obsyConfig *ObsyConfig - securityContext *SecurityContext - BitTwister *btConfig -} - -// NewInstance creates a new instance of the Instance struct -func NewInstance(name string) (*Instance, error) { - // Generate a UUID for this instance - - k8sName, err := generateK8sName(name) - if err != nil { - return nil, ErrGeneratingK8sName.WithParams(name).Wrap(err) - } - obsyConfig := &ObsyConfig{ - otelCollectorVersion: "0.83.0", - otlpPort: 0, - prometheusEndpointPort: 0, - prometheusEndpointJobName: "", - prometheusEndpointScrapeInterval: "", - jaegerGrpcPort: 0, - jaegerThriftCompactPort: 0, - jaegerThriftHttpPort: 0, - otlpEndpoint: "", - otlpUsername: "", - otlpPassword: "", - jaegerEndpoint: "", - prometheusExporterEndpoint: "", - prometheusRemoteWriteExporterEndpoint: "", - } - securityContext := &SecurityContext{ - privileged: false, - capabilitiesAdd: make([]string, 0), - } - - // Create the instance - return &Instance{ - name: name, - k8sName: k8sName, - imageName: "", - state: None, - instanceType: BasicInstance, - portsTCP: make([]int, 0), - portsUDP: make([]int, 0), - command: make([]string, 0), - args: make([]string, 0), - env: make(map[string]string), - volumes: make([]*k8s.Volume, 0), - memoryRequest: "", - memoryLimit: "", - cpuRequest: "", - policyRules: make([]rbacv1.PolicyRule, 0), - livenessProbe: nil, - readinessProbe: nil, - startupProbe: nil, - files: make([]*k8s.File, 0), - isSidecar: false, - parentInstance: nil, - sidecars: make([]*Instance, 0), - obsyConfig: obsyConfig, - securityContext: securityContext, - BitTwister: getBitTwisterDefaultConfig(), - }, nil -} - -func (i *Instance) EnableBitTwister() error { - if i.IsInState(Started) { - return ErrEnablingBitTwister - } - i.BitTwister.enable() - return nil -} - -func (i *Instance) DisableBitTwister() error { - i.BitTwister.disable() - return nil -} - -// Name returns the name of the instance -func (i *Instance) Name() string { - return i.name -} - -// SetImage sets the image of the instance. -// When calling in state 'Started', make sure to call AddVolume() before. -// It is only allowed in the 'None' and 'Started' states. -func (i *Instance) SetImage(image string) error { - // Check if setting the image is allowed in the current state - if !i.IsInState(None, Started) { - return ErrSettingImageNotAllowed.WithParams(i.state.String()) - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - // Handle each state accordingly - switch i.state { - case None: - // Use the builder to build a new image - factory, err := container.NewBuilderFactory(image, i.getBuildDir(), ImageBuilder()) - if err != nil { - return ErrCreatingBuilder.Wrap(err) - } - i.builderFactory = factory - i.state = Preparing - case Started: - - if i.isSidecar { - return ErrSettingImageNotAllowedForSidecarsStarted - } - - if err := i.setImageWithGracePeriod(ctx, image, nil); err != nil { - return err - } - } - - return nil -} - -// SetGitRepo builds the image from the given git repo, pushes it -// to the registry under the given name and sets the image of the instance. -func (i *Instance) SetGitRepo(ctx context.Context, gitContext builder.GitContext) error { - if !i.IsInState(None) { - return ErrSettingGitRepo.WithParams(i.state.String()) - } - - bCtx, err := gitContext.BuildContext() - if err != nil { - return ErrGettingBuildContext.Wrap(err) - } - imageName, err := builder.DefaultImageName(bCtx) - if err != nil { - return ErrGettingImageName.Wrap(err) - } - - factory, err := container.NewBuilderFactory(imageName, i.getBuildDir(), ImageBuilder()) - if err != nil { - return ErrCreatingBuilder.Wrap(err) - } - i.builderFactory = factory - i.state = Preparing - - return i.builderFactory.BuildImageFromGitRepo(ctx, gitContext, imageName) -} - -// SetImageInstant sets the image of the instance without a grace period. -// Instant means that the pod is replaced without a grace period of 1 second. -// It is only allowed in the 'Running' state. -func (i *Instance) SetImageInstant(image string) error { - // Check if setting the image is allowed in the current state - if !i.IsInState(Started) { - return ErrSettingImageNotAllowedForSidecarsStarted.WithParams(i.state.String()) - } - - if i.isSidecar { - return ErrSettingImageNotAllowedForSidecars - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - gracePeriod := int64(0) - if err := i.setImageWithGracePeriod(ctx, image, &gracePeriod); err != nil { - return err - } - - return nil -} - -// SetCommand sets the command to run in the instance -// This function can only be called when the instance is in state 'Preparing' or 'Committed' -func (i *Instance) SetCommand(command ...string) error { - if !i.IsInState(Preparing, Committed) { - return ErrSettingCommand.WithParams(i.state.String()) - } - i.command = command - return nil -} - -// SetArgs sets the arguments passed to the instance -// This function can only be called in the states 'Preparing' or 'Committed' -func (i *Instance) SetArgs(args ...string) error { - if !i.IsInState(Preparing, Committed) { - return ErrSettingArgsNotAllowed.WithParams(i.state.String()) - } - i.args = args - return nil -} - -// AddPortTCP adds a TCP port to the instance -// This function can be called in the states 'Preparing' and 'Committed' -func (i *Instance) AddPortTCP(port int) error { - if !i.IsInState(Preparing, Committed) { - return ErrAddingPortNotAllowed.WithParams(i.state.String()) - } - err := validatePort(port) - if err != nil { - return err - } - if i.isTCPPortRegistered(port) { - return ErrPortAlreadyRegistered.WithParams(port) - } - i.portsTCP = append(i.portsTCP, port) - logrus.Debugf("Added TCP port '%d' to instance '%s'", port, i.name) - return nil -} - -// PortForwardTCP forwards the given port to a random port on the host -// This function can only be called in the state 'Started' -func (i *Instance) PortForwardTCP(port int) (int, error) { - if !i.IsInState(Started) { - return -1, ErrRandomPortForwardingNotAllowed.WithParams(i.state.String()) - } - err := validatePort(port) - if err != nil { - return 0, err - } - if !i.isTCPPortRegistered(port) { - return -1, ErrPortNotRegistered.WithParams(port) - } - // Get a random port on the host - localPort, err := getFreePortTCP() - if err != nil { - return -1, ErrGettingFreePort.WithParams(port) - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - // Forward the port - pod, err := k8sClient.GetFirstPodFromReplicaSet(ctx, i.k8sName) - if err != nil { - return -1, ErrGettingPodFromReplicaSet.WithParams(i.k8sName).Wrap(err) - } - // We need to retry here because the port forwarding might fail as getFreePortTCP() might not free the port fast enough - retries := 5 - wait := 5 * time.Second - for r := 0; r < retries; r++ { - err = k8sClient.PortForwardPod(ctx, pod.Name, localPort, port) - if err == nil { - break - } - if retries == r+1 { - return -1, ErrForwardingPort.WithParams(retries) - } - logrus.Debugf("Forwaring port %d failed, cause: %v, retrying after %v (retry %d/%d)", port, err, wait, r+1, retries) - time.Sleep(wait) - } - return localPort, nil -} - -// AddPortUDP adds a UDP port to the instance -// This function can be called in the states 'Preparing' and 'Committed' -func (i *Instance) AddPortUDP(port int) error { - if !i.IsInState(Preparing, Committed) { - return ErrAddingPortNotAllowed.WithParams(i.state.String()) - } - err := validatePort(port) - if err != nil { - return err - } - if i.isUDPPortRegistered(port) { - return ErrUDPPortAlreadyRegistered.WithParams(port) - } - i.portsUDP = append(i.portsUDP, port) - logrus.Debugf("Added UDP port '%d' to instance '%s'", port, i.k8sName) - return nil -} - -// ExecuteCommand executes the given command in the instance -// This function can only be called in the states 'Preparing' and 'Started' -func (i *Instance) ExecuteCommand(command ...string) (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - return i.ExecuteCommandWithContext(ctx, command...) -} - -// ExecuteCommandWithContext executes the given command in the instance -// This function can only be called in the states 'Preparing' and 'Started' -// The context can be used to cancel the command and it is only possible in start state -func (i *Instance) ExecuteCommandWithContext(ctx context.Context, command ...string) (string, error) { - if !i.IsInState(Preparing, Started) { - return "", ErrExecutingCommandNotAllowed.WithParams(i.state.String()) - } - - if i.IsInState(Preparing) { - output, err := i.builderFactory.ExecuteCmdInBuilder(command) - if err != nil { - return "", ErrExecutingCommandInInstance.WithParams(command, i.name).Wrap(err) - } - return output, nil - } - - var ( - instanceName string - eErr *Error - containerName = i.k8sName - ) - - if i.isSidecar { - instanceName = i.parentInstance.k8sName - eErr = ErrExecutingCommandInSidecar.WithParams(command, i.k8sName, i.parentInstance.k8sName) - } else { - instanceName = i.k8sName - eErr = ErrExecutingCommandInInstance.WithParams(command, i.k8sName) - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - pod, err := k8sClient.GetFirstPodFromReplicaSet(ctx, instanceName) - if err != nil { - return "", ErrGettingPodFromReplicaSet.WithParams(i.k8sName).Wrap(err) - } - - commandWithShell := []string{"/bin/sh", "-c", strings.Join(command, " ")} - output, err := k8sClient.RunCommandInPod(ctx, pod.Name, containerName, commandWithShell) - if err != nil { - return "", eErr.Wrap(err) - } - return output, nil -} - -// checkStateForAddingFile checks if the current state allows adding a file -func (i *Instance) checkStateForAddingFile() error { - if !i.IsInState(Preparing, Committed) { - return ErrAddingFileNotAllowed.WithParams(i.state.String()) - } - return nil -} - -// AddFile adds a file to the instance -// This function can only be called in the state 'Preparing' -func (i *Instance) AddFile(src string, dest string, chown string) error { - if err := i.checkStateForAddingFile(); err != nil { - return err - } - - err := i.validateFileArgs(src, dest, chown) - if err != nil { - return err - } - - // check if src exists (either as file or as folder) - if _, err := os.Stat(src); os.IsNotExist(err) { - return ErrSrcDoesNotExist.WithParams(src).Wrap(err) - } - - // copy file to build dir - dstPath := filepath.Join(i.getBuildDir(), dest) - - // make sure dir exists - err = os.MkdirAll(filepath.Dir(dstPath), os.ModePerm) - if err != nil { - return ErrCreatingDirectory.Wrap(err) - } - // Create destination file making sure the path is writeable. - dst, err := os.Create(dstPath) - if err != nil { - return ErrFailedToCreateDestFile.WithParams(dstPath).Wrap(err) - } - defer dst.Close() - - // Open source file for reading. - srcFile, err := os.Open(src) - if err != nil { - return ErrFailedToOpenSrcFile.WithParams(src).Wrap(err) - } - defer srcFile.Close() - - // Copy the contents from source file to destination file - _, err = io.Copy(dst, srcFile) - if err != nil { - return ErrFailedToCopyFile.WithParams(src, dstPath).Wrap(err) - } - - switch i.state { - case Preparing: - err := i.addFileToBuilder(src, dest, chown) - if err != nil { - return err - } - case Committed: - // check if the dest is a sub folder of added volumes and print a warning if not - if !i.isSubFolderOfVolumes(dest) { - return ErrFileIsNotSubFolderOfVolumes.WithParams(dest) - } - - // only allow files, not folders - srcInfo, err := os.Stat(src) - if os.IsNotExist(err) || srcInfo.IsDir() { - return ErrSrcDoesNotExistOrIsDirectory.WithParams(src).Wrap(err) - } - file := k8sClient.NewFile(dstPath, dest) - - // the user provided a chown string (e.g. "10001:10001") and we only need the group (second part) - parts := strings.Split(chown, ":") - if len(parts) != 2 { - return ErrInvalidFormat - } - - // second part of array, base of number is 10, and we want a 64-bit integer - group, err := strconv.ParseInt(parts[1], 10, 64) - if err != nil { - return ErrFailedToConvertToInt64.Wrap(err) - } - - if i.fsGroup != 0 && i.fsGroup != group { - return ErrAllFilesMustHaveSameGroup - } else { - i.fsGroup = group - } - - i.files = append(i.files, file) - } - - logrus.Debugf("Added file '%s' to instance '%s'", dest, i.name) - return nil -} - -// AddFolder adds a folder to the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) AddFolder(src string, dest string, chown string) error { - if !i.IsInState(Preparing, Committed) { - return ErrAddingFolderNotAllowed.WithParams(i.state.String()) - } - - i.validateFileArgs(src, dest, chown) - - // check if src exists (should be a folder) - srcInfo, err := os.Stat(src) - if os.IsNotExist(err) || !srcInfo.IsDir() { - return ErrSrcDoesNotExistOrIsNotDirectory.WithParams(src).Wrap(err) - } - - // iterate over the files/directories in the src - err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // create the destination path - relPath, err := filepath.Rel(src, path) - if err != nil { - return err - } - dstPath := filepath.Join(i.getBuildDir(), dest, relPath) - - if info.IsDir() { - // create directory at destination path - return os.MkdirAll(dstPath, os.ModePerm) - } - // copy file to destination path - return i.AddFile(path, filepath.Join(dest, relPath), chown) - }) - - if err != nil { - return ErrCopyingFolderToInstance.WithParams(src, i.name).Wrap(err) - } - - logrus.Debugf("Added folder '%s' to instance '%s'", dest, i.name) - return nil -} - -// AddFileBytes adds a file with the given content to the instance -// This function can only be called in the state 'Preparing' -func (i *Instance) AddFileBytes(bytes []byte, dest string, chown string) error { - if err := i.checkStateForAddingFile(); err != nil { - return err - } - - // create a temporary file - tmpfile, err := os.CreateTemp("", "temp") - if err != nil { - return err - } - defer os.Remove(tmpfile.Name()) // clean up - - // write bytes to the temporary file - if _, err := tmpfile.Write(bytes); err != nil { - return err - } - if err := tmpfile.Close(); err != nil { - return err - } - - // use AddFile to copy the temp file to the destination - return i.AddFile(tmpfile.Name(), dest, chown) -} - -// SetUser sets the user for the instance -// This function can only be called in the state 'Preparing' -func (i *Instance) SetUser(user string) error { - if !i.IsInState(Preparing) { - return ErrSettingUserNotAllowed.WithParams(i.state.String()) - } - err := i.builderFactory.SetUser(user) - if err != nil { - return ErrSettingUser.WithParams(user, i.name).Wrap(err) - } - logrus.Debugf("Set user '%s' for instance '%s'", user, i.name) - return nil -} - -// imageCache maps image hash values to image names -var imageCache = make(map[string]string) - -// checkImageHashInCache checks if the given image hash exists in the cache. -func checkImageHashInCache(imageHash string) (imageName string, exists bool) { - imageName, exists = imageCache[imageHash] - return imageName, exists -} - -// updateImageCacheWithHash adds or updates the image cache with the given hash and image name. -func updateImageCacheWithHash(imageHash, imageName string) { - imageCache[imageHash] = imageName // Update the cache with the new hash and image name -} - -// Commit commits the instance -// This function can only be called in the state 'Preparing' -func (i *Instance) Commit() error { - if !i.IsInState(Preparing) { - return ErrCommittingNotAllowed.WithParams(i.state.String()) - } - // in the case ports are already set before commit, we already can deploy the service - if len(i.portsTCP) > 0 || len(i.portsUDP) > 0 { - go func() { - i.deployOrPatchService(context.TODO(), i.portsTCP, i.portsUDP) - }() - } - if i.builderFactory.Changed() { - // TODO: To speed up the process, the image name could be dependent on the hash of the image - imageName, err := i.getImageRegistry() - if err != nil { - return ErrGettingImageRegistry.Wrap(err) - } - - // Generate a hash for the current image - imageHash, err := i.builderFactory.GenerateImageHash() - if err != nil { - return ErrGeneratingImageHash.Wrap(err) - } - - // Check if the generated image hash already exists in the cache, otherwise, we build it. - cachedImageName, exists := checkImageHashInCache(imageHash) - if exists { - i.imageName = cachedImageName - logrus.Debugf("Using cached image for instance '%s'", i.name) - } else { - logrus.Debugf("Cannot use any cached image for instance '%s'", i.name) - err = i.builderFactory.PushBuilderImage(imageName) - if err != nil { - return ErrPushingImage.WithParams(i.name).Wrap(err) - } - updateImageCacheWithHash(imageHash, imageName) - i.imageName = imageName - logrus.Debugf("Pushed new image for instance '%s'", i.name) - } - } else { - i.imageName = i.builderFactory.ImageNameFrom() - logrus.Debugf("No need to build and push image for instance '%s'", i.name) - } - i.state = Committed - logrus.Debugf("Set state of instance '%s' to '%s'", i.name, i.state.String()) - - return nil -} - -// AddVolume adds a volume to the instance -// The owner of the volume is set to 0, if you want to set a custom owner use AddVolumeWithOwner -// This function can only be called in the states 'Preparing' and 'Committed' -func (i *Instance) AddVolume(path, size string) error { - // temporary feat, we will remove it once we can add multiple volumes - if len(i.volumes) > 0 { - logrus.Debugf("Maximum volumes exceeded for instance '%s', volumes: %d", i.name, len(i.volumes)) - return ErrMaximumVolumesExceeded.WithParams(i.name) - } - i.AddVolumeWithOwner(path, size, 0) - return nil -} - -// AddVolumeWithOwner adds a volume to the instance with the given owner -// This function can only be called in the states 'Preparing' and 'Committed' -func (i *Instance) AddVolumeWithOwner(path, size string, owner int64) error { - if !i.IsInState(Preparing, Committed) { - return ErrAddingVolumeNotAllowed.WithParams(i.state.String()) - } - // temporary feat, we will remove it once we can add multiple volumes - if len(i.volumes) > 0 { - logrus.Debugf("Maximum volumes exceeded for instance '%s', volumes: %d", i.name, len(i.volumes)) - return ErrMaximumVolumesExceeded.WithParams(i.name) - } - volume := k8sClient.NewVolume(path, size, owner) - i.volumes = append(i.volumes, volume) - logrus.Debugf("Added volume '%s' with size '%s' and owner '%d' to instance '%s'", path, size, owner, i.name) - return nil -} - -// SetMemory sets the memory of the instance -// This function can only be called in the states 'Preparing' and 'Committed' -func (i *Instance) SetMemory(request, limit string) error { - if !i.IsInState(Preparing, Committed) { - return ErrSettingMemoryNotAllowed.WithParams(i.state.String()) - } - i.memoryRequest = request - i.memoryLimit = limit - logrus.Debugf("Set memory to '%s' and limit to '%s' in instance '%s'", request, limit, i.name) - return nil -} - -// SetCPU sets the CPU of the instance -// This function can only be called in the states 'Preparing' and 'Committed' -func (i *Instance) SetCPU(request string) error { - if !i.IsInState(Preparing, Committed) { - return ErrSettingCPUNotAllowed.WithParams(i.state.String()) - } - i.cpuRequest = request - logrus.Debugf("Set cpu to '%s' in instance '%s'", request, i.name) - return nil -} - -// SetEnvironmentVariable sets the given environment variable in the instance -// This function can only be called in the states 'Preparing' and 'Committed' -func (i *Instance) SetEnvironmentVariable(key, value string) error { - if !i.IsInState(Preparing, Committed) { - return ErrSettingEnvNotAllowed.WithParams(i.state.String()) - } - if i.state == Preparing { - err := i.builderFactory.SetEnvVar(key, value) - if err != nil { - return err - } - } else if i.state == Committed { - i.env[key] = value - } - logrus.Debugf("Set environment variable '%s' to '%s' in instance '%s'", key, value, i.name) - return nil -} - -// GetIP returns the IP of the instance -// This function can only be called in the states 'Preparing' and 'Started' -func (i *Instance) GetIP() (string, error) { - // Check if i.kubernetesService already has the IP - if i.kubernetesService != nil && i.kubernetesService.Spec.ClusterIP != "" { - return i.kubernetesService.Spec.ClusterIP, nil - } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - // If not, proceed with the existing logic to deploy the service and get the IP - svc, err := k8sClient.GetService(ctx, i.k8sName) - if err != nil || svc == nil { - // Service does not exist, so we need to deploy it - err := i.deployService(ctx, i.portsTCP, i.portsUDP) - if err != nil { - return "", ErrDeployingServiceForInstance.WithParams(i.k8sName).Wrap(err) - } - svc, err = k8sClient.GetService(ctx, i.k8sName) - if err != nil { - return "", ErrGettingServiceForInstance.WithParams(i.k8sName).Wrap(err) - } - } - - ip := svc.Spec.ClusterIP - if ip == "" { - return "", ErrGettingServiceIP.WithParams(i.k8sName) - } - - // Update i.kubernetesService for future reference - i.kubernetesService = svc - - return ip, nil -} - -// GetFileBytes returns the content of the given file -// This function can only be called in the states 'Preparing' and 'Committed' -func (i *Instance) GetFileBytes(file string) ([]byte, error) { - if !i.IsInState(Preparing, Committed, Started) { - return nil, ErrGettingFileNotAllowed.WithParams(i.state.String()) - } - - if i.state != Started { - bytes, err := i.builderFactory.ReadFileFromBuilder(file) - if err != nil { - return nil, ErrGettingFile.WithParams(file, i.name).Wrap(err) - } - return bytes, nil - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - rc, err := i.ReadFileFromRunningInstance(ctx, file) - if err != nil { - return nil, ErrReadingFile.WithParams(file, i.name).Wrap(err) - } - - defer rc.Close() - return io.ReadAll(rc) -} - -func (i *Instance) ReadFileFromRunningInstance(ctx context.Context, filePath string) (io.ReadCloser, error) { - if !i.IsInState(Started) { - return nil, ErrReadingFileNotAllowed.WithParams(i.state.String()) - } - - // Not the best solution, we need to find a better one. - // Tested with a 110MB+ file and it worked. - fileContent, err := i.ExecuteCommandWithContext(ctx, "cat", filePath) - if err != nil { - return nil, ErrReadingFileFromInstance.WithParams(filePath, i.name).Wrap(err) - } - return io.NopCloser(strings.NewReader(fileContent)), nil -} - -// AddPolicyRule adds a policy rule to the instance -// This function can only be called in the states 'Preparing' and 'Committed' -func (i *Instance) AddPolicyRule(rule rbacv1.PolicyRule) error { - if !i.IsInState(Preparing, Committed) { - return ErrAddingPolicyRuleNotAllowed.WithParams(i.state.String()) - } - i.policyRules = append(i.policyRules, rule) - return nil -} - -// checkStateForProbe checks if the current state is allowed for setting a probe -func (i *Instance) checkStateForProbe() error { - if !i.IsInState(Preparing, Committed) { - return ErrSettingProbeNotAllowed.WithParams(i.state.String()) - } - return nil -} - -// SetLivenessProbe sets the liveness probe of the instance -// A live probe is a probe that is used to determine if the instance is still alive, and should be restarted if not -// See usage documentation: https://pkg.go.dev/k8sClient.io/api/core/v1@v0.27.3#Probe -// This function can only be called in the states 'Preparing' and 'Committed' -func (i *Instance) SetLivenessProbe(livenessProbe *v1.Probe) error { - if err := i.checkStateForProbe(); err != nil { - return err - } - i.livenessProbe = livenessProbe - logrus.Debugf("Set liveness probe to '%s' in instance '%s'", livenessProbe, i.name) - return nil -} - -// SetReadinessProbe sets the readiness probe of the instance -// A readiness probe is a probe that is used to determine if the instance is ready to receive traffic -// See usage documentation: https://pkg.go.dev/k8sClient.io/api/core/v1@v0.27.3#Probe -// This function can only be called in the states 'Preparing' and 'Committed' -func (i *Instance) SetReadinessProbe(readinessProbe *v1.Probe) error { - if err := i.checkStateForProbe(); err != nil { - return err - } - i.readinessProbe = readinessProbe - logrus.Debugf("Set readiness probe to '%s' in instance '%s'", readinessProbe, i.name) - return nil -} - -// SetStartupProbe sets the startup probe of the instance -// A startup probe is a probe that is used to determine if the instance is ready to receive traffic after a startup -// See usage documentation: https://pkg.go.dev/k8sClient.io/api/core/v1@v0.27.3#Probe -// This function can only be called in the states 'Preparing' and 'Committed' -func (i *Instance) SetStartupProbe(startupProbe *v1.Probe) error { - if err := i.checkStateForProbe(); err != nil { - return err - } - i.startupProbe = startupProbe - logrus.Debugf("Set startup probe to '%s' in instance '%s'", startupProbe, i.name) - return nil -} - -// AddSidecar adds a sidecar to the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) AddSidecar(sidecar *Instance) error { - - if !i.IsInState(Preparing, Committed) { - return ErrAddingSidecarNotAllowed.WithParams(i.state.String()) - } - if sidecar == nil { - return ErrSidecarIsNil - } - if sidecar == i { - return ErrSidecarCannotBeSameInstance - } - if sidecar.state != Committed { - return ErrSidecarNotCommitted.WithParams(sidecar.name) - } - if i.isSidecar { - return ErrSidecarCannotHaveSidecar.WithParams(i.name) - } - if sidecar.isSidecar { - return ErrSidecarAlreadySidecar.WithParams(sidecar.name) - } - - i.sidecars = append(i.sidecars, sidecar) - sidecar.isSidecar = true - sidecar.parentInstance = i - logrus.Debugf("Added sidecar '%s' to instance '%s'", sidecar.name, i.name) - return nil -} - -// SetOtelCollectorVersion sets the OpenTelemetry collector version for the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) SetOtelCollectorVersion(version string) error { - if err := i.validateStateForObsy("OpenTelemetry collector version"); err != nil { - return err - } - i.obsyConfig.otelCollectorVersion = version - logrus.Debugf("Set OpenTelemetry collector version '%s' for instance '%s'", version, i.name) - return nil -} - -// SetOtelEndpoint sets the OpenTelemetry endpoint for the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) SetOtelEndpoint(port int) error { - if err := i.validateStateForObsy("OpenTelemetry endpoint"); err != nil { - return err - } - i.obsyConfig.otlpPort = port - logrus.Debugf("Set OpenTelemetry endpoint '%d' for instance '%s'", port, i.name) - return nil -} - -// SetPrometheusEndpoint sets the Prometheus endpoint for the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) SetPrometheusEndpoint(port int, jobName, scapeInterval string) error { - if err := i.validateStateForObsy("Prometheus endpoint"); err != nil { - return err - } - i.obsyConfig.prometheusEndpointPort = port - i.obsyConfig.prometheusEndpointJobName = jobName - i.obsyConfig.prometheusEndpointScrapeInterval = scapeInterval - logrus.Debugf("Set Prometheus endpoint '%d' for instance '%s'", port, i.name) - return nil -} - -// SetJaegerEndpoint sets the Jaeger endpoint for the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) SetJaegerEndpoint(grpcPort, thriftCompactPort, thriftHttpPort int) error { - if err := i.validateStateForObsy("Jaeger endpoint"); err != nil { - return err - } - i.obsyConfig.jaegerGrpcPort = grpcPort - i.obsyConfig.jaegerThriftCompactPort = thriftCompactPort - i.obsyConfig.jaegerThriftHttpPort = thriftHttpPort - logrus.Debugf("Set Jaeger endpoints '%d', '%d' and '%d' for instance '%s'", grpcPort, thriftCompactPort, thriftHttpPort, i.name) - return nil -} - -// SetOtlpExporter sets the OTLP exporter for the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) SetOtlpExporter(endpoint, username, password string) error { - if err := i.validateStateForObsy("OTLP exporter"); err != nil { - return err - } - i.obsyConfig.otlpEndpoint = endpoint - i.obsyConfig.otlpUsername = username - i.obsyConfig.otlpPassword = password - logrus.Debugf("Set OTLP exporter '%s' for instance '%s'", endpoint, i.name) - return nil -} - -// SetJaegerExporter sets the Jaeger exporter for the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) SetJaegerExporter(endpoint string) error { - if err := i.validateStateForObsy("Jaeger exporter"); err != nil { - return err - } - i.obsyConfig.jaegerEndpoint = endpoint - logrus.Debugf("Set Jaeger exporter '%s' for instance '%s'", endpoint, i.name) - return nil -} - -// SetPrometheusExporter sets the Prometheus exporter for the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) SetPrometheusExporter(endpoint string) error { - if err := i.validateStateForObsy("Prometheus exporter"); err != nil { - return err - } - i.obsyConfig.prometheusExporterEndpoint = endpoint - logrus.Debugf("Set Prometheus exporter '%s' for instance '%s'", endpoint, i.name) - return nil -} - -// SetPrometheusRemoteWriteExporter sets the Prometheus remote write exporter for the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) SetPrometheusRemoteWriteExporter(endpoint string) error { - if err := i.validateStateForObsy("Prometheus remote write exporter"); err != nil { - return err - } - i.obsyConfig.prometheusRemoteWriteExporterEndpoint = endpoint - logrus.Debugf("Set Prometheus remote write exporter '%s' for instance '%s'", endpoint, i.name) - return nil -} - -// SetPrivileged sets the privileged status for the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) SetPrivileged(privileged bool) error { - if !i.IsInState(Preparing, Committed) { - return ErrSettingPrivilegedNotAllowed.WithParams(i.state.String()) - } - i.securityContext.privileged = privileged - logrus.Debugf("Set privileged to '%t' for instance '%s'", privileged, i.name) - return nil -} - -// AddCapability adds a capability to the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) AddCapability(capability string) error { - if !i.IsInState(Preparing, Committed) { - return ErrAddingCapabilityNotAllowed.WithParams(i.state.String()) - } - i.securityContext.capabilitiesAdd = append(i.securityContext.capabilitiesAdd, capability) - logrus.Debugf("Added capability '%s' to instance '%s'", capability, i.name) - return nil -} - -// AddCapabilities adds multiple capabilities to the instance -// This function can only be called in the state 'Preparing' or 'Committed' -func (i *Instance) AddCapabilities(capabilities []string) error { - if !i.IsInState(Preparing, Committed) { - return ErrAddingCapabilitiesNotAllowed.WithParams(i.state.String()) - } - for _, capability := range capabilities { - i.securityContext.capabilitiesAdd = append(i.securityContext.capabilitiesAdd, capability) - logrus.Debugf("Added capability '%s' to instance '%s'", capability, i.name) - } - return nil -} - -// StartAsync starts the instance without waiting for it to be ready -// This function can only be called in the state 'Committed' or 'Stopped' -// This function will replace StartWithoutWait -func (i *Instance) StartAsync() error { - if err := i.StartWithoutWait(); err != nil { - return err - } - return nil -} - -// StartWithoutWait starts the instance without waiting for it to be ready -// This function can only be called in the state 'Committed' or 'Stopped' -func (i *Instance) StartWithoutWait() error { - if !i.IsInState(Committed, Stopped) { - return ErrStartingNotAllowed.WithParams(i.state.String()) - } - if err := applyFunctionToInstances(i.sidecars, func(sidecar Instance) error { - if !sidecar.IsInState(Committed, Stopped) { - return ErrStartingNotAllowedForSidecar.WithParams(sidecar.name, sidecar.state.String()) - } - return nil - }); err != nil { - return err - } - if i.isSidecar { - return ErrStartingSidecarNotAllowed - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - if i.state == Committed { - // deploy otel collector if observability is enabled - if i.isObservabilityEnabled() { - if err := i.addOtelCollectorSidecar(); err != nil { - return ErrAddingOtelCollectorSidecar.WithParams(i.k8sName).Wrap(err) - } - } - - if i.BitTwister.Enabled() { - if err := i.addBitTwisterSidecar(); err != nil { - return ErrAddingNetworkSidecar.WithParams(i.k8sName).Wrap(err) - } - } - - if err := i.deployResources(ctx); err != nil { - return ErrDeployingResourcesForInstance.WithParams(i.k8sName).Wrap(err) - } - if err := applyFunctionToInstances(i.sidecars, func(sidecar Instance) error { - return sidecar.deployResources(ctx) - }); err != nil { - return ErrDeployingResourcesForSidecars.WithParams(i.k8sName).Wrap(err) - } - } - - err := i.deployPod(ctx) - if err != nil { - return ErrDeployingPodForInstance.WithParams(i.k8sName).Wrap(err) - } - i.state = Started - setStateForSidecars(i.sidecars, Started) - logrus.Debugf("Set state of instance '%s' to '%s'", i.k8sName, i.state.String()) - - return nil -} - -// Start starts the instance and waits for it to be ready -// This function can only be called in the state 'Committed' and 'Stopped' -func (i *Instance) Start() error { - if err := i.StartWithoutWait(); err != nil { - return err - } - - err := i.WaitInstanceIsRunning() - if err != nil { - return ErrWaitingForInstanceRunning.WithParams(i.k8sName).Wrap(err) - } - - return nil -} - -// IsRunning returns true if the instance is running -// This function can only be called in the state 'Started' -func (i *Instance) IsRunning() (bool, error) { - if !i.IsInState(Started, Stopped) { - return false, ErrCheckingIfInstanceRunningNotAllowed.WithParams(i.state.String()) - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return k8sClient.IsReplicaSetRunning(ctx, i.k8sName) -} - -// WaitInstanceIsRunning waits until the instance is running -// This function can only be called in the state 'Started' -func (i *Instance) WaitInstanceIsRunning() error { - if !i.IsInState(Started) { - return ErrWaitingForInstanceNotAllowed.WithParams(i.state.String()) - } - timeout := time.After(1 * time.Minute) - tick := time.Tick(1 * time.Second) - - for { - select { - case <-timeout: - return ErrWaitingForInstanceTimeout.WithParams(i.k8sName) - case <-tick: - running, err := i.IsRunning() - if err != nil { - return ErrCheckingIfInstanceRunning.WithParams(i.k8sName).Wrap(err) - } - if running { - return nil - } - } - } -} - -// DisableNetwork disables the network of the instance -// This does not apply to executor instances -// This function can only be called in the state 'Started' -func (i *Instance) DisableNetwork() error { - if !i.IsInState(Started) { - return ErrDisablingNetworkNotAllowed.WithParams(i.state.String()) - } - executorSelectorMap := map[string]string{ - "knuu.sh/type": ExecutorInstance.String(), - } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - err := k8sClient.CreateNetworkPolicy(ctx, i.k8sName, i.getLabels(), executorSelectorMap, executorSelectorMap) - if err != nil { - return ErrDisablingNetwork.WithParams(i.k8sName).Wrap(err) - } - return nil -} - -// SetBandwidthLimit sets the bandwidth limit of the instance -// bandwidth limit in bps (e.g. 1000 for 1Kbps) -// Currently, only one of bandwidth, jitter, latency or packet loss can be set -// This function can only be called in the state 'Commited' -func (i *Instance) SetBandwidthLimit(limit int64) error { - if !i.IsInState(Started) { - return ErrSettingBandwidthLimitNotAllowed.WithParams(i.state.String()) - } - if !i.BitTwister.Enabled() { - return ErrSettingBandwidthLimitNotAllowedBitTwister - } - - // We first need to stop it, otherwise we get an error - if err := i.BitTwister.Client().BandwidthStop(); err != nil { - if !sdk.IsErrorServiceNotInitialized(err) && - !sdk.IsErrorServiceNotReady(err) && - !sdk.IsErrorServiceNotStarted(err) { - return ErrStoppingBandwidthLimit.WithParams(i.k8sName).Wrap(err) - } - } - - err := i.BitTwister.Client().BandwidthStart(sdk.BandwidthStartRequest{ - NetworkInterfaceName: i.BitTwister.NetworkInterface(), - Limit: limit, - }) - if err != nil { - return ErrSettingBandwidthLimit.WithParams(i.k8sName).Wrap(err) - } - - logrus.Debugf("Set bandwidth limit to '%d' in instance '%s'", limit, i.name) - return nil -} - -// SetLatencyAndJitter simulates variable network conditions by applying both latency and jitter to packet transmission. -// Latency establishes a base delay, while jitter adds variability that can lead to packet reordering, mimicking real-world network behavior. -// Example: -// To set a latency of 100ms and jitter of 75ms: -// err := instance.SetLatencyAndJitter(100, 75) -// if err != nil { -// log.Fatalf("Error: %v", err) -// } -// With this configuration, if one packet is assigned a 100ms delay and the following packet has a delay of only 50ms, -// the second packet may be transmitted first. This is due to the jitter, which can cause variability in delay times, -// allowing subsequent packets to potentially be sent earlier than those queued before them. -func (i *Instance) SetLatencyAndJitter(latency, jitter int64) error { - if !i.IsInState(Started) { - return ErrSettingLatencyJitterNotAllowed.WithParams(i.state.String()) - } - if !i.BitTwister.Enabled() { - return ErrSettingLatencyJitterNotAllowedBitTwister - } - - // We first need to stop it, otherwise we get an error - if err := i.BitTwister.Client().LatencyStop(); err != nil { - if !sdk.IsErrorServiceNotInitialized(err) && - !sdk.IsErrorServiceNotReady(err) && - !sdk.IsErrorServiceNotStarted(err) { - return ErrStoppingLatencyJitter.WithParams(i.k8sName).Wrap(err) - } - } - - err := i.BitTwister.Client().LatencyStart(sdk.LatencyStartRequest{ - NetworkInterfaceName: i.BitTwister.NetworkInterface(), - Latency: latency, - Jitter: jitter, - }) - if err != nil { - return ErrSettingLatencyJitter.WithParams(i.k8sName).Wrap(err) - } - - logrus.Debugf("Set latency to '%d' and jitter to '%d' in instance '%s'", latency, jitter, i.name) - return nil -} - -// SetPacketLoss sets the packet loss of the instance -// packet loss in percent (e.g. 10 for 10%) -// Currently, only one of bandwidth, jitter, latency or packet loss can be set -// This function can only be called in the state 'Commited' -func (i *Instance) SetPacketLoss(packetLoss int32) error { - if !i.IsInState(Started) { - return ErrSettingPacketLossNotAllowed.WithParams(i.state.String()) - } - if !i.BitTwister.Enabled() { - return ErrSettingPacketLossNotAllowedBitTwister - } - - // We first need to stop it, otherwise we get an error - if err := i.BitTwister.Client().PacketlossStop(); err != nil { - if !sdk.IsErrorServiceNotInitialized(err) && - !sdk.IsErrorServiceNotReady(err) && - !sdk.IsErrorServiceNotStarted(err) { - return ErrStoppingPacketLoss.WithParams(i.k8sName).Wrap(err) - } - } - - err := i.BitTwister.Client().PacketlossStart(sdk.PacketLossStartRequest{ - NetworkInterfaceName: i.BitTwister.NetworkInterface(), - PacketLossRate: packetLoss, - }) - if err != nil { - return ErrSettingPacketLoss.WithParams(i.k8sName).Wrap(err) - } - - logrus.Debugf("Set packet loss to '%d' in instance '%s'", packetLoss, i.name) - return nil -} - -// EnableNetwork enables the network of the instance -// This function can only be called in the state 'Started' -func (i *Instance) EnableNetwork() error { - if !i.IsInState(Started) { - return ErrEnablingNetworkNotAllowed.WithParams(i.state.String()) - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - err := k8sClient.DeleteNetworkPolicy(ctx, i.k8sName) - if err != nil { - return ErrEnablingNetwork.WithParams(i.k8sName).Wrap(err) - } - return nil +func (k *Knuu) NewInstance(name string) (*instance.Instance, error) { + return instance.New(name, k.SystemDependencies) } -// NetworkIsDisabled returns true if the network of the instance is disabled -// This function can only be called in the state 'Started' -func (i *Instance) NetworkIsDisabled() (bool, error) { - if !i.IsInState(Started) { - return false, ErrCheckingIfNetworkDisabledNotAllowed.WithParams(i.state.String()) - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - return k8sClient.NetworkPolicyExists(ctx, i.k8sName), nil +func (k *Knuu) NewExecutor(ctx context.Context) (*instance.Executor, error) { + return instance.NewExecutor(ctx, k.SystemDependencies) } -// WaitInstanceIsStopped waits until the instance is not running anymore -// This function can only be called in the state 'Stopped' -func (i *Instance) WaitInstanceIsStopped() error { - if !i.IsInState(Stopped) { - return ErrWaitingForInstanceStoppedNotAllowed.WithParams(i.state.String()) - } - for { - running, err := i.IsRunning() - if !running { - break - } - if err != nil { - return ErrCheckingIfInstanceStopped.WithParams(i.k8sName).Wrap(err) - } - } - - return nil -} - -// Stop stops the instance -// CAUTION: In order to keep data of the instance, you need to use AddVolume() before. -// This function can only be called in the state 'Started' -func (i *Instance) Stop() error { - if !i.IsInState(Started) { - return ErrStoppingNotAllowed.WithParams(i.state.String()) - - } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - err := i.destroyPod(ctx) - if err != nil { - return ErrDestroyingPod.WithParams(i.k8sName).Wrap(err) - } - i.state = Stopped - setStateForSidecars(i.sidecars, Stopped) - logrus.Debugf("Set state of instance '%s' to '%s'", i.k8sName, i.state.String()) - - return nil -} - -// Clone creates a clone of the instance -// This function can only be called in the state 'Committed' -// When cloning an instance that is a sidecar, the clone will be not a sidecar -// When cloning an instance with sidecars, the sidecars will be cloned as well -func (i *Instance) Clone() (*Instance, error) { - if !i.IsInState(Committed) { - return nil, ErrCloningNotAllowed.WithParams(i.state.String()) - } - - newK8sName, err := generateK8sName(i.name) - if err != nil { - return nil, ErrGeneratingK8sName.WithParams(i.name).Wrap(err) - } - // Create a new instance with the same attributes as the original instance - ins := i.cloneWithSuffix("") - ins.k8sName = newK8sName - return ins, nil -} - -// CloneWithName creates a clone of the instance with a given name -// This function can only be called in the state 'Committed' -// When cloning an instance that is a sidecar, the clone will be not a sidecar -// When cloning an instance with sidecars, the sidecars will be cloned as well -func (i *Instance) CloneWithName(name string) (*Instance, error) { - if !i.IsInState(Committed) { - return nil, ErrCloningNotAllowedForSidecar.WithParams(i.state.String()) - } - - newK8sName, err := generateK8sName(name) - if err != nil { - return nil, ErrGeneratingK8sNameForSidecar.WithParams(name).Wrap(err) - } - // Create a new instance with the same attributes as the original instance - ins := i.cloneWithSuffix("") - ins.name = name - ins.k8sName = newK8sName - return ins, nil -} - -// CreateCustomResource creates a custom resource for the instance -// The names and namespace are set and overridden by knuu -func (i *Instance) CreateCustomResource(gvr *schema.GroupVersionResource, obj *map[string]interface{}) error { - - crdExists, err := i.CustomResourceDefinitionExists(gvr) - if err != nil { - return err - } - if !crdExists { - return ErrCustomResourceDefinitionDoesNotExist.WithParams(gvr.Resource) - } - - return k8sClient.CreateCustomResource(context.TODO(), i.k8sName, gvr, obj) -} - -// CustomResourceDefinitionExists checks if the custom resource definition exists -func (i *Instance) CustomResourceDefinitionExists(gvr *schema.GroupVersionResource) (bool, error) { - return k8sClient.CustomResourceDefinitionExists(context.TODO(), gvr), nil -} - -func (i *Instance) AddHost(port int) (err error, host string) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - prefix := fmt.Sprintf("%s-%d", i.k8sName, port) - if err := traefikClient.AddHost(ctx, i.k8sName, prefix, port); err != nil { - return ErrAddingToProxy.WithParams(i.k8sName).Wrap(err), "" - } - host, err = traefikClient.URL(ctx, prefix) - if err != nil { - return ErrGettingProxyURL.WithParams(i.k8sName).Wrap(err), "" - } - return nil, host +func (k *Knuu) NewPreloader() (*preloader.Preloader, error) { + return preloader.New(k.SystemDependencies) } diff --git a/pkg/knuu/instance_old.go b/pkg/knuu/instance_old.go new file mode 100644 index 0000000..ee0b026 --- /dev/null +++ b/pkg/knuu/instance_old.go @@ -0,0 +1,444 @@ +/* +* This file is deprecated. +* Please use the new package knuu instead. +* This file keeps the old functionality of knuu for backward compatibility. +* A global variable is defined, tmpKnuu, which is used to hold the knuu instance. + */ + +package knuu + +import ( + "context" + "errors" + "io" + + "github.com/celestiaorg/knuu/pkg/builder" + "github.com/celestiaorg/knuu/pkg/instance" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type Instance struct { + instance.Instance +} + +type Executor struct { + *Instance +} + +type InstancePool struct { + instance.InstancePool +} + +type InstanceState instance.InstanceState + +const ( + None InstanceState = iota + Preparing + Committed + Started + Stopped + Destroyed +) + +// Deprecated: Use the new package knuu instead. +func NewInstance(name string) (*Instance, error) { + if tmpKnuu == nil { + return nil, errors.New("tmpKnuu is not initialized") + } + + i, err := tmpKnuu.NewInstance(name) + if err != nil { + return nil, err + } + return &Instance{*i}, nil +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetImage(image string) error { + if tmpKnuu == nil { + return errors.New("tmpKnuu is not initialized") + } + ctx, cancel := context.WithTimeout(context.Background(), tmpKnuu.timeout) + defer cancel() + return i.Instance.SetImage(ctx, image) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetGitRepo(ctx context.Context, gitContext builder.GitContext) error { + return i.Instance.SetGitRepo(ctx, gitContext) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetImageInstant(image string) error { + return i.Instance.SetImageInstant(context.Background(), image) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetCommand(command ...string) error { + return i.Instance.SetCommand(command...) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetArgs(args ...string) error { + return i.Instance.SetArgs(args...) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddPortTCP(port int) error { + return i.Instance.AddPortTCP(port) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) PortForwardTCP(port int) (int, error) { + return i.Instance.PortForwardTCP(context.Background(), port) +} + +// AddPortUDP adds a UDP port to the instance +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddPortUDP(port int) error { + return i.Instance.AddPortUDP(port) +} + +// Deprecated: Use the new package knuu instead. +// ExecuteCommand executes a command in the instance +func (i *Instance) ExecuteCommand(command ...string) (string, error) { + return i.Instance.ExecuteCommand(context.Background(), command...) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) ExecuteCommandWithContext(ctx context.Context, command ...string) (string, error) { + return i.Instance.ExecuteCommand(ctx, command...) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddFile(srcPath, dstPath string, chown string) error { + return i.Instance.AddFile(srcPath, dstPath, chown) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddFolder(srcPath, dstPath string, chown string) error { + return i.Instance.AddFolder(srcPath, dstPath, chown) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddFileBytes(bytes []byte, dest string, chown string) error { + return i.Instance.AddFileBytes(bytes, dest, chown) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetUser(user string) error { + return i.Instance.SetUser(user) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) Commit() error { + return i.Instance.Commit() +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddVolume(path, size string) error { + return i.Instance.AddVolume(path, size) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddVolumeWithOwner(path, size string, owner int64) error { + return i.Instance.AddVolumeWithOwner(path, size, owner) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetMemory(request, limit string) error { + return i.Instance.SetMemory(request, limit) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetCPU(request string) error { + return i.Instance.SetCPU(request) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetEnvironmentVariable(key, value string) error { + return i.Instance.SetEnvironmentVariable(key, value) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) GetIP() (string, error) { + return i.Instance.GetIP(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) GetFileBytes(file string) ([]byte, error) { + return i.Instance.GetFileBytes(context.Background(), file) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) ReadFileFromRunningInstance(ctx context.Context, filePath string) (io.ReadCloser, error) { + return i.Instance.ReadFileFromRunningInstance(ctx, filePath) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddPolicyRule(rule rbacv1.PolicyRule) error { + return i.Instance.AddPolicyRule(rule) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetLivenessProbe(livenessProbe *v1.Probe) error { + return i.Instance.SetLivenessProbe(livenessProbe) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetReadinessProbe(readinessProbe *v1.Probe) error { + return i.Instance.SetReadinessProbe(readinessProbe) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetStartupProbe(startupProbe *v1.Probe) error { + return i.Instance.SetStartupProbe(startupProbe) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddSidecar(sidecar *Instance) error { + return i.Instance.AddSidecar(&sidecar.Instance) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetOtelCollectorVersion(version string) error { + return i.Instance.SetOtelCollectorVersion(version) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetOtelEndpoint(port int) error { + return i.Instance.SetOtelEndpoint(port) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetPrometheusEndpoint(port int, jobName, scapeInterval string) error { + return i.Instance.SetPrometheusEndpoint(port, jobName, scapeInterval) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetJaegerEndpoint(grpcPort, thriftCompactPort, thriftHttpPort int) error { + return i.Instance.SetJaegerEndpoint(grpcPort, thriftCompactPort, thriftHttpPort) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetOtlpExporter(endpoint, username, password string) error { + return i.Instance.SetOtlpExporter(endpoint, username, password) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetJaegerExporter(endpoint string) error { + return i.Instance.SetJaegerExporter(endpoint) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetPrometheusExporter(endpoint string) error { + return i.Instance.SetPrometheusExporter(endpoint) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetPrometheusRemoteWriteExporter(endpoint string) error { + return i.Instance.SetPrometheusRemoteWriteExporter(endpoint) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetPrivileged(privileged bool) error { + return i.Instance.SetPrivileged(privileged) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddCapability(capability string) error { + return i.Instance.AddCapability(capability) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) AddCapabilities(capabilities []string) error { + return i.Instance.AddCapabilities(capabilities) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) StartAsync() error { + return i.Instance.StartAsync(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) StartWithoutWait() error { + return i.Instance.StartWithoutWait(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) Start() error { + return i.Instance.Start(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) IsRunning() (bool, error) { + return i.Instance.IsRunning(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) WaitInstanceIsRunning() error { + return i.Instance.WaitInstanceIsRunning(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) DisableNetwork() error { + return i.Instance.DisableNetwork(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetBandwidthLimit(limit int64) error { + return i.Instance.SetBandwidthLimit(limit) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetLatencyAndJitter(latency, jitter int64) error { + return i.Instance.SetLatencyAndJitter(latency, jitter) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) SetPacketLoss(packetLoss int32) error { + return i.Instance.SetPacketLoss(packetLoss) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) EnableNetwork() error { + return i.Instance.EnableNetwork(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) NetworkIsDisabled() (bool, error) { + return i.Instance.NetworkIsDisabled(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) WaitInstanceIsStopped() error { + return i.Instance.WaitInstanceIsStopped(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) Stop() error { + return i.Instance.Stop(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) Clone() (*Instance, error) { + newInst, err := i.Instance.Clone() + if err != nil { + return nil, err + } + return &Instance{Instance: *newInst}, nil +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) CloneWithName(name string) (*Instance, error) { + newInst, err := i.Instance.CloneWithName(name) + if err != nil { + return nil, err + } + return &Instance{*newInst}, nil +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) CreateCustomResource(gvr *schema.GroupVersionResource, obj *map[string]interface{}) error { + return i.Instance.CreateCustomResource(context.Background(), gvr, obj) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) CustomResourceDefinitionExists(gvr *schema.GroupVersionResource) (bool, error) { + return i.Instance.CustomResourceDefinitionExists(context.Background(), gvr) +} + +// Deprecated: Use the new package knuu instead. +func NewExecutor() (*Executor, error) { + if tmpKnuu == nil { + return nil, errors.New("tmpKnuu is not initialized") + } + e, err := tmpKnuu.NewExecutor(context.Background()) + if err != nil { + return nil, err + } + return &Executor{ + Instance: &Instance{ + Instance: *e.Instance, + }, + }, nil +} + +// Deprecated: Use the new package knuu instead. +func (e *Executor) Destroy() error { + return e.Instance.Destroy() +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) Destroy() error { + return i.Instance.Destroy(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func BatchDestroy(instances ...*Instance) error { + ins := make([]*instance.Instance, len(instances)) + for i, instance := range instances { + ins[i] = &instance.Instance + } + return instance.BatchDestroy(context.Background(), ins...) +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) CreatePool(amount int) (*InstancePool, error) { + pool, err := i.Instance.NewPool(amount) + if err != nil { + return nil, err + } + return &InstancePool{*pool}, nil +} + +// Deprecated: Use the new package knuu instead. +func (i *InstancePool) StartWithoutWait() error { + return i.InstancePool.StartWithoutWait(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *InstancePool) Start() error { + return i.InstancePool.Start(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *InstancePool) Destroy() error { + return i.InstancePool.Destroy(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *InstancePool) WaitInstancePoolIsRunning() error { + return i.InstancePool.WaitInstancePoolIsRunning(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func (i *InstancePool) Instances() []*Instance { + instances := i.InstancePool.Instances() + newInstances := make([]*Instance, len(instances)) + for i, instance := range instances { + newInstances[i] = &Instance{*instance} + } + return newInstances +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) Labels() map[string]string { + return i.Instance.Labels() +} + +// Deprecated: Use the new package knuu instead. +func (i *Instance) IsInState(states ...InstanceState) bool { + statesNew := make([]instance.InstanceState, len(states)) + for i, state := range states { + statesNew[i] = instance.InstanceState(state) + } + return i.Instance.IsInState(statesNew...) +} + +func (i *Instance) AddHost(port int) (err error, host string) { + host, err = i.Instance.AddHost(context.Background(), port) + return err, host +} diff --git a/pkg/knuu/knuu.go b/pkg/knuu/knuu.go index 7f9f068..c74d3cc 100644 --- a/pkg/knuu/knuu.go +++ b/pkg/knuu/knuu.go @@ -18,199 +18,158 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "github.com/celestiaorg/knuu/pkg/builder" - "github.com/celestiaorg/knuu/pkg/builder/docker" "github.com/celestiaorg/knuu/pkg/builder/kaniko" + "github.com/celestiaorg/knuu/pkg/instance" "github.com/celestiaorg/knuu/pkg/k8s" "github.com/celestiaorg/knuu/pkg/minio" + "github.com/celestiaorg/knuu/pkg/system" "github.com/celestiaorg/knuu/pkg/traefik" ) -var ( - // testScope is the testScope of the current knuu instance - testScope string - startTime string - timeout time.Duration - imageBuilder builder.Builder +const ( + defaultTimeout = 60 * time.Minute + timeoutHandlerName = "timeout-handler" + // FIXME: use supported kubernetes version images (use of latest could break) (https://github.com/celestiaorg/knuu/issues/116) + timeoutHandlerImage = "docker.io/bitnami/kubectl:latest" - // TODO: these are temporary until we refactor knuu pkg - k8sClient *k8s.Client - traefikClient *traefik.Traefik + TimeFormat = "20060102T150405Z" ) -// Initialize initializes knuu with a unique scope -func Initialize() error { - t := time.Now() - scope := fmt.Sprintf("%s-%03d", t.Format("20060102-150405"), t.Nanosecond()/1e6) - return InitializeWithScope(scope) +type Knuu struct { + system.SystemDependencies + timeout time.Duration + proxyEnabled bool } -// Scope returns the scope of the current knuu instance -func Scope() string { - return testScope -} +type Option func(*Knuu) -// InitializeWithScope initializes knuu with a given scope -func InitializeWithScope(scope string) error { - var err error - err = godotenv.Load() - if err != nil { - if os.IsNotExist(err) { - logrus.Info("The .env file does not exist, continuing without loading environment variables.") - } else { - return ErrCannotLoadEnv.Wrap(err) - } +func WithImageBuilder(builder builder.Builder) Option { + return func(k *Knuu) { + k.ImageBuilder = builder } - if scope == "" { - return ErrCannotInitializeKnuuWithEmptyScope +} + +func WithTestScope(scope string) Option { + return func(k *Knuu) { + k.TestScope = k8s.SanitizeName(scope) } +} - // Sanitize the scope to ensure it is a valid Kubernetes name and match the namespace - testScope = k8s.SanitizeName(scope) +// This timeout indicates how long the test will run before it is considered failed. +func WithTimeout(timeout time.Duration) Option { + return func(k *Knuu) { + k.timeout = timeout + } +} - t := time.Now() - startTime = fmt.Sprintf("%s-%03d", t.Format("20060102-150405"), t.Nanosecond()/1e6) +func WithMinio(minio *minio.Minio) Option { + return func(k *Knuu) { + k.MinioCli = minio + } +} - setupLogging() +func WithK8s(k8s k8s.KubeManager) Option { + return func(k *Knuu) { + k.K8sCli = k8s + } +} - // Override scope if KNUU_NAMESPACE is set - namespaceEnv := os.Getenv("KNUU_NAMESPACE") - if namespaceEnv != "" { - scope = namespaceEnv - logrus.Warnf("KNUU_NAMESPACE is deprecated. Scope overridden to: %s", scope) +func WithLogger(logger *logrus.Logger) Option { + return func(k *Knuu) { + k.Logger = logger } +} - logrus.Infof("Initializing knuu with scope: %s", testScope) +func WithProxyEnabled() Option { + return func(k *Knuu) { + k.proxyEnabled = true + } +} - // read timeout from env - timeoutString := os.Getenv("KNUU_TIMEOUT") - if timeoutString == "" { - timeout = 60 * time.Minute - } else { - parsedTimeout, err := time.ParseDuration(timeoutString) - if err != nil { - return ErrCannotParseTimeout.Wrap(err) +func New(ctx context.Context, opts ...Option) (*Knuu, error) { + if err := godotenv.Load(); err != nil { + if !os.IsNotExist(err) { + return nil, ErrCannotLoadEnv.Wrap(err) } - timeout = parsedTimeout + logrus.Info("The .env file does not exist, continuing without loading environment variables.") } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - k8sClient, err = k8s.New(ctx, scope) - if err != nil { - return ErrCannotInitializeK8s.Wrap(err) + k := &Knuu{} + for _, opt := range opts { + opt(k) } - traefikClient = &traefik.Traefik{ - K8s: k8sClient, - } - if err := traefikClient.Deploy(ctx); err != nil { - return ErrCannotDeployTraefik.Wrap(err) - } + k.StartTime = time.Now().UTC().Format(TimeFormat) - publicIP, err := traefikClient.Endpoint(ctx) - if err != nil { - return ErrCannotGetTraefikEndpoint.Wrap(err) + // handle default values + if k.Logger == nil { + k.Logger = defaultLogger() } - logrus.Debugf("Traefik publicIP: %v\n", publicIP) - minioClient = &minio.Minio{ - Clientset: k8sClient.Clientset(), - Namespace: k8sClient.Namespace(), + if k.TestScope == "" { + t := time.Now() + k.TestScope = fmt.Sprintf("%s-%03d", t.Format("20060102-150405"), t.Nanosecond()/1e6) } - if err := handleTimeout(); err != nil { - return ErrCannotHandleTimeout.Wrap(err) + if k.timeout == 0 { + k.timeout = defaultTimeout } - builderType := os.Getenv("KNUU_BUILDER") - switch builderType { - case "kubernetes": - SetImageBuilder(&kaniko.Kaniko{ - K8sClientset: k8sClient.Clientset(), - K8sNamespace: k8sClient.Namespace(), - Minio: minioClient, // same client is used to make the best use of the resources - }) - case "docker", "": - SetImageBuilder(&docker.Docker{ - K8sClientset: k8sClient.Clientset(), - K8sNamespace: k8sClient.Namespace(), - }) - default: - return ErrInvalidKnuuBuilder.WithParams(builderType) + if k.K8sCli == nil { + var err error + k.K8sCli, err = k8s.New(ctx, k.TestScope) + if err != nil { + return nil, ErrCannotInitializeK8s.Wrap(err) + } } - HandleStopSignal() - return nil -} - -// Deprecated: Identifier is deprecated, use Scope() instead. -func Identifier() string { - logrus.Warn("Identifier() is deprecated, use Scope() instead.") - return Scope() -} - -// Deprecated: InitializeWithIdentifier is deprecated, use InitializeWithScope(scope string) instead. -func InitializeWithIdentifier(uniqueIdentifier string) error { - logrus.Warn("InitializeWithIdentifier is deprecated, use InitializeWithScope(scope string) instead.") - return InitializeWithScope(uniqueIdentifier) -} - -// setupLogging Configures the log -func setupLogging() { - // Set the default log level - logrus.SetLevel(logrus.InfoLevel) - - // Set the custom formatter - logrus.SetFormatter(&logrus.TextFormatter{ - FullTimestamp: true, - CallerPrettyfier: func(f *runtime.Frame) (string, string) { - filename := path.Base(f.File) - directory := path.Base(path.Dir(f.File)) - return "", directory + "/" + filename + ":" + strconv.Itoa(f.Line) - }, - }) + if k.MinioCli == nil { + // TODO: minio also needs a little refactor to accept k8s obj instead + k.MinioCli = &minio.Minio{ + Clientset: k.K8sCli.Clientset(), + Namespace: k.K8sCli.Namespace(), + } + } - // Enable reporting the file and line - logrus.SetReportCaller(true) - - switch os.Getenv("LOG_LEVEL") { - case "debug": - logrus.SetLevel(logrus.DebugLevel) - case "info": - logrus.SetLevel(logrus.InfoLevel) - case "warn": - logrus.SetLevel(logrus.WarnLevel) - case "error": - logrus.SetLevel(logrus.ErrorLevel) - default: - logrus.SetLevel(logrus.InfoLevel) + if k.ImageBuilder == nil { + // TODO: Also here for kaniko + k.ImageBuilder = &kaniko.Kaniko{ + K8sClientset: k.K8sCli.Clientset(), + K8sNamespace: k.K8sCli.Namespace(), + Minio: k.MinioCli, + } } - logrus.Info("LOG_LEVEL: ", logrus.GetLevel()) -} + if k.proxyEnabled { + k.Proxy = &traefik.Traefik{ + K8s: k.K8sCli, + } + if err := k.Proxy.Deploy(ctx); err != nil { + return nil, ErrCannotDeployTraefik.Wrap(err) + } + endpoint, err := k.Proxy.Endpoint(ctx) + if err != nil { + return nil, ErrCannotGetTraefikEndpoint.Wrap(err) + } + k.Logger.Debugf("Proxy endpoint: %s", endpoint) + } -func SetImageBuilder(b builder.Builder) { - imageBuilder = b -} + if err := k.handleTimeout(ctx); err != nil { + return nil, ErrCannotHandleTimeout.Wrap(err) + } -func ImageBuilder() builder.Builder { - return imageBuilder + return k, nil } -// IsInitialized returns true if knuu is initialized, and false otherwise -func IsInitialized() bool { - return k8sClient != nil +func (k *Knuu) Scope() string { + return k.TestScope } -func CleanUp() error { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - return k8sClient.DeleteNamespace(ctx, testScope) +func (k *Knuu) CleanUp(ctx context.Context) error { + return k.K8sCli.DeleteNamespace(ctx, k.TestScope) } -func HandleStopSignal() { +func (k *Knuu) HandleStopSignal() { stop := make(chan os.Signal, 1) signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM, os.Interrupt) go func() { @@ -223,17 +182,17 @@ func HandleStopSignal() { } // handleTimeout creates a timeout handler that will delete all resources with the scope after the timeout -func handleTimeout() error { - instance, err := NewInstance("timeout-handler") +func (k *Knuu) handleTimeout(ctx context.Context) error { + inst, err := k.NewInstance(timeoutHandlerName) if err != nil { return ErrCannotCreateInstance.Wrap(err) } - instance.instanceType = TimeoutHandlerInstance - // FIXME: use supported kubernetes version images (use of latest could break) (https://github.com/celestiaorg/knuu/issues/116) - if err := instance.SetImage("docker.io/bitnami/kubectl:latest"); err != nil { + inst.SetInstanceType(instance.TimeoutHandlerInstance) + + if err := inst.SetImage(ctx, timeoutHandlerImage); err != nil { return ErrCannotSetImage.Wrap(err) } - if err := instance.Commit(); err != nil { + if err := inst.Commit(); err != nil { return ErrCannotCommitInstance.Wrap(err) } @@ -241,24 +200,26 @@ func handleTimeout() error { // Wait for a specific period before executing the next operation. // This is useful to ensure that any previous operation has time to complete. - commands = append(commands, fmt.Sprintf("sleep %d", int64(timeout.Seconds()))) + commands = append(commands, fmt.Sprintf("sleep %d", int64(k.timeout.Seconds()))) // Collects all resources (pods, services, etc.) within the specified namespace that match a specific label, excluding certain types, // and then deletes them. This is useful for cleaning up specific test resources before proceeding to delete the namespace. - commands = append(commands, fmt.Sprintf("kubectl get all,pvc,netpol,roles,serviceaccounts,rolebindings,configmaps -l knuu.sh/scope=%s -n %s -o json | jq -r '.items[] | select(.metadata.labels.\"knuu.sh/type\" != \"%s\") | \"\\(.kind)/\\(.metadata.name)\"' | xargs -r kubectl delete -n %s", testScope, k8sClient.Namespace(), TimeoutHandlerInstance.String(), k8sClient.Namespace())) + commands = append(commands, + fmt.Sprintf("kubectl get all,pvc,netpol,roles,serviceaccounts,rolebindings,configmaps -l knuu.sh/scope=%s -n %s -o json | jq -r '.items[] | select(.metadata.labels.\"knuu.sh/type\" != \"%s\") | \"\\(.kind)/\\(.metadata.name)\"' | xargs -r kubectl delete -n %s", + k.TestScope, k.K8sCli.Namespace(), instance.TimeoutHandlerInstance.String(), k.K8sCli.Namespace())) // Delete the namespace as it was created by knuu. - logrus.Debugf("The namespace generated [%s] will be deleted", k8sClient.Namespace()) - commands = append(commands, fmt.Sprintf("kubectl delete namespace %s", k8sClient.Namespace())) + k.Logger.Debugf("The namespace generated [%s] will be deleted", k.K8sCli.Namespace()) + commands = append(commands, fmt.Sprintf("kubectl delete namespace %s", k.K8sCli.Namespace())) // Delete all labeled resources within the namespace. // Unlike the previous command that excludes certain types, this command ensures that everything remaining is deleted. - commands = append(commands, fmt.Sprintf("kubectl delete all,pvc,netpol,roles,serviceaccounts,rolebindings,configmaps -l knuu.sh/scope=%s -n %s", testScope, k8sClient.Namespace())) + commands = append(commands, fmt.Sprintf("kubectl delete all,pvc,netpol,roles,serviceaccounts,rolebindings,configmaps -l knuu.sh/scope=%s -n %s", k.TestScope, k.K8sCli.Namespace())) finalCmd := strings.Join(commands, " && ") // Run the command - if err := instance.SetCommand("sh", "-c", finalCmd); err != nil { - logrus.Debugf("The full command generated is [%s]", finalCmd) + if err := inst.SetCommand("sh", "-c", finalCmd); err != nil { + k.Logger.Debugf("The full command generated is [%s]", finalCmd) return ErrCannotSetCommand.Wrap(err) } @@ -268,12 +229,39 @@ func handleTimeout() error { Resources: []string{"*"}, } - if err := instance.AddPolicyRule(rule); err != nil { + if err := inst.AddPolicyRule(rule); err != nil { return ErrCannotAddPolicyRule.Wrap(err) } - if err := instance.StartWithoutWait(); err != nil { + if err := inst.Start(ctx); err != nil { return ErrCannotStartInstance.Wrap(err) } return nil } + +func defaultLogger() *logrus.Logger { + logger := logrus.New() + + logger.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + CallerPrettyfier: func(f *runtime.Frame) (string, string) { + filename := path.Base(f.File) + directory := path.Base(path.Dir(f.File)) + return "", directory + "/" + filename + ":" + strconv.Itoa(f.Line) + }, + }) + + // Enable reporting the file and line + logger.SetReportCaller(true) + + customLevel := os.Getenv("LOG_LEVEL") + if customLevel != "" { + err := logger.Level.UnmarshalText([]byte(customLevel)) + if err != nil { + logger.Warnf("Failed to parse LOG_LEVEL: %v, defaulting to INFO", err) + } + } + logger.Info("LOG_LEVEL: ", logger.GetLevel()) + + return logger +} diff --git a/pkg/knuu/knuu_old.go b/pkg/knuu/knuu_old.go new file mode 100644 index 0000000..84c4e40 --- /dev/null +++ b/pkg/knuu/knuu_old.go @@ -0,0 +1,145 @@ +/* +* This file is deprecated. +* Please use the new package knuu instead. +* This file keeps the old functionality of knuu for backward compatibility. +* A global variable is defined, tmpKnuu, which is used to hold the knuu instance. + */ +package knuu + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "time" + + "github.com/sirupsen/logrus" + + "github.com/celestiaorg/knuu/pkg/builder" + "github.com/celestiaorg/knuu/pkg/builder/docker" + "github.com/celestiaorg/knuu/pkg/builder/kaniko" +) + +// This is a temporary variable to hold the knuu instance until we refactor knuu pkg +// TODO: remove this temporary variable +var tmpKnuu *Knuu + +// Deprecated: Use the new package knuu instead. +// Initialize initializes knuu with a unique scope +func Initialize() error { + t := time.Now() + scope := fmt.Sprintf("%s-%03d", t.Format("20060102-150405"), t.Nanosecond()/1e6) + return InitializeWithScope(scope) +} + +// Deprecated: Use the new package knuu instead. +func Scope() string { + if tmpKnuu == nil { + return "" + } + return tmpKnuu.Scope() +} + +// Deprecated: Use the new package knuu instead. +// InitializeWithScope initializes knuu with a given scope +func InitializeWithScope(testScope string) error { + // Override scope if KNUU_NAMESPACE is set + namespaceEnv := os.Getenv("KNUU_NAMESPACE") + if namespaceEnv != "" { + testScope = namespaceEnv + logrus.Warnf("KNUU_NAMESPACE is deprecated. Scope overridden to: %s", testScope) + } + + logrus.Infof("Initializing knuu with scope: %s", testScope) + + // read timeout from env + var ( + timeoutString = os.Getenv("KNUU_TIMEOUT") + timeout = 60 * time.Minute + ) + if timeoutString != "" { + parsedTimeout, err := time.ParseDuration(timeoutString) + if err != nil { + return ErrCannotParseTimeout.Wrap(err) + } + timeout = parsedTimeout + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + var err error + tmpKnuu, err = New(ctx, + WithTestScope(testScope), + WithTimeout(timeout), + WithProxyEnabled(), + ) + if err != nil { + return ErrCannotInitializeKnuu.Wrap(err) + } + + builderType := os.Getenv("KNUU_BUILDER") + switch builderType { + case "kubernetes": + tmpKnuu.ImageBuilder = &kaniko.Kaniko{ + K8sClientset: tmpKnuu.K8sCli.Clientset(), + K8sNamespace: tmpKnuu.K8sCli.Namespace(), + Minio: tmpKnuu.MinioCli, + } + case "docker", "": + tmpKnuu.ImageBuilder = &docker.Docker{ + K8sClientset: tmpKnuu.K8sCli.Clientset(), + K8sNamespace: tmpKnuu.K8sCli.Namespace(), + } + default: + return ErrInvalidKnuuBuilder.WithParams(builderType) + } + + // TODO: this must be moved to somewhere more meaningful + tmpKnuu.HandleStopSignal() + return nil +} + +// Deprecated: Identifier is deprecated, use Scope() instead. +func Identifier() string { + logrus.Warn("Identifier() is deprecated, use Scope() instead.") + return Scope() +} + +// Deprecated: InitializeWithIdentifier is deprecated, use InitializeWithScope(scope string) instead. +func InitializeWithIdentifier(uniqueIdentifier string) error { + logrus.Warn("InitializeWithIdentifier is deprecated, use InitializeWithScope(scope string) instead.") + return InitializeWithScope(uniqueIdentifier) +} + +// Deprecated: Use the new package knuu instead. +func ImageBuilder() builder.Builder { + if tmpKnuu == nil { + return nil + } + return tmpKnuu.ImageBuilder +} + +// Deprecated: Use the new package knuu instead. +// IsInitialized returns true if knuu is initialized, and false otherwise +func IsInitialized() bool { + return tmpKnuu != nil +} + +// Deprecated: Use the new package knuu instead. +func CleanUp() error { + if tmpKnuu == nil { + return errors.New("tmpKnuu is not initialized") + } + return tmpKnuu.CleanUp(context.Background()) +} + +// Deprecated: Use the new package knuu instead. +func PushFileToMinio(ctx context.Context, contentName string, reader io.Reader) error { + return tmpKnuu.PushFileToMinio(ctx, contentName, reader) +} + +// Deprecated: Use the new package knuu instead. +func GetMinioURL(ctx context.Context, contentName string) (string, error) { + return tmpKnuu.GetMinioURL(ctx, contentName) +} diff --git a/pkg/knuu/knuu_test.go b/pkg/knuu/knuu_test.go new file mode 100644 index 0000000..e1232e4 --- /dev/null +++ b/pkg/knuu/knuu_test.go @@ -0,0 +1,148 @@ +package knuu + +import ( + "context" + "testing" + "time" + + "github.com/celestiaorg/knuu/pkg/builder/kaniko" + "github.com/celestiaorg/knuu/pkg/k8s" + "github.com/celestiaorg/knuu/pkg/minio" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + appv1 "k8s.io/api/apps/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + testTimeout = 5 * time.Minute +) + +type mockK8s struct { + k8s.KubeManager + mock.Mock +} + +func (m *mockK8s) Clientset() *kubernetes.Clientset { + return &kubernetes.Clientset{} +} + +func (m *mockK8s) Namespace() string { + return "test" +} + +func (m *mockK8s) CreateServiceAccount(ctx context.Context, name string, labels map[string]string) error { + return nil +} + +func (m *mockK8s) CreateRole(ctx context.Context, name string, labels map[string]string, policyRules []rbacv1.PolicyRule) error { + return nil +} + +func (m *mockK8s) CreateRoleBinding(ctx context.Context, name string, labels map[string]string, role, serviceAccount string) error { + return nil +} + +func (m *mockK8s) CreateReplicaSet(ctx context.Context, rsConfig k8s.ReplicaSetConfig, init bool) (*appv1.ReplicaSet, error) { + return &appv1.ReplicaSet{}, nil +} + +func (m *mockK8s) IsReplicaSetRunning(ctx context.Context, name string) (bool, error) { + return true, nil +} + +func TestNew(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + tt := []struct { + name string + options []Option + expectError bool + validateFunc func(*testing.T, *Knuu) + }{ + { + name: "Default initialization", + options: nil, + expectError: false, + validateFunc: func(t *testing.T, k *Knuu) { + assert.NotNil(t, k) + assert.NotNil(t, k.Logger) + assert.NotNil(t, k.K8sCli) + assert.NotNil(t, k.MinioCli) + assert.NotNil(t, k.ImageBuilder) + assert.Equal(t, defaultTimeout, k.timeout) + }, + }, + { + name: "With custom Logger", + options: []Option{ + WithLogger(&logrus.Logger{}), + }, + expectError: false, + validateFunc: func(t *testing.T, k *Knuu) { + assert.NotNil(t, k) + assert.NotNil(t, k.Logger) + }, + }, + { + name: "With custom Timeout", + options: []Option{ + WithTimeout(30 * time.Minute), + }, + expectError: false, + validateFunc: func(t *testing.T, k *Knuu) { + assert.NotNil(t, k) + assert.Equal(t, 30*time.Minute, k.timeout) + }, + }, + { + name: "With custom K8s client", + options: []Option{ + WithK8s(&mockK8s{}), + }, + expectError: false, + validateFunc: func(t *testing.T, k *Knuu) { + assert.NotNil(t, k) + assert.NotNil(t, k.K8sCli) + }, + }, + { + name: "With custom Minio client", + options: []Option{ + WithMinio(&minio.Minio{}), + }, + expectError: false, + validateFunc: func(t *testing.T, k *Knuu) { + assert.NotNil(t, k) + assert.NotNil(t, k.MinioCli) + }, + }, + { + name: "With custom Image Builder", + options: []Option{ + WithImageBuilder(&kaniko.Kaniko{}), + }, + expectError: false, + validateFunc: func(t *testing.T, k *Knuu) { + assert.NotNil(t, k) + assert.NotNil(t, k.ImageBuilder) + }, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + k, err := New(ctx, tc.options...) + if tc.expectError { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + tc.validateFunc(t, k) + }) + } +} diff --git a/pkg/knuu/minio.go b/pkg/knuu/minio.go index ad841b0..b91e1ec 100644 --- a/pkg/knuu/minio.go +++ b/pkg/knuu/minio.go @@ -3,40 +3,36 @@ package knuu import ( "context" "io" - - "github.com/celestiaorg/knuu/pkg/minio" ) const minioBucketName = "knuu" -var minioClient *minio.Minio - -func initMinio(ctx context.Context) error { - if minioClient == nil { +func (k *Knuu) initMinio(ctx context.Context) error { + if k.MinioCli == nil { return ErrMinioNotInitialized } - ok, err := minioClient.IsMinioDeployed(ctx) + ok, err := k.MinioCli.IsMinioDeployed(ctx) if err != nil { return err } if ok { return nil } - return minioClient.DeployMinio(ctx) + return k.MinioCli.DeployMinio(ctx) } // contentName is a unique string to identify the content in Minio -func PushFileToMinio(ctx context.Context, contentName string, reader io.Reader) error { - if err := initMinio(ctx); err != nil { +func (k *Knuu) PushFileToMinio(ctx context.Context, contentName string, reader io.Reader) error { + if err := k.initMinio(ctx); err != nil { return err } - return minioClient.PushToMinio(ctx, reader, contentName, minioBucketName) + return k.MinioCli.PushToMinio(ctx, reader, contentName, minioBucketName) } -func GetMinioURL(ctx context.Context, contentName string) (string, error) { - if err := initMinio(ctx); err != nil { +func (k *Knuu) GetMinioURL(ctx context.Context, contentName string) (string, error) { + if err := k.initMinio(ctx); err != nil { return "", err } - return minioClient.GetMinioURL(ctx, contentName, minioBucketName) + return k.MinioCli.GetMinioURL(ctx, contentName, minioBucketName) } diff --git a/pkg/knuu/preloader_old.go b/pkg/knuu/preloader_old.go new file mode 100644 index 0000000..953325a --- /dev/null +++ b/pkg/knuu/preloader_old.go @@ -0,0 +1,42 @@ +/* +* This file is deprecated. +* Please use the new package knuu instead. +* This file keeps the old functionality of knuu for backward compatibility. +* A global variable is defined, tmpKnuu, which is used to hold the knuu instance. + */ + +package knuu + +import ( + "context" + + "github.com/celestiaorg/knuu/pkg/preloader" +) + +type Preloader struct { + preloader.Preloader +} + +// Deprecated: Use the new package knuu instead. +func NewPreloader() (*Preloader, error) { + p, err := tmpKnuu.NewPreloader() + if err != nil { + return nil, err + } + return &Preloader{Preloader: *p}, nil +} + +// Deprecated: Use the new package knuu instead. +func (p *Preloader) AddImage(image string) error { + return p.Preloader.AddImage(context.Background(), image) +} + +// Deprecated: Use the new package knuu instead. +func (p *Preloader) RemoveImage(image string) error { + return p.Preloader.RemoveImage(context.Background(), image) +} + +// Deprecated: Use the new package knuu instead. +func (p *Preloader) EmptyImages() error { + return p.Preloader.EmptyImages(context.Background()) +} diff --git a/pkg/preloader/errors.go b/pkg/preloader/errors.go new file mode 100644 index 0000000..3f4ddf4 --- /dev/null +++ b/pkg/preloader/errors.go @@ -0,0 +1,38 @@ +package preloader + +import ( + "fmt" +) + +type Error struct { + Code string + Message string + Err error + Params []interface{} +} + +func (e *Error) Error() string { + if e.Err == e { + return e.Message + } + + msg := fmt.Sprintf(e.Message, e.Params...) + if e.Err != nil { + return fmt.Sprintf("%s: %v", msg, e.Err) + } + return msg +} + +func (e *Error) Wrap(err error) error { + e.Err = err + return e +} + +func (e *Error) WithParams(params ...interface{}) *Error { + e.Params = params + return e +} + +var ( + ErrGeneratingK8sNameForPreloader = &Error{Code: "GeneratingK8sNameForPreloader", Message: "error generating k8s name for preloader"} +) diff --git a/pkg/knuu/preloader.go b/pkg/preloader/preloader.go similarity index 59% rename from pkg/knuu/preloader.go rename to pkg/preloader/preloader.go index c69df04..78fe933 100644 --- a/pkg/knuu/preloader.go +++ b/pkg/preloader/preloader.go @@ -1,29 +1,42 @@ -package knuu +package preloader import ( "context" "fmt" + "github.com/celestiaorg/knuu/pkg/names" + "github.com/celestiaorg/knuu/pkg/system" v1 "k8s.io/api/core/v1" ) +const ( + preloaderName = "knuu-preloader" + managedByLabel = "knuu" + pauseContainerImage = "k8s.gcr.io/pause" + preloaderCommand = "/bin/sh" + preloaderCommandArgs = "-c" + preloaderCommandExit = "exit 0" +) + // Preloader is a struct that contains the list of preloaded images. // A preloader makes sure that the images are preloaded before the test suite starts. // Hint: If you use a Preloader per test suite, you can save resources type Preloader struct { K8sName string `json:"k8sName"` Images []string `json:"images"` + system.SystemDependencies } -// NewPreloader creates a new preloader -func NewPreloader() (*Preloader, error) { - k8sName, err := generateK8sName("knuu-preloader") +// New creates a new preloader +func New(sysDeps system.SystemDependencies) (*Preloader, error) { + k8sName, err := names.NewRandomK8(preloaderName) if err != nil { return nil, ErrGeneratingK8sNameForPreloader.Wrap(err) } return &Preloader{ - K8sName: k8sName, - Images: []string{}, + K8sName: k8sName, + Images: []string{}, + SystemDependencies: sysDeps, }, nil } @@ -33,8 +46,8 @@ func (p *Preloader) GetImages() []string { } // AddImage adds an image to the list of preloaded images -func (p *Preloader) AddImage(image string) error { - // dont add duplicates +func (p *Preloader) AddImage(ctx context.Context, image string) error { + // don't add duplicates for _, v := range p.Images { if v == image { return nil @@ -42,31 +55,23 @@ func (p *Preloader) AddImage(image string) error { } p.Images = append(p.Images, image) - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return p.preloadImages(ctx) } // RemoveImage removes an image from the list of preloaded images -func (p *Preloader) RemoveImage(image string) error { +func (p *Preloader) RemoveImage(ctx context.Context, image string) error { for i, v := range p.Images { if v == image { p.Images = append(p.Images[:i], p.Images[i+1:]...) } } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() return p.preloadImages(ctx) } // EmptyImages empties the list of preloaded images -func (p *Preloader) EmptyImages() error { +func (p *Preloader) EmptyImages(ctx context.Context) error { p.Images = []string{} - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() return p.preloadImages(ctx) } @@ -74,7 +79,7 @@ func (p *Preloader) EmptyImages() error { func (p *Preloader) preloadImages(ctx context.Context) error { // delete the daemonset if no images are preloaded if len(p.Images) == 0 { - return k8sClient.DeleteDaemonSet(ctx, p.K8sName) + return p.K8sCli.DeleteDaemonSet(ctx, p.K8sName) } var initContainers []v1.Container @@ -83,9 +88,9 @@ func (p *Preloader) preloadImages(ctx context.Context) error { Name: fmt.Sprintf("image%d-preloader", i), Image: image, Command: []string{ - "/bin/sh", - "-c", - "exit 0", + preloaderCommand, + preloaderCommandArgs, + preloaderCommandExit, }, }) } @@ -94,28 +99,28 @@ func (p *Preloader) preloadImages(ctx context.Context) error { containers = append(containers, v1.Container{ Name: "pause-container", - Image: "k8s.gcr.io/pause", + Image: pauseContainerImage, }) labels := map[string]string{ "app": p.K8sName, - "k8s.kubernetes.io/managed-by": "knuu", - "knuu.sh/scope": testScope, - "knuu.sh/test-started": startTime, + "k8s.kubernetes.io/managed-by": managedByLabel, + "knuu.sh/scope": p.TestScope, + "knuu.sh/test-started": p.StartTime, } - exists, err := k8sClient.DaemonSetExists(ctx, p.K8sName) + exists, err := p.K8sCli.DaemonSetExists(ctx, p.K8sName) if err != nil { return err } // update the daemonset if it already exists if exists { - _, err = k8sClient.UpdateDaemonSet(ctx, p.K8sName, labels, initContainers, containers) + _, err = p.K8sCli.UpdateDaemonSet(ctx, p.K8sName, labels, initContainers, containers) return err } // create the daemonset if it doesn't exist - _, err = k8sClient.CreateDaemonSet(ctx, p.K8sName, labels, initContainers, containers) + _, err = p.K8sCli.CreateDaemonSet(ctx, p.K8sName, labels, initContainers, containers) return err } diff --git a/pkg/system/dependencies.go b/pkg/system/dependencies.go new file mode 100644 index 0000000..0b28f73 --- /dev/null +++ b/pkg/system/dependencies.go @@ -0,0 +1,19 @@ +package system + +import ( + "github.com/celestiaorg/knuu/pkg/builder" + "github.com/celestiaorg/knuu/pkg/k8s" + "github.com/celestiaorg/knuu/pkg/minio" + "github.com/celestiaorg/knuu/pkg/traefik" + "github.com/sirupsen/logrus" +) + +type SystemDependencies struct { + ImageBuilder builder.Builder + K8sCli k8s.KubeManager + MinioCli *minio.Minio + Logger *logrus.Logger + Proxy *traefik.Traefik + TestScope string + StartTime string +} diff --git a/pkg/traefik/traefik.go b/pkg/traefik/traefik.go index 7b5e98c..55da22b 100644 --- a/pkg/traefik/traefik.go +++ b/pkg/traefik/traefik.go @@ -20,17 +20,18 @@ import ( ) const ( - traefikServiceName = "traefik" - Port = 80 - PortSecure = 443 - deploymentName = "traefik-deployment" - roleName = "traefik-role" - containerName = "traefik" - image = "traefik:v3.0" - appLabel = "app" - appLabelValue = "traefik" - replicas = 1 - waitRetry = 5 * time.Second + traefikServiceName = "traefik" + traefikAPIGroupVersion = "traefik.io/v1alpha1" + Port = 80 + PortSecure = 443 + deploymentName = "traefik-deployment" + roleName = "traefik-role" + containerName = "traefik" + image = "traefik:v3.0" + appLabel = "app" + appLabelValue = "traefik" + replicas = 1 + waitRetry = 5 * time.Second defaultCPURequest = "500m" defaultMemoryRequest = "500Mi" @@ -39,7 +40,7 @@ const ( ) type Traefik struct { - K8s *k8s.Client + K8s k8s.KubeManager endpoint string } @@ -341,3 +342,30 @@ func (t *Traefik) createIngressRoute( return nil } + +// IsTraefikAPIAvailable checks if the Traefik API is available in the cluster. +func (t *Traefik) IsTraefikAPIAvailable(ctx context.Context) bool { + apiResourceList, err := t.K8s.Clientset().Discovery().ServerResourcesForGroupVersion(traefikAPIGroupVersion) + if err != nil { + logrus.Errorf("Failed to discover Traefik API resources: %v", err) + return false + } + + requiredResources := []string{"middlewares", "ingressroutes"} + + for _, resource := range apiResourceList.APIResources { + for i, requiredResource := range requiredResources { + if resource.Name == requiredResource { + requiredResources = append(requiredResources[:i], requiredResources[i+1:]...) + break + } + } + } + + if len(requiredResources) == 0 { + return true + } + + logrus.Warnf("Missing Traefik API resources: %v", requiredResources) + return false +}