Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: add ability to pass down storage class to apic/engine #1928

Closed
wants to merge 10 commits into from
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func getSuppliers(clusterId string, clusterType KurtosisClusterType, kubernetesC
}

backendSupplier = func(ctx context.Context) (backend_interface.KurtosisBackend, error) {
backend, err := kubernetes_kurtosis_backend.GetCLIBackend(ctx)
backend, err := kubernetes_kurtosis_backend.GetCLIBackend(ctx, *kubernetesConfig.StorageClass)
if err != nil {
return nil, stacktrace.Propagate(
err,
Expand Down
4 changes: 3 additions & 1 deletion cli/cli/kurtosis_gateway/connection/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
const (
grpcPortIdStr = "grpc"
httpApplicationProtocol = "http"
// this doesn't have any effect as this is just the gateway
emptyStorageClassName = ""
)

var noWait *port_spec.Wait = nil
Expand All @@ -42,7 +44,7 @@ func NewGatewayConnectionProvider(ctx context.Context, kubernetesConfig *restcli
if err != nil {
return nil, stacktrace.Propagate(err, "Expected to be able to get config for Kubernetes client set, instead a non nil error was returned")
}
kubernetesManager := kubernetes_manager.NewKubernetesManager(clientSet, kubernetesConfig)
kubernetesManager := kubernetes_manager.NewKubernetesManager(clientSet, kubernetesConfig, emptyStorageClassName)

return &GatewayConnectionProvider{
config: kubernetesConfig,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,9 @@ func NewAPIContainerKubernetesKurtosisBackend(
kubernetesManager *kubernetes_manager.KubernetesManager,
ownEnclaveUuid enclave.EnclaveUUID,
ownNamespaceName string,
storageClassName string,
) *KubernetesKurtosisBackend {
modeArgs := shared_helpers.NewApiContainerModeArgs(ownEnclaveUuid, ownNamespaceName)
modeArgs := shared_helpers.NewApiContainerModeArgs(ownEnclaveUuid, ownNamespaceName, storageClassName)
return newKubernetesKurtosisBackend(
kubernetesManager,
nil,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ var kubeConfigFileFilepath = filepath.Join(
os.Getenv("HOME"), ".kube", "config",
)

func GetCLIBackend(ctx context.Context) (backend_interface.KurtosisBackend, error) {
func GetCLIBackend(ctx context.Context, storageClass string) (backend_interface.KurtosisBackend, error) {
kubernetesConfig, err := clientcmd.BuildConfigFromFlags(emptyMasterURL, kubeConfigFileFilepath)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred creating kubernetes configuration from flags in file '%v'", kubeConfigFileFilepath)
Expand All @@ -35,6 +35,7 @@ func GetCLIBackend(ctx context.Context) (backend_interface.KurtosisBackend, erro
ctx,
kubernetesConfig,
backendSupplier,
storageClass,
)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred wrapping the CLI Kubernetes backend")
Expand All @@ -44,7 +45,7 @@ func GetCLIBackend(ctx context.Context) (backend_interface.KurtosisBackend, erro
}

func GetEngineServerBackend(
ctx context.Context,
ctx context.Context, storageClass string,
) (backend_interface.KurtosisBackend, error) {
kubernetesConfig, err := rest.InClusterConfig()
if err != nil {
Expand All @@ -61,6 +62,7 @@ func GetEngineServerBackend(
ctx,
kubernetesConfig,
backendSupplier,
storageClass,
)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred wrapping the Kurtosis Engine Kubernetes backend")
Expand All @@ -71,6 +73,7 @@ func GetEngineServerBackend(

func GetApiContainerBackend(
ctx context.Context,
storageClass string,
) (backend_interface.KurtosisBackend, error) {
kubernetesConfig, err := rest.InClusterConfig()
if err != nil {
Expand Down Expand Up @@ -103,13 +106,15 @@ func GetApiContainerBackend(
kubernetesManager,
enclaveId,
namespaceName,
storageClass,
), nil
}

wrappedBackend, err := getWrappedKubernetesKurtosisBackend(
ctx,
kubernetesConfig,
backendSupplier,
storageClass,
)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred wrapping the APIC Kubernetes backend")
Expand All @@ -127,13 +132,14 @@ func getWrappedKubernetesKurtosisBackend(
ctx context.Context,
kubernetesConfig *rest.Config,
kurtosisBackendSupplier func(context.Context, *kubernetes_manager.KubernetesManager) (*KubernetesKurtosisBackend, error),
storageClass string,
) (*metrics_reporting.MetricsReportingKurtosisBackend, error) {
clientSet, err := kubernetes.NewForConfig(kubernetesConfig)
if err != nil {
return nil, stacktrace.Propagate(err, "Expected to be able to create kubernetes client set using Kubernetes config '%+v', instead a non nil error was returned", kubernetesConfig)
}

kubernetesManager := kubernetes_manager.NewKubernetesManager(clientSet, kubernetesConfig)
kubernetesManager := kubernetes_manager.NewKubernetesManager(clientSet, kubernetesConfig, storageClass)

kubernetesBackend, err := kurtosisBackendSupplier(ctx, kubernetesManager)
if err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,11 @@ type dumpPodResult struct {

func NewApiContainerModeArgs(
ownEnclaveId enclave.EnclaveUUID,
ownNamespaceName string) *ApiContainerModeArgs {
ownNamespaceName string, storageClassName string) *ApiContainerModeArgs {
return &ApiContainerModeArgs{
ownEnclaveId: ownEnclaveId,
ownNamespaceName: ownNamespaceName,
storageClassName: "",
storageClassName: storageClassName,
filesArtifactExpansionVolumeSizeInMegabytes: 0,
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,14 +145,17 @@ type KubernetesManager struct {
kubernetesClientSet *kubernetes.Clientset
// Underlying restClient configuration
kuberneteRestConfig *rest.Config
// The storage class name as specified in the `kurtosis-config.yaml`
storageClass string
}

func int64Ptr(i int64) *int64 { return &i }

func NewKubernetesManager(kubernetesClientSet *kubernetes.Clientset, kuberneteRestConfig *rest.Config) *KubernetesManager {
func NewKubernetesManager(kubernetesClientSet *kubernetes.Clientset, kuberneteRestConfig *rest.Config, storageClass string) *KubernetesManager {
return &KubernetesManager{
kubernetesClientSet: kubernetesClientSet,
kuberneteRestConfig: kuberneteRestConfig,
storageClass: storageClass,
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,15 @@ import (
)

type KubernetesBackendConfigSupplier struct {
storageClass string
}

func NewKubernetesKurtosisBackendConfigSupplier() KubernetesBackendConfigSupplier {
func NewKubernetesKurtosisBackendConfigSupplier(storageClass string) KubernetesBackendConfigSupplier {
return KubernetesBackendConfigSupplier{
// More fields here when needed
storageClass: storageClass,
}
}

func (backendConfigSupplier KubernetesBackendConfigSupplier) getKurtosisBackendConfig() (args.KurtosisBackendType, interface{}) {
return args.KurtosisBackendType_Kubernetes, kurtosis_backend_config.KubernetesBackendConfig{}
return args.KurtosisBackendType_Kubernetes, kurtosis_backend_config.KubernetesBackendConfig{StorageClass: backendConfigSupplier.storageClass}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@
package kurtosis_backend_config

type KubernetesBackendConfig struct {
// More things here as needed
StorageClass string
}
4 changes: 2 additions & 2 deletions core/server/api_container/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ func runMain() error {
}
case args.KurtosisBackendType_Kubernetes:
// TODO Use this value when we have fields for the API container
_, ok := (clusterConfig).(kurtosis_backend_config.KubernetesBackendConfig)
clusterConfigK8s, ok := (clusterConfig).(kurtosis_backend_config.KubernetesBackendConfig)
if !ok {
return stacktrace.NewError(
"Failed to cast untyped cluster configuration object '%+v' to the appropriate type, even though "+
Expand All @@ -155,7 +155,7 @@ func runMain() error {
args.KurtosisBackendType_Kubernetes.String(),
)
}
kurtosisBackend, err = kubernetes_kurtosis_backend.GetApiContainerBackend(ctx)
kurtosisBackend, err = kubernetes_kurtosis_backend.GetApiContainerBackend(ctx, clusterConfigK8s.StorageClass)
if err != nil {
return stacktrace.Propagate(
err,
Expand Down
7 changes: 6 additions & 1 deletion docs/docs/cli-reference/cluster-get.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,9 @@ kurtosis-clusters:
kubernetes-cluster-name: "NAME-OF-YOUR-CLUSTER"
storage-class: "standard"
enclave-size-in-megabytes: 10
```
```

:::tip Storage Class
The Storage Class specified in the configuration above will be used for spinning up persistent volumes. Make sure you have the right
value in case you are using persistent directories.
:::
7 changes: 6 additions & 1 deletion docs/docs/cli-reference/cluster-ls.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,9 @@ kurtosis-clusters:
kubernetes-cluster-name: "NAME-OF-YOUR-CLUSTER"
storage-class: "standard"
enclave-size-in-megabytes: 10
```
```

:::tip Storage Class
The Storage Class specified in the configuration above will be used for spinning up persistent volumes. Make sure you have the right
value in case you are using persistent directories.
:::
7 changes: 6 additions & 1 deletion docs/docs/cli-reference/cluster-set.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,9 @@ kurtosis-clusters:
kubernetes-cluster-name: "NAME-OF-YOUR-CLUSTER"
storage-class: "standard"
enclave-size-in-megabytes: 10
```
```

:::tip Storage Class
The Storage Class specified in the configuration above will be used for spinning up persistent volumes. Make sure you have the right
value in case you are using persistent directories.
:::
5 changes: 5 additions & 0 deletions docs/docs/guides/running-in-k8s.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ kurtosis-clusters:
enclave-size-in-megabytes: 10
```

:::tip Storage Class
The Storage Class specified in the configuration above will be used for spinning up persistent volumes. Make sure you have the right
value in case you are using persistent directories.
:::

IV. Configure Kurtosis
--------------------------------

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@
package kurtosis_backend_config

type KubernetesBackendConfig struct {
StorageClass string
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,5 @@ func NewKubernetesKurtosisBackendConfigSupplier(storageClass string, enclaveSize
}

func (backendConfigSupplier KubernetesBackendConfigSupplier) getKurtosisBackendConfig() (args.KurtosisBackendType, interface{}) {
return args.KurtosisBackendType_Kubernetes, kurtosis_backend_config.KubernetesBackendConfig{}
return args.KurtosisBackendType_Kubernetes, kurtosis_backend_config.KubernetesBackendConfig{StorageClass: backendConfigSupplier.storageClass}
}
13 changes: 9 additions & 4 deletions engine/server/engine/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ func runMain() error {
logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, osFs, realTime)
logFileManager.StartLogFileManagement(ctx)

enclaveManager, err := getEnclaveManager(kurtosisBackend, serverArgs.KurtosisBackendType, serverArgs.ImageVersionTag, serverArgs.PoolSize, serverArgs.EnclaveEnvVars, logFileManager, serverArgs.MetricsUserID, serverArgs.DidUserAcceptSendingMetrics, serverArgs.IsCI, serverArgs.CloudUserID, serverArgs.CloudInstanceID)
enclaveManager, err := getEnclaveManager(kurtosisBackend, serverArgs.KurtosisBackendType, serverArgs.ImageVersionTag, serverArgs.PoolSize, serverArgs.EnclaveEnvVars, logFileManager, serverArgs.MetricsUserID, serverArgs.DidUserAcceptSendingMetrics, serverArgs.IsCI, serverArgs.CloudUserID, serverArgs.CloudInstanceID, serverArgs.KurtosisLocalBackendConfig)
if err != nil {
return stacktrace.Propagate(err, "Failed to create an enclave manager for backend type '%v' and config '%+v'", serverArgs.KurtosisBackendType, backendConfig)
}
Expand Down Expand Up @@ -299,13 +299,18 @@ func getEnclaveManager(
isCI bool,
cloudUserId metrics_client.CloudUserID,
cloudInstanceId metrics_client.CloudInstanceID,
kurtosisLocalBackendConfig interface{},
) (*enclave_manager.EnclaveManager, error) {
var apiContainerKurtosisBackendConfigSupplier api_container_launcher.KurtosisBackendConfigSupplier
switch kurtosisBackendType {
case args.KurtosisBackendType_Docker:
apiContainerKurtosisBackendConfigSupplier = api_container_launcher.NewDockerKurtosisBackendConfigSupplier()
case args.KurtosisBackendType_Kubernetes:
apiContainerKurtosisBackendConfigSupplier = api_container_launcher.NewKubernetesKurtosisBackendConfigSupplier()
kurtosisLocalBackendConfigKubernetesType, ok := kurtosisLocalBackendConfig.(kurtosis_backend_config.KubernetesBackendConfig)
if !ok {
return nil, stacktrace.NewError("Failed to cast cluster configuration interface to the appropriate type, even though Kurtosis backend type is '%v'", args.KurtosisBackendType_Kubernetes.String())
}
apiContainerKurtosisBackendConfigSupplier = api_container_launcher.NewKubernetesKurtosisBackendConfigSupplier(kurtosisLocalBackendConfigKubernetesType.StorageClass)
default:
return nil, stacktrace.NewError("Backend type '%v' was not recognized by engine server.", kurtosisBackendType.String())
}
Expand Down Expand Up @@ -347,11 +352,11 @@ func getKurtosisBackend(ctx context.Context, kurtosisBackendType args.KurtosisBa
"connect to a remote Kurtosis backend")
}
// Use this with more properties
_, ok := (backendConfig).(kurtosis_backend_config.KubernetesBackendConfig)
clusterConfigK8s, ok := (backendConfig).(kurtosis_backend_config.KubernetesBackendConfig)
if !ok {
return nil, stacktrace.NewError("Failed to cast cluster configuration interface to the appropriate type, even though Kurtosis backend type is '%v'", args.KurtosisBackendType_Kubernetes.String())
}
kurtosisBackend, err = kubernetes_kurtosis_backend.GetEngineServerBackend(ctx)
kurtosisBackend, err = kubernetes_kurtosis_backend.GetEngineServerBackend(ctx, clusterConfigK8s.StorageClass)
if err != nil {
return nil, stacktrace.Propagate(
err,
Expand Down
Loading