diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7c9afafe..68610c7ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,6 @@ repos: rev: v5.0.0 hooks: - id: check-executables-have-shebangs - - id: check-yaml - id: double-quote-string-fixer - id: end-of-file-fixer - id: trailing-whitespace diff --git a/Makefile b/Makefile index 6f926028f..7eb205287 100644 --- a/Makefile +++ b/Makefile @@ -72,8 +72,8 @@ generate: controller-gen SRC_ROOT = $(shell git rev-parse --show-toplevel) helm-controller-version: - $(eval VERSION := $(shell grep 'appVersion:' charts/capsule/Chart.yaml | awk '{print "v"$$2}')) - $(eval KO_TAGS := $(shell grep 'appVersion:' charts/capsule/Chart.yaml | awk '{print "v"$$2}')) + $(eval VERSION := $(shell grep 'appVersion:' charts/capsule/Chart.yaml | awk '{print $$2}')) + $(eval KO_TAGS := $(shell grep 'appVersion:' charts/capsule/Chart.yaml | awk '{print $$2}')) helm-docs: helm-doc $(HELM_DOCS) --chart-search-root ./charts diff --git a/PROJECT b/PROJECT index bd679d02b..494138390 100644 --- a/PROJECT +++ b/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: clastix.io layout: - go.kubebuilder.io/v3 @@ -44,4 +48,11 @@ resources: kind: GlobalTenantResource path: github.com/projectcapsule/capsule/api/v1beta2 version: v1beta2 +- api: + crdVersion: v1 + domain: clastix.io + group: capsule + kind: GlobalResourceQuota + path: github.com/projectcapsule/capsule/api/v1beta2 + version: v1beta2 version: "3" diff --git a/api/v1beta2/globalresourcequota_func.go b/api/v1beta2/globalresourcequota_func.go new file mode 100644 index 000000000..6b8037137 --- /dev/null +++ b/api/v1beta2/globalresourcequota_func.go @@ -0,0 +1,112 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + "fmt" + "sort" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/projectcapsule/capsule/pkg/api" +) + +func (g *GlobalResourceQuota) GetQuotaSpace(index api.Name) (corev1.ResourceList, error) { + quotaSpace := corev1.ResourceList{} + + // First, check if quota exists in the status + if quotaStatus, exists := g.Status.Quota[index]; exists { + // Iterate over all resources in the status + for resourceName, hardValue := range quotaStatus.Hard { + usedValue, usedExists := quotaStatus.Used[resourceName] + if !usedExists { + usedValue = resource.MustParse("0") // Default to zero if no used value is found + } + + // Compute remaining quota (hard - used) + remaining := hardValue.DeepCopy() + remaining.Sub(usedValue) + + // Ensure we don't set negative values + if remaining.Sign() == -1 { + remaining.Set(0) + } + + quotaSpace[resourceName] = remaining + } + + return quotaSpace, nil + } + + // If not in status, fall back to spec.Hard + if quotaSpec, exists := g.Spec.Items[index]; exists { + for resourceName, hardValue := range quotaSpec.Hard { + quotaSpace[resourceName] = hardValue.DeepCopy() + } + + return quotaSpace, nil + } + + return nil, fmt.Errorf("no item found") +} + +func (g *GlobalResourceQuota) GetAggregatedQuotaSpace(index api.Name, used corev1.ResourceList) (corev1.ResourceList, error) { + quotaSpace := corev1.ResourceList{} + + // First, check if quota exists in the status + if quotaStatus, exists := g.Status.Quota[index]; exists { + // Iterate over all resources in the status + for resourceName, hardValue := range quotaStatus.Hard { + usedValue, usedExists := quotaStatus.Used[resourceName] + if !usedExists { + usedValue = resource.MustParse("0") // Default to zero if no used value is found + } + + // Compute remaining quota (hard - used) + remaining := hardValue.DeepCopy() + remaining.Sub(usedValue) + + // Ensure we don't set negative values + if remaining.Sign() == -1 { + remaining.Set(0) + } + + /// Add the remaining Quota with the used quota + if currentUsed, exists := used[resourceName]; exists { + remaining.Add(currentUsed) + } + + quotaSpace[resourceName] = remaining + } + + return quotaSpace, nil + } + + // If not in status, fall back to spec.Hard + if quotaSpec, exists := g.Spec.Items[index]; exists { + for resourceName, hardValue := range quotaSpec.Hard { + quotaSpace[resourceName] = hardValue.DeepCopy() + } + + return quotaSpace, nil + } + + return nil, fmt.Errorf("no item found") +} + +func (in *GlobalResourceQuota) AssignNamespaces(namespaces []corev1.Namespace) { + var l []string + + for _, ns := range namespaces { + if ns.Status.Phase == corev1.NamespaceActive { + l = append(l, ns.GetName()) + } + } + + sort.Strings(l) + + in.Status.Namespaces = l + in.Status.Size = uint(len(l)) +} diff --git a/api/v1beta2/globalresourcequota_func_test.go b/api/v1beta2/globalresourcequota_func_test.go new file mode 100644 index 000000000..fdf2e0f26 --- /dev/null +++ b/api/v1beta2/globalresourcequota_func_test.go @@ -0,0 +1,148 @@ +package v1beta2_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("GlobalResourceQuota", func() { + + Context("GetQuotaSpace", func() { + var grq *capsulev1beta2.GlobalResourceQuota + + BeforeEach(func() { + grq = &capsulev1beta2.GlobalResourceQuota{ + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + Items: map[api.Name]corev1.ResourceQuotaSpec{ + "compute": { + Hard: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceMemory: resource.MustParse("16Gi"), + }, + }, + }, + }, + Status: capsulev1beta2.GlobalResourceQuotaStatus{ + Quota: map[api.Name]*corev1.ResourceQuotaStatus{ + "compute": { + Hard: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10"), + corev1.ResourceMemory: resource.MustParse("32Gi"), + }, + Used: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + corev1.ResourceMemory: resource.MustParse("10Gi"), + }, + }, + }, + }, + } + }) + + It("should calculate available quota correctly when status exists", func() { + expected := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("6"), // 10 - 4 + corev1.ResourceMemory: resource.MustParse("22Gi"), // 32Gi - 10Gi + } + + quotaSpace, _ := grq.GetQuotaSpace("compute") + Expect(quotaSpace).To(Equal(expected)) + }) + + It("should return spec quota if status does not exist", func() { + expected := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceMemory: resource.MustParse("16Gi"), + } + + quotaSpace, _ := grq.GetQuotaSpace("network") // "network" is not in Status + Expect(quotaSpace).To(Equal(expected)) + }) + + It("should handle cases where used quota is missing (default to 0)", func() { + grq.Status.Quota["compute"].Used = nil + + expected := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10"), // 10 - 0 + corev1.ResourceMemory: resource.MustParse("32Gi"), // 32Gi - 0 + } + + quotaSpace, _ := grq.GetQuotaSpace("compute") + Expect(quotaSpace).To(Equal(expected)) + }) + + It("should return 0 quota if used exceeds hard limit", func() { + grq.Status.Quota["compute"].Used = corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("12"), + corev1.ResourceMemory: resource.MustParse("40Gi"), + } + + expected := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0"), // Hard 10, Used 12 → should be 0 + corev1.ResourceMemory: resource.MustParse("0"), // Hard 32, Used 40 → should be 0 + } + + quotaSpace, _ := grq.GetQuotaSpace("compute") + Expect(quotaSpace).To(Equal(expected)) + }) + }) + + Context("AssignNamespaces", func() { + var grq *capsulev1beta2.GlobalResourceQuota + + BeforeEach(func() { + grq = &capsulev1beta2.GlobalResourceQuota{} + }) + + It("should assign only active namespaces and update status", func() { + namespaces := []corev1.Namespace{ + {ObjectMeta: metav1.ObjectMeta{Name: "dev"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "staging"}, Status: corev1.NamespaceStatus{Phase: corev1.NamespaceActive}}, + {ObjectMeta: metav1.ObjectMeta{Name: "prod"}, Status: corev1.NamespaceStatus{Phase: corev1.NamespaceActive}}, + } + + grq.AssignNamespaces(namespaces) + + Expect(grq.Status.Namespaces).To(Equal([]string{"prod", "staging"})) // Sorted order + Expect(grq.Status.Size).To(Equal(uint(2))) + }) + + It("should handle empty namespace list", func() { + grq.AssignNamespaces([]corev1.Namespace{}) + + Expect(grq.Status.Namespaces).To(BeEmpty()) + Expect(grq.Status.Size).To(Equal(uint(0))) + }) + + It("should ignore inactive namespaces", func() { + namespaces := []corev1.Namespace{ + {ObjectMeta: metav1.ObjectMeta{Name: "inactive"}, Status: corev1.NamespaceStatus{Phase: corev1.NamespaceTerminating}}, + {ObjectMeta: metav1.ObjectMeta{Name: "active"}, Status: corev1.NamespaceStatus{Phase: corev1.NamespaceActive}}, + } + + grq.AssignNamespaces(namespaces) + + Expect(grq.Status.Namespaces).To(Equal([]string{"active"})) // Only active namespaces are assigned + Expect(grq.Status.Size).To(Equal(uint(1))) + }) + + It("should sort namespaces alphabetically", func() { + namespaces := []corev1.Namespace{ + {ObjectMeta: metav1.ObjectMeta{Name: "zeta"}, Status: corev1.NamespaceStatus{Phase: corev1.NamespaceActive}}, + {ObjectMeta: metav1.ObjectMeta{Name: "alpha"}, Status: corev1.NamespaceStatus{Phase: corev1.NamespaceActive}}, + {ObjectMeta: metav1.ObjectMeta{Name: "beta"}, Status: corev1.NamespaceStatus{Phase: corev1.NamespaceActive}}, + } + + grq.AssignNamespaces(namespaces) + + Expect(grq.Status.Namespaces).To(Equal([]string{"alpha", "beta", "zeta"})) + Expect(grq.Status.Size).To(Equal(uint(3))) + }) + }) +}) diff --git a/api/v1beta2/globalresourcequota_status.go b/api/v1beta2/globalresourcequota_status.go new file mode 100644 index 000000000..d56556cfd --- /dev/null +++ b/api/v1beta2/globalresourcequota_status.go @@ -0,0 +1,25 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + "github.com/projectcapsule/capsule/pkg/api" + corev1 "k8s.io/api/core/v1" +) + +// GlobalResourceQuotaStatus defines the observed state of GlobalResourceQuota +type GlobalResourceQuotaStatus struct { + // If this quota is active or not. + // +kubebuilder:default=true + Active bool `json:"active"` + // How many namespaces are assigned to the Tenant. + // +kubebuilder:default=0 + Size uint `json:"size"` + // List of namespaces assigned to the Tenant. + Namespaces []string `json:"namespaces,omitempty"` + // Tracks the quotas for the Resource. + Quota GlobalResourceQuotaStatusQuota `json:"quotas,omitempty"` +} + +type GlobalResourceQuotaStatusQuota map[api.Name]*corev1.ResourceQuotaStatus diff --git a/api/v1beta2/globalresourcequota_types.go b/api/v1beta2/globalresourcequota_types.go new file mode 100644 index 000000000..fe0f97748 --- /dev/null +++ b/api/v1beta2/globalresourcequota_types.go @@ -0,0 +1,64 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package v1beta2 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/projectcapsule/capsule/pkg/api" +) + +// GlobalResourceQuotaSpec defines the desired state of GlobalResourceQuota +type GlobalResourceQuotaSpec struct { + // When a quota is active it's checking for the resources in the cluster + // If not active the resourcequotas are removed and the webhook no longer blocks updates + // +kubebuilder:default=true + Active bool `json:"active,omitempty"` + + // Selector to match the namespaces that should be managed by the GlobalResourceQuota + Selectors []GlobalResourceQuotaSelector `json:"selectors,omitempty"` + + // Define resourcequotas for the namespaces + Items map[api.Name]corev1.ResourceQuotaSpec `json:"quotas,omitempty"` +} + +type GlobalResourceQuotaSelector struct { + // Only considers namespaces which are part of a tenant, other namespaces which might match + // the label, but do not have a tenant, are ignored. + // +kubebuilder:default=true + MustTenantNamespace bool `json:"tenant,omitempty"` + + // Selector to match the namespaces that should be managed by the GlobalResourceQuota + api.NamespaceSelector `json:",inline"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=globalquota +// +kubebuilder:printcolumn:name="Active",type="boolean",JSONPath=".status.active",description="Active status of the GlobalResourceQuota" +// +kubebuilder:printcolumn:name="Namespaces",type="integer",JSONPath=".status.size",description="The total amount of Namespaces spanned across" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Age" + +// GlobalResourceQuota is the Schema for the globalresourcequotas API +type GlobalResourceQuota struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GlobalResourceQuotaSpec `json:"spec,omitempty"` + Status GlobalResourceQuotaStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// GlobalResourceQuotaList contains a list of GlobalResourceQuota +type GlobalResourceQuotaList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GlobalResourceQuota `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GlobalResourceQuota{}, &GlobalResourceQuotaList{}) +} diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 79ab4abaa..931ce2438 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -9,6 +9,7 @@ package v1beta2 import ( "github.com/projectcapsule/capsule/pkg/api" + corev1 "k8s.io/api/core/v1" "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -154,6 +155,176 @@ func (in *CapsuleResources) DeepCopy() *CapsuleResources { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuota) DeepCopyInto(out *GlobalResourceQuota) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuota. +func (in *GlobalResourceQuota) DeepCopy() *GlobalResourceQuota { + if in == nil { + return nil + } + out := new(GlobalResourceQuota) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GlobalResourceQuota) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuotaList) DeepCopyInto(out *GlobalResourceQuotaList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GlobalResourceQuota, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaList. +func (in *GlobalResourceQuotaList) DeepCopy() *GlobalResourceQuotaList { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GlobalResourceQuotaList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuotaSelector) DeepCopyInto(out *GlobalResourceQuotaSelector) { + *out = *in + in.NamespaceSelector.DeepCopyInto(&out.NamespaceSelector) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaSelector. +func (in *GlobalResourceQuotaSelector) DeepCopy() *GlobalResourceQuotaSelector { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuotaSpec) DeepCopyInto(out *GlobalResourceQuotaSpec) { + *out = *in + if in.Selectors != nil { + in, out := &in.Selectors, &out.Selectors + *out = make([]GlobalResourceQuotaSelector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make(map[api.Name]corev1.ResourceQuotaSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaSpec. +func (in *GlobalResourceQuotaSpec) DeepCopy() *GlobalResourceQuotaSpec { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalResourceQuotaStatus) DeepCopyInto(out *GlobalResourceQuotaStatus) { + *out = *in + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Quota != nil { + in, out := &in.Quota, &out.Quota + *out = make(GlobalResourceQuotaStatusQuota, len(*in)) + for key, val := range *in { + var outVal *corev1.ResourceQuotaStatus + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = new(corev1.ResourceQuotaStatus) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaStatus. +func (in *GlobalResourceQuotaStatus) DeepCopy() *GlobalResourceQuotaStatus { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in GlobalResourceQuotaStatusQuota) DeepCopyInto(out *GlobalResourceQuotaStatusQuota) { + { + in := &in + *out = make(GlobalResourceQuotaStatusQuota, len(*in)) + for key, val := range *in { + var outVal *corev1.ResourceQuotaStatus + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = new(corev1.ResourceQuotaStatus) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalResourceQuotaStatusQuota. +func (in GlobalResourceQuotaStatusQuota) DeepCopy() GlobalResourceQuotaStatusQuota { + if in == nil { + return nil + } + out := new(GlobalResourceQuotaStatusQuota) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GlobalTenantResource) DeepCopyInto(out *GlobalTenantResource) { *out = *in diff --git a/busy-deploy.yaml b/busy-deploy.yaml new file mode 100644 index 000000000..30c473875 --- /dev/null +++ b/busy-deploy.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: busybox-deployment + labels: + app: busybox +spec: + replicas: 1 + selector: + matchLabels: + app: busybox + template: + metadata: + labels: + app: busybox + spec: + containers: + - name: busybox + image: alpine:latest + resources: + limits: + cpu: 0.3 + memory: 128Mi + requests: + cpu: 0.25 + memory: 128Mi + command: + - sh + - -c + - | + apk add busybox-extras && sleep infinity diff --git a/busy-pod.yaml b/busy-pod.yaml new file mode 100644 index 000000000..ff315413d --- /dev/null +++ b/busy-pod.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: busbox +spec: + containers: + - name: busybox + image: alpine:latest + resources: + limits: + cpu: 0.5 + memory: 128Mi + requests: + cpu: 0.25 + memory: 128Mi + command: + - sh + - -c + - | + apk add busybox-extras && sleep infinity diff --git a/charts/capsule/README.md b/charts/capsule/README.md index 5a9a2d330..ac576cffe 100644 --- a/charts/capsule/README.md +++ b/charts/capsule/README.md @@ -221,6 +221,9 @@ Here the values you can override: | webhooks.hooks.pods.failurePolicy | string | `"Fail"` | | | webhooks.hooks.pods.namespaceSelector.matchExpressions[0].key | string | `"capsule.clastix.io/tenant"` | | | webhooks.hooks.pods.namespaceSelector.matchExpressions[0].operator | string | `"Exists"` | | +| webhooks.hooks.quotas.failurePolicy | string | `"Fail"` | | +| webhooks.hooks.quotas.namespaceSelector.matchExpressions[0].key | string | `"capsule.clastix.io/tenant"` | | +| webhooks.hooks.quotas.namespaceSelector.matchExpressions[0].operator | string | `"Exists"` | | | webhooks.hooks.services.failurePolicy | string | `"Fail"` | | | webhooks.hooks.services.namespaceSelector.matchExpressions[0].key | string | `"capsule.clastix.io/tenant"` | | | webhooks.hooks.services.namespaceSelector.matchExpressions[0].operator | string | `"Exists"` | | diff --git a/charts/capsule/crds/capsule.clastix.io_globalresourcequotas.yaml b/charts/capsule/crds/capsule.clastix.io_globalresourcequotas.yaml new file mode 100644 index 000000000..99a2e0670 --- /dev/null +++ b/charts/capsule/crds/capsule.clastix.io_globalresourcequotas.yaml @@ -0,0 +1,243 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.2 + name: globalresourcequotas.capsule.clastix.io +spec: + group: capsule.clastix.io + names: + kind: GlobalResourceQuota + listKind: GlobalResourceQuotaList + plural: globalresourcequotas + shortNames: + - globalquota + singular: globalresourcequota + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Active status of the GlobalResourceQuota + jsonPath: .status.active + name: Active + type: boolean + - description: The total amount of Namespaces spanned across + jsonPath: .status.size + name: Namespaces + type: integer + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta2 + schema: + openAPIV3Schema: + description: GlobalResourceQuota is the Schema for the globalresourcequotas + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GlobalResourceQuotaSpec defines the desired state of GlobalResourceQuota + properties: + active: + default: true + description: |- + When a quota is active it's checking for the resources in the cluster + If not active the resourcequotas are removed and the webhook no longer blocks updates + type: boolean + quotas: + additionalProperties: + description: ResourceQuotaSpec defines the desired hard limits to + enforce for Quota. + properties: + hard: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + hard is the set of desired hard limits for each named resource. + More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/ + type: object + scopeSelector: + description: |- + scopeSelector is also a collection of filters like scopes that must match each object tracked by a quota + but expressed using ScopeSelectorOperator in combination with possible values. + For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. + properties: + matchExpressions: + description: A list of scope selector requirements by scope + of the resources. + items: + description: |- + A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator + that relates the scope name and values. + properties: + operator: + description: |- + Represents a scope's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. + type: string + scopeName: + description: The name of the scope that the selector + applies to. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - operator + - scopeName + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + scopes: + description: |- + A collection of filters that must match each object tracked by a quota. + If not specified, the quota matches all objects. + items: + description: A ResourceQuotaScope defines a filter that must + match each object tracked by a quota + type: string + type: array + x-kubernetes-list-type: atomic + type: object + description: Define resourcequotas for the namespaces + type: object + selectors: + description: Selector to match the namespaces that should be managed + by the GlobalResourceQuota + items: + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + tenant: + default: true + description: |- + Only considers namespaces which are part of a tenant, other namespaces which might match + the label, but do not have a tenant, are ignored. + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + type: object + status: + description: GlobalResourceQuotaStatus defines the observed state of GlobalResourceQuota + properties: + active: + default: true + description: If this quota is active or not. + type: boolean + namespaces: + description: List of namespaces assigned to the Tenant. + items: + type: string + type: array + quotas: + additionalProperties: + description: ResourceQuotaStatus defines the enforced hard limits + and observed use. + properties: + hard: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Hard is the set of enforced hard limits for each named resource. + More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/ + type: object + used: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Used is the current observed total usage of the + resource in the namespace. + type: object + type: object + description: Tracks the quotas for the Resource. + type: object + size: + default: 0 + description: How many namespaces are assigned to the Tenant. + type: integer + required: + - active + - size + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/capsule/templates/mutatingwebhookconfiguration.yaml b/charts/capsule/templates/mutatingwebhookconfiguration.yaml index 68ab405f5..2f7169041 100644 --- a/charts/capsule/templates/mutatingwebhookconfiguration.yaml +++ b/charts/capsule/templates/mutatingwebhookconfiguration.yaml @@ -31,7 +31,7 @@ webhooks: - pods scope: "Namespaced" namespaceSelector: - {{- toYaml .namespaceSelector | nindent 4}} + {{- toYaml .namespaceSelector | nindent 4}} sideEffects: None timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} {{- end }} @@ -53,11 +53,11 @@ webhooks: - persistentvolumeclaims scope: "Namespaced" namespaceSelector: - {{- toYaml .namespaceSelector | nindent 4}} + {{- toYaml .namespaceSelector | nindent 4}} sideEffects: None timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} {{- end }} -{{- with .Values.webhooks.hooks.defaults.ingress }} +{{- with .Values.webhooks.hooks.defaults.ingress }} - admissionReviewVersions: - v1 clientConfig: @@ -81,7 +81,7 @@ webhooks: sideEffects: None timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} {{- end }} -{{- with .Values.webhooks.hooks.namespaceOwnerReference }} + {{- with .Values.webhooks.hooks.namespaceOwnerReference }} - admissionReviewVersions: - v1 - v1beta1 @@ -106,5 +106,33 @@ webhooks: scope: '*' sideEffects: NoneOnDryRun timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} -{{- end }} + {{- end }} + {{- with .Values.webhooks.hooks.quotas }} +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + {{- include "capsule.webhooks.service" (dict "path" "/globalquota/mutation" "ctx" $) | nindent 4 }} + failurePolicy: {{ .failurePolicy }} + matchPolicy: Equivalent + name: quotas.projectcapsule.dev + namespaceSelector: {} + objectSelector: {} + reinvocationPolicy: Never + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - resourcequotas/status + scope: 'Namespaced' + sideEffects: None + namespaceSelector: + {{- toYaml .namespaceSelector | nindent 4}} + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} {{- end }} diff --git a/charts/capsule/templates/validatingwebhookconfiguration.yaml b/charts/capsule/templates/validatingwebhookconfiguration.yaml index d0f35b89a..f58852d02 100644 --- a/charts/capsule/templates/validatingwebhookconfiguration.yaml +++ b/charts/capsule/templates/validatingwebhookconfiguration.yaml @@ -248,7 +248,7 @@ webhooks: sideEffects: None timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} {{- end }} -{{- with .Values.webhooks.hooks.tenants }} + {{- with .Values.webhooks.hooks.tenants }} - admissionReviewVersions: - v1 - v1beta1 @@ -273,5 +273,32 @@ webhooks: scope: '*' sideEffects: None timeoutSeconds: {{ $.Values.webhooks.validatingWebhooksTimeoutSeconds }} + {{- end }} + {{- with .Values.webhooks.hooks.quotas }} +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + {{- include "capsule.webhooks.service" (dict "path" "/globalquota/validation" "ctx" $) | nindent 4 }} + failurePolicy: {{ .failurePolicy }} + matchPolicy: Equivalent + name: quotas.projectcapsule.dev + reinvocationPolicy: Never + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - UPDATE + - DELETE + resources: + - resourcequotas + - resourcequotas/status + scope: 'Namespaced' + sideEffects: None + namespaceSelector: + {{- toYaml .namespaceSelector | nindent 4}} + timeoutSeconds: {{ $.Values.webhooks.mutatingWebhooksTimeoutSeconds }} + {{- end }} {{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/capsule/values.schema.json b/charts/capsule/values.schema.json index 252236b3a..b99e5bf0d 100644 --- a/charts/capsule/values.schema.json +++ b/charts/capsule/values.schema.json @@ -730,6 +730,33 @@ }, "type": "object" }, + "quotas": { + "properties": { + "failurePolicy": { + "type": "string" + }, + "namespaceSelector": { + "properties": { + "matchExpressions": { + "items": { + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + } + }, + "type": "object" + }, "services": { "properties": { "failurePolicy": { diff --git a/charts/capsule/values.yaml b/charts/capsule/values.yaml index c1d70eaa7..c5864047c 100644 --- a/charts/capsule/values.yaml +++ b/charts/capsule/values.yaml @@ -311,6 +311,13 @@ webhooks: operator: Exists nodes: failurePolicy: Fail + + quotas: + failurePolicy: Fail + namespaceSelector: + matchExpressions: + - key: capsule.clastix.io/tenant + operator: Exists defaults: ingress: failurePolicy: Fail @@ -331,6 +338,7 @@ webhooks: - key: capsule.clastix.io/tenant operator: Exists + # ServiceMonitor serviceMonitor: # -- Enable ServiceMonitor diff --git a/controllers/globalquota/manager.go b/controllers/globalquota/manager.go new file mode 100644 index 000000000..79ba9e169 --- /dev/null +++ b/controllers/globalquota/manager.go @@ -0,0 +1,183 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package globalquota + +import ( + "context" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/metrics" + capsuleutils "github.com/projectcapsule/capsule/pkg/utils" +) + +type Manager struct { + client.Client + Log logr.Logger + Recorder record.EventRecorder + RESTConfig *rest.Config +} + +func (r *Manager) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&capsulev1beta2.GlobalResourceQuota{}). + Owns(&corev1.ResourceQuota{}). + Watches(&corev1.Namespace{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + // Fetch all GlobalResourceQuota objects + grqList := &capsulev1beta2.GlobalResourceQuotaList{} + if err := mgr.GetClient().List(ctx, grqList); err != nil { + // Log the error and return no requests to reconcile + r.Log.Error(err, "Failed to list GlobalResourceQuota objects") + return nil + } + + // Enqueue a reconcile request for each GlobalResourceQuota + var requests []reconcile.Request + for _, grq := range grqList.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&grq), + }) + } + + return requests + }), + ). + Complete(r) +} + +//nolint:nakedret +func (r Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { + r.Log = r.Log.WithValues("Request.Name", request.Name) + // Fetch the Tenant instance + instance := &capsulev1beta2.GlobalResourceQuota{} + if err = r.Get(ctx, request.NamespacedName, instance); err != nil { + if apierrors.IsNotFound(err) { + r.Log.Info("Request object not found, could have been deleted after reconcile request") + + // If tenant was deleted or cannot be found, clean up metrics + metrics.GlobalResourceUsage.DeletePartialMatch(map[string]string{"quota": request.Name}) + metrics.GlobalResourceLimit.DeletePartialMatch(map[string]string{"quota": request.Name}) + + return reconcile.Result{}, nil + } + + r.Log.Error(err, "Error reading the object") + + return + } + + // Ensuring the Quota Status + if err = r.updateQuotaStatus(ctx, instance); err != nil { + r.Log.Error(err, "Cannot update Tenant status") + + return + } + + if !instance.Spec.Active { + r.Log.Info("GlobalResourceQuota is not active, skipping reconciliation") + + return + } + + // Get Item within Resource Quota + objectLabel, err := capsuleutils.GetTypeLabel(&capsulev1beta2.Tenant{}) + if err != nil { + return + } + + // Collect Namespaces (Matching) + namespaces := make([]corev1.Namespace, 0) + seenNamespaces := make(map[string]struct{}) + + for _, selector := range instance.Spec.Selectors { + selected, serr := selector.GetMatchingNamespaces(ctx, r.Client) + if serr != nil { + r.Log.Error(err, "Cannot get matching namespaces") + + continue + } + + for _, ns := range selected { + // Skip if namespace is being deleted + if !ns.ObjectMeta.DeletionTimestamp.IsZero() { + continue + } + + if _, exists := seenNamespaces[ns.Name]; exists { + continue // Skip duplicates + } + + if selector.MustTenantNamespace { + if _, ok := ns.Labels[objectLabel]; !ok { + continue + } + } + + seenNamespaces[ns.Name] = struct{}{} + namespaces = append(namespaces, ns) + } + } + + nsNames := make([]string, 0, len(namespaces)) + for _, ns := range namespaces { + nsNames = append(nsNames, ns.Name) + } + + // ResourceQuota Reconcilation + err = r.syncResourceQuotas(ctx, instance, nsNames) + if err != nil { + r.Log.Error(err, "Cannot sync ResourceQuotas") + } + + // Collect Namespaces for Status + if err = r.statusNamespaces(ctx, instance, namespaces); err != nil { + r.Log.Error(err, "Cannot update Tenant status") + + return + } + + return ctrl.Result{}, err +} + +// Update tracking namespaces +func (r *Manager) statusNamespaces(ctx context.Context, quota *capsulev1beta2.GlobalResourceQuota, ns []corev1.Namespace) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + latest := &capsulev1beta2.GlobalResourceQuota{} + if err := r.Client.Get(ctx, client.ObjectKey{Name: quota.Name}, latest); err != nil { + r.Log.Error(err, "Failed to fetch the latest Tenant object during retry") + + return err + } + + latest.AssignNamespaces(ns) + + return r.Client.Status().Update(ctx, latest, &client.SubResourceUpdateOptions{}) + }) +} + +func (r *Manager) updateQuotaStatus(ctx context.Context, quota *capsulev1beta2.GlobalResourceQuota) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + latest := &capsulev1beta2.GlobalResourceQuota{} + if err := r.Client.Get(ctx, client.ObjectKey{Name: quota.Name}, latest); err != nil { + r.Log.Error(err, "Failed to fetch the latest Tenant object during retry") + + return err + } + // Update the state based on the latest spec + latest.Status.Active = latest.Spec.Active + + return r.Client.Status().Update(ctx, latest) + }) +} diff --git a/controllers/globalquota/resourcequotas.go b/controllers/globalquota/resourcequotas.go new file mode 100644 index 000000000..594361f5a --- /dev/null +++ b/controllers/globalquota/resourcequotas.go @@ -0,0 +1,335 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package globalquota + +import ( + "context" + "fmt" + + "golang.org/x/sync/errgroup" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/metrics" + "github.com/projectcapsule/capsule/pkg/utils" +) + +// When the Resource Budget assigned to a Tenant is Tenant-scoped we have to rely on the ResourceQuota resources to +// represent the resource quota for the single Tenant rather than the single Namespace, +// so abusing of this API although its Namespaced scope. +// +// Since a Namespace could take-up all the available resource quota, the Namespace ResourceQuota will be a 1:1 mapping +// to the Tenant one: in first time Capsule is going to sum all the analogous ResourceQuota resources on other Tenant +// namespaces to check if the Tenant quota has been exceeded or not, reusing the native Kubernetes policy putting the +// .Status.Used value as the .Hard value. +// This will trigger following reconciliations but that's ok: the mutateFn will re-use the same business logic, letting +// the mutateFn along with the CreateOrUpdate to don't perform the update since resources are identical. +// +// In case of Namespace-scoped Resource Budget, we're just replicating the resources across all registered Namespaces. + +//nolint:nakedret, gocognit +func (r *Manager) syncResourceQuotas( + ctx context.Context, + quota *capsulev1beta2.GlobalResourceQuota, + matchingNamespaces []string, +) (err error) { + // getting ResourceQuota labels for the mutateFn + var quotaLabel, typeLabel string + + if quotaLabel, err = utils.GetTypeLabel(&capsulev1beta2.GlobalResourceQuota{}); err != nil { + return err + } + + typeLabel = utils.GetGlobalResourceQuotaTypeLabel() + + // Keep original status to verify if we need to change anything + originalStatus := quota.Status.DeepCopy() + + // Initialize on empty status + if quota.Status.Quota == nil { + quota.Status.Quota = make(capsulev1beta2.GlobalResourceQuotaStatusQuota) + } + + // Process each item (quota index) + for index, resourceQuota := range quota.Spec.Items { + // Fetch the latest tenant quota status + itemUsage, exists := quota.Status.Quota[index] + if !exists { + // Initialize Object + quota.Status.Quota[index] = &corev1.ResourceQuotaStatus{ + Used: corev1.ResourceList{}, + Hard: corev1.ResourceList{}, + } + + itemUsage = &corev1.ResourceQuotaStatus{ + Used: corev1.ResourceList{}, + Hard: resourceQuota.Hard, + } + } + + // ✅ Update the Used state in the global quota + quota.Status.Quota[index] = itemUsage + } + + // Update the tenant's status with the computed quota information + // We only want to update the status if we really have to, resulting in less + // conflicts because the usage status is updated by the webhook + if !equality.Semantic.DeepEqual(quota.Status, *originalStatus) { + if err := r.Status().Update(ctx, quota); err != nil { + r.Log.Info("updating status", "quota", quota.Status) + + r.Log.Error(err, "Failed to update tenant status") + + return err + } + } + + // Remove prior metrics, to avoid cleaning up for metrics of deleted ResourceQuotas + metrics.TenantResourceUsage.DeletePartialMatch(map[string]string{"quota": quota.Name}) + metrics.TenantResourceLimit.DeletePartialMatch(map[string]string{"quota": quota.Name}) + + // Remove Quotas which are no longer mentioned in spec + for existingIndex := range quota.Status.Quota { + if _, exists := quota.Spec.Items[api.Name(existingIndex)]; !exists { + + r.Log.V(7).Info("Orphaned quota index detected", "quotaIndex", existingIndex) + + for _, ns := range append(matchingNamespaces, quota.Status.Namespaces...) { + selector := labels.SelectorFromSet(map[string]string{ + quotaLabel: quota.Name, + typeLabel: existingIndex.String(), + }) + + r.Log.V(7).Info("Searching for ResourceQuotas to delete", "namespace", ns, "selector", selector.String()) + + // Query and delete all ResourceQuotas with matching labels in the namespace + rqList := &corev1.ResourceQuotaList{} + if err := r.Client.List(ctx, rqList, &client.ListOptions{ + Namespace: ns, + LabelSelector: selector, + }); err != nil { + r.Log.Error(err, "Failed to list ResourceQuotas", "namespace", ns, "quotaName", quota.Name, "index", existingIndex) + return err + } + + r.Log.V(7).Info("Found ResourceQuotas for deletion", "count", len(rqList.Items), "namespace", ns, "quotaIndex", existingIndex) + + for _, rq := range rqList.Items { + if err := r.Client.Delete(ctx, &rq); err != nil { + r.Log.Error(err, "Failed to delete ResourceQuota", "name", rq.Name, "namespace", ns) + return err + } + + r.Log.V(7).Info("Deleted orphaned ResourceQuota", "name", rq.Name, "namespace", ns) + } + } + + // Only Remove from status if the ResourceQuota has been deleted + // Remove the orphaned quota from status + delete(quota.Status.Quota, existingIndex) + r.Log.Info("Removed orphaned quota from status", "quotaIndex", existingIndex) + } else { + r.Log.V(7).Info("no lifecycle", "quotaIndex", existingIndex) + } + } + + // Convert matchingNamespaces to a map for quick lookup + matchingNamespaceSet := make(map[string]struct{}, len(matchingNamespaces)) + for _, ns := range matchingNamespaces { + matchingNamespaceSet[ns] = struct{}{} + } + + // Garbage collect namespaces which no longer match selector + for _, existingNamespace := range quota.Status.Namespaces { + if _, exists := matchingNamespaceSet[existingNamespace]; !exists { + if err := r.gcResourceQuotas(ctx, quota, existingNamespace); err != nil { + r.Log.Error(err, "Failed to garbage collect resource quota", "namespace", existingNamespace) + return err + } + } + } + + return SyncResourceQuotas(ctx, r.Client, quota, matchingNamespaces) +} + +// Synchronize resources quotas in all the given namespaces (routines) +func SyncResourceQuotas( + ctx context.Context, + c client.Client, + quota *capsulev1beta2.GlobalResourceQuota, + namespaces []string, +) (err error) { + group := new(errgroup.Group) + + // Sync resource quotas for matching namespaces + for _, ns := range namespaces { + namespace := ns + + group.Go(func() error { + return SyncResourceQuota(ctx, c, quota, namespace) + }) + } + + return group.Wait() +} + +// Synchronize a single resourcequota +// +//nolint:nakedret +func SyncResourceQuota( + ctx context.Context, + c client.Client, + quota *capsulev1beta2.GlobalResourceQuota, + namespace string, +) (err error) { + // getting ResourceQuota labels for the mutateFn + var quotaLabel, typeLabel string + + if quotaLabel, err = utils.GetTypeLabel(&capsulev1beta2.GlobalResourceQuota{}); err != nil { + return err + } + + typeLabel = utils.GetGlobalResourceQuotaTypeLabel() + + for index, resQuota := range quota.Spec.Items { + target := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: ItemObjectName(index, quota), + Namespace: namespace, + }, + } + + // Verify if quota is present + if err := c.Get(ctx, types.NamespacedName{Name: target.Name, Namespace: target.Namespace}, target); err != nil && !apierrors.IsNotFound(err) { + return err + } + + err = retry.RetryOnConflict(retry.DefaultBackoff, func() (retryErr error) { + _, retryErr = controllerutil.CreateOrUpdate(ctx, c, target, func() (err error) { + targetLabels := target.GetLabels() + if targetLabels == nil { + targetLabels = map[string]string{} + } + + targetLabels[quotaLabel] = quota.Name + targetLabels[typeLabel] = index.String() + + target.SetLabels(targetLabels) + target.Spec.Scopes = resQuota.Scopes + target.Spec.ScopeSelector = resQuota.ScopeSelector + + // Gather what's left in quota + space, err := quota.GetAggregatedQuotaSpace(index, target.Status.Used) + if err != nil { + return err + } + + // This is important when a resourcequota is newly added (new namespace) + // We don't want to have a racing condition and wait until the elements are synced to + // the quota. But we take what's left (or when first namespace then hard 1:1) and assign it. + // It may be further reduced by the limits reconciler + target.Spec.Hard = space + + return controllerutil.SetControllerReference(quota, target, c.Scheme()) + }) + + return retryErr + }) + + if err != nil { + return + } + } + + return nil +} + +// Attempts to garbage collect a ResourceQuota resource. +func (r *Manager) gcResourceQuotas(ctx context.Context, quota *capsulev1beta2.GlobalResourceQuota, namespace string) error { + // Check if the namespace still exists + ns := &corev1.Namespace{} + if err := r.Client.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { + if errors.IsNotFound(err) { + r.Log.V(5).Info("Namespace does not exist, skipping garbage collection", "namespace", namespace) + return nil + } + return fmt.Errorf("failed to check namespace existence: %w", err) + } + + // Attempt to delete the ResourceQuota + for index, _ := range quota.Spec.Items { + target := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: ItemObjectName(index, quota), + Namespace: namespace, + }, + } + err := r.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: target.GetName()}, target) + if err != nil { + if errors.IsNotFound(err) { + r.Log.V(5).Info("ResourceQuota already deleted", "namespace", namespace, "name", ItemObjectName(index, quota)) + continue + } + return err + } + + // Delete the ResourceQuota + if err := r.Client.Delete(ctx, target); err != nil { + return fmt.Errorf("failed to delete ResourceQuota %s in namespace %s: %w", ItemObjectName(index, quota), namespace, err) + } + } + + r.Log.Info("Deleted ResourceQuota", "namespace", namespace) + return nil +} + +// Serial ResourceQuota processing is expensive: using Go routines we can speed it up. +// In case of multiple errors these are logged properly, returning a generic error since we have to repush back the +// reconciliation loop. +func (r *Manager) resourceQuotasUpdate(ctx context.Context, resourceName corev1.ResourceName, actual resource.Quantity, toKeep sets.Set[corev1.ResourceName], limit resource.Quantity, list ...corev1.ResourceQuota) (err error) { + group := new(errgroup.Group) + + for _, item := range list { + rq := item + + group.Go(func() (err error) { + found := &corev1.ResourceQuota{} + if err = r.Get(ctx, types.NamespacedName{Namespace: rq.Namespace, Name: rq.Name}, found); err != nil { + return + } + + return retry.RetryOnConflict(retry.DefaultBackoff, func() (retryErr error) { + _, retryErr = controllerutil.CreateOrUpdate(ctx, r.Client, found, func() error { + // Updating the Resource according to the actual.Cmp result + found.Spec.Hard = rq.Spec.Hard + + return nil + }) + + return retryErr + }) + }) + } + + if err = group.Wait(); err != nil { + // We had an error and we mark the whole transaction as failed + // to process it another time according to the Tenant controller back-off factor. + r.Log.Error(err, "Cannot update outer ResourceQuotas", "resourceName", resourceName.String()) + err = fmt.Errorf("update of outer ResourceQuota items has failed: %w", err) + } + + return err +} diff --git a/controllers/globalquota/utils.go b/controllers/globalquota/utils.go new file mode 100644 index 000000000..204652a0f --- /dev/null +++ b/controllers/globalquota/utils.go @@ -0,0 +1,97 @@ +package globalquota + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + capsuleutils "github.com/projectcapsule/capsule/pkg/utils" +) + +// Get all matching namespaces (just names) +func GetMatchingGlobalQuotaNamespacesByName( + ctx context.Context, + c client.Client, + quota *capsulev1beta2.GlobalResourceQuota, +) (nsNames []string, err error) { + namespaces, err := GetMatchingGlobalQuotaNamespaces(ctx, c, quota) + if err != nil { + return + } + + nsNames = make([]string, 0, len(namespaces)) + for _, ns := range namespaces { + nsNames = append(nsNames, ns.Name) + } + + return +} + +// Get all matching namespaces +func GetMatchingGlobalQuotaNamespaces( + ctx context.Context, + c client.Client, + quota *capsulev1beta2.GlobalResourceQuota, +) (namespaces []corev1.Namespace, err error) { + // Collect Namespaces (Matching) + namespaces = make([]corev1.Namespace, 0) + seenNamespaces := make(map[string]struct{}) + + // Get Item within Resource Quota + objectLabel, err := capsuleutils.GetTypeLabel(&capsulev1beta2.Tenant{}) + if err != nil { + return + } + + for _, selector := range quota.Spec.Selectors { + selected, err := selector.GetMatchingNamespaces(ctx, c) + if err != nil { + continue + } + + for _, ns := range selected { + // Skip if namespace is being deleted + if !ns.ObjectMeta.DeletionTimestamp.IsZero() { + continue + } + + if _, exists := seenNamespaces[ns.Name]; exists { + continue // Skip duplicates + } + + if selector.MustTenantNamespace { + if _, ok := ns.Labels[objectLabel]; !ok { + continue + } + } + + seenNamespaces[ns.Name] = struct{}{} + namespaces = append(namespaces, ns) + } + } + + return +} + +// Returns for an item it's name as Kubernetes object +func ItemObjectName(itemName api.Name, quota *capsulev1beta2.GlobalResourceQuota) string { + // Generate a name using the tenant name and item name + return fmt.Sprintf("capsule-%s-%s", quota.Name, itemName) +} + +func (r *Manager) emitEvent(object runtime.Object, namespace string, res controllerutil.OperationResult, msg string, err error) { + eventType := corev1.EventTypeNormal + + if err != nil { + eventType = corev1.EventTypeWarning + res = "Error" + } + + r.Recorder.AnnotatedEventf(object, map[string]string{"OperationResult": string(res)}, eventType, namespace, msg) +} diff --git a/controllers/tenant/manager.go b/controllers/tenant/manager.go index 1ccb3e8b1..67e1fce71 100644 --- a/controllers/tenant/manager.go +++ b/controllers/tenant/manager.go @@ -146,12 +146,20 @@ func (r Manager) Reconcile(ctx context.Context, request ctrl.Request) (result ct func (r *Manager) updateTenantStatus(ctx context.Context, tnt *capsulev1beta2.Tenant) error { return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { - if tnt.Spec.Cordoned { - tnt.Status.State = capsulev1beta2.TenantStateCordoned + // Re-fetch the tenant to get the latest state (avoid conflicts by quota webhook) + latestTenant := &capsulev1beta2.Tenant{} + if err := r.Client.Get(ctx, client.ObjectKey{Name: tnt.Name, Namespace: tnt.Namespace}, latestTenant); err != nil { + r.Log.Error(err, "Failed to fetch the latest Tenant object during retry") + return err + } + + // Update the state based on the latest spec + if latestTenant.Spec.Cordoned { + latestTenant.Status.State = capsulev1beta2.TenantStateCordoned } else { - tnt.Status.State = capsulev1beta2.TenantStateActive + latestTenant.Status.State = capsulev1beta2.TenantStateActive } - return r.Client.Status().Update(ctx, tnt) + return r.Client.Status().Update(ctx, latestTenant) }) } diff --git a/controllers/tenant/resourcequotas.go b/controllers/tenant/resourcequotas.go index 2e5174040..ed7ffbae2 100644 --- a/controllers/tenant/resourcequotas.go +++ b/controllers/tenant/resourcequotas.go @@ -99,6 +99,7 @@ func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2 return scopeErr } + // Iterating over all the options declared for the ResourceQuota, // summing all the used quota across different Namespaces to determinate // if we're hitting a Hard quota at Tenant level. @@ -111,6 +112,7 @@ func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2 var quantity resource.Quantity for _, item := range list.Items { quantity.Add(item.Status.Used[name]) + //tenantQuotaStatus[strconv.Itoa(index)].Quotas[item.Namespace] = item.Status } r.Log.Info("Computed " + name.String() + " quota for the whole Tenant is " + quantity.String()) @@ -188,6 +190,7 @@ func (r *Manager) syncResourceQuotas(ctx context.Context, tenant *capsulev1beta2 if err = group.Wait(); err != nil { return } + } // getting requested ResourceQuota keys keys := make([]string, 0, len(tenant.Spec.ResourceQuota.Items)) diff --git a/e2e/globalresourcequota_test.go b/e2e/globalresourcequota_test.go new file mode 100644 index 000000000..63f44f74c --- /dev/null +++ b/e2e/globalresourcequota_test.go @@ -0,0 +1,569 @@ +//go:build e2e + +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" + "github.com/projectcapsule/capsule/pkg/utils" +) + +var _ = Describe("Global ResourceQuotas", func() { + solar := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "solar-quota", + Labels: map[string]string{ + "customer-resource-pool": "dev", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: capsulev1beta2.OwnerListSpec{ + { + Name: "solar-user", + Kind: "User", + }, + }, + }, + } + + wind := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "wind-quota", + Labels: map[string]string{ + "customer-resource-pool": "dev", + }, + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: capsulev1beta2.OwnerListSpec{ + { + Name: "wind-user", + Kind: "User", + }, + }, + }, + } + + grq := &capsulev1beta2.GlobalResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "global-quota", + Labels: map[string]string{ + "replicate": "true", + }, + }, + Spec: capsulev1beta2.GlobalResourceQuotaSpec{ + Selectors: []capsulev1beta2.GlobalResourceQuotaSelector{ + { + MustTenantNamespace: true, // Only namespaces belonging to a tenant are considered + NamespaceSelector: api.NamespaceSelector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "capsule.clastix.io/tenant": "solar-quota", + }, + }, + }, + }, + { + MustTenantNamespace: true, // Only namespaces belonging to a tenant are considered + NamespaceSelector: api.NamespaceSelector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "capsule.clastix.io/tenant": "wind-quota", + }, + }, + }, + }, + { + MustTenantNamespace: false, // Allow non-tenant namespaces + NamespaceSelector: api.NamespaceSelector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "loose-quota": "any", + }, + }, + }, + }, + }, + Items: map[api.Name]corev1.ResourceQuotaSpec{ + "scheduling": { + Hard: corev1.ResourceList{ + corev1.ResourceLimitsCPU: resource.MustParse("2"), + corev1.ResourceLimitsMemory: resource.MustParse("2Gi"), + corev1.ResourceRequestsCPU: resource.MustParse("2"), + corev1.ResourceRequestsMemory: resource.MustParse("2Gi"), + }, + }, + "pods": { + Hard: corev1.ResourceList{ + corev1.ResourcePods: resource.MustParse("5"), + }, + }, + "connectivity": { + Hard: corev1.ResourceList{ + corev1.ResourceServices: resource.MustParse("2"), + }, + }, + }, + }, + } + + JustBeforeEach(func() { + EventuallyCreation(func() error { + solar.ResourceVersion = "" + return k8sClient.Create(context.TODO(), solar) + }).Should(Succeed()) + + EventuallyCreation(func() error { + wind.ResourceVersion = "" + return k8sClient.Create(context.TODO(), wind) + }).Should(Succeed()) + + EventuallyCreation(func() error { + grq.ResourceVersion = "" + return k8sClient.Create(context.TODO(), grq) + }).Should(Succeed()) + }) + + JustAfterEach(func() { + Expect(k8sClient.Delete(context.TODO(), solar)).Should(Succeed()) + Expect(k8sClient.Delete(context.TODO(), wind)).Should(Succeed()) + Expect(k8sClient.Delete(context.TODO(), grq)).Should(Succeed()) + Eventually(func() error { + deploymentList := &appsv1.DeploymentList{} + labelSelector := client.MatchingLabels{"test-label": "to-delete"} + if err := k8sClient.List(context.TODO(), deploymentList, labelSelector); err != nil { + return err + } + + for _, deployment := range deploymentList.Items { + if err := k8sClient.Delete(context.TODO(), &deployment); err != nil { + return err + } + } + + return nil + }, "30s", "5s").Should(Succeed()) + }) + + It("handle overprovisioning (eventually)", func() { + solarNs := []string{"solar-one", "solar-two", "solar-three"} + + By("creating solar Namespaces", func() { + for _, ns := range solarNs { + NamespaceCreation(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, solar.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed()) + } + }) + + By("Scheduling services simultaneously in all namespaces", func() { + wg := sync.WaitGroup{} // Use WaitGroup for concurrency + for _, ns := range solarNs { + wg.Add(1) + go func(namespace string) { // Run in parallel + defer wg.Done() + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + Namespace: namespace, + Labels: map[string]string{ + "test-label": "to-delete", + }, + }, + Spec: corev1.ServiceSpec{ + // Select pods with this label (ensure these pods exist in the namespace) + Selector: map[string]string{"app": "test"}, + Ports: []corev1.ServicePort{ + { + Port: 80, + TargetPort: intstr.FromInt(8080), + Protocol: corev1.ProtocolTCP, + }, + }, + Type: corev1.ServiceTypeClusterIP, + }, + } + err := k8sClient.Create(context.TODO(), service) + Expect(err).Should(Succeed(), "Failed to create Service in namespace %s", namespace) + }(ns) + } + wg.Wait() // Ensure all services are scheduled at the same time + }) + + By("Scheduling deployments simultaneously in all namespaces", func() { + wg := sync.WaitGroup{} // Use WaitGroup for concurrency + for _, ns := range solarNs { + wg.Add(1) + go func(namespace string) { // Run in parallel + defer wg.Done() + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + Namespace: namespace, + Labels: map[string]string{ + "test-label": "to-delete", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(3)), // Adjust the replica count if needed + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "test"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "busybox", + Image: "busybox", + Args: []string{"sleep", "3600"}, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0.3"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("0.25"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + }, + }, + }, + }, + }, + }, + } + err := k8sClient.Create(context.TODO(), deployment) + Expect(err).Should(Succeed(), "Failed to create Deployment in namespace %s", namespace) + }(ns) + } + wg.Wait() // Ensure all deployments are scheduled at the same time + }) + + By("Waiting for at least 5 pods with label app=test to be scheduled and in Running state", func() { + Eventually(func() int { + podList := &corev1.PodList{} + err := k8sClient.List(context.TODO(), podList, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{"app": "test"}), + }) + if err != nil { + return 0 + } + + // Count only pods that are in Running phase + runningPods := 0 + for _, pod := range podList.Items { + if pod.Status.Phase == corev1.PodRunning { + runningPods++ + } + } + return runningPods + }, defaultTimeoutInterval, defaultPollInterval).Should(BeNumerically(">=", 5), "Expected at least 5 running pods with label app=test") + }) + + By("Sleeping for 10 minutes", func() { + time.Sleep(10 * time.Minute) + }) + + By("Collecting and logging ResourceQuota statuses across namespaces (pods item)", func() { + totalHard := corev1.ResourceList{} + totalUsed := corev1.ResourceList{} + + // Construct first label requirement (e.g., object type) + r1, err := labels.NewRequirement(utils.GetGlobalResourceQuotaTypeLabel(), selection.Equals, []string{"pods"}) + Expect(err).Should(Succeed(), "❌ Error creating label requirement for %s: %v\n", utils.GetGlobalResourceQuotaTypeLabel(), err) + + // List ResourceQuotas in the namespace + quotaList := corev1.ResourceQuotaList{} + err = k8sClient.List(context.TODO(), "aList, &client.ListOptions{ + LabelSelector: labels.NewSelector().Add(*r1), + }) + Expect(err).Should(Succeed(), "Failed to list resourcequotas: %v", err) + + for _, quota := range quotaList.Items { + fmt.Printf("Processing ResourceQuota: %s in namespace %s status: %v\n", quota.Name, quota.Namespace, quota.Status) + + // Aggregate Status Used values + for resourceName, usedValue := range quota.Status.Used { + if existing, exists := totalUsed[resourceName]; exists { + existing.Add(usedValue) + totalUsed[resourceName] = existing + } else { + totalUsed[resourceName] = usedValue.DeepCopy() + } + } + } + + fmt.Println("✅ Aggregated ResourceQuotas:") + fmt.Println("Total Spec Hard Limits:", totalHard) + fmt.Println("Total Used Resources:", totalUsed) + }) + }) + + It("should replicate resourcequotas to relevant namespaces", func() { + solarNs := []string{"solar-one", "solar-two", "solar-three"} + + By("creating solar Namespaces", func() { + for _, ns := range solarNs { + NamespaceCreation(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, solar.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed()) + } + }) + + // Fetch the GlobalResourceQuota object + globalQuota := &capsulev1beta2.GlobalResourceQuota{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: grq.Name}, globalQuota) + Expect(err).Should(Succeed()) + + for _, ns := range solarNs { + By(fmt.Sprintf("waiting resourcequotas in %s Namespace", ns), func() { + Eventually(func() []corev1.ResourceQuota { + // List ResourceQuotas in the namespace + quotaList := corev1.ResourceQuotaList{} + err = k8sClient.List(context.TODO(), "aList, &client.ListOptions{ + Namespace: ns, + }) + if err != nil { + fmt.Printf("Error listing ResourceQuotas in namespace %s: %v\n", ns, err) + return nil + } + + // Filter ResourceQuotas based on GlobalResourceQuota validation + var matchingQuotas []corev1.ResourceQuota + for _, rq := range quotaList.Items { + // Validate against GlobalResourceQuota + if validateQuotaAgainstGlobal(rq, globalQuota) { + matchingQuotas = append(matchingQuotas, rq) + } else { + fmt.Printf("❌ ResourceQuota %s does not match GlobalResourceQuota %s\n", rq.Name, grq.Name) + } + } + + return matchingQuotas + }, defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(2)) + }) + } + + By("Verify General Status", func() { + // Fetch the GlobalResourceQuota object + globalQuota := &capsulev1beta2.GlobalResourceQuota{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: grq.Name}, globalQuota) + Expect(err).Should(Succeed()) + + // Expected values + expectedNamespaces := []string{"solar-one", "solar-two", "solar-three"} + expectedSize := 3 + + // Verify `active` field + Expect(globalQuota.Status.Active).To(BeTrue(), "❌ GlobalResourceQuota should be active") + + // Verify `size` field + Expect(int(globalQuota.Status.Size)).To(Equal(expectedSize), "❌ GlobalResourceQuota size should be %d", expectedSize) + + // Verify `namespaces` field (ensuring it matches exactly) + Expect(globalQuota.Status.Namespaces).To(ConsistOf(expectedNamespaces), + "❌ GlobalResourceQuota namespaces should match %v", expectedNamespaces) + }) + + windNs := []string{"wind-one", "wind-two", "wind-three"} + + By("creating wind Namespaces", func() { + for _, ns := range windNs { + NamespaceCreation(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, wind.Spec.Owners[0], defaultTimeoutInterval).Should(Succeed()) + } + }) + + for _, ns := range windNs { + By(fmt.Sprintf("waiting resourcequotas in %s Namespace", ns), func() { + Eventually(func() []corev1.ResourceQuota { + // List ResourceQuotas in the namespace + quotaList := corev1.ResourceQuotaList{} + err = k8sClient.List(context.TODO(), "aList, &client.ListOptions{ + Namespace: ns, + }) + if err != nil { + fmt.Printf("Error listing ResourceQuotas in namespace %s: %v\n", ns, err) + return nil + } + + // Filter ResourceQuotas based on GlobalResourceQuota validation + var matchingQuotas []corev1.ResourceQuota + for _, rq := range quotaList.Items { + // Validate against GlobalResourceQuota + if validateQuotaAgainstGlobal(rq, globalQuota) { + matchingQuotas = append(matchingQuotas, rq) + } else { + fmt.Printf("❌ ResourceQuota %s does not match GlobalResourceQuota %s\n", rq.Name, grq.Name) + } + } + + return matchingQuotas + }, defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(2)) + }) + } + + By("Verify General Status", func() { + // Fetch the GlobalResourceQuota object + globalQuota := &capsulev1beta2.GlobalResourceQuota{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: grq.Name}, globalQuota) + Expect(err).Should(Succeed()) + + // Expected values + expectedNamespaces := append(solarNs, windNs...) + expectedSize := 6 + + // Verify `active` field + Expect(globalQuota.Status.Active).To(BeTrue(), "❌ GlobalResourceQuota should be active") + + // Verify `size` field + Expect(int(globalQuota.Status.Size)).To(Equal(expectedSize), "❌ GlobalResourceQuota size should be %d", expectedSize) + + // Verify `namespaces` field (ensuring it matches exactly) + Expect(globalQuota.Status.Namespaces).To(ConsistOf(expectedNamespaces), + "❌ GlobalResourceQuota namespaces should match %v", expectedNamespaces) + }) + + By("Updating GlobalResourceQuota selectors", func() { + // Fetch the GlobalResourceQuota object + globalQuota := &capsulev1beta2.GlobalResourceQuota{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: grq.Name}, globalQuota) + Expect(err).Should(Succeed()) + + // Modify the `spec.selectors` field with new values + globalQuota.Spec.Selectors = []capsulev1beta2.GlobalResourceQuotaSelector{ + { + MustTenantNamespace: true, // Only namespaces belonging to a tenant are considered + NamespaceSelector: api.NamespaceSelector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "capsule.clastix.io/tenant": "wind-quota", + }, + }, + }, + }, + } + + // Update the GlobalResourceQuota object in Kubernetes + err = k8sClient.Update(context.TODO(), globalQuota) + Expect(err).Should(Succeed(), "Failed to update GlobalResourceQuota selectors") + }) + + By("Verify General Status", func() { + // Fetch the GlobalResourceQuota object + globalQuota := &capsulev1beta2.GlobalResourceQuota{} + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: grq.Name}, globalQuota) + Expect(err).Should(Succeed()) + + // Expected values + expectedSize := 3 + + // Verify `active` field + Expect(globalQuota.Status.Active).To(BeTrue(), "❌ GlobalResourceQuota should be active") + + // Verify `size` field + Expect(int(globalQuota.Status.Size)).To(Equal(expectedSize), "❌ GlobalResourceQuota size should be %d", expectedSize) + + // Verify `namespaces` field (ensuring it matches exactly) + Expect(globalQuota.Status.Namespaces).To(ConsistOf(windNs), + "❌ GlobalResourceQuota namespaces should match %v", windNs) + }) + + objectLabel, _ := utils.GetTypeLabel(&capsulev1beta2.GlobalResourceQuota{}) + + for _, ns := range solarNs { + By(fmt.Sprintf("verify resourcequotas in %s Namespace absent", ns), func() { + Eventually(func() []corev1.ResourceQuota { + // Construct first label requirement (e.g., object type) + r1, err := labels.NewRequirement(objectLabel, selection.Equals, []string{grq.Name}) + if err != nil { + fmt.Printf("❌ Error creating label requirement for %s: %v\n", objectLabel, err) + return nil + } + + // List ResourceQuotas in the namespace + quotaList := corev1.ResourceQuotaList{} + err = k8sClient.List(context.TODO(), "aList, &client.ListOptions{ + LabelSelector: labels.NewSelector().Add(*r1), + Namespace: ns, + }) + if err != nil { + fmt.Printf("❌ Error listing ResourceQuotas in namespace %s: %v\n", ns, err) + return nil + } + + return quotaList.Items + }, defaultTimeoutInterval, defaultPollInterval).Should(HaveLen(0)) + }) + } + + }) + +}) + +func validateQuotaAgainstGlobal(rq corev1.ResourceQuota, grq *capsulev1beta2.GlobalResourceQuota) bool { + objectLabel, _ := utils.GetTypeLabel(&capsulev1beta2.GlobalResourceQuota{}) + + // Fetch the GlobalResourceQuota object + err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: grq.Name}, grq) + Expect(err).Should(Succeed()) + + // Verify GlobalQuotaReference + globalQuotaName, exists := rq.ObjectMeta.Labels[objectLabel] + if !exists { + fmt.Printf("Skipping ResourceQuota %s: Missing label %s\n", rq.Name, objectLabel) + return false + } + if globalQuotaName != grq.Name { + fmt.Printf("Skipping ResourceQuota %s: Label mismatch (expected: %s, found: %s)\n", rq.Name, grq.Name, globalQuotaName) + return false + } + + // Verify Item is correctly labeled + itemName, exists := rq.ObjectMeta.Labels[utils.GetGlobalResourceQuotaTypeLabel()] + if !exists { + fmt.Printf("Skipping ResourceQuota %s: Missing label %s\n", rq.Name, utils.GetGlobalResourceQuotaTypeLabel()) + return false + } + + // Check if the GlobalResourceQuota has a matching entry + _, exists = grq.Spec.Items[api.Name(itemName)] + if !exists { + fmt.Printf("Skipping ResourceQuota %s: Item %s: Missing Item in Spec\n", grq.Name, itemName) + return false + } + + // Validate that ResourceQuota.Spec.Hard matches GlobalResourceQuota.Status.Quota[labelValue].Hard + globalQuotaStatus, statusExists := grq.Status.Quota[api.Name(itemName)] + if !statusExists { + fmt.Printf("Skipping ResourceQuota %s: Item %s: Missing Item in Status\n", grq.Name, itemName) + return false + } + + // Compare ResourceQuota.Spec.Hard with GlobalResourceQuota.Status.Quota[labelValue].Hard + for resourceName, specHardValue := range rq.Spec.Hard { + if statusHardValue, exists := globalQuotaStatus.Hard[resourceName]; !exists || specHardValue.Cmp(statusHardValue) != 0 { + fmt.Printf("❌ ResourceQuota difference state %v\n", specHardValue.Cmp(statusHardValue)) + return false + } + } + + return true +} diff --git a/e2e/monitoring/grafana.flux.yaml b/e2e/monitoring/grafana.flux.yaml new file mode 100644 index 000000000..4a0464974 --- /dev/null +++ b/e2e/monitoring/grafana.flux.yaml @@ -0,0 +1,94 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: grafana + namespace: flux-system +spec: + interval: 15m + timeout: 10m + targetNamespace: observability-system + releaseName: "grafana" + chart: + spec: + chart: grafana + version: "8.3.4" + sourceRef: + kind: HelmRepository + name: grafana + interval: 24h + install: + remediation: + retries: -1 + upgrade: + remediation: + remediateLastFailure: true + retries: -1 + driftDetection: + mode: enabled + values: + global: + dnsService: "kube-dns" + dnsNamespace: "kube-system" + assertNoLeakedSecrets: false + deploymentStrategy: + type: Recreate + persistence: + enabled: true + initChownData: + enabled: false + plugins: + - grafana-pyroscope-app + env: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" + GF_DIAGNOSTICS_PROFILING_ENABLED: "true" + GF_DIAGNOSTICS_PROFILING_ADDR: "0.0.0.0" + GF_DIAGNOSTICS_PROFILING_PORT: "6060" + sidecar: + securityContext: + runAsUser: 472 + runAsGroup: 472 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + enableUniqueFilenames: true + datasources: + enabled: false + dashboards: + enabled: false + # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/ + grafana.ini: + analytics: + reporting_enabled: false + check_for_updates: false + check_for_plugin_updates: false + security: + disable_gravatar: true + cookie_secure: true + cookie_samesite: lax + strict_transport_security: true + strict_transport_security_preload: true + strict_transport_security_subdomains: true + content_security_policy: true + auth: + disable_login_form: false + users: + allow_sign_up: true + auto_assign_org: true + ingress: + enabled: false +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: grafana + namespace: flux-system +spec: + interval: 24h0m0s + url: https://grafana.github.io/helm-charts diff --git a/e2e/monitoring/kustomization.yaml b/e2e/monitoring/kustomization.yaml new file mode 100644 index 000000000..5aa542eb2 --- /dev/null +++ b/e2e/monitoring/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - grafana.flux.yaml + - pyroscope.flux.yaml + - prometheus.flux.yaml + - repo.yaml diff --git a/e2e/monitoring/prometheus.flux.yaml b/e2e/monitoring/prometheus.flux.yaml new file mode 100644 index 000000000..a74b3a4be --- /dev/null +++ b/e2e/monitoring/prometheus.flux.yaml @@ -0,0 +1,104 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: prometheus + namespace: flux-system +spec: + interval: 24h0m0s + url: https://prometheus-community.github.io/helm-charts +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: kube-prometheus-stack + namespace: flux-system +spec: + interval: 15m + timeout: 10m + targetNamespace: observability-system + releaseName: "kube-prometheus-stack" + chart: + spec: + chart: kube-prometheus-stack + version: "61.3.2" + sourceRef: + kind: HelmRepository + name: prometheus + interval: 24h + install: + remediation: + retries: -1 + upgrade: + remediation: + remediateLastFailure: true + retries: -1 + driftDetection: + mode: enabled + values: + crds.enabled: true + grafana: + enabled: false + prometheusOperator: + enabled: true + alertmanager: + enabled: true + alertmanagerSpec: + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + limits: + cpu: 300m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + + kubeStateMetrics: + enabled: true + kube-state-metrics: + securityContext: + runAsGroup: 65534 + runAsUser: 65534 + runAsNonRoot: true + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + limits: + cpu: 300m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + prometheus: + monitor: + metricRelabelings: + - action: keep + regex: .*system.*|argocd|cert-manager|external-dns|ingress-controllers|kube-node-lease|kube-public + sourceLabels: + - namespace + rbac: + extraRules: + - apiGroups: + - "capsule.clastix.io" + resources: + - "tenants" + verbs: + - "list" + - "watch" diff --git a/e2e/monitoring/pyroscope.flux.yaml b/e2e/monitoring/pyroscope.flux.yaml new file mode 100644 index 000000000..c11633d9e --- /dev/null +++ b/e2e/monitoring/pyroscope.flux.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: pyroscope + namespace: flux-system +spec: + interval: 15m + timeout: 10m + targetNamespace: observability-system + releaseName: "pyroscope" + chart: + spec: + chart: pyroscope + version: "1.9.1" + sourceRef: + kind: HelmRepository + name: lgtm + interval: 24h + install: + remediation: + retries: -1 + upgrade: + remediation: + remediateLastFailure: true + retries: -1 + driftDetection: + mode: enabled + values: + global: + dnsService: "kube-dns" + dnsNamespace: "kube-system" + clusterLabelOverride: "kind" + pyroscope: + structuredConfig: + multitenancy_enabled: true diff --git a/e2e/monitoring/repo.yaml b/e2e/monitoring/repo.yaml new file mode 100644 index 000000000..a7b8c43fb --- /dev/null +++ b/e2e/monitoring/repo.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: lgtm + namespace: flux-system +spec: + interval: 24h0m0s + url: https://grafana.github.io/helm-charts diff --git a/e2e/utils_test.go b/e2e/utils_test.go index 10a43693b..25536f738 100644 --- a/e2e/utils_test.go +++ b/e2e/utils_test.go @@ -27,7 +27,7 @@ import ( ) const ( - defaultTimeoutInterval = 20 * time.Second + defaultTimeoutInterval = 40 * time.Second defaultPollInterval = time.Second ) diff --git a/main.go b/main.go index f4918fbd9..1c1a61b07 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,7 @@ import ( capsulev1beta1 "github.com/projectcapsule/capsule/api/v1beta1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" configcontroller "github.com/projectcapsule/capsule/controllers/config" + "github.com/projectcapsule/capsule/controllers/globalquota" podlabelscontroller "github.com/projectcapsule/capsule/controllers/pod" "github.com/projectcapsule/capsule/controllers/pv" rbaccontroller "github.com/projectcapsule/capsule/controllers/rbac" @@ -42,6 +43,7 @@ import ( "github.com/projectcapsule/capsule/pkg/indexer" "github.com/projectcapsule/capsule/pkg/webhook" "github.com/projectcapsule/capsule/pkg/webhook/defaults" + globalquotahook "github.com/projectcapsule/capsule/pkg/webhook/globalquota" "github.com/projectcapsule/capsule/pkg/webhook/ingress" namespacewebhook "github.com/projectcapsule/capsule/pkg/webhook/namespace" "github.com/projectcapsule/capsule/pkg/webhook/networkpolicy" @@ -231,6 +233,7 @@ func main() { route.Cordoning(tenant.CordoningHandler(cfg), tenant.ResourceCounterHandler(manager.GetClient())), route.Node(utils.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))), route.Defaults(defaults.Handler(cfg, kubeVersion)), + route.QuotaValidation(globalquotahook.StatusHandler(ctrl.Log.WithName("controllers").WithName("Webhook")), utils.InCapsuleGroups(cfg, globalquotahook.ValidationHandler()), globalquotahook.DeletionHandler(ctrl.Log.WithName("controllers").WithName("Webhook"))), ) nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion) @@ -308,6 +311,15 @@ func main() { os.Exit(1) } + if err = (&globalquota.Manager{ + Log: ctrl.Log.WithName("controllers").WithName("GlobalResourceQuotas"), + Client: manager.GetClient(), + Recorder: manager.GetEventRecorderFor("global-quota-ctrl"), + }).SetupWithManager(manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "globalquota") + os.Exit(1) + } + setupLog.Info("starting manager") if err = manager.Start(ctx); err != nil { diff --git a/mount-pod.yaml b/mount-pod.yaml new file mode 100644 index 000000000..69a365e16 --- /dev/null +++ b/mount-pod.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: mount-pod +spec: + containers: + - name: my-container + image: alpine + command: + - sh + - -c + - | + sleep infinity + volumeMounts: + - mountPath: /data + name: my-volume + volumes: + - name: my-volume + persistentVolumeClaim: + claimName: k41-minio diff --git a/e2e/additional_role_bindings_test.go b/old/additional_role_bindings_test.go similarity index 100% rename from e2e/additional_role_bindings_test.go rename to old/additional_role_bindings_test.go diff --git a/e2e/allowed_external_ips_test.go b/old/allowed_external_ips_test.go similarity index 100% rename from e2e/allowed_external_ips_test.go rename to old/allowed_external_ips_test.go diff --git a/e2e/container_registry_test.go b/old/container_registry_test.go similarity index 100% rename from e2e/container_registry_test.go rename to old/container_registry_test.go diff --git a/e2e/custom_capsule_group_test.go b/old/custom_capsule_group_test.go similarity index 100% rename from e2e/custom_capsule_group_test.go rename to old/custom_capsule_group_test.go diff --git a/e2e/custom_resource_quota_test.go b/old/custom_resource_quota_test.go similarity index 100% rename from e2e/custom_resource_quota_test.go rename to old/custom_resource_quota_test.go diff --git a/e2e/disable_externalname_test.go b/old/disable_externalname_test.go similarity index 100% rename from e2e/disable_externalname_test.go rename to old/disable_externalname_test.go diff --git a/e2e/disable_ingress_wildcard_test.go b/old/disable_ingress_wildcard_test.go similarity index 100% rename from e2e/disable_ingress_wildcard_test.go rename to old/disable_ingress_wildcard_test.go diff --git a/e2e/disable_loadbalancer_test.go b/old/disable_loadbalancer_test.go similarity index 100% rename from e2e/disable_loadbalancer_test.go rename to old/disable_loadbalancer_test.go diff --git a/e2e/disable_node_ports_test.go b/old/disable_node_ports_test.go similarity index 100% rename from e2e/disable_node_ports_test.go rename to old/disable_node_ports_test.go diff --git a/e2e/dynamic_tenant_owner_clusterroles_test.go b/old/dynamic_tenant_owner_clusterroles_test.go similarity index 100% rename from e2e/dynamic_tenant_owner_clusterroles_test.go rename to old/dynamic_tenant_owner_clusterroles_test.go diff --git a/e2e/enable_loadbalancer_test.go b/old/enable_loadbalancer_test.go similarity index 100% rename from e2e/enable_loadbalancer_test.go rename to old/enable_loadbalancer_test.go diff --git a/e2e/enable_node_ports_test.go b/old/enable_node_ports_test.go similarity index 100% rename from e2e/enable_node_ports_test.go rename to old/enable_node_ports_test.go diff --git a/e2e/forbidden_annotations_regex_test.go b/old/forbidden_annotations_regex_test.go similarity index 100% rename from e2e/forbidden_annotations_regex_test.go rename to old/forbidden_annotations_regex_test.go diff --git a/e2e/force_tenant_prefix_tenant_scope_test.go b/old/force_tenant_prefix_tenant_scope_test.go similarity index 100% rename from e2e/force_tenant_prefix_tenant_scope_test.go rename to old/force_tenant_prefix_tenant_scope_test.go diff --git a/e2e/force_tenant_prefix_test.go b/old/force_tenant_prefix_test.go similarity index 100% rename from e2e/force_tenant_prefix_test.go rename to old/force_tenant_prefix_test.go diff --git a/e2e/globaltenantresource_test.go b/old/globaltenantresource_test.go similarity index 100% rename from e2e/globaltenantresource_test.go rename to old/globaltenantresource_test.go diff --git a/e2e/imagepullpolicy_multiple_test.go b/old/imagepullpolicy_multiple_test.go similarity index 100% rename from e2e/imagepullpolicy_multiple_test.go rename to old/imagepullpolicy_multiple_test.go diff --git a/e2e/imagepullpolicy_single_test.go b/old/imagepullpolicy_single_test.go similarity index 100% rename from e2e/imagepullpolicy_single_test.go rename to old/imagepullpolicy_single_test.go diff --git a/e2e/ingress_class_extensions_test.go b/old/ingress_class_extensions_test.go similarity index 100% rename from e2e/ingress_class_extensions_test.go rename to old/ingress_class_extensions_test.go diff --git a/e2e/ingress_class_networking_test.go b/old/ingress_class_networking_test.go similarity index 100% rename from e2e/ingress_class_networking_test.go rename to old/ingress_class_networking_test.go diff --git a/e2e/ingress_hostnames_collision_cluster_scope_test.go b/old/ingress_hostnames_collision_cluster_scope_test.go similarity index 100% rename from e2e/ingress_hostnames_collision_cluster_scope_test.go rename to old/ingress_hostnames_collision_cluster_scope_test.go diff --git a/e2e/ingress_hostnames_collision_disabled_test.go b/old/ingress_hostnames_collision_disabled_test.go similarity index 100% rename from e2e/ingress_hostnames_collision_disabled_test.go rename to old/ingress_hostnames_collision_disabled_test.go diff --git a/e2e/ingress_hostnames_collision_namespace_scope_test.go b/old/ingress_hostnames_collision_namespace_scope_test.go similarity index 100% rename from e2e/ingress_hostnames_collision_namespace_scope_test.go rename to old/ingress_hostnames_collision_namespace_scope_test.go diff --git a/e2e/ingress_hostnames_collision_tenant_scope_test.go b/old/ingress_hostnames_collision_tenant_scope_test.go similarity index 100% rename from e2e/ingress_hostnames_collision_tenant_scope_test.go rename to old/ingress_hostnames_collision_tenant_scope_test.go diff --git a/e2e/ingress_hostnames_test.go b/old/ingress_hostnames_test.go similarity index 100% rename from e2e/ingress_hostnames_test.go rename to old/ingress_hostnames_test.go diff --git a/e2e/missing_tenant_test.go b/old/missing_tenant_test.go similarity index 100% rename from e2e/missing_tenant_test.go rename to old/missing_tenant_test.go diff --git a/e2e/namespace_additional_metadata_test.go b/old/namespace_additional_metadata_test.go similarity index 100% rename from e2e/namespace_additional_metadata_test.go rename to old/namespace_additional_metadata_test.go diff --git a/e2e/namespace_capsule_label_test.go b/old/namespace_capsule_label_test.go similarity index 100% rename from e2e/namespace_capsule_label_test.go rename to old/namespace_capsule_label_test.go diff --git a/e2e/namespace_hijacking_test.go b/old/namespace_hijacking_test.go similarity index 100% rename from e2e/namespace_hijacking_test.go rename to old/namespace_hijacking_test.go diff --git a/e2e/namespace_user_metadata_test.go b/old/namespace_user_metadata_test.go similarity index 100% rename from e2e/namespace_user_metadata_test.go rename to old/namespace_user_metadata_test.go diff --git a/e2e/new_namespace_test.go b/old/new_namespace_test.go similarity index 100% rename from e2e/new_namespace_test.go rename to old/new_namespace_test.go diff --git a/e2e/node_user_metadata_test.go b/old/node_user_metadata_test.go similarity index 100% rename from e2e/node_user_metadata_test.go rename to old/node_user_metadata_test.go diff --git a/e2e/overquota_namespace_test.go b/old/overquota_namespace_test.go similarity index 100% rename from e2e/overquota_namespace_test.go rename to old/overquota_namespace_test.go diff --git a/e2e/owner_webhooks_test.go b/old/owner_webhooks_test.go similarity index 100% rename from e2e/owner_webhooks_test.go rename to old/owner_webhooks_test.go diff --git a/e2e/pod_metadata_test.go b/old/pod_metadata_test.go similarity index 100% rename from e2e/pod_metadata_test.go rename to old/pod_metadata_test.go diff --git a/e2e/pod_priority_class_test.go b/old/pod_priority_class_test.go similarity index 100% rename from e2e/pod_priority_class_test.go rename to old/pod_priority_class_test.go diff --git a/e2e/pod_runtime_class_test.go b/old/pod_runtime_class_test.go similarity index 100% rename from e2e/pod_runtime_class_test.go rename to old/pod_runtime_class_test.go diff --git a/e2e/preventing_pv_cross_tenant_mount_test.go b/old/preventing_pv_cross_tenant_mount_test.go similarity index 100% rename from e2e/preventing_pv_cross_tenant_mount_test.go rename to old/preventing_pv_cross_tenant_mount_test.go diff --git a/e2e/protected_namespace_regex_test.go b/old/protected_namespace_regex_test.go similarity index 100% rename from e2e/protected_namespace_regex_test.go rename to old/protected_namespace_regex_test.go diff --git a/e2e/resource_quota_exceeded_test.go b/old/resource_quota_exceeded_test.go similarity index 100% rename from e2e/resource_quota_exceeded_test.go rename to old/resource_quota_exceeded_test.go diff --git a/e2e/sa_prevent_privilege_escalation_test.go b/old/sa_prevent_privilege_escalation_test.go similarity index 100% rename from e2e/sa_prevent_privilege_escalation_test.go rename to old/sa_prevent_privilege_escalation_test.go diff --git a/e2e/selecting_non_owned_tenant_test.go b/old/selecting_non_owned_tenant_test.go similarity index 100% rename from e2e/selecting_non_owned_tenant_test.go rename to old/selecting_non_owned_tenant_test.go diff --git a/e2e/selecting_tenant_fail_test.go b/old/selecting_tenant_fail_test.go similarity index 100% rename from e2e/selecting_tenant_fail_test.go rename to old/selecting_tenant_fail_test.go diff --git a/e2e/selecting_tenant_with_label_test.go b/old/selecting_tenant_with_label_test.go similarity index 100% rename from e2e/selecting_tenant_with_label_test.go rename to old/selecting_tenant_with_label_test.go diff --git a/e2e/service_forbidden_metadata_test.go b/old/service_forbidden_metadata_test.go similarity index 100% rename from e2e/service_forbidden_metadata_test.go rename to old/service_forbidden_metadata_test.go diff --git a/e2e/service_metadata_test.go b/old/service_metadata_test.go similarity index 100% rename from e2e/service_metadata_test.go rename to old/service_metadata_test.go diff --git a/e2e/storage_class_test.go b/old/storage_class_test.go similarity index 100% rename from e2e/storage_class_test.go rename to old/storage_class_test.go diff --git a/e2e/tenant_cordoning_test.go b/old/tenant_cordoning_test.go similarity index 100% rename from e2e/tenant_cordoning_test.go rename to old/tenant_cordoning_test.go diff --git a/e2e/tenant_metadata.go b/old/tenant_metadata.go similarity index 100% rename from e2e/tenant_metadata.go rename to old/tenant_metadata.go diff --git a/e2e/tenant_name_webhook_test.go b/old/tenant_name_webhook_test.go similarity index 100% rename from e2e/tenant_name_webhook_test.go rename to old/tenant_name_webhook_test.go diff --git a/e2e/tenant_protected_webhook_test.go b/old/tenant_protected_webhook_test.go similarity index 100% rename from e2e/tenant_protected_webhook_test.go rename to old/tenant_protected_webhook_test.go diff --git a/e2e/tenant_resources_changes_test.go b/old/tenant_resources_changes_test.go similarity index 100% rename from e2e/tenant_resources_changes_test.go rename to old/tenant_resources_changes_test.go diff --git a/e2e/tenant_resources_test.go b/old/tenant_resources_test.go similarity index 100% rename from e2e/tenant_resources_test.go rename to old/tenant_resources_test.go diff --git a/e2e/tenantresource_test.go b/old/tenantresource_test.go similarity index 100% rename from e2e/tenantresource_test.go rename to old/tenantresource_test.go diff --git a/pkg/api/name.go b/pkg/api/name.go new file mode 100644 index 000000000..e013f9a3a --- /dev/null +++ b/pkg/api/name.go @@ -0,0 +1,16 @@ +package api + +// Name must be unique within a namespace. Is required when creating resources, although +// some resources may allow a client to request the generation of an appropriate name +// automatically. Name is primarily intended for creation idempotence and configuration +// definition. +// Cannot be updated. +// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names +// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` +// +kubebuilder:validation:MaxLength=20 +// +kubebuilder:object:generate=true +type Name string + +func (n Name) String() string { + return string(n) +} diff --git a/pkg/api/selectors.go b/pkg/api/selectors.go new file mode 100644 index 000000000..e4474bc94 --- /dev/null +++ b/pkg/api/selectors.go @@ -0,0 +1,45 @@ +package api + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Selector for resources and their labels or selecting origin namespaces +// +kubebuilder:object:generate=true +type NamespaceSelector struct { + // Select Items based on their labels. If the namespaceSelector is also set, the selector is applied + // to items within the selected namespaces. Otherwise for all the items. + *metav1.LabelSelector `json:",inline"` +} + +// GetMatchingNamespaces retrieves the list of namespaces that match the NamespaceSelector. +func (s *NamespaceSelector) GetMatchingNamespaces(ctx context.Context, client client.Client) ([]corev1.Namespace, error) { + if s.LabelSelector == nil { + return nil, nil // No namespace selector means all namespaces + } + + nsSelector, err := metav1.LabelSelectorAsSelector(s.LabelSelector) + if err != nil { + return nil, fmt.Errorf("invalid namespace selector: %w", err) + } + + namespaceList := &corev1.NamespaceList{} + if err := client.List(context.TODO(), namespaceList); err != nil { + return nil, fmt.Errorf("failed to list namespaces: %w", err) + } + + var matchingNamespaces []corev1.Namespace + for _, ns := range namespaceList.Items { + if nsSelector.Matches(labels.Set(ns.Labels)) { + matchingNamespaces = append(matchingNamespaces, ns) + } + } + + return matchingNamespaces, nil +} diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index e3ac0fb27..00757bdb5 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -190,6 +191,26 @@ func (in *LimitRangesSpec) DeepCopy() *LimitRangesSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamespaceSelector) DeepCopyInto(out *NamespaceSelector) { + *out = *in + if in.LabelSelector != nil { + in, out := &in.LabelSelector, &out.LabelSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespaceSelector. +func (in *NamespaceSelector) DeepCopy() *NamespaceSelector { + if in == nil { + return nil + } + out := new(NamespaceSelector) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkPolicySpec) DeepCopyInto(out *NetworkPolicySpec) { *out = *in diff --git a/pkg/indexer/globalquota/namespaces.go b/pkg/indexer/globalquota/namespaces.go new file mode 100644 index 000000000..767db3048 --- /dev/null +++ b/pkg/indexer/globalquota/namespaces.go @@ -0,0 +1,31 @@ +package globalquota + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" +) + +// NamespacesReference defines the indexer logic for GlobalResourceQuota namespaces. +type NamespacesReference struct { + Obj client.Object +} + +func (o NamespacesReference) Object() client.Object { + return o.Obj +} + +func (o NamespacesReference) Field() string { + return ".status.namespaces" +} + +//nolint:forcetypeassert +func (o NamespacesReference) Func() client.IndexerFunc { + return func(object client.Object) []string { + grq, ok := object.(*capsulev1beta2.GlobalResourceQuota) + if !ok { + return nil + } + return grq.Status.Namespaces + } +} diff --git a/pkg/indexer/indexer.go b/pkg/indexer/indexer.go index 9d7b0fa8d..0cffd7112 100644 --- a/pkg/indexer/indexer.go +++ b/pkg/indexer/indexer.go @@ -15,6 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/indexer/globalquota" "github.com/projectcapsule/capsule/pkg/indexer/ingress" "github.com/projectcapsule/capsule/pkg/indexer/namespace" "github.com/projectcapsule/capsule/pkg/indexer/tenant" @@ -31,6 +32,7 @@ type CustomIndexer interface { func AddToManager(ctx context.Context, log logr.Logger, mgr manager.Manager) error { indexers := []CustomIndexer{ tenant.NamespacesReference{Obj: &capsulev1beta2.Tenant{}}, + globalquota.NamespacesReference{Obj: &capsulev1beta2.GlobalResourceQuota{}}, tenant.OwnerReference{}, namespace.OwnerReference{}, ingress.HostnamePath{Obj: &extensionsv1beta1.Ingress{}}, diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index b9b06044a..7d55efc90 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -20,6 +20,16 @@ var ( Name: metricsPrefix + "tenant_resource_limit", Help: "Current resource limit for a given resource in a tenant", }, []string{"tenant", "resource", "resourcequotaindex"}) + + GlobalResourceUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: metricsPrefix + "global_resource_usage", + Help: "Current resource usage for a given resource in a global resource quota", + }, []string{"quota", "resource", "resourcequotaindex"}) + + GlobalResourceLimit = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: metricsPrefix + "global_resource_limit", + Help: "Current resource limit for a given resource in a global resource quota", + }, []string{"quota", "resource", "resourcequotaindex"}) ) func init() { diff --git a/pkg/utils/tenant_labels.go b/pkg/utils/tenant_labels.go index b236c5f89..85bbc7788 100644 --- a/pkg/utils/tenant_labels.go +++ b/pkg/utils/tenant_labels.go @@ -19,6 +19,8 @@ func GetTypeLabel(t runtime.Object) (label string, err error) { switch v := t.(type) { case *v1beta1.Tenant, *v1beta2.Tenant: return "capsule.clastix.io/tenant", nil + case *v1beta2.GlobalResourceQuota: + return "capsule.clastix.io/global-quota", nil case *corev1.LimitRange: return "capsule.clastix.io/limit-range", nil case *networkingv1.NetworkPolicy: @@ -33,3 +35,7 @@ func GetTypeLabel(t runtime.Object) (label string, err error) { return } + +func GetGlobalResourceQuotaTypeLabel() (label string) { + return "capsule.clastix.io/global-quota-item" +} diff --git a/pkg/webhook/globalquota/calculation.go b/pkg/webhook/globalquota/calculation.go new file mode 100644 index 000000000..030908d40 --- /dev/null +++ b/pkg/webhook/globalquota/calculation.go @@ -0,0 +1,241 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package globalquota + +import ( + "context" + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/go-logr/logr" + "github.com/projectcapsule/capsule/pkg/api" + capsuleutils "github.com/projectcapsule/capsule/pkg/utils" + capsulewebhook "github.com/projectcapsule/capsule/pkg/webhook" + "github.com/projectcapsule/capsule/pkg/webhook/utils" +) + +type statusHandler struct { + log logr.Logger +} + +func StatusHandler(log logr.Logger) capsulewebhook.Handler { + return &statusHandler{log: log} +} + +func (h *statusHandler) OnCreate(c client.Client, decoder admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return nil + } +} + +func (h *statusHandler) OnDelete(c client.Client, decoder admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return nil + } +} + +func (h *statusHandler) OnUpdate(c client.Client, decoder admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return h.calculate(ctx, c, decoder, recorder, req) + } +} + +func (h *statusHandler) calculate(ctx context.Context, c client.Client, decoder admission.Decoder, recorder record.EventRecorder, req admission.Request) *admission.Response { + h.log.V(3).Info("loggign request", "REQUEST", req) + + return utils.ErroredResponse(fmt.Errorf("meowie")) + + // Decode the incoming object + quota := &corev1.ResourceQuota{} + if err := decoder.Decode(req, quota); err != nil { + return utils.ErroredResponse(fmt.Errorf("failed to decode new ResourceQuota object: %w", err)) + } + + // Decode the old object (previous state before update) + oldQuota := &corev1.ResourceQuota{} + if err := decoder.DecodeRaw(req.OldObject, oldQuota); err != nil { + return utils.ErroredResponse(fmt.Errorf("failed to decode old ResourceQuota object: %w", err)) + } + + h.log.V(3).Info("loggign request", "REQUEST", req) + + // Get Item within Resource Quota + indexLabel := capsuleutils.GetGlobalResourceQuotaTypeLabel() + item, ok := quota.GetLabels()[indexLabel] + + if !ok || item == "" { + return nil + } + + // Get Item within Resource Quota + globalQuota, err := GetGlobalQuota(ctx, c, quota) + if err != nil { + return utils.ErroredResponse(err) + } + + if globalQuota == nil { + return nil + } + + // Skip if quota not active + if !globalQuota.Spec.Active { + h.log.V(3).Info("GlobalQuota is not active", "quota", globalQuota.Name) + + return nil + } + + // Skip Directly when the Status has not changed + //if quota.Status.Hard == oldQuota.Status.Hard { + // return nil + //} + + h.log.V(7).Info("selected quota", "quota", globalQuota.Name, "item", item) + + zero := resource.MustParse("0") + + // Fetch the latest tenant quota status + tenantQuota, exists := globalQuota.Status.Quota[api.Name(item)] + if !exists { + h.log.V(5).Info("No quota entry found in tenant status; initializing", "item", api.Name(item)) + + return nil + } + + // Calculate remaining available space for this item + availableSpace, _ := globalQuota.GetQuotaSpace(api.Name(item)) + if availableSpace == nil { + availableSpace = corev1.ResourceList{} + } + + // Fetch current used quota + tenantUsed := tenantQuota.Used + if tenantUsed == nil { + tenantUsed = corev1.ResourceList{} + } + + h.log.V(3).Info("Available space calculated", "space", availableSpace) + + // Process each resource and enforce allocation limits + for resourceName, avail := range availableSpace { + rlog := h.log.WithValues("resource", resourceName) + + rlog.V(3).Info("AVAILABLE", "avail", avail, "USED", tenantUsed[resourceName], "HARD", tenantQuota.Hard[resourceName]) + + if avail.Cmp(zero) == 0 { + rlog.V(3).Info("NO SPACE AVAILABLE") + quota.Status.Hard[resourceName] = oldQuota.Status.Hard[resourceName] + continue + } + + // Get From the status whet's currently Used + var globalUsage resource.Quantity + if currentUsed, exists := tenantUsed[resourceName]; exists { + globalUsage = currentUsed.DeepCopy() + } else { + globalUsage = resource.MustParse("0") + } + + // Calculate Ingestion Size + oldAllocated, exists := oldQuota.Status.Used[resourceName] + if !exists { + oldAllocated = resource.Quantity{} // default to zero + } + // + //// Get the newly requested limit from the updated quota + newRequested, exists := quota.Status.Used[resourceName] + if !exists { + quota.Status.Hard[resourceName] = resource.Quantity{} + newRequested = oldAllocated.DeepCopy() // assume no change if missing + } + + // Calculate Difference in Usage + diff := newRequested.DeepCopy() + diff.Sub(oldAllocated) + + rlog.V(3).Info("calculate ingestion", "diff", diff, "old", oldAllocated, "new", newRequested) + + // Compare how the newly ingested resources compare against empty resources + // This is the quickest way to find out, how the status must be updated + stat := diff.Cmp(zero) + + switch { + // Resources are eual + case stat == 0: + continue + // Resource Consumtion Increased + case stat > 0: + rlog.V(3).Info("increase") + // Validate Space + // Overprovisioned, allocate what's left + if avail.Cmp(diff) < 0 { + // Overprovisioned, allocate what's left + globalUsage.Add(avail) + + // Here we cap overprovisioning, we add what's left to the + // old status and update the item status. For the other operations that's ensured + // because of this webhook. + + //oldAllocated.Add(avail) + rlog.V(5).Info("PREVENT OVERPROVISING", "allocation", oldAllocated) + quota.Status.Hard[resourceName] = oldQuota.Status.Hard[resourceName] + + } else { + // Adding, since requested resources have space + globalUsage.Add(diff) + + oldAllocated.Add(diff) + quota.Status.Hard[resourceName] = oldAllocated + + } + // Resource Consumption decreased + default: + rlog.V(3).Info("negate") + // SUbstract Difference from available + // Negative values also combine correctly with the Add() operation + globalUsage.Add(diff) + + // Prevent Usage from going to negative + stat := globalUsage.Cmp(zero) + if stat < 0 { + globalUsage = zero + } + } + + rlog.V(3).Info("caclulated total usage", "global", globalUsage, "diff", diff, "usage", avail, "hard", quota.Status.Hard[resourceName], "usage", quota.Status.Used[resourceName]) + tenantUsed[resourceName] = globalUsage + } + + // Persist the updated usage in globalQuota.Status.Quota + tenantQuota.Used = tenantUsed.DeepCopy() + globalQuota.Status.Quota[api.Name(item)] = tenantQuota + + // Ensure the status is updated immediately + if err := c.Status().Update(ctx, globalQuota); err != nil { + if apierrors.IsConflict(err) { + h.log.Info("GlobalQuota status update conflict detected: object was updated concurrently", "error", err.Error()) + } + + h.log.Info("failed to update GlobalQuota status", "error", err.Error(), "global", globalQuota.Name, "quota", api.Name(item), "namespace", quota.Namespace) + + return utils.ErroredResponse(err) + } + + marshaled, err := json.Marshal(quota) + if err != nil { + h.log.Error(err, "Failed to marshal mutated ResourceQuota object") + + return utils.ErroredResponse(err) + } + + response := admission.PatchResponseFromRaw(req.Object.Raw, marshaled) + + return &response +} diff --git a/pkg/webhook/globalquota/change-guard.go b/pkg/webhook/globalquota/change-guard.go new file mode 100644 index 000000000..28f059346 --- /dev/null +++ b/pkg/webhook/globalquota/change-guard.go @@ -0,0 +1,81 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 +package globalquota + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + capsuleutils "github.com/projectcapsule/capsule/pkg/utils" + capsulewebhook "github.com/projectcapsule/capsule/pkg/webhook" + "github.com/projectcapsule/capsule/pkg/webhook/utils" +) + +type validationHandler struct{} + +func ValidationHandler() capsulewebhook.Handler { + return &validationHandler{} +} + +func (r *validationHandler) OnCreate(client.Client, admission.Decoder, record.EventRecorder) capsulewebhook.Func { + return func(context.Context, admission.Request) *admission.Response { + return nil + } +} + +func (r *validationHandler) OnDelete(client client.Client, decoder admission.Decoder, _ record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + allowed, err := r.handle(ctx, req, client, decoder) + if err != nil { + return utils.ErroredResponse(err) + } + + if !allowed { + response := admission.Denied("Capsule Resource Quotas cannot be deleted") + + return &response + } + + return nil + } +} + +func (r *validationHandler) OnUpdate(client client.Client, decoder admission.Decoder, _ record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + allowed, err := r.handle(ctx, req, client, decoder) + if err != nil { + return utils.ErroredResponse(err) + } + + if !allowed { + response := admission.Denied("Capsule ResourceQuotas cannot be updated") + + return &response + } + + return nil + } +} + +func (r *validationHandler) handle(ctx context.Context, req admission.Request, client client.Client, _ admission.Decoder) (allowed bool, err error) { + allowed = true + + np := &corev1.ResourceQuota{} + if err = client.Get(ctx, types.NamespacedName{Namespace: req.AdmissionRequest.Namespace, Name: req.AdmissionRequest.Name}, np); err != nil { + return false, err + } + + objectLabel := capsuleutils.GetGlobalResourceQuotaTypeLabel() + + labels := np.GetLabels() + if _, ok := labels[objectLabel]; ok { + allowed = false + } + + return +} diff --git a/pkg/webhook/globalquota/deletion.go b/pkg/webhook/globalquota/deletion.go new file mode 100644 index 000000000..9f5341e69 --- /dev/null +++ b/pkg/webhook/globalquota/deletion.go @@ -0,0 +1,155 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package globalquota + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/go-logr/logr" + "github.com/projectcapsule/capsule/pkg/api" + capsuleutils "github.com/projectcapsule/capsule/pkg/utils" + capsulewebhook "github.com/projectcapsule/capsule/pkg/webhook" + "github.com/projectcapsule/capsule/pkg/webhook/utils" +) + +type deletionHandler struct { + log logr.Logger +} + +func DeletionHandler(log logr.Logger) capsulewebhook.Handler { + return &deletionHandler{log: log} +} + +func (h *deletionHandler) OnCreate(c client.Client, decoder admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return nil + } +} + +// Substract a ResourceQuota (Usage) when it's deleted +// In normal operations this covers the case, when a namespace no longer get's selected and therefor +// The quota is being terminated /Maybe not working on status subresource +func (h *deletionHandler) OnDelete(c client.Client, decoder admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + h.log.V(5).Info("loggign request", "REQUEST", req) + + // Decode the incoming object (Always old object) + quota := &corev1.ResourceQuota{} + if err := decoder.DecodeRaw(req.OldObject, quota); err != nil { + return utils.ErroredResponse(fmt.Errorf("failed to decode new ResourceQuota object: %w", err)) + } + + // Get Item within Resource Quota + indexLabel := capsuleutils.GetGlobalResourceQuotaTypeLabel() + item, ok := quota.GetLabels()[indexLabel] + + if !ok || item == "" { + return nil + } + + // Get Item within Resource Quota + globalQuota, err := GetGlobalQuota(ctx, c, quota) + // Just delete the quopta when the globalquota was delete + if apierrors.IsNotFound(err) { + return nil + } + + if err != nil { + return utils.ErroredResponse(err) + } + + if globalQuota == nil { + return nil + } + + zero := resource.MustParse("0") + + // Use retry to handle concurrent updates + err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { + // Re-fetch the tenant to get the latest status + if err := c.Get(ctx, client.ObjectKey{Name: globalQuota.Name}, globalQuota); err != nil { + h.log.Error(err, "Failed to fetch globalquota during retry", "quota", globalQuota.Name) + + return err + } + // Fetch the latest tenant quota status + tenantQuota, exists := globalQuota.Status.Quota[api.Name(item)] + if !exists { + h.log.V(5).Info("No quota entry found in tenant status; initializing", "item", api.Name(item)) + + return nil + } + + // Fetch current used quota + tenantUsed := tenantQuota.Used + if tenantUsed == nil { + tenantUsed = corev1.ResourceList{} + } + + // Remove all resources from the used property on the global quota + for resourceName, used := range quota.Status.Used { + rlog := h.log.WithValues("resource", resourceName) + + // Get From the status whet's currently Used + var globalUsage resource.Quantity + if currentUsed, exists := tenantUsed[resourceName]; exists { + globalUsage = currentUsed.DeepCopy() + } else { + continue + } + + // Remove + globalUsage.Sub(used) + + // Avoid being below 0 (negative) + stat := globalUsage.Cmp(zero) + if stat < 0 { + globalUsage = zero + } + + rlog.V(7).Info("decreasing global usage", "decrease", used, "status", globalUsage) + + tenantUsed[resourceName] = globalUsage + + } + + h.log.V(7).Info("calculated status", "used", tenantUsed) + + // Persist the updated usage in globalQuota.Status.Qcuota + globalQuota.Status.Quota[api.Name(item)].Used = tenantUsed.DeepCopy() + + // Ensure the status is updated immediately + if err := c.Status().Update(ctx, globalQuota); err != nil { + h.log.Info("Failed to update GlobalQuota status", "error", err.Error()) + + return fmt.Errorf("failed to update GlobalQuota status: %w", err) + } + + return nil + }) + + if err != nil { + h.log.Error(err, "Failed to process ResourceQuota update", "quota", quota.Name) + + return utils.ErroredResponse(err) + } + + return nil + } +} + +func (h *deletionHandler) OnUpdate(c client.Client, decoder admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) *admission.Response { + return nil + } +} diff --git a/pkg/webhook/globalquota/utils.go b/pkg/webhook/globalquota/utils.go new file mode 100644 index 000000000..28307b251 --- /dev/null +++ b/pkg/webhook/globalquota/utils.go @@ -0,0 +1,38 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package globalquota + +import ( + "context" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + capsuleutils "github.com/projectcapsule/capsule/pkg/utils" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func GetGlobalQuota(ctx context.Context, c client.Client, quota *corev1.ResourceQuota) (q *capsulev1beta2.GlobalResourceQuota, err error) { + q = &capsulev1beta2.GlobalResourceQuota{} + + // Get Item within Resource Quota + objectLabel, err := capsuleutils.GetTypeLabel(&capsulev1beta2.GlobalResourceQuota{}) + if err != nil { + return + } + + // Not a global quota resourcequota + labels := quota.GetLabels() + + globalQuotaName, ok := labels[objectLabel] + if !ok { + return + } + + if err = c.Get(ctx, types.NamespacedName{Name: globalQuotaName}, q); err != nil { + return + } + + return +} diff --git a/pkg/webhook/route/globalquota.go b/pkg/webhook/route/globalquota.go new file mode 100644 index 000000000..2c00806ce --- /dev/null +++ b/pkg/webhook/route/globalquota.go @@ -0,0 +1,40 @@ +// Copyright 2020-2023 Project Capsule Authors. +// SPDX-License-Identifier: Apache-2.0 + +package route + +import ( + capsulewebhook "github.com/projectcapsule/capsule/pkg/webhook" +) + +type quotamutation struct { + handlers []capsulewebhook.Handler +} + +func QuotaMutation(handler ...capsulewebhook.Handler) capsulewebhook.Webhook { + return "amutation{handlers: handler} +} + +func (w *quotamutation) GetHandlers() []capsulewebhook.Handler { + return w.handlers +} + +func (w *quotamutation) GetPath() string { + return "/globalquota/mutation" +} + +type quotaValidation struct { + handlers []capsulewebhook.Handler +} + +func QuotaValidation(handler ...capsulewebhook.Handler) capsulewebhook.Webhook { + return "aValidation{handlers: handler} +} + +func (w *quotaValidation) GetHandlers() []capsulewebhook.Handler { + return w.handlers +} + +func (w *quotaValidation) GetPath() string { + return "/globalquota/validation" +} diff --git a/quota.yaml b/quota.yaml new file mode 100644 index 000000000..0d8cdb6b0 --- /dev/null +++ b/quota.yaml @@ -0,0 +1,20 @@ +apiVersion: capsule.clastix.io/v1beta2 +kind: GlobalResourceQuota +metadata: + name: sampler +spec: + selectors: + - matchLabels: + capsule.clastix.io/tenant: solar + - matchLabels: + capsule.clastix.io/tenant: solar + quotas: + scheduling: + hard: + limits.cpu: "2" + limits.memory: 2Gi + requests.cpu: "2" + requests.memory: 2Gi + pods: + hard: + pods: "3" diff --git a/test.yaml b/test.yaml new file mode 100644 index 000000000..e3d0903b2 --- /dev/null +++ b/test.yaml @@ -0,0 +1,20 @@ +apiVersion: capsule.clastix.io/v1beta2 +kind: Tenant +metadata: + name: solar +spec: + owners: + - name: alice + kind: User + namespaceOptions: + quota: 3 + #resourceQuotas: + # scope: Tenant + # items: + # - hard: + # limits.cpu: "2" + # limits.memory: 2Gi + # requests.cpu: "2" + # requests.memory: 2Gi + # - hard: + # pods: "3" diff --git a/tnt.yaml b/tnt.yaml new file mode 100644 index 000000000..84ad461a5 --- /dev/null +++ b/tnt.yaml @@ -0,0 +1,26 @@ +apiVersion: capsule.clastix.io/v1beta2 +kind: Tenant +metadata: + creationTimestamp: "2025-02-18T17:38:52Z" + generation: 1 + labels: + customer-resource-pool: dev + kubernetes.io/metadata.name: solar-quota + name: solar-quota + resourceVersion: "28140" + uid: 81c4ca40-550c-4dca-97f7-6f0ca98ad88a +spec: + cordoned: false + ingressOptions: + hostnameCollisionScope: Disabled + limitRanges: {} + networkPolicies: {} + owners: + - clusterRoles: + - admin + - capsule-namespace-deleter + kind: User + name: solar-user + preventDeletion: false + resourceQuotas: + scope: Tenant diff --git a/zero-quota.yaml b/zero-quota.yaml new file mode 100644 index 000000000..a72b2ffff --- /dev/null +++ b/zero-quota.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: compute-resources +spec: + hard: + pods: "0"