Skip to content

Commit

Permalink
feat: Use PartialObjectMetadata for Configmaps
Browse files Browse the repository at this point in the history
  • Loading branch information
mrueg committed Aug 7, 2024
1 parent f8aa7d9 commit 151ede1
Show file tree
Hide file tree
Showing 9 changed files with 150 additions and 21 deletions.
70 changes: 65 additions & 5 deletions internal/store/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@ import (
policyv1 "k8s.io/api/policy/v1"
rbacv1 "k8s.io/api/rbac/v1"
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/metadata"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"

Expand All @@ -65,9 +67,10 @@ var _ ksmtypes.BuilderInterface = &Builder{}
// Builder helps to build store. It follows the builder pattern
// (https://en.wikipedia.org/wiki/Builder_pattern).
type Builder struct {
kubeClient clientset.Interface
customResourceClients map[string]interface{}
namespaces options.NamespaceList
kubeClient clientset.Interface
metadataOnlyKubeClient metadata.Interface
customResourceClients map[string]interface{}
namespaces options.NamespaceList
// namespaceFilter is inside fieldSelectorFilter
fieldSelectorFilter string
ctx context.Context
Expand All @@ -78,6 +81,7 @@ type Builder struct {
shard int32
totalShards int
buildStoresFunc ksmtypes.BuildStoresFunc
buildMetadataOnlyStoresFunc ksmtypes.BuildMetadataOnlyStoresFunc
buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc
allowAnnotationsList map[string][]string
allowLabelsList map[string][]string
Expand Down Expand Up @@ -157,6 +161,11 @@ func (b *Builder) WithKubeClient(c clientset.Interface) {
b.kubeClient = c
}

// WithMetadataOnlyKubeClient sets the metadataOnlyKubeClient property of a Builder.
func (b *Builder) WithMetadataOnlyKubeClient(c metadata.Interface) {
b.metadataOnlyKubeClient = c
}

// WithCustomResourceClients sets the customResourceClients property of a Builder.
func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) {
b.customResourceClients = cs
Expand All @@ -178,6 +187,11 @@ func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) {
b.buildStoresFunc = f
}

// WithGenerateMetadataOnlyStoresFunc configures a custom generate custom resource store function
func (b *Builder) WithGenerateMetadataOnlyStoresFunc(f ksmtypes.BuildMetadataOnlyStoresFunc) {
b.buildMetadataOnlyStoresFunc = f
}

// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function
func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) {
b.buildCustomResourceStoresFunc = f
Expand All @@ -188,6 +202,11 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc {
return b.buildStores
}

// DefaultGenerateMetadataOnlyStoresFunc returns default buildStores function
func (b *Builder) DefaultGenerateMetadataOnlyStoresFunc() ksmtypes.BuildMetadataOnlyStoresFunc {
return b.buildMetadataOnlyStores
}

// DefaultGenerateCustomResourceStoresFunc returns default buildCustomResourceStores function
func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc {
return b.buildCustomResourceStores
Expand Down Expand Up @@ -362,7 +381,7 @@ func availableResources() []string {
}

func (b *Builder) buildConfigMapStores() []cache.Store {
return b.buildStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &v1.ConfigMap{}, createConfigMapListWatch, b.useAPIServerCache)
return b.buildMetadataOnlyStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &metav1.PartialObjectMetadata{}, createConfigMapListWatch, b.useAPIServerCache)
}

func (b *Builder) buildCronJobStores() []cache.Store {
Expand Down Expand Up @@ -519,7 +538,8 @@ func (b *Builder) buildStores(
if b.fieldSelectorFilter != "" {
klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
}
listWatcher := listWatchFunc(b.kubeClient, v1.NamespaceAll, b.fieldSelectorFilter)
kubeClient := b.kubeClient
listWatcher := listWatchFunc(kubeClient, v1.NamespaceAll, b.fieldSelectorFilter)
b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
return []cache.Store{store}
}
Expand All @@ -541,6 +561,46 @@ func (b *Builder) buildStores(
return stores
}

func (b *Builder) buildMetadataOnlyStores(
metricFamilies []generator.FamilyGenerator,
expectedType interface{},
listWatchFunc func(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher,
useAPIServerCache bool,
) []cache.Store {
metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies)
composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies)
familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies)

if b.namespaces.IsAllNamespaces() {
store := metricsstore.NewMetricsStore(
familyHeaders,
composedMetricGenFuncs,
)
if b.fieldSelectorFilter != "" {
klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
}
listWatcher := listWatchFunc(b.metadataOnlyKubeClient, v1.NamespaceAll, b.fieldSelectorFilter)
b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
return []cache.Store{store}
}

stores := make([]cache.Store, 0, len(b.namespaces))
for _, ns := range b.namespaces {
store := metricsstore.NewMetricsStore(
familyHeaders,
composedMetricGenFuncs,
)
if b.fieldSelectorFilter != "" {
klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
}
listWatcher := listWatchFunc(b.metadataOnlyKubeClient, ns, b.fieldSelectorFilter)
b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
stores = append(stores, store)
}

return stores
}

// TODO(Garrybest): Merge `buildStores` and `buildCustomResourceStores`
func (b *Builder) buildCustomResourceStores(resourceName string,
metricFamilies []generator.FamilyGenerator,
Expand Down
24 changes: 12 additions & 12 deletions internal/store/configmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ package store
import (
"context"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/watch"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/metadata"
"k8s.io/client-go/tools/cache"
basemetrics "k8s.io/component-base/metrics"

Expand All @@ -43,7 +43,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
metric.Gauge,
basemetrics.ALPHA,
"",
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
if len(allowAnnotationsList) == 0 {
return &metric.Family{}
}
Expand All @@ -65,7 +65,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
metric.Gauge,
basemetrics.STABLE,
"",
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
if len(allowLabelsList) == 0 {
return &metric.Family{}
}
Expand All @@ -87,7 +87,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
metric.Gauge,
basemetrics.STABLE,
"",
wrapConfigMapFunc(func(_ *v1.ConfigMap) *metric.Family {
wrapConfigMapFunc(func(_ *metav1.PartialObjectMetadata) *metric.Family {
return &metric.Family{
Metrics: []*metric.Metric{{
LabelKeys: []string{},
Expand All @@ -103,7 +103,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
metric.Gauge,
basemetrics.STABLE,
"",
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
ms := []*metric.Metric{}

if !c.CreationTimestamp.IsZero() {
Expand All @@ -125,7 +125,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
metric.Gauge,
basemetrics.ALPHA,
"",
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
return &metric.Family{
Metrics: resourceVersionMetric(c.ObjectMeta.ResourceVersion),
}
Expand All @@ -134,22 +134,22 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
}
}

func createConfigMapListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher {
func createConfigMapListWatch(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher {
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
opts.FieldSelector = fieldSelector
return kubeClient.CoreV1().ConfigMaps(ns).List(context.TODO(), opts)
return kubeClient.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}).Namespace(ns).List(context.TODO(), opts)
},
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
opts.FieldSelector = fieldSelector
return kubeClient.CoreV1().ConfigMaps(ns).Watch(context.TODO(), opts)
return kubeClient.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}).Namespace(ns).Watch(context.TODO(), opts)
},
}
}

func wrapConfigMapFunc(f func(*v1.ConfigMap) *metric.Family) func(interface{}) *metric.Family {
func wrapConfigMapFunc(f func(*metav1.PartialObjectMetadata) *metric.Family) func(interface{}) *metric.Family {
return func(obj interface{}) *metric.Family {
configMap := obj.(*v1.ConfigMap)
configMap := obj.(*metav1.PartialObjectMetadata)

metricFamily := f(configMap)

Expand Down
5 changes: 2 additions & 3 deletions internal/store/configmap_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package store
import (
"testing"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
Expand All @@ -37,7 +36,7 @@ func TestConfigMapStore(t *testing.T) {
AllowLabelsList: []string{
"app",
},
Obj: &v1.ConfigMap{
Obj: &metav1.PartialObjectMetadata{
ObjectMeta: metav1.ObjectMeta{
Name: "configmap1",
Namespace: "ns1",
Expand Down Expand Up @@ -73,7 +72,7 @@ func TestConfigMapStore(t *testing.T) {
},
},
{
Obj: &v1.ConfigMap{
Obj: &metav1.PartialObjectMetadata{
ObjectMeta: metav1.ObjectMeta{
Name: "configmap2",
Namespace: "ns2",
Expand Down
7 changes: 7 additions & 0 deletions pkg/app/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,7 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {

storeBuilder.WithUsingAPIServerCache(opts.UseAPIServerCache)
storeBuilder.WithGenerateStoresFunc(storeBuilder.DefaultGenerateStoresFunc())
storeBuilder.WithGenerateMetadataOnlyStoresFunc(storeBuilder.DefaultGenerateMetadataOnlyStoresFunc())
proc.StartReaper()

storeBuilder.WithUtilOptions(opts)
Expand All @@ -275,6 +276,12 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
}
storeBuilder.WithKubeClient(kubeClient)

metadataOnlyKubeClient, err := util.CreateMetadataOnlyKubeClient(opts.Apiserver, opts.Kubeconfig)
if err != nil {
return fmt.Errorf("failed to create metadata-only client: %v", err)
}
storeBuilder.WithMetadataOnlyKubeClient(metadataOnlyKubeClient)

storeBuilder.WithSharding(opts.Shard, opts.TotalShards)
if err := storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList); err != nil {
return fmt.Errorf("failed to set up annotations allowlist: %v", err)
Expand Down
6 changes: 6 additions & 0 deletions pkg/app/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func BenchmarkKubeStateMetrics(b *testing.B) {
builder.WithContext(ctx)
builder.WithNamespaces(options.DefaultNamespaces)
builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
builder.WithGenerateMetadataOnlyStoresFunc(builder.DefaultGenerateMetadataOnlyStoresFunc())

allowDenyListFilter, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
if err != nil {
Expand Down Expand Up @@ -159,6 +160,7 @@ func TestFullScrapeCycle(t *testing.T) {
builder.WithKubeClient(kubeClient)
builder.WithNamespaces(options.DefaultNamespaces)
builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
builder.WithGenerateMetadataOnlyStoresFunc(builder.DefaultGenerateMetadataOnlyStoresFunc())

l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
if err != nil {
Expand Down Expand Up @@ -463,6 +465,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
unshardedBuilder.WithFamilyGeneratorFilter(l)
unshardedBuilder.WithAllowLabels(map[string][]string{})
unshardedBuilder.WithGenerateStoresFunc(unshardedBuilder.DefaultGenerateStoresFunc())
unshardedBuilder.WithGenerateMetadataOnlyStoresFunc(unshardedBuilder.DefaultGenerateMetadataOnlyStoresFunc())

unshardedHandler := metricshandler.New(&options.Options{}, kubeClient, unshardedBuilder, false)
unshardedHandler.ConfigureSharding(ctx, 0, 1)
Expand All @@ -479,6 +482,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
shardedBuilder1.WithFamilyGeneratorFilter(l)
shardedBuilder1.WithAllowLabels(map[string][]string{})
shardedBuilder1.WithGenerateStoresFunc(shardedBuilder1.DefaultGenerateStoresFunc())
shardedBuilder1.WithGenerateMetadataOnlyStoresFunc(shardedBuilder1.DefaultGenerateMetadataOnlyStoresFunc())

shardedHandler1 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder1, false)
shardedHandler1.ConfigureSharding(ctx, 0, 2)
Expand All @@ -495,6 +499,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
shardedBuilder2.WithFamilyGeneratorFilter(l)
shardedBuilder2.WithAllowLabels(map[string][]string{})
shardedBuilder2.WithGenerateStoresFunc(shardedBuilder2.DefaultGenerateStoresFunc())
shardedBuilder2.WithGenerateMetadataOnlyStoresFunc(shardedBuilder2.DefaultGenerateMetadataOnlyStoresFunc())

shardedHandler2 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder2, false)
shardedHandler2.ConfigureSharding(ctx, 1, 2)
Expand Down Expand Up @@ -635,6 +640,7 @@ func TestCustomResourceExtension(t *testing.T) {
builder.WithCustomResourceClients(customResourceClients)
builder.WithNamespaces(options.DefaultNamespaces)
builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
builder.WithGenerateMetadataOnlyStoresFunc(builder.DefaultGenerateMetadataOnlyStoresFunc())
builder.WithGenerateCustomResourceStoresFunc(builder.DefaultGenerateCustomResourceStoresFunc())

l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
Expand Down
16 changes: 16 additions & 0 deletions pkg/builder/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/prometheus/client_golang/prometheus"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/metadata"
"k8s.io/client-go/tools/cache"

internalstore "k8s.io/kube-state-metrics/v2/internal/store"
Expand Down Expand Up @@ -84,6 +85,11 @@ func (b *Builder) WithKubeClient(c clientset.Interface) {
b.internal.WithKubeClient(c)
}

// WithMetadataOnlyKubeClient sets the metadataOnlyKubeClient property of a Builder.
func (b *Builder) WithMetadataOnlyKubeClient(c metadata.Interface) {
b.internal.WithMetadataOnlyKubeClient(c)
}

// WithCustomResourceClients sets the customResourceClients property of a Builder.
func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) {
b.internal.WithCustomResourceClients(cs)
Expand Down Expand Up @@ -120,6 +126,16 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc {
return b.internal.DefaultGenerateStoresFunc()
}

// WithGenerateMetadataOnlyStoresFunc configures a custom generate metadataonly store function
func (b *Builder) WithGenerateMetadataOnlyStoresFunc(f ksmtypes.BuildMetadataOnlyStoresFunc) {
b.internal.WithGenerateMetadataOnlyStoresFunc(f)
}

// DefaultGenerateMetadataOnlyStoresFunc returns default buildMetadataOnlyStore function
func (b *Builder) DefaultGenerateMetadataOnlyStoresFunc() ksmtypes.BuildMetadataOnlyStoresFunc {
return b.internal.DefaultGenerateMetadataOnlyStoresFunc()
}

// DefaultGenerateCustomResourceStoresFunc returns default buildStores function
func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc {
return b.internal.DefaultGenerateCustomResourceStoresFunc()
Expand Down
11 changes: 11 additions & 0 deletions pkg/builder/types/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

"github.com/prometheus/client_golang/prometheus"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/metadata"
"k8s.io/client-go/tools/cache"

"k8s.io/kube-state-metrics/v2/pkg/customresource"
Expand All @@ -38,6 +39,7 @@ type BuilderInterface interface {
WithFieldSelectorFilter(fieldSelectors string)
WithSharding(shard int32, totalShards int)
WithContext(ctx context.Context)
WithMetadataOnlyKubeClient(c metadata.Interface)
WithKubeClient(c clientset.Interface)
WithCustomResourceClients(cs map[string]interface{})
WithUsingAPIServerCache(u bool)
Expand All @@ -46,6 +48,8 @@ type BuilderInterface interface {
WithAllowLabels(l map[string][]string) error
WithGenerateStoresFunc(f BuildStoresFunc)
DefaultGenerateStoresFunc() BuildStoresFunc
WithGenerateMetadataOnlyStoresFunc(f BuildMetadataOnlyStoresFunc)
DefaultGenerateMetadataOnlyStoresFunc() BuildMetadataOnlyStoresFunc
DefaultGenerateCustomResourceStoresFunc() BuildCustomResourceStoresFunc
WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory)
Build() metricsstore.MetricsWriterList
Expand All @@ -60,6 +64,13 @@ type BuildStoresFunc func(metricFamilies []generator.FamilyGenerator,
useAPIServerCache bool,
) []cache.Store

// BuildMetadataOnlyStoresFunc function signature that is used to return a list of cache.Store
type BuildMetadataOnlyStoresFunc func(metricFamilies []generator.FamilyGenerator,
expectedType interface{},
listWatchFunc func(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher,
useAPIServerCache bool,
) []cache.Store

// BuildCustomResourceStoresFunc function signature that is used to return a list of custom resource cache.Store
type BuildCustomResourceStoresFunc func(resourceName string,
metricFamilies []generator.FamilyGenerator,
Expand Down
Loading

0 comments on commit 151ede1

Please sign in to comment.