From 52d84eb7c17fbbdd5542d4b5ce4c632a4b8e65a0 Mon Sep 17 00:00:00 2001 From: didukh86 <78904472+didukh86@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:29:52 +0000 Subject: [PATCH 01/21] Query-frontend: Fix connection to Redis with TLS. Issue: https://github.com/thanos-io/thanos/issues/7672 Signed-off-by: didukh86 Signed-off-by: didukh86 <78904472+didukh86@users.noreply.github.com> Signed-off-by: didukh86 Signed-off-by: Taras Didukh --- pkg/queryfrontend/config.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/queryfrontend/config.go b/pkg/queryfrontend/config.go index 437f0cf9ad..aabb841405 100644 --- a/pkg/queryfrontend/config.go +++ b/pkg/queryfrontend/config.go @@ -162,13 +162,15 @@ func NewCacheConfig(logger log.Logger, confContentYaml []byte) (*cortexcache.Con } return &cortexcache.Config{ Redis: cortexcache.RedisConfig{ - Endpoint: config.Redis.Addr, - Timeout: config.Redis.ReadTimeout, - MasterName: config.Redis.MasterName, - Expiration: config.Expiration, - DB: config.Redis.DB, - Password: flagext.Secret{Value: config.Redis.Password}, - Username: config.Redis.Username, + Endpoint: config.Redis.Addr, + Timeout: config.Redis.ReadTimeout, + MasterName: config.Redis.MasterName, + Expiration: config.Expiration, + DB: config.Redis.DB, + Password: flagext.Secret{Value: config.Redis.Password}, + Username: config.Redis.Username, + EnableTLS: config.Redis.TLSEnabled, + InsecureSkipVerify: config.Redis.TLSConfig.InsecureSkipVerify, }, Background: cortexcache.BackgroundConfig{ WriteBackBuffer: config.Redis.MaxSetMultiConcurrency * config.Redis.SetMultiBatchSize, From c89dee19e5af272bc08c6925bf56ebb72f9083d2 Mon Sep 17 00:00:00 2001 From: Taras Didukh Date: Mon, 2 Sep 2024 13:04:03 +0300 Subject: [PATCH 02/21] Add record to the changelog Signed-off-by: Taras Didukh --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cd17a2cf6..bb27f67410 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#7614](https://github.com/thanos-io/thanos/pull/7614) *: fix debug log formatting. - [#7492](https://github.com/thanos-io/thanos/pull/7492) Compactor: update filtered blocks list before second downsample pass. - [#7644](https://github.com/thanos-io/thanos/pull/7644) fix(ui): add null check to find overlapping blocks logic +- [#7674](https://github.com/thanos-io/thanos/pull/7674) Query-frontend: Fix connection to Redis cluster with TLS. ### Added From a230123399a8bb932f4c7f00c2f8919c14464cbb Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Mon, 2 Sep 2024 08:55:16 +0200 Subject: [PATCH 03/21] query: queryable is not respecting limits (#7679) Signed-off-by: Michael Hoffmann --- CHANGELOG.md | 1 + cmd/thanos/query.go | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb27f67410..e5e75ac814 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#7492](https://github.com/thanos-io/thanos/pull/7492) Compactor: update filtered blocks list before second downsample pass. - [#7644](https://github.com/thanos-io/thanos/pull/7644) fix(ui): add null check to find overlapping blocks logic - [#7674](https://github.com/thanos-io/thanos/pull/7674) Query-frontend: Fix connection to Redis cluster with TLS. +- [#7679](https://github.com/thanos-io/thanos/pull/7679) Query: respect store.limit.* flags when evaluating queries ### Added diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index e4311647e1..32f0a95f77 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -561,7 +561,8 @@ func runQuery( queryConnMetricLabels..., ) - proxy = store.NewProxyStore(logger, reg, endpoints.GetStoreClients, component.Query, selectorLset, storeResponseTimeout, store.RetrievalStrategy(grpcProxyStrategy), options...) + proxyStore = store.NewProxyStore(logger, reg, endpoints.GetStoreClients, component.Query, selectorLset, storeResponseTimeout, store.RetrievalStrategy(grpcProxyStrategy), options...) + seriesProxy = store.NewLimitedStoreServer(store.NewInstrumentedStoreServer(reg, proxyStore), reg, storeRateLimits) rulesProxy = rules.NewProxy(logger, endpoints.GetRulesClients) targetsProxy = targets.NewProxy(logger, endpoints.GetTargetsClients) metadataProxy = metadata.NewProxy(logger, endpoints.GetMetricMetadataClients) @@ -569,7 +570,7 @@ func runQuery( queryableCreator = query.NewQueryableCreator( logger, extprom.WrapRegistererWithPrefix("thanos_query_", reg), - proxy, + seriesProxy, maxConcurrentSelects, queryTimeout, ) @@ -792,16 +793,16 @@ func runQuery( infoSrv := info.NewInfoServer( component.Query.String(), - info.WithLabelSetFunc(func() []labelpb.ZLabelSet { return proxy.LabelSet() }), + info.WithLabelSetFunc(func() []labelpb.ZLabelSet { return proxyStore.LabelSet() }), info.WithStoreInfoFunc(func() (*infopb.StoreInfo, error) { if httpProbe.IsReady() { - mint, maxt := proxy.TimeRange() + mint, maxt := proxyStore.TimeRange() return &infopb.StoreInfo{ MinTime: mint, MaxTime: maxt, SupportsSharding: true, SupportsWithoutReplicaLabels: true, - TsdbInfos: proxy.TSDBInfos(), + TsdbInfos: proxyStore.TSDBInfos(), }, nil } return nil, errors.New("Not ready") @@ -815,10 +816,9 @@ func runQuery( defaultEngineType := querypb.EngineType(querypb.EngineType_value[defaultEngine]) grpcAPI := apiv1.NewGRPCAPI(time.Now, queryReplicaLabels, queryableCreator, engineFactory, defaultEngineType, lookbackDeltaCreator, instantDefaultMaxSourceResolution) - storeServer := store.NewLimitedStoreServer(store.NewInstrumentedStoreServer(reg, proxy), reg, storeRateLimits) s := grpcserver.New(logger, reg, tracer, grpcLogOpts, logFilterMethods, comp, grpcProbe, grpcserver.WithServer(apiv1.RegisterQueryServer(grpcAPI)), - grpcserver.WithServer(store.RegisterStoreServer(storeServer, logger)), + grpcserver.WithServer(store.RegisterStoreServer(seriesProxy, logger)), grpcserver.WithServer(rules.RegisterRulesServer(rulesProxy)), grpcserver.WithServer(targets.RegisterTargetsServer(targetsProxy)), grpcserver.WithServer(metadata.RegisterMetadataServer(metadataProxy)), From 65c3a0e2e76d780775bb769214c3bb62bb9450bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 09:09:59 +0100 Subject: [PATCH 04/21] build(deps): bump go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp (#7666) Bumps [go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp](https://github.com/open-telemetry/opentelemetry-go) from 1.27.0 to 1.29.0. - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.27.0...v1.29.0) --- updated-dependencies: - dependency-name: go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Taras Didukh --- go.mod | 32 ++++++++++++++--------------- go.sum | 64 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index ca2651a74b..46a41e211e 100644 --- a/go.mod +++ b/go.mod @@ -72,24 +72,24 @@ require ( go.elastic.co/apm v1.15.0 go.elastic.co/apm/module/apmot v1.15.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect - go.opentelemetry.io/otel v1.28.0 + go.opentelemetry.io/otel v1.29.0 go.opentelemetry.io/otel/bridge/opentracing v1.28.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 - go.opentelemetry.io/otel/sdk v1.28.0 - go.opentelemetry.io/otel/trace v1.28.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 + go.opentelemetry.io/otel/sdk v1.29.0 + go.opentelemetry.io/otel/trace v1.29.0 go.uber.org/atomic v1.11.0 go.uber.org/automaxprocs v1.5.3 go.uber.org/goleak v1.3.0 - golang.org/x/crypto v0.25.0 - golang.org/x/net v0.27.0 - golang.org/x/sync v0.7.0 - golang.org/x/text v0.16.0 + golang.org/x/crypto v0.26.0 + golang.org/x/net v0.28.0 + golang.org/x/sync v0.8.0 + golang.org/x/text v0.17.0 golang.org/x/time v0.5.0 google.golang.org/api v0.183.0 // indirect google.golang.org/genproto v0.0.0-20240528184218-531527333157 // indirect - google.golang.org/grpc v1.64.0 + google.golang.org/grpc v1.65.0 google.golang.org/grpc/examples v0.0.0-20211119005141-f45e61797429 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 @@ -154,8 +154,8 @@ require ( go.opentelemetry.io/contrib/propagators/ot v1.28.0 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect k8s.io/apimachinery v0.30.2 // indirect k8s.io/client-go v0.30.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect @@ -218,7 +218,7 @@ require ( github.com/google/uuid v1.6.0 github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.4 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -261,12 +261,12 @@ require ( go.opentelemetry.io/contrib/propagators/aws v1.28.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.28.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.18.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/oauth2 v0.22.0 // indirect + golang.org/x/sys v0.24.0 // indirect golang.org/x/tools v0.22.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/protobuf v1.34.2 diff --git a/go.sum b/go.sum index c18911e7cc..11dcdfd0e6 100644 --- a/go.sum +++ b/go.sum @@ -1846,8 +1846,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/hashicorp/consul/api v1.29.1 h1:UEwOjYJrd3lG1x5w7HxDRMGiAUPrb3f103EoeKuuEcc= github.com/hashicorp/consul/api v1.29.1/go.mod h1:lumfRkY/coLuqMICkI7Fh3ylMG31mQSRZyef2c5YvJI= @@ -2314,32 +2314,32 @@ go.opentelemetry.io/contrib/samplers/jaegerremote v0.22.0/go.mod h1:2tZTRqCbvx7n go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= go.opentelemetry.io/otel/bridge/opentracing v1.28.0 h1:erHvOxIUFnSXj/HuS5SqaKe2CbWSBskONXm2bEBxYgc= go.opentelemetry.io/otel/bridge/opentracing v1.28.0/go.mod h1:ZMOFThPtIKYiVqzKrU53s41j25Cj27KySyu5Az5jRPU= go.opentelemetry.io/otel/exporters/jaeger v1.16.0 h1:YhxxmXZ011C0aDZKoNw+juVWAmEfv/0W2XBOv9aHTaA= go.opentelemetry.io/otel/exporters/jaeger v1.16.0/go.mod h1:grYbBo/5afWlPpdPZYhyn78Bk04hnvxn2+hvxQhKIQM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 h1:dIIDULZJpgdiHz5tXrTgKIMLkus6jEFa7x5SOKcyR7E= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0/go.mod h1:jlRVBe7+Z1wyxFSUs48L6OBQZ5JwH2Hg/Vbl+t9rAgI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 h1:JAv0Jwtl01UFiyWZEMiJZBiTlv5A50zNs8lsthXqIio= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0/go.mod h1:QNKLmUEAq2QUbPQUfvw4fmv0bgbK7UlOSFCnXyfvSNc= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -2384,8 +2384,8 @@ golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2530,8 +2530,8 @@ golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2565,8 +2565,8 @@ golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2588,8 +2588,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2701,8 +2701,8 @@ golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2721,8 +2721,8 @@ golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2743,8 +2743,8 @@ golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -3117,8 +3117,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe/go. google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.mod h1:rbHMSEDyoYX62nRVLOCc4Qt1HbsdytAYoVwgjiOhF3I= google.golang.org/genproto/googleapis/api v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:PVreiBMirk8ypES6aw9d4p6iiBNSIfZEBqr3UGoAi2E= google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd h1:BBOTEWLuuEGQy9n1y9MhVJ9Qt0BDu21X8qZs71/uPZo= +google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:fO8wJzT2zbQbAjbIoos1285VfEIYKDDY+Dt+WpTkh6g= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230807174057-1744710a1577/go.mod h1:NjCQG/D8JandXxM57PZbAJL1DCNL6EypA0vPPwfsc7c= google.golang.org/genproto/googleapis/bytestream v0.0.0-20231030173426-d783a09b4405/go.mod h1:GRUCuLdzVqZte8+Dl/D4N25yLzcGqqWaYkeVOwulFqw= @@ -3152,8 +3152,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014/go.mod h1:SaPjaZGWb0lPqs6Ittu0spdfrOArqji4ZdeP5IC/9N4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:YUWgXUFRPfoYK1IHMuxH5K6nPEXSCzIMljnQ59lLRCk= google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd h1:6TEm2ZxXoQmFWFlt1vNxvVOa1Q0dXFQD1m/rYjXmS0E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= From 93200ea437507ea0581e36dc142549d4320cb176 Mon Sep 17 00:00:00 2001 From: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> Date: Mon, 2 Sep 2024 13:41:54 +0530 Subject: [PATCH 05/21] website: max-height for version-picker dropdown (#7642) Signed-off-by: milinddethe15 Signed-off-by: Taras Didukh --- website/static/main.css | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/website/static/main.css b/website/static/main.css index a68e0ea0be..5305af10b7 100644 --- a/website/static/main.css +++ b/website/static/main.css @@ -80,7 +80,7 @@ pre { .header-anchor { font-size: 100%; visibility: hidden;} h1:hover a, h2:hover a, h3:hover a, h4:hover a { visibility: visible} -/* seaarchBox */ +/* searchBox */ .searchBox { position: relative; height: 100%; @@ -126,6 +126,13 @@ input[type="search"]::placeholder { /* Chrome, Firefox, Opera, Safari 10.1+ */ color: #fff; } +/* version-picker */ +.dropdown-menu { + max-height: 600px; + overflow-y: auto; + position: absolute; +} + /* SideMenu Collapse */ .docs-menu{ text-transform: capitalize; From 1c2d6ca60b5393395f6f1e4679f0f33656956e98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 08:27:21 +0000 Subject: [PATCH 06/21] build(deps): bump github/codeql-action from 3.26.2 to 3.26.5 (#7667) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.26.2 to 3.26.5. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/429e1977040da7a23b6822b13c129cd1ba93dbb2...2c779ab0d087cd7fe7b826087247c2c81f27bfa6) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Taras Didukh --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7115aedb7c..ec99cd8642 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -47,7 +47,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@429e1977040da7a23b6822b13c129cd1ba93dbb2 # v3.26.2 + uses: github/codeql-action/init@2c779ab0d087cd7fe7b826087247c2c81f27bfa6 # v3.26.5 with: languages: ${{ matrix.language }} config-file: ./.github/codeql/codeql-config.yml @@ -59,7 +59,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@429e1977040da7a23b6822b13c129cd1ba93dbb2 # v3.26.2 + uses: github/codeql-action/autobuild@2c779ab0d087cd7fe7b826087247c2c81f27bfa6 # v3.26.5 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -73,4 +73,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@429e1977040da7a23b6822b13c129cd1ba93dbb2 # v3.26.2 + uses: github/codeql-action/analyze@2c779ab0d087cd7fe7b826087247c2c81f27bfa6 # v3.26.5 From 75f0328ebb1d07b0e8e4e1670dff9871b19d448b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 09:23:41 +0000 Subject: [PATCH 07/21] Bump golang.org/x/time from 0.5.0 to 0.6.0 (#7601) Bumps [golang.org/x/time](https://github.com/golang/time) from 0.5.0 to 0.6.0. - [Commits](https://github.com/golang/time/compare/v0.5.0...v0.6.0) --- updated-dependencies: - dependency-name: golang.org/x/time dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Taras Didukh --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 46a41e211e..dcce4ecbf2 100644 --- a/go.mod +++ b/go.mod @@ -86,7 +86,7 @@ require ( golang.org/x/net v0.28.0 golang.org/x/sync v0.8.0 golang.org/x/text v0.17.0 - golang.org/x/time v0.5.0 + golang.org/x/time v0.6.0 google.golang.org/api v0.183.0 // indirect google.golang.org/genproto v0.0.0-20240528184218-531527333157 // indirect google.golang.org/grpc v1.65.0 diff --git a/go.sum b/go.sum index 11dcdfd0e6..51ef2f04da 100644 --- a/go.sum +++ b/go.sum @@ -2751,8 +2751,9 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 53250395e22889d0cba3cab9d846b30cb2ad2927 Mon Sep 17 00:00:00 2001 From: Taras Didukh Date: Wed, 4 Sep 2024 20:02:01 +0300 Subject: [PATCH 08/21] Remove empty lines Signed-off-by: Taras Didukh --- .github/workflows/codeql-analysis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 8b6563e751..2bc98d128b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,7 +49,6 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6 - with: languages: ${{ matrix.language }} config-file: ./.github/codeql/codeql-config.yml @@ -63,7 +62,6 @@ jobs: - name: Autobuild uses: github/codeql-action/autobuild@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6 - # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -77,4 +75,3 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6 - From b144ebac208fb0eb1171d39e29ffb37e51f589c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Thu, 21 Nov 2024 11:27:53 +0200 Subject: [PATCH 09/21] receive: port expanded postings cache from Cortex (#7914) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port expanded postings cache from Cortex. Huge kudos to @alanprot for the implementation. I added a TODO item to convert our whole internal caching infra to be promise based. Signed-off-by: Giedrius Statkevičius --- cmd/thanos/receive.go | 11 +- docs/components/receive.md | 6 + pkg/receive/expandedpostingscache/cache.go | 414 ++++++++++++++++++ .../expandedpostingscache/cache_test.go | 168 +++++++ pkg/receive/expandedpostingscache/tsdb.go | 146 ++++++ pkg/receive/multitsdb.go | 35 ++ pkg/store/storepb/inprocess.go | 8 + test/e2e/e2ethanos/services.go | 39 +- test/e2e/receive_test.go | 14 +- 9 files changed, 819 insertions(+), 22 deletions(-) create mode 100644 pkg/receive/expandedpostingscache/cache.go create mode 100644 pkg/receive/expandedpostingscache/cache_test.go create mode 100644 pkg/receive/expandedpostingscache/tsdb.go diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index fffcf6bbde..299634546a 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -141,7 +141,10 @@ func runReceive( level.Info(logger).Log("mode", receiveMode, "msg", "running receive") - multiTSDBOptions := []receive.MultiTSDBOption{} + multiTSDBOptions := []receive.MultiTSDBOption{ + receive.WithHeadExpandedPostingsCacheSize(conf.headExpandedPostingsCacheSize), + receive.WithBlockExpandedPostingsCacheSize(conf.compactedBlocksExpandedPostingsCacheSize), + } for _, feature := range *conf.featureList { if feature == metricNamesFilter { multiTSDBOptions = append(multiTSDBOptions, receive.WithMetricNameFilterEnabled()) @@ -886,6 +889,9 @@ type receiveConfig struct { asyncForwardWorkerCount uint featureList *[]string + + headExpandedPostingsCacheSize uint64 + compactedBlocksExpandedPostingsCacheSize uint64 } func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) { @@ -996,6 +1002,9 @@ func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) { cmd.Flag("tsdb.no-lockfile", "Do not create lockfile in TSDB data directory. In any case, the lockfiles will be deleted on next startup.").Default("false").BoolVar(&rc.noLockFile) + cmd.Flag("tsdb.head.expanded-postings-cache-size", "[EXPERIMENTAL] If non-zero, enables expanded postings cache for the head block.").Default("0").Uint64Var(&rc.headExpandedPostingsCacheSize) + cmd.Flag("tsdb.block.expanded-postings-cache-size", "[EXPERIMENTAL] If non-zero, enables expanded postings cache for compacted blocks.").Default("0").Uint64Var(&rc.compactedBlocksExpandedPostingsCacheSize) + cmd.Flag("tsdb.max-exemplars", "Enables support for ingesting exemplars and sets the maximum number of exemplars that will be stored per tenant."+ " In case the exemplar storage becomes full (number of stored exemplars becomes equal to max-exemplars),"+ diff --git a/docs/components/receive.md b/docs/components/receive.md index cd0dc322c7..dfaddd73b1 100644 --- a/docs/components/receive.md +++ b/docs/components/receive.md @@ -552,6 +552,12 @@ Flags: Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge. Does not do anything, enabled all the time. + --tsdb.block.expanded-postings-cache-size=0 + [EXPERIMENTAL] If non-zero, enables expanded + postings cache for compacted blocks. + --tsdb.head.expanded-postings-cache-size=0 + [EXPERIMENTAL] If non-zero, enables expanded + postings cache for the head block. --tsdb.max-exemplars=0 Enables support for ingesting exemplars and sets the maximum number of exemplars that will be stored per tenant. In case the exemplar diff --git a/pkg/receive/expandedpostingscache/cache.go b/pkg/receive/expandedpostingscache/cache.go new file mode 100644 index 0000000000..d5c7069735 --- /dev/null +++ b/pkg/receive/expandedpostingscache/cache.go @@ -0,0 +1,414 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +// Original code by Alan Protasio (https://github.com/alanprot) in the Cortex project. + +package expandedpostingscache + +import ( + "container/list" + "context" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/cespare/xxhash" + "github.com/oklog/ulid" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/chunks" + "github.com/prometheus/prometheus/tsdb/index" +) + +type ExpandedPostingsCache interface { + PostingsForMatchers(ctx context.Context, blockID ulid.ULID, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error) + ExpireSeries(metric labels.Labels) + tsdb.SeriesLifecycleCallback +} + +type BlocksPostingsForMatchersCache struct { + strippedLock []sync.RWMutex + + headCache *fifoCache[[]storage.SeriesRef] + blocksCache *fifoCache[[]storage.SeriesRef] + + headSeedByMetricName []int + postingsForMatchersFunc func(ctx context.Context, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error) + timeNow func() time.Time + + metrics *ExpandedPostingsCacheMetrics +} + +var ( + rangeHeadULID = ulid.MustParse("0000000000XXXXXXXRANGEHEAD") + headULID = ulid.MustParse("0000000000XXXXXXXXXXXXHEAD") +) + +const ( + // size of the seed array. Each seed is a 64bits int (8 bytes) + // totaling 8mb. + seedArraySize = 1024 * 1024 + + numOfSeedsStripes = 512 +) + +type ExpandedPostingsCacheMetrics struct { + CacheRequests *prometheus.CounterVec + CacheHits *prometheus.CounterVec + CacheEvicts *prometheus.CounterVec + NonCacheableQueries *prometheus.CounterVec +} + +func NewPostingCacheMetrics(r prometheus.Registerer) *ExpandedPostingsCacheMetrics { + return &ExpandedPostingsCacheMetrics{ + CacheRequests: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Name: "expanded_postings_cache_requests_total", + Help: "Total number of requests to the cache.", + }, []string{"cache"}), + CacheHits: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Name: "expanded_postings_cache_hits_total", + Help: "Total number of hit requests to the cache.", + }, []string{"cache"}), + CacheEvicts: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Name: "expanded_postings_cache_evicts_total", + Help: "Total number of evictions in the cache, excluding items that got evicted.", + }, []string{"cache", "reason"}), + NonCacheableQueries: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Name: "expanded_postings_non_cacheable_queries_total", + Help: "Total number of non cacheable queries.", + }, []string{"cache"}), + } +} + +func NewBlocksPostingsForMatchersCache(metrics *ExpandedPostingsCacheMetrics, headExpandedPostingsCacheSize uint64, blockExpandedPostingsCacheSize uint64) ExpandedPostingsCache { + return &BlocksPostingsForMatchersCache{ + headCache: newFifoCache[[]storage.SeriesRef]("head", metrics, time.Now, headExpandedPostingsCacheSize), + blocksCache: newFifoCache[[]storage.SeriesRef]("block", metrics, time.Now, blockExpandedPostingsCacheSize), + headSeedByMetricName: make([]int, seedArraySize), + strippedLock: make([]sync.RWMutex, numOfSeedsStripes), + postingsForMatchersFunc: tsdb.PostingsForMatchers, + timeNow: time.Now, + metrics: metrics, + } +} + +func (c *BlocksPostingsForMatchersCache) PostCreation(metric labels.Labels) { + c.ExpireSeries(metric) +} + +func (c *BlocksPostingsForMatchersCache) PostDeletion(metrics map[chunks.HeadSeriesRef]labels.Labels) { + for _, metric := range metrics { + c.ExpireSeries(metric) + } +} + +func (c *BlocksPostingsForMatchersCache) PreCreation(labels.Labels) error { + return nil +} + +func (c *BlocksPostingsForMatchersCache) ExpireSeries(metric labels.Labels) { + var metricName string + + metric.Range(func(l labels.Label) { + if l.Name != model.MetricNameLabel { + return + } + metricName = l.Value + }) + + if metricName == "" { + return + } + + h := MemHashString(metricName) + i := h % uint64(len(c.headSeedByMetricName)) + l := h % uint64(len(c.strippedLock)) + c.strippedLock[l].Lock() + defer c.strippedLock[l].Unlock() + c.headSeedByMetricName[i]++ +} + +func (c *BlocksPostingsForMatchersCache) PostingsForMatchers(ctx context.Context, blockID ulid.ULID, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error) { + return c.fetchPostings(blockID, ix, ms...)(ctx) +} + +func (c *BlocksPostingsForMatchersCache) fetchPostings(blockID ulid.ULID, ix tsdb.IndexReader, ms ...*labels.Matcher) func(context.Context) (index.Postings, error) { + var seed string + cache := c.blocksCache + + // If is a head block, lets add the seed on the cache key so we can + // invalidate the cache when new series are created for this metric name + if isHeadBlock(blockID) { + cache = c.headCache + + metricName, ok := metricNameFromMatcher(ms) + // Lets not cache head if we don;t find an equal matcher for the label __name__ + if !ok { + c.metrics.NonCacheableQueries.WithLabelValues(cache.name).Inc() + return func(ctx context.Context) (index.Postings, error) { + return tsdb.PostingsForMatchers(ctx, ix, ms...) + } + } + + seed = c.getSeedForMetricName(metricName) + } + + c.metrics.CacheRequests.WithLabelValues(cache.name).Inc() + + fetch := func() ([]storage.SeriesRef, int64, error) { + // Use context.Background() as this promise is maybe shared across calls + postings, err := c.postingsForMatchersFunc(context.Background(), ix, ms...) + + if err == nil { + ids, err := index.ExpandPostings(postings) + return ids, int64(len(ids) * 8), err + } + + return nil, 0, err + } + + key := c.cacheKey(seed, blockID, ms...) + promise, loaded := cache.getPromiseForKey(key, fetch) + if loaded { + c.metrics.CacheHits.WithLabelValues(cache.name).Inc() + } + + return c.result(promise) +} + +func (c *BlocksPostingsForMatchersCache) result(ce *cacheEntryPromise[[]storage.SeriesRef]) func(ctx context.Context) (index.Postings, error) { + return func(ctx context.Context) (index.Postings, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-ce.done: + if ctx.Err() != nil { + return nil, ctx.Err() + } + return index.NewListPostings(ce.v), ce.err + } + } +} + +func (c *BlocksPostingsForMatchersCache) getSeedForMetricName(metricName string) string { + h := MemHashString(metricName) + i := h % uint64(len(c.headSeedByMetricName)) + l := h % uint64(len(c.strippedLock)) + c.strippedLock[l].RLock() + defer c.strippedLock[l].RUnlock() + return strconv.Itoa(c.headSeedByMetricName[i]) +} + +func (c *BlocksPostingsForMatchersCache) cacheKey(seed string, blockID ulid.ULID, ms ...*labels.Matcher) string { + slices.SortFunc(ms, func(i, j *labels.Matcher) int { + if i.Type != j.Type { + return int(i.Type - j.Type) + } + if i.Name != j.Name { + return strings.Compare(i.Name, j.Name) + } + if i.Value != j.Value { + return strings.Compare(i.Value, j.Value) + } + return 0 + }) + + const ( + typeLen = 2 + sepLen = 1 + ) + + var size int + for _, m := range ms { + size += len(seed) + len(blockID.String()) + len(m.Name) + len(m.Value) + typeLen + 2*sepLen + } + sb := strings.Builder{} + sb.Grow(size) + sb.WriteString(seed) + sb.WriteByte('|') + sb.WriteString(blockID.String()) + for _, m := range ms { + sb.WriteString(m.Name) + sb.WriteString(m.Type.String()) + sb.WriteString(m.Value) + sb.WriteByte('|') + } + key := sb.String() + return key +} + +func isHeadBlock(blockID ulid.ULID) bool { + return blockID == rangeHeadULID || blockID == headULID +} + +func metricNameFromMatcher(ms []*labels.Matcher) (string, bool) { + for _, m := range ms { + if m.Name == labels.MetricName && m.Type == labels.MatchEqual { + return m.Value, true + } + } + + return "", false +} + +// TODO(GiedriusS): convert Thanos caching system to be promised-based +// i.e. avoid multiple loads for same item. This is a copy from Cortex. +// Use as an inspiration. +type fifoCache[V any] struct { + cachedValues *sync.Map + timeNow func() time.Time + name string + metrics ExpandedPostingsCacheMetrics + + ttl time.Duration + maxBytes int64 + + // Fields from here should be locked + cachedMtx sync.RWMutex + cached *list.List + cachedBytes int64 +} + +func newFifoCache[V any](name string, metrics *ExpandedPostingsCacheMetrics, timeNow func() time.Time, maxBytes uint64) *fifoCache[V] { + return &fifoCache[V]{ + cachedValues: new(sync.Map), + cached: list.New(), + timeNow: timeNow, + name: name, + metrics: *metrics, + ttl: 10 * time.Minute, + maxBytes: int64(maxBytes), + } +} + +func (c *fifoCache[V]) expire() { + if c.ttl.Seconds() <= 0 { + return + } + c.cachedMtx.RLock() + if _, r := c.shouldEvictHead(); !r { + c.cachedMtx.RUnlock() + return + } + c.cachedMtx.RUnlock() + c.cachedMtx.Lock() + defer c.cachedMtx.Unlock() + for reason, r := c.shouldEvictHead(); r; reason, r = c.shouldEvictHead() { + c.metrics.CacheEvicts.WithLabelValues(c.name, reason).Inc() + c.evictHead() + } +} + +func (c *fifoCache[V]) getPromiseForKey(k string, fetch func() (V, int64, error)) (*cacheEntryPromise[V], bool) { + r := &cacheEntryPromise[V]{ + done: make(chan struct{}), + } + defer close(r.done) + + loaded, ok := c.cachedValues.LoadOrStore(k, r) + + if !ok { + r.v, r.sizeBytes, r.err = fetch() + r.sizeBytes += int64(len(k)) + r.ts = c.timeNow() + c.created(k, r.sizeBytes) + c.expire() + } + + if ok { + // If the promise is already in the cache, lets wait it to fetch the data. + <-loaded.(*cacheEntryPromise[V]).done + + // If is cached but is expired, lets try to replace the cache value. + if loaded.(*cacheEntryPromise[V]).isExpired(c.ttl, c.timeNow()) && c.cachedValues.CompareAndSwap(k, loaded, r) { + c.metrics.CacheEvicts.WithLabelValues(c.name, "expired").Inc() + r.v, r.sizeBytes, r.err = fetch() + r.sizeBytes += int64(len(k)) + c.updateSize(loaded.(*cacheEntryPromise[V]).sizeBytes, r.sizeBytes) + loaded = r + r.ts = c.timeNow() + ok = false + } + } + + return loaded.(*cacheEntryPromise[V]), ok +} + +func (c *fifoCache[V]) shouldEvictHead() (string, bool) { + h := c.cached.Front() + if h == nil { + return "", false + } + + if c.cachedBytes > c.maxBytes { + return "full", true + } + key := h.Value.(string) + + if l, ok := c.cachedValues.Load(key); ok { + return "expired", l.(*cacheEntryPromise[V]).isExpired(c.ttl, c.timeNow()) + } + + return "", false +} + +func (c *fifoCache[V]) evictHead() { + front := c.cached.Front() + c.cached.Remove(front) + oldestKey := front.Value.(string) + if oldest, loaded := c.cachedValues.LoadAndDelete(oldestKey); loaded { + c.cachedBytes -= oldest.(*cacheEntryPromise[V]).sizeBytes + } +} + +func (c *fifoCache[V]) created(key string, sizeBytes int64) { + if c.ttl <= 0 { + c.cachedValues.Delete(key) + return + } + c.cachedMtx.Lock() + defer c.cachedMtx.Unlock() + c.cached.PushBack(key) + c.cachedBytes += sizeBytes +} + +func (c *fifoCache[V]) updateSize(oldSize, newSizeBytes int64) { + if oldSize == newSizeBytes { + return + } + + c.cachedMtx.Lock() + defer c.cachedMtx.Unlock() + c.cachedBytes += newSizeBytes - oldSize +} + +func (c *fifoCache[V]) contains(k string) bool { + _, ok := c.cachedValues.Load(k) + return ok +} + +type cacheEntryPromise[V any] struct { + ts time.Time + sizeBytes int64 + + done chan struct{} + v V + err error +} + +func (ce *cacheEntryPromise[V]) isExpired(ttl time.Duration, now time.Time) bool { + ts := ce.ts + r := now.Sub(ts) + return r >= ttl +} + +func MemHashString(str string) uint64 { + return xxhash.Sum64String(str) +} diff --git a/pkg/receive/expandedpostingscache/cache_test.go b/pkg/receive/expandedpostingscache/cache_test.go new file mode 100644 index 0000000000..a2ba2dafd2 --- /dev/null +++ b/pkg/receive/expandedpostingscache/cache_test.go @@ -0,0 +1,168 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +// Original code by Alan Protasio (https://github.com/alanprot) in the Cortex project. +// +//nolint:unparam +package expandedpostingscache + +import ( + "bytes" + "fmt" + "strings" + "sync" + "testing" + "time" + + "go.uber.org/atomic" + + "github.com/prometheus/client_golang/prometheus" + promtest "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func Test_ShouldFetchPromiseOnlyOnce(t *testing.T) { + m := NewPostingCacheMetrics(prometheus.NewPedanticRegistry()) + cache := newFifoCache[int]("test", m, time.Now, 10<<20) + calls := atomic.Int64{} + concurrency := 100 + wg := sync.WaitGroup{} + wg.Add(concurrency) + + fetchFunc := func() (int, int64, error) { + calls.Inc() + time.Sleep(100 * time.Millisecond) + return 0, 0, nil //nolint:unparam + } + + for i := 0; i < 100; i++ { + go func() { + defer wg.Done() + cache.getPromiseForKey("key1", fetchFunc) + }() + } + + wg.Wait() + require.Equal(t, int64(1), calls.Load()) + +} + +func TestFifoCacheExpire(t *testing.T) { + + keySize := 20 + numberOfKeys := 100 + + tc := map[string]struct { + ttl time.Duration + maxBytes uint64 + expectedFinalItems int + ttlExpire bool + }{ + "MaxBytes": { + expectedFinalItems: 10, + ttl: time.Hour, + maxBytes: uint64(10 * (8 + keySize)), + }, + "TTL": { + expectedFinalItems: numberOfKeys, + ttlExpire: true, + ttl: time.Hour, + maxBytes: 10 << 20, + }, + } + + for name, c := range tc { + t.Run(name, func(t *testing.T) { + r := prometheus.NewPedanticRegistry() + m := NewPostingCacheMetrics(r) + timeNow := time.Now + cache := newFifoCache[int]("test", m, timeNow, c.maxBytes) + + for i := 0; i < numberOfKeys; i++ { + key := repeatStringIfNeeded(fmt.Sprintf("key%d", i), keySize) + p, loaded := cache.getPromiseForKey(key, func() (int, int64, error) { + return 1, 8, nil + }) + require.False(t, loaded) + require.Equal(t, 1, p.v) + require.True(t, cache.contains(key)) + p, loaded = cache.getPromiseForKey(key, func() (int, int64, error) { + return 1, 0, nil + }) + require.True(t, loaded) + require.Equal(t, 1, p.v) + } + + totalCacheSize := 0 + + for i := 0; i < numberOfKeys; i++ { + key := repeatStringIfNeeded(fmt.Sprintf("key%d", i), keySize) + if cache.contains(key) { + totalCacheSize++ + } + } + + require.Equal(t, c.expectedFinalItems, totalCacheSize) + + if c.expectedFinalItems != numberOfKeys { + err := promtest.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` + # HELP expanded_postings_cache_evicts_total Total number of evictions in the cache, excluding items that got evicted. + # TYPE expanded_postings_cache_evicts_total counter + expanded_postings_cache_evicts_total{cache="test",reason="full"} %v +`, numberOfKeys-c.expectedFinalItems)), "expanded_postings_cache_evicts_total") + require.NoError(t, err) + + } + + if c.ttlExpire { + cache.timeNow = func() time.Time { + return timeNow().Add(2 * c.ttl) + } + + for i := 0; i < numberOfKeys; i++ { + key := repeatStringIfNeeded(fmt.Sprintf("key%d", i), keySize) + originalSize := cache.cachedBytes + p, loaded := cache.getPromiseForKey(key, func() (int, int64, error) { + return 2, 18, nil + }) + require.False(t, loaded) + // New value + require.Equal(t, 2, p.v) + // Total Size Updated + require.Equal(t, originalSize+10, cache.cachedBytes) + } + + err := promtest.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` + # HELP expanded_postings_cache_evicts_total Total number of evictions in the cache, excluding items that got evicted. + # TYPE expanded_postings_cache_evicts_total counter + expanded_postings_cache_evicts_total{cache="test",reason="expired"} %v +`, numberOfKeys)), "expanded_postings_cache_evicts_total") + require.NoError(t, err) + + cache.timeNow = func() time.Time { + return timeNow().Add(5 * c.ttl) + } + + cache.getPromiseForKey("newKwy", func() (int, int64, error) { + return 2, 18, nil + }) + + // Should expire all keys again as ttl is expired + err = promtest.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` + # HELP expanded_postings_cache_evicts_total Total number of evictions in the cache, excluding items that got evicted. + # TYPE expanded_postings_cache_evicts_total counter + expanded_postings_cache_evicts_total{cache="test",reason="expired"} %v +`, numberOfKeys*2)), "expanded_postings_cache_evicts_total") + require.NoError(t, err) + } + }) + } +} + +func repeatStringIfNeeded(seed string, length int) string { + if len(seed) > length { + return seed + } + + return strings.Repeat(seed, 1+length/len(seed))[:max(length, len(seed))] +} diff --git a/pkg/receive/expandedpostingscache/tsdb.go b/pkg/receive/expandedpostingscache/tsdb.go new file mode 100644 index 0000000000..09d9276575 --- /dev/null +++ b/pkg/receive/expandedpostingscache/tsdb.go @@ -0,0 +1,146 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +// Original code by Alan Protasio (https://github.com/alanprot) in the Cortex project. + +package expandedpostingscache + +import ( + "context" + "errors" + "fmt" + + "github.com/oklog/ulid" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" + prom_tsdb "github.com/prometheus/prometheus/tsdb" + tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" + "github.com/prometheus/prometheus/tsdb/index" + "github.com/prometheus/prometheus/tsdb/tombstones" + "github.com/prometheus/prometheus/util/annotations" +) + +/* + This file is basically a copy from https://github.com/prometheus/prometheus/blob/e2e01c1cffbfc4f26f5e9fe6138af87d7ff16122/tsdb/querier.go + with the difference that the PostingsForMatchers function is called from the Postings Cache +*/ + +type blockBaseQuerier struct { + blockID ulid.ULID + index prom_tsdb.IndexReader + chunks prom_tsdb.ChunkReader + tombstones tombstones.Reader + + closed bool + + mint, maxt int64 +} + +func newBlockBaseQuerier(b prom_tsdb.BlockReader, mint, maxt int64) (*blockBaseQuerier, error) { + indexr, err := b.Index() + if err != nil { + return nil, fmt.Errorf("open index reader: %w", err) + } + chunkr, err := b.Chunks() + if err != nil { + indexr.Close() + return nil, fmt.Errorf("open chunk reader: %w", err) + } + tombsr, err := b.Tombstones() + if err != nil { + indexr.Close() + chunkr.Close() + return nil, fmt.Errorf("open tombstone reader: %w", err) + } + + if tombsr == nil { + tombsr = tombstones.NewMemTombstones() + } + return &blockBaseQuerier{ + blockID: b.Meta().ULID, + mint: mint, + maxt: maxt, + index: indexr, + chunks: chunkr, + tombstones: tombsr, + }, nil +} + +func (q *blockBaseQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + res, err := q.index.SortedLabelValues(ctx, name, matchers...) + return res, nil, err +} + +func (q *blockBaseQuerier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + res, err := q.index.LabelNames(ctx, matchers...) + return res, nil, err +} + +func (q *blockBaseQuerier) Close() error { + if q.closed { + return errors.New("block querier already closed") + } + + errs := tsdb_errors.NewMulti( + q.index.Close(), + q.chunks.Close(), + q.tombstones.Close(), + ) + q.closed = true + return errs.Err() +} + +type cachedBlockChunkQuerier struct { + *blockBaseQuerier + + cache ExpandedPostingsCache +} + +func NewCachedBlockChunkQuerier(cache ExpandedPostingsCache, b prom_tsdb.BlockReader, mint, maxt int64) (storage.ChunkQuerier, error) { + q, err := newBlockBaseQuerier(b, mint, maxt) + if err != nil { + return nil, err + } + return &cachedBlockChunkQuerier{blockBaseQuerier: q, cache: cache}, nil +} + +func (q *cachedBlockChunkQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) storage.ChunkSeriesSet { + return selectChunkSeriesSet(ctx, sortSeries, hints, ms, q.blockID, q.index, q.chunks, q.tombstones, q.mint, q.maxt, q.cache) +} + +func selectChunkSeriesSet(ctx context.Context, sortSeries bool, hints *storage.SelectHints, ms []*labels.Matcher, + blockID ulid.ULID, ir prom_tsdb.IndexReader, chunks prom_tsdb.ChunkReader, tombstones tombstones.Reader, mint, maxt int64, + cache ExpandedPostingsCache, +) storage.ChunkSeriesSet { + disableTrimming := false + sharded := hints != nil && hints.ShardCount > 0 + + if hints != nil { + mint = hints.Start + maxt = hints.End + disableTrimming = hints.DisableTrimming + } + + var postings index.Postings + if cache != nil { + p, err := cache.PostingsForMatchers(ctx, blockID, ir, ms...) + if err != nil { + return storage.ErrChunkSeriesSet(err) + } + postings = p + } else { + p, err := prom_tsdb.PostingsForMatchers(ctx, ir, ms...) + if err != nil { + return storage.ErrChunkSeriesSet(err) + } + postings = p + } + + if sharded { + postings = ir.ShardedPostings(postings, hints.ShardIndex, hints.ShardCount) + } + if sortSeries { + postings = ir.SortedPostings(postings) + } + return prom_tsdb.NewBlockChunkSeriesSet(blockID, ir, chunks, tombstones, postings, mint, maxt, disableTrimming) +} diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index 26e6284b73..1997e0fd38 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -34,7 +34,9 @@ import ( "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/errutil" "github.com/thanos-io/thanos/pkg/exemplars" + "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/info/infopb" + "github.com/thanos-io/thanos/pkg/receive/expandedpostingscache" "github.com/thanos-io/thanos/pkg/shipper" "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/store/labelpb" @@ -69,6 +71,9 @@ type MultiTSDB struct { exemplarClientsNeedUpdate bool metricNameFilterEnabled bool + + headExpandedPostingsCacheSize uint64 + blockExpandedPostingsCacheSize uint64 } // MultiTSDBOption is a functional option for MultiTSDB. @@ -81,6 +86,18 @@ func WithMetricNameFilterEnabled() MultiTSDBOption { } } +func WithHeadExpandedPostingsCacheSize(size uint64) MultiTSDBOption { + return func(s *MultiTSDB) { + s.headExpandedPostingsCacheSize = size + } +} + +func WithBlockExpandedPostingsCacheSize(size uint64) MultiTSDBOption { + return func(s *MultiTSDB) { + s.blockExpandedPostingsCacheSize = size + } +} + // NewMultiTSDB creates new MultiTSDB. // NOTE: Passed labels must be sorted lexicographically (alphabetically). func NewMultiTSDB( @@ -687,9 +704,27 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant dataDir := t.defaultTenantDataDir(tenantID) level.Info(logger).Log("msg", "opening TSDB") + + var expandedPostingsCache expandedpostingscache.ExpandedPostingsCache + if t.headExpandedPostingsCacheSize > 0 || t.blockExpandedPostingsCacheSize > 0 { + var expandedPostingsCacheMetrics = expandedpostingscache.NewPostingCacheMetrics(extprom.WrapRegistererWithPrefix("thanos_", reg)) + + expandedPostingsCache = expandedpostingscache.NewBlocksPostingsForMatchersCache(expandedPostingsCacheMetrics, t.headExpandedPostingsCacheSize, t.blockExpandedPostingsCacheSize) + } + opts := *t.tsdbOpts opts.BlocksToDelete = tenant.blocksToDelete opts.EnableDelayedCompaction = true + + opts.BlockChunkQuerierFunc = func(b tsdb.BlockReader, mint, maxt int64) (storage.ChunkQuerier, error) { + if expandedPostingsCache != nil { + return expandedpostingscache.NewCachedBlockChunkQuerier(expandedPostingsCache, b, mint, maxt) + } + return tsdb.NewBlockChunkQuerier(b, mint, maxt) + } + if expandedPostingsCache != nil { + opts.SeriesLifecycleCallback = expandedPostingsCache + } tenant.blocksToDeleteFn = tsdb.DefaultBlocksToDelete // NOTE(GiedriusS): always set to false to properly handle OOO samples - OOO samples are written into the WBL diff --git a/pkg/store/storepb/inprocess.go b/pkg/store/storepb/inprocess.go index 0c3e7641ba..7e960845f3 100644 --- a/pkg/store/storepb/inprocess.go +++ b/pkg/store/storepb/inprocess.go @@ -5,8 +5,10 @@ package storepb import ( "context" + "fmt" "io" "iter" + "runtime/debug" "google.golang.org/grpc" ) @@ -92,6 +94,12 @@ func (s serverAsClient) LabelValues(ctx context.Context, in *LabelValuesRequest, func (s serverAsClient) Series(ctx context.Context, in *SeriesRequest, _ ...grpc.CallOption) (Store_SeriesClient, error) { var srvIter iter.Seq2[*SeriesResponse, error] = func(yield func(*SeriesResponse, error) bool) { + defer func() { + if r := recover(); r != nil { + st := debug.Stack() + panic(fmt.Sprintf("panic %v in server iterator: %s", r, st)) + } + }() srv := newInProcessServer(ctx, yield) err := s.srv.Series(in, srv) if err != nil { diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index c8a9e7fc62..b4448aa633 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -538,20 +538,21 @@ type ReceiveBuilder struct { f e2e.FutureRunnable - maxExemplars int - capnp bool - ingestion bool - limit int - tenantsLimits receive.TenantsWriteLimitsConfig - metaMonitoring string - metaMonitoringQuery string - hashringConfigs []receive.HashringConfig - relabelConfigs []*relabel.Config - replication int - image string - nativeHistograms bool - labels []string - tenantSplitLabel string + maxExemplars int + capnp bool + ingestion bool + expandedPostingsCache bool + limit int + tenantsLimits receive.TenantsWriteLimitsConfig + metaMonitoring string + metaMonitoringQuery string + hashringConfigs []receive.HashringConfig + relabelConfigs []*relabel.Config + replication int + image string + nativeHistograms bool + labels []string + tenantSplitLabel string } func NewReceiveBuilder(e e2e.Environment, name string) *ReceiveBuilder { @@ -582,6 +583,11 @@ func (r *ReceiveBuilder) WithIngestionEnabled() *ReceiveBuilder { return r } +func (r *ReceiveBuilder) WithExpandedPostingsCache() *ReceiveBuilder { + r.expandedPostingsCache = true + return r +} + func (r *ReceiveBuilder) WithLabel(name, value string) *ReceiveBuilder { r.labels = append(r.labels, fmt.Sprintf(`%s="%s"`, name, value)) return r @@ -661,6 +667,11 @@ func (r *ReceiveBuilder) Init() *e2eobs.Observable { args["--receive.local-endpoint"] = r.InternalEndpoint("grpc") } + if r.expandedPostingsCache { + args["--tsdb.head.expanded-postings-cache-size"] = "1000" + args["--tsdb.block.expanded-postings-cache-size"] = "1000" + } + if r.limit != 0 && r.metaMonitoring != "" { cfg := receive.RootLimitsConfig{ WriteLimits: receive.WriteLimitsConfig{ diff --git a/test/e2e/receive_test.go b/test/e2e/receive_test.go index c938a4f040..7d841e9a7e 100644 --- a/test/e2e/receive_test.go +++ b/test/e2e/receive_test.go @@ -80,7 +80,7 @@ func TestReceive(t *testing.T) { t.Cleanup(e2ethanos.CleanScenario(t, e)) // Setup Router Ingestor. - i := e2ethanos.NewReceiveBuilder(e, "ingestor").WithIngestionEnabled().Init() + i := e2ethanos.NewReceiveBuilder(e, "ingestor").WithIngestionEnabled().WithExpandedPostingsCache().Init() testutil.Ok(t, e2e.StartAndWaitReady(i)) // Setup Prometheus @@ -135,9 +135,9 @@ func TestReceive(t *testing.T) { t.Cleanup(e2ethanos.CleanScenario(t, e)) // Setup Receives - r1 := e2ethanos.NewReceiveBuilder(e, "1").WithIngestionEnabled().Init() - r2 := e2ethanos.NewReceiveBuilder(e, "2").WithIngestionEnabled().Init() - r3 := e2ethanos.NewReceiveBuilder(e, "3").WithIngestionEnabled().Init() + r1 := e2ethanos.NewReceiveBuilder(e, "1").WithIngestionEnabled().WithExpandedPostingsCache().Init() + r2 := e2ethanos.NewReceiveBuilder(e, "2").WithIngestionEnabled().WithExpandedPostingsCache().Init() + r3 := e2ethanos.NewReceiveBuilder(e, "3").WithIngestionEnabled().WithExpandedPostingsCache().Init() testutil.Ok(t, e2e.StartAndWaitReady(r1, r2, r3)) @@ -291,9 +291,9 @@ test_metric{a="2", b="2"} 1`) t.Cleanup(e2ethanos.CleanScenario(t, e)) // Setup 3 ingestors. - i1 := e2ethanos.NewReceiveBuilder(e, "i1").WithIngestionEnabled().Init() - i2 := e2ethanos.NewReceiveBuilder(e, "i2").WithIngestionEnabled().Init() - i3 := e2ethanos.NewReceiveBuilder(e, "i3").WithIngestionEnabled().Init() + i1 := e2ethanos.NewReceiveBuilder(e, "i1").WithIngestionEnabled().WithExpandedPostingsCache().Init() + i2 := e2ethanos.NewReceiveBuilder(e, "i2").WithIngestionEnabled().WithExpandedPostingsCache().Init() + i3 := e2ethanos.NewReceiveBuilder(e, "i3").WithIngestionEnabled().WithExpandedPostingsCache().Init() h := receive.HashringConfig{ Endpoints: []receive.Endpoint{ From a55844d52a8193d2a411fcd7a027445c5821c0cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Mon, 25 Nov 2024 12:14:08 +0200 Subject: [PATCH 10/21] receive/expandedpostingscache: fix race (#7937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Porting https://github.com/cortexproject/cortex/pull/6369 to our code base. Add test that fails without the fix. Signed-off-by: Giedrius Statkevičius --- pkg/receive/expandedpostingscache/cache.go | 22 ++++++---- .../expandedpostingscache/cache_test.go | 42 +++++++++++++++++++ pkg/receive/multitsdb.go | 2 +- 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/pkg/receive/expandedpostingscache/cache.go b/pkg/receive/expandedpostingscache/cache.go index d5c7069735..2e38d1971c 100644 --- a/pkg/receive/expandedpostingscache/cache.go +++ b/pkg/receive/expandedpostingscache/cache.go @@ -43,7 +43,7 @@ type BlocksPostingsForMatchersCache struct { postingsForMatchersFunc func(ctx context.Context, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error) timeNow func() time.Time - metrics *ExpandedPostingsCacheMetrics + metrics ExpandedPostingsCacheMetrics } var ( @@ -66,8 +66,8 @@ type ExpandedPostingsCacheMetrics struct { NonCacheableQueries *prometheus.CounterVec } -func NewPostingCacheMetrics(r prometheus.Registerer) *ExpandedPostingsCacheMetrics { - return &ExpandedPostingsCacheMetrics{ +func NewPostingCacheMetrics(r prometheus.Registerer) ExpandedPostingsCacheMetrics { + return ExpandedPostingsCacheMetrics{ CacheRequests: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ Name: "expanded_postings_cache_requests_total", Help: "Total number of requests to the cache.", @@ -87,11 +87,15 @@ func NewPostingCacheMetrics(r prometheus.Registerer) *ExpandedPostingsCacheMetri } } -func NewBlocksPostingsForMatchersCache(metrics *ExpandedPostingsCacheMetrics, headExpandedPostingsCacheSize uint64, blockExpandedPostingsCacheSize uint64) ExpandedPostingsCache { +func NewBlocksPostingsForMatchersCache(metrics ExpandedPostingsCacheMetrics, headExpandedPostingsCacheSize uint64, blockExpandedPostingsCacheSize uint64, seedSize int64) *BlocksPostingsForMatchersCache { + if seedSize <= 0 { + seedSize = seedArraySize + } + return &BlocksPostingsForMatchersCache{ headCache: newFifoCache[[]storage.SeriesRef]("head", metrics, time.Now, headExpandedPostingsCacheSize), blocksCache: newFifoCache[[]storage.SeriesRef]("block", metrics, time.Now, blockExpandedPostingsCacheSize), - headSeedByMetricName: make([]int, seedArraySize), + headSeedByMetricName: make([]int, seedSize), strippedLock: make([]sync.RWMutex, numOfSeedsStripes), postingsForMatchersFunc: tsdb.PostingsForMatchers, timeNow: time.Now, @@ -129,7 +133,7 @@ func (c *BlocksPostingsForMatchersCache) ExpireSeries(metric labels.Labels) { h := MemHashString(metricName) i := h % uint64(len(c.headSeedByMetricName)) - l := h % uint64(len(c.strippedLock)) + l := i % uint64(len(c.strippedLock)) c.strippedLock[l].Lock() defer c.strippedLock[l].Unlock() c.headSeedByMetricName[i]++ @@ -200,7 +204,7 @@ func (c *BlocksPostingsForMatchersCache) result(ce *cacheEntryPromise[[]storage. func (c *BlocksPostingsForMatchersCache) getSeedForMetricName(metricName string) string { h := MemHashString(metricName) i := h % uint64(len(c.headSeedByMetricName)) - l := h % uint64(len(c.strippedLock)) + l := i % uint64(len(c.strippedLock)) c.strippedLock[l].RLock() defer c.strippedLock[l].RUnlock() return strconv.Itoa(c.headSeedByMetricName[i]) @@ -276,13 +280,13 @@ type fifoCache[V any] struct { cachedBytes int64 } -func newFifoCache[V any](name string, metrics *ExpandedPostingsCacheMetrics, timeNow func() time.Time, maxBytes uint64) *fifoCache[V] { +func newFifoCache[V any](name string, metrics ExpandedPostingsCacheMetrics, timeNow func() time.Time, maxBytes uint64) *fifoCache[V] { return &fifoCache[V]{ cachedValues: new(sync.Map), cached: list.New(), timeNow: timeNow, name: name, - metrics: *metrics, + metrics: metrics, ttl: 10 * time.Minute, maxBytes: int64(maxBytes), } diff --git a/pkg/receive/expandedpostingscache/cache_test.go b/pkg/receive/expandedpostingscache/cache_test.go index a2ba2dafd2..86c7573e99 100644 --- a/pkg/receive/expandedpostingscache/cache_test.go +++ b/pkg/receive/expandedpostingscache/cache_test.go @@ -15,9 +15,11 @@ import ( "time" "go.uber.org/atomic" + "golang.org/x/exp/rand" "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" ) @@ -166,3 +168,43 @@ func repeatStringIfNeeded(seed string, length int) string { return strings.Repeat(seed, 1+length/len(seed))[:max(length, len(seed))] } + +func TestLockRaceExpireSeries(t *testing.T) { + for j := 0; j < 10; j++ { + wg := &sync.WaitGroup{} + + c := NewBlocksPostingsForMatchersCache(ExpandedPostingsCacheMetrics{}, 1<<7, 1<<7, 3) + for i := 0; i < 1000; i++ { + wg.Add(2) + + go func() { + defer wg.Done() + for i := 0; i < 10; i++ { + c.ExpireSeries( + labels.FromMap(map[string]string{"__name__": randSeq(10)}), + ) + } + }() + + go func() { + defer wg.Done() + + for i := 0; i < 10; i++ { + c.getSeedForMetricName(randSeq(10)) + } + }() + } + wg.Wait() + } +} + +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func randSeq(n int) string { + b := make([]rune, n) + rand.Seed(uint64(time.Now().UnixNano())) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index 1997e0fd38..10c41d32ba 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -709,7 +709,7 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant if t.headExpandedPostingsCacheSize > 0 || t.blockExpandedPostingsCacheSize > 0 { var expandedPostingsCacheMetrics = expandedpostingscache.NewPostingCacheMetrics(extprom.WrapRegistererWithPrefix("thanos_", reg)) - expandedPostingsCache = expandedpostingscache.NewBlocksPostingsForMatchersCache(expandedPostingsCacheMetrics, t.headExpandedPostingsCacheSize, t.blockExpandedPostingsCacheSize) + expandedPostingsCache = expandedpostingscache.NewBlocksPostingsForMatchersCache(expandedPostingsCacheMetrics, t.headExpandedPostingsCacheSize, t.blockExpandedPostingsCacheSize, 0) } opts := *t.tsdbOpts From 1d763356112841231b447fd5c6fd4c4e252f3c00 Mon Sep 17 00:00:00 2001 From: Philip Gough Date: Mon, 25 Nov 2024 13:00:26 +0000 Subject: [PATCH 11/21] receive: Allow specifying a custom gRPC service config via flag (#7907) Signed-off-by: Philip Gough --- CHANGELOG.md | 1 + cmd/thanos/receive.go | 7 +++++++ docs/components/receive.md | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f89c47c855..d50aa427b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#7855](https://github.com/thanos-io/thanos/pull/7855) Compcat/Query: Add support for comma separated replica labels. - [#7654](https://github.com/thanos-io/thanos/pull/7654) *: Add '--grpc-server-tls-min-version' flag to allow user to specify TLS version, otherwise default to TLS 1.3 - [#7854](https://github.com/thanos-io/thanos/pull/7854) Query Frontend: Add `--query-frontend.force-query-stats` flag to force collection of query statistics from upstream queriers. +- [#7907](https://github.com/thanos-io/thanos/pull/7907) Receive: Add `--receive.grpc-service-config` flag to configure gRPC service config for the receivers. ### Changed diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 299634546a..0372e83690 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -175,6 +175,10 @@ func runReceive( dialOpts = append(dialOpts, grpc.WithDefaultCallOptions(grpc.UseCompressor(conf.compression))) } + if conf.grpcServiceConfig != "" { + dialOpts = append(dialOpts, grpc.WithDefaultServiceConfig(conf.grpcServiceConfig)) + } + var bkt objstore.Bucket confContentYaml, err := conf.objStoreConfig.Content() if err != nil { @@ -856,6 +860,7 @@ type receiveConfig struct { maxBackoff *model.Duration compression string replicationProtocol string + grpcServiceConfig string tsdbMinBlockDuration *model.Duration tsdbMaxBlockDuration *model.Duration @@ -970,6 +975,8 @@ func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) { cmd.Flag("receive.capnproto-address", "Address for the Cap'n Proto server.").Default(fmt.Sprintf("0.0.0.0:%s", receive.DefaultCapNProtoPort)).StringVar(&rc.replicationAddr) + cmd.Flag("receive.grpc-service-config", "gRPC service configuration file or content in JSON format. See https://github.com/grpc/grpc/blob/master/doc/service_config.md").PlaceHolder("").Default("").StringVar(&rc.grpcServiceConfig) + rc.forwardTimeout = extkingpin.ModelDuration(cmd.Flag("receive-forward-timeout", "Timeout for each forward request.").Default("5s").Hidden()) rc.maxBackoff = extkingpin.ModelDuration(cmd.Flag("receive-forward-max-backoff", "Maximum backoff for each forward fan-out request").Default("5s").Hidden()) diff --git a/docs/components/receive.md b/docs/components/receive.md index dfaddd73b1..38906489ba 100644 --- a/docs/components/receive.md +++ b/docs/components/receive.md @@ -429,6 +429,10 @@ Flags: Compression algorithm to use for gRPC requests to other receivers. Must be one of: snappy, none + --receive.grpc-service-config= + gRPC service configuration file + or content in JSON format. See + https://github.com/grpc/grpc/blob/master/doc/service_config.md --receive.hashrings= Alternative to 'receive.hashrings-file' flag (lower priority). Content of file that contains From e133849d3872594c0b2ea4c008e3093fdee5b3ea Mon Sep 17 00:00:00 2001 From: bluesky6529 Date: Fri, 29 Nov 2024 01:12:34 +0800 Subject: [PATCH 12/21] remove redundant redis config (#7942) Signed-off-by: Helen Tseng --- docs/components/store.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/components/store.md b/docs/components/store.md index 3425000146..dfab1855cc 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -467,10 +467,6 @@ Here is an example of what effect client-side caching could have: Example of client-side in action - reduced network usage by a lot -- `pool_size`: maximum number of socket connections. -- `min_idle_conns`: specifies the minimum number of idle connections which is useful when establishing new connection is slow. -- `idle_timeout`: amount of time after which client closes idle connections. Should be less than server's timeout. -- `max_conn_age`: connection age at which client retires (closes) the connection. - `max_get_multi_concurrency`: specifies the maximum number of concurrent GetMulti() operations. - `get_multi_batch_size`: specifies the maximum size per batch for mget. - `max_set_multi_concurrency`: specifies the maximum number of concurrent SetMulti() operations. From dd86ec8d0abd5c0ac2e027e3a98a7dd7dd73df8b Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Thu, 28 Nov 2024 18:15:20 +0100 Subject: [PATCH 13/21] Capnp: Use segment from existing message (#7945) * Capnp: Use segment from existing message Signed-off-by: Filip Petkovski * Downgrade capnproto Signed-off-by: Filip Petkovski --------- Signed-off-by: Filip Petkovski --- go.mod | 4 ++-- go.sum | 18 ++++++++---------- pkg/receive/writecapnp/client.go | 15 +-------------- pkg/receive/writecapnp/marshal.go | 21 +++++++++++++-------- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 309e0f6432..726fc4683d 100644 --- a/go.mod +++ b/go.mod @@ -112,7 +112,7 @@ require ( ) require ( - capnproto.org/go/capnp/v3 v3.0.1-alpha.2.0.20240830165715-46ccd63a72af + capnproto.org/go/capnp/v3 v3.0.0-alpha.30 github.com/cortexproject/promqlsmith v0.0.0-20240506042652-6cfdd9739a5e github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/hashicorp/golang-lru/v2 v2.0.7 @@ -133,7 +133,6 @@ require ( github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect github.com/cilium/ebpf v0.11.0 // indirect - github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381 // indirect github.com/containerd/cgroups/v3 v3.0.3 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elastic/go-licenser v0.3.1 // indirect @@ -167,6 +166,7 @@ require ( k8s.io/client-go v0.31.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + zenhack.net/go/util v0.0.0-20230414204917-531d38494cf5 // indirect ) require ( diff --git a/go.sum b/go.sum index 9250fff8b8..743002878c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -capnproto.org/go/capnp/v3 v3.0.1-alpha.2.0.20240830165715-46ccd63a72af h1:A5wxH0ZidOtYYUGjhtBaRuB87M73bGfc06uWB8sHpg0= -capnproto.org/go/capnp/v3 v3.0.1-alpha.2.0.20240830165715-46ccd63a72af/go.mod h1:2vT5D2dtG8sJGEoEKU17e+j7shdaYp1Myl8X03B3hmc= +capnproto.org/go/capnp/v3 v3.0.0-alpha.30 h1:iABQan/YiHFCgSXym5aNj27osapnEgAk4WaWYqb4sQM= +capnproto.org/go/capnp/v3 v3.0.0-alpha.30/go.mod h1:+ysMHvOh1EWNOyorxJWs1omhRFiDoKxKkWQACp54jKM= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= @@ -1500,8 +1500,6 @@ github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381 h1:d5EKgQfRQvO97jnISfR89AiCCCJMwMFoSxUiU0OGCRU= -github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381/go.mod h1:OU76gHeRo8xrzGJU3F3I1CqX1ekM8dfJw0+wPeMwnp0= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -2106,8 +2104,8 @@ github.com/ovh/go-ovh v1.6.0 h1:ixLOwxQdzYDx296sXcgS35TOPEahJkpjMGtzPadCjQI= github.com/ovh/go-ovh v1.6.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= -github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= -github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= +github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= @@ -2263,10 +2261,8 @@ github.com/thanos-io/promql-engine v0.0.0-20241106100125-097e6e9f425a h1:BhWU58V github.com/thanos-io/promql-engine v0.0.0-20241106100125-097e6e9f425a/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY= -github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= -github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k= -github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= -github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0= +github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= @@ -3337,3 +3333,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+s sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +zenhack.net/go/util v0.0.0-20230414204917-531d38494cf5 h1:yksDCGMVzyn3vlyf0GZ3huiF5FFaMGQpQ3UJvR0EoGA= +zenhack.net/go/util v0.0.0-20230414204917-531d38494cf5/go.mod h1:1LtNdPAs8WH+BTcQiZAOo2MIKD/5jyK/u7sZ9ZPe5SE= diff --git a/pkg/receive/writecapnp/client.go b/pkg/receive/writecapnp/client.go index 0a20d90d44..3cd9f2d082 100644 --- a/pkg/receive/writecapnp/client.go +++ b/pkg/receive/writecapnp/client.go @@ -69,22 +69,9 @@ func (r *RemoteWriteClient) writeWithReconnect(ctx context.Context, numReconnect if err := r.connect(ctx); err != nil { return nil, err } - arena := capnp.SingleSegment(nil) - defer arena.Release() result, release := r.writer.Write(ctx, func(params Writer_write_Params) error { - _, seg, err := capnp.NewMessage(arena) - if err != nil { - return err - } - wr, err := NewRootWriteRequest(seg) - if err != nil { - return err - } - if err := params.SetWr(wr); err != nil { - return err - } - wr, err = params.Wr() + wr, err := params.NewWr() if err != nil { return err } diff --git a/pkg/receive/writecapnp/marshal.go b/pkg/receive/writecapnp/marshal.go index 2d42d60b84..efc1a8ef03 100644 --- a/pkg/receive/writecapnp/marshal.go +++ b/pkg/receive/writecapnp/marshal.go @@ -6,6 +6,8 @@ package writecapnp import ( "capnproto.org/go/capnp/v3" + "github.com/pkg/errors" + "github.com/thanos-io/thanos/pkg/store/labelpb" "github.com/thanos-io/thanos/pkg/store/storepb/prompb" ) @@ -46,7 +48,7 @@ func Build(tenant string, tsreq []prompb.TimeSeries) (WriteRequest, error) { func BuildInto(wr WriteRequest, tenant string, tsreq []prompb.TimeSeries) error { if err := wr.SetTenant(tenant); err != nil { - return err + return errors.Wrap(err, "set tenant") } series, err := wr.NewTimeSeries(int32(len(tsreq))) @@ -59,27 +61,30 @@ func BuildInto(wr WriteRequest, tenant string, tsreq []prompb.TimeSeries) error lblsc, err := tsc.NewLabels(int32(len(ts.Labels))) if err != nil { - return err + return errors.Wrap(err, "new labels") } if err := marshalLabels(lblsc, ts.Labels, builder); err != nil { - return err + return errors.Wrap(err, "marshal labels") } if err := marshalSamples(tsc, ts.Samples); err != nil { - return err + return errors.Wrap(err, "marshal samples") } if err := marshalHistograms(tsc, ts.Histograms); err != nil { - return err + return errors.Wrap(err, "marshal histograms") } if err := marshalExemplars(tsc, ts.Exemplars, builder); err != nil { - return err + return errors.Wrap(err, "marshal exemplars") } } symbols, err := wr.NewSymbols() if err != nil { - return err + return errors.Wrap(err, "new symbols") } - return marshalSymbols(builder, symbols) + if err := marshalSymbols(builder, symbols); err != nil { + return errors.Wrap(err, "marshal symbols") + } + return nil } func marshalSymbols(builder *symbolsBuilder, symbols Symbols) error { From 96cc4f17ff2127c2cff5eb77602a171057b43da5 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Thu, 28 Nov 2024 17:22:20 +0000 Subject: [PATCH 14/21] Fix ver Signed-off-by: Saswata Mukherjee --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0f1a7dfc7c..c05b0f0786 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.37.0 +0.38.0-dev From 1ea4e699083eeed7c13fde052b64878c8d3e7e5a Mon Sep 17 00:00:00 2001 From: Yi Jin <96499497+jnyi@users.noreply.github.com> Date: Tue, 3 Dec 2024 00:52:06 -0800 Subject: [PATCH 15/21] [Receive] Fix race condition when adding multiple new tenants at once (#7941) * [Receive] fix race condition Signed-off-by: Yi Jin * add a change log Signed-off-by: Yi Jin * memorize tsdb local clients without race condition Signed-off-by: Yi Jin * fix data race in testing with some concurrent safe helper functions Signed-off-by: Yi Jin * address comments Signed-off-by: Yi Jin --------- Signed-off-by: Yi Jin --- CHANGELOG.md | 1 + docs/sharding.md | 2 +- pkg/receive/multitsdb.go | 140 ++++++++++++++++------------------ pkg/receive/multitsdb_test.go | 49 +++++++++++- pkg/receive/receive_test.go | 12 +-- 5 files changed, 121 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2956561a..b9fb86439c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#7893](https://github.com/thanos-io/thanos/pull/7893) Sidecar: Fix retrieval of external labels for Prometheus v3.0.0. - [#7903](https://github.com/thanos-io/thanos/pull/7903) Query: Fix panic on regex store matchers. - [#7915](https://github.com/thanos-io/thanos/pull/7915) Store: Close block series client at the end to not reuse chunk buffer +- [#7941](https://github.com/thanos-io/thanos/pull/7941) Receive: Fix race condition when adding multiple new tenants, see [issue-7892](https://github.com/thanos-io/thanos/issues/7892). ### Added diff --git a/docs/sharding.md b/docs/sharding.md index 9cfa0fbf8a..f943ec071b 100644 --- a/docs/sharding.md +++ b/docs/sharding.md @@ -18,7 +18,7 @@ Queries against store gateway which are touching large number of blocks (no matt # Relabelling -Similar to [promtail](https://grafana.com/docs/loki/latest/send-data/promtail/configuration/#relabel_configs) this config follows native [Prometheus relabel-config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) syntax. +Similar to [promtail](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#relabel_configs) this config follows native [Prometheus relabel-config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) syntax. Currently, thanos only supports the following relabel actions: diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index 10c41d32ba..526e3c6ec9 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -64,11 +64,8 @@ type MultiTSDB struct { hashFunc metadata.HashFunc hashringConfigs []HashringConfig - tsdbClients []store.Client - tsdbClientsNeedUpdate bool - - exemplarClients map[string]*exemplars.TSDB - exemplarClientsNeedUpdate bool + tsdbClients []store.Client + exemplarClients map[string]*exemplars.TSDB metricNameFilterEnabled bool @@ -117,19 +114,19 @@ func NewMultiTSDB( } mt := &MultiTSDB{ - dataDir: dataDir, - logger: log.With(l, "component", "multi-tsdb"), - reg: reg, - tsdbOpts: tsdbOpts, - mtx: &sync.RWMutex{}, - tenants: map[string]*tenant{}, - labels: labels, - tsdbClientsNeedUpdate: true, - exemplarClientsNeedUpdate: true, - tenantLabelName: tenantLabelName, - bucket: bucket, - allowOutOfOrderUpload: allowOutOfOrderUpload, - hashFunc: hashFunc, + dataDir: dataDir, + logger: log.With(l, "component", "multi-tsdb"), + reg: reg, + tsdbOpts: tsdbOpts, + mtx: &sync.RWMutex{}, + tenants: map[string]*tenant{}, + labels: labels, + tsdbClients: make([]store.Client, 0), + exemplarClients: map[string]*exemplars.TSDB{}, + tenantLabelName: tenantLabelName, + bucket: bucket, + allowOutOfOrderUpload: allowOutOfOrderUpload, + hashFunc: hashFunc, } for _, option := range options { @@ -139,6 +136,49 @@ func NewMultiTSDB( return mt } +// testGetTenant returns the tenant with the given tenantID for testing purposes. +func (t *MultiTSDB) testGetTenant(tenantID string) *tenant { + t.mtx.RLock() + defer t.mtx.RUnlock() + return t.tenants[tenantID] +} + +func (t *MultiTSDB) updateTSDBClients() { + t.tsdbClients = t.tsdbClients[:0] + for _, tenant := range t.tenants { + client := tenant.client() + if client != nil { + t.tsdbClients = append(t.tsdbClients, client) + } + } +} + +func (t *MultiTSDB) addTenantUnlocked(tenantID string, newTenant *tenant) { + t.tenants[tenantID] = newTenant + t.updateTSDBClients() + if newTenant.exemplars() != nil { + t.exemplarClients[tenantID] = newTenant.exemplars() + } +} + +func (t *MultiTSDB) addTenantLocked(tenantID string, newTenant *tenant) { + t.mtx.Lock() + defer t.mtx.Unlock() + t.addTenantUnlocked(tenantID, newTenant) +} + +func (t *MultiTSDB) removeTenantUnlocked(tenantID string) { + delete(t.tenants, tenantID) + delete(t.exemplarClients, tenantID) + t.updateTSDBClients() +} + +func (t *MultiTSDB) removeTenantLocked(tenantID string) { + t.mtx.Lock() + defer t.mtx.Unlock() + t.removeTenantUnlocked(tenantID) +} + type localClient struct { store *store.TSDBStore @@ -433,9 +473,7 @@ func (t *MultiTSDB) Prune(ctx context.Context) error { } level.Info(t.logger).Log("msg", "Pruned tenant", "tenant", tenantID) - delete(t.tenants, tenantID) - t.tsdbClientsNeedUpdate = true - t.exemplarClientsNeedUpdate = true + t.removeTenantUnlocked(tenantID) } return merr.Err() @@ -595,58 +633,17 @@ func (t *MultiTSDB) RemoveLockFilesIfAny() error { return merr.Err() } +// TSDBLocalClients should be used as read-only. func (t *MultiTSDB) TSDBLocalClients() []store.Client { t.mtx.RLock() - if !t.tsdbClientsNeedUpdate { - t.mtx.RUnlock() - return t.tsdbClients - } - - t.mtx.RUnlock() - t.mtx.Lock() - defer t.mtx.Unlock() - if !t.tsdbClientsNeedUpdate { - return t.tsdbClients - } - - res := make([]store.Client, 0, len(t.tenants)) - for _, tenant := range t.tenants { - client := tenant.client() - if client != nil { - res = append(res, client) - } - } - - t.tsdbClientsNeedUpdate = false - t.tsdbClients = res - + defer t.mtx.RUnlock() return t.tsdbClients } +// TSDBExemplars should be used as read-only. func (t *MultiTSDB) TSDBExemplars() map[string]*exemplars.TSDB { t.mtx.RLock() - if !t.exemplarClientsNeedUpdate { - t.mtx.RUnlock() - return t.exemplarClients - } - t.mtx.RUnlock() - t.mtx.Lock() - defer t.mtx.Unlock() - - if !t.exemplarClientsNeedUpdate { - return t.exemplarClients - } - - res := make(map[string]*exemplars.TSDB, len(t.tenants)) - for k, tenant := range t.tenants { - e := tenant.exemplars() - if e != nil { - res[k] = e - } - } - - t.exemplarClientsNeedUpdate = false - t.exemplarClients = res + defer t.mtx.RUnlock() return t.exemplarClients } @@ -740,11 +737,7 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant nil, ) if err != nil { - t.mtx.Lock() - delete(t.tenants, tenantID) - t.tsdbClientsNeedUpdate = true - t.exemplarClientsNeedUpdate = true - t.mtx.Unlock() + t.removeTenantLocked(tenantID) return err } var ship *shipper.Shipper @@ -767,6 +760,7 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant options = append(options, store.WithCuckooMetricNameStoreFilter()) } tenant.set(store.NewTSDBStore(logger, s, component.Receive, lset, options...), s, ship, exemplars.NewTSDB(s, lset)) + t.addTenantLocked(tenantID, tenant) // need to update the client list once store is ready & client != nil level.Info(logger).Log("msg", "TSDB is now ready") return nil } @@ -795,9 +789,7 @@ func (t *MultiTSDB) getOrLoadTenant(tenantID string, blockingStart bool) (*tenan } tenant = newTenant() - t.tenants[tenantID] = tenant - t.tsdbClientsNeedUpdate = true - t.exemplarClientsNeedUpdate = true + t.addTenantUnlocked(tenantID, tenant) t.mtx.Unlock() logger := log.With(t.logger, "tenant", tenantID) diff --git a/pkg/receive/multitsdb_test.go b/pkg/receive/multitsdb_test.go index 4bee9c0514..a36db4b402 100644 --- a/pkg/receive/multitsdb_test.go +++ b/pkg/receive/multitsdb_test.go @@ -5,6 +5,7 @@ package receive import ( "context" + "fmt" "io" "math" "os" @@ -193,7 +194,7 @@ func TestMultiTSDB(t *testing.T) { testutil.Ok(t, m.Open()) testutil.Ok(t, appendSample(m, testTenant, time.Now())) - tenant := m.tenants[testTenant] + tenant := m.testGetTenant(testTenant) db := tenant.readyStorage().Get() testutil.Equals(t, 0, len(db.Blocks())) @@ -541,6 +542,47 @@ func TestMultiTSDBRecreatePrunedTenant(t *testing.T) { testutil.Equals(t, 1, len(m.TSDBLocalClients())) } +func TestMultiTSDBAddNewTenant(t *testing.T) { + t.Parallel() + const iterations = 10 + // This test detects race conditions, so we run it multiple times to increase the chance of catching the issue. + for i := 0; i < iterations; i++ { + t.Run(fmt.Sprintf("iteration-%d", i), func(t *testing.T) { + dir := t.TempDir() + m := NewMultiTSDB(dir, log.NewNopLogger(), prometheus.NewRegistry(), + &tsdb.Options{ + MinBlockDuration: (2 * time.Hour).Milliseconds(), + MaxBlockDuration: (2 * time.Hour).Milliseconds(), + RetentionDuration: (6 * time.Hour).Milliseconds(), + }, + labels.FromStrings("replica", "test"), + "tenant_id", + objstore.NewInMemBucket(), + false, + metadata.NoneFunc, + ) + defer func() { testutil.Ok(t, m.Close()) }() + + concurrency := 50 + var wg sync.WaitGroup + for i := 0; i < concurrency; i++ { + wg.Add(1) + // simulate remote write with new tenant concurrently + go func(i int) { + defer wg.Done() + testutil.Ok(t, appendSample(m, fmt.Sprintf("tenant-%d", i), time.UnixMilli(int64(10)))) + }(i) + // simulate read request concurrently + go func() { + m.TSDBLocalClients() + }() + } + wg.Wait() + testutil.Equals(t, concurrency, len(m.TSDBLocalClients())) + }) + } +} + func TestAlignedHeadFlush(t *testing.T) { t.Parallel() @@ -801,7 +843,10 @@ func appendSampleWithLabels(m *MultiTSDB, tenant string, lbls labels.Labels, tim func queryLabelValues(ctx context.Context, m *MultiTSDB) error { proxy := store.NewProxyStore(nil, nil, func() []store.Client { - clients := m.TSDBLocalClients() + m.mtx.Lock() + defer m.mtx.Unlock() + clients := make([]store.Client, len(m.tsdbClients)) + copy(clients, m.tsdbClients) if len(clients) > 0 { clients[0] = &slowClient{clients[0]} } diff --git a/pkg/receive/receive_test.go b/pkg/receive/receive_test.go index 7e90c3c204..bf38cb06ed 100644 --- a/pkg/receive/receive_test.go +++ b/pkg/receive/receive_test.go @@ -210,7 +210,7 @@ func TestAddingExternalLabelsForTenants(t *testing.T) { for _, c := range tc.cfg { for _, tenantId := range c.Tenants { - if m.tenants[tenantId] == nil { + if m.testGetTenant(tenantId) == nil { err = appendSample(m, tenantId, time.Now()) require.NoError(t, err) } @@ -294,7 +294,7 @@ func TestLabelSetsOfTenantsWhenAddingTenants(t *testing.T) { for _, c := range initialConfig { for _, tenantId := range c.Tenants { - if m.tenants[tenantId] == nil { + if m.testGetTenant(tenantId) == nil { err = appendSample(m, tenantId, time.Now()) require.NoError(t, err) } @@ -319,7 +319,7 @@ func TestLabelSetsOfTenantsWhenAddingTenants(t *testing.T) { for _, c := range changedConfig { for _, tenantId := range c.Tenants { - if m.tenants[tenantId] == nil { + if m.testGetTenant(tenantId) == nil { err = appendSample(m, tenantId, time.Now()) require.NoError(t, err) } @@ -534,7 +534,7 @@ func TestLabelSetsOfTenantsWhenChangingLabels(t *testing.T) { for _, c := range initialConfig { for _, tenantId := range c.Tenants { - if m.tenants[tenantId] == nil { + if m.testGetTenant(tenantId) == nil { err = appendSample(m, tenantId, time.Now()) require.NoError(t, err) } @@ -704,7 +704,7 @@ func TestAddingLabelsWhenTenantAppearsInMultipleHashrings(t *testing.T) { for _, c := range initialConfig { for _, tenantId := range c.Tenants { - if m.tenants[tenantId] == nil { + if m.testGetTenant(tenantId) == nil { err = appendSample(m, tenantId, time.Now()) require.NoError(t, err) } @@ -778,7 +778,7 @@ func TestReceiverLabelsNotOverwrittenByExternalLabels(t *testing.T) { for _, c := range cfg { for _, tenantId := range c.Tenants { - if m.tenants[tenantId] == nil { + if m.testGetTenant(tenantId) == nil { err = appendSample(m, tenantId, time.Now()) require.NoError(t, err) } From 1a328c124b664b91fea8fc8b54a161cc3e725b46 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Tue, 3 Dec 2024 13:33:10 +0000 Subject: [PATCH 16/21] Update promql-engine for subquery fix (#7953) Signed-off-by: Saswata Mukherjee --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 726fc4683d..9a725cb167 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.9.0 github.com/thanos-io/objstore v0.0.0-20241111205755-d1dd89d41f97 - github.com/thanos-io/promql-engine v0.0.0-20241106100125-097e6e9f425a + github.com/thanos-io/promql-engine v0.0.0-20241203103240-2f49f80c7c68 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a diff --git a/go.sum b/go.sum index 743002878c..9933abda40 100644 --- a/go.sum +++ b/go.sum @@ -2257,8 +2257,8 @@ github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1 github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= github.com/thanos-io/objstore v0.0.0-20241111205755-d1dd89d41f97 h1:VjG0mwhN1DkncwDHFvrpd12/2TLfgYNRmEQA48ikp+0= github.com/thanos-io/objstore v0.0.0-20241111205755-d1dd89d41f97/go.mod h1:vyzFrBXgP+fGNG2FopEGWOO/zrIuoy7zt3LpLeezRsw= -github.com/thanos-io/promql-engine v0.0.0-20241106100125-097e6e9f425a h1:BhWU58VHOxkxQEMByih9fM2WwuwCGtk5AulIcSRSr0A= -github.com/thanos-io/promql-engine v0.0.0-20241106100125-097e6e9f425a/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= +github.com/thanos-io/promql-engine v0.0.0-20241203103240-2f49f80c7c68 h1:cChM/FbpXeYmrSmXO1/MmmSlONviLVxWAWCB0/g4JrY= +github.com/thanos-io/promql-engine v0.0.0-20241203103240-2f49f80c7c68/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY= github.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0= From dec2686f994d57bddea4aad7530b97948a9a370e Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Tue, 3 Dec 2024 18:55:46 +0000 Subject: [PATCH 17/21] Sidecar: Ensure limit param is positive for compatibility with older Prometheus (#7954) Signed-off-by: Saswata Mukherjee --- pkg/promclient/promclient.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index eeca44db92..b655ea1ab1 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -787,7 +787,9 @@ func (c *Client) SeriesInGRPC(ctx context.Context, base *url.URL, matchers []*la q.Add("match[]", storepb.PromMatchersToString(matchers...)) q.Add("start", formatTime(timestamp.Time(startTime))) q.Add("end", formatTime(timestamp.Time(endTime))) - q.Add("limit", strconv.Itoa(limit)) + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } u.RawQuery = q.Encode() var m struct { @@ -809,7 +811,9 @@ func (c *Client) LabelNamesInGRPC(ctx context.Context, base *url.URL, matchers [ } q.Add("start", formatTime(timestamp.Time(startTime))) q.Add("end", formatTime(timestamp.Time(endTime))) - q.Add("limit", strconv.Itoa(limit)) + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } u.RawQuery = q.Encode() var m struct { @@ -830,7 +834,9 @@ func (c *Client) LabelValuesInGRPC(ctx context.Context, base *url.URL, label str } q.Add("start", formatTime(timestamp.Time(startTime))) q.Add("end", formatTime(timestamp.Time(endTime))) - q.Add("limit", strconv.Itoa(limit)) + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } u.RawQuery = q.Encode() var m struct { @@ -898,7 +904,6 @@ func (c *Client) MetricMetadataInGRPC(ctx context.Context, base *url.URL, metric if metric != "" { q.Add("metric", metric) } - // We only set limit when it is >= 0. if limit >= 0 { q.Add("limit", strconv.Itoa(limit)) } From d0d93dbf3efcf11cce28b81cb1392eb5f88aa34f Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Thu, 5 Dec 2024 21:39:58 +0900 Subject: [PATCH 18/21] Fix potential deadlock in hedging request (#7962) Signed-off-by: kade.lee --- pkg/exthttp/hedging.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/exthttp/hedging.go b/pkg/exthttp/hedging.go index 09a1b3e8a2..af200bd92e 100644 --- a/pkg/exthttp/hedging.go +++ b/pkg/exthttp/hedging.go @@ -48,11 +48,11 @@ func (hrt *hedgingRoundTripper) RoundTrip(req *http.Request) (*http.Response, er } duration := float64(time.Since(start).Milliseconds()) hrt.mu.Lock() + defer hrt.mu.Unlock() err = hrt.TDigest.Add(duration) if err != nil { return nil, err } - hrt.mu.Unlock() return resp, err } From 51c7dcd8c27814fd4f3d64e5d6294cdd1772ccb5 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 9 Dec 2024 23:13:11 -0800 Subject: [PATCH 19/21] Mark posting group lazy if it has a lot of keys (#7961) * mark posting group lazy if it has a lot of add keys Signed-off-by: Ben Ye * update docs Signed-off-by: Ben Ye * rename labels Signed-off-by: Ben Ye * changelog Signed-off-by: Ben Ye * change to use max key series ratio Signed-off-by: Ben Ye * update docs Signed-off-by: Ben Ye * mention metrics Signed-off-by: Ben Ye * update docs Signed-off-by: Ben Ye --------- Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + cmd/thanos/store.go | 73 +++++----- docs/components/store.md | 12 ++ pkg/store/bucket.go | 61 +++++++-- pkg/store/bucket_test.go | 30 ++-- pkg/store/lazy_postings.go | 68 +++++++--- pkg/store/lazy_postings_test.go | 233 +++++++++++++++++++++++++++----- 7 files changed, 368 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 114eef55d7..3bf3378b5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Added - [#7907](https://github.com/thanos-io/thanos/pull/7907) Receive: Add `--receive.grpc-service-config` flag to configure gRPC service config for the receivers. +- [#7961](https://github.com/thanos-io/thanos/pull/7961) Store Gateway: Add `--store.posting-group-max-keys` flag to mark posting group as lazy if it exceeds number of keys limit. Added `thanos_bucket_store_lazy_expanded_posting_groups_total` for total number of lazy posting groups and corresponding reasons. ### Changed diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 795c108cd3..1cdc12679c 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -67,40 +67,41 @@ const ( ) type storeConfig struct { - indexCacheConfigs extflag.PathOrContent - objStoreConfig extflag.PathOrContent - dataDir string - cacheIndexHeader bool - grpcConfig grpcConfig - httpConfig httpConfig - indexCacheSizeBytes units.Base2Bytes - chunkPoolSize units.Base2Bytes - estimatedMaxSeriesSize uint64 - estimatedMaxChunkSize uint64 - seriesBatchSize int - storeRateLimits store.SeriesSelectLimits - maxDownloadedBytes units.Base2Bytes - maxConcurrency int - component component.StoreAPI - debugLogging bool - syncInterval time.Duration - blockListStrategy string - blockSyncConcurrency int - blockMetaFetchConcurrency int - filterConf *store.FilterConfig - selectorRelabelConf extflag.PathOrContent - advertiseCompatibilityLabel bool - consistencyDelay commonmodel.Duration - ignoreDeletionMarksDelay commonmodel.Duration - disableWeb bool - webConfig webConfig - label string - postingOffsetsInMemSampling int - cachingBucketConfig extflag.PathOrContent - reqLogConfig *extflag.PathOrContent - lazyIndexReaderEnabled bool - lazyIndexReaderIdleTimeout time.Duration - lazyExpandedPostingsEnabled bool + indexCacheConfigs extflag.PathOrContent + objStoreConfig extflag.PathOrContent + dataDir string + cacheIndexHeader bool + grpcConfig grpcConfig + httpConfig httpConfig + indexCacheSizeBytes units.Base2Bytes + chunkPoolSize units.Base2Bytes + estimatedMaxSeriesSize uint64 + estimatedMaxChunkSize uint64 + seriesBatchSize int + storeRateLimits store.SeriesSelectLimits + maxDownloadedBytes units.Base2Bytes + maxConcurrency int + component component.StoreAPI + debugLogging bool + syncInterval time.Duration + blockListStrategy string + blockSyncConcurrency int + blockMetaFetchConcurrency int + filterConf *store.FilterConfig + selectorRelabelConf extflag.PathOrContent + advertiseCompatibilityLabel bool + consistencyDelay commonmodel.Duration + ignoreDeletionMarksDelay commonmodel.Duration + disableWeb bool + webConfig webConfig + label string + postingOffsetsInMemSampling int + cachingBucketConfig extflag.PathOrContent + reqLogConfig *extflag.PathOrContent + lazyIndexReaderEnabled bool + lazyIndexReaderIdleTimeout time.Duration + lazyExpandedPostingsEnabled bool + postingGroupMaxKeySeriesRatio float64 indexHeaderLazyDownloadStrategy string } @@ -204,6 +205,9 @@ func (sc *storeConfig) registerFlag(cmd extkingpin.FlagClause) { cmd.Flag("store.enable-lazy-expanded-postings", "If true, Store Gateway will estimate postings size and try to lazily expand postings if it downloads less data than expanding all postings."). Default("false").BoolVar(&sc.lazyExpandedPostingsEnabled) + cmd.Flag("store.posting-group-max-key-series-ratio", "Mark posting group as lazy if it fetches more keys than R * max series the query should fetch. With R set to 100, a posting group which fetches 100K keys will be marked as lazy if the current query only fetches 1000 series. thanos_bucket_store_lazy_expanded_posting_groups_total shows lazy expanded postings groups with reasons and you can tune this config accordingly. This config is only valid if lazy expanded posting is enabled. 0 disables the limit."). + Default("100").Float64Var(&sc.postingGroupMaxKeySeriesRatio) + cmd.Flag("store.index-header-lazy-download-strategy", "Strategy of how to download index headers lazily. Supported values: eager, lazy. If eager, always download index header during initial load. If lazy, download index header during query time."). Default(string(indexheader.EagerDownloadStrategy)). EnumVar(&sc.indexHeaderLazyDownloadStrategy, string(indexheader.EagerDownloadStrategy), string(indexheader.LazyDownloadStrategy)) @@ -429,6 +433,7 @@ func runStore( return conf.estimatedMaxChunkSize }), store.WithLazyExpandedPostings(conf.lazyExpandedPostingsEnabled), + store.WithPostingGroupMaxKeySeriesRatio(conf.postingGroupMaxKeySeriesRatio), store.WithIndexHeaderLazyDownloadStrategy( indexheader.IndexHeaderLazyDownloadStrategy(conf.indexHeaderLazyDownloadStrategy).StrategyToDownloadFunc(), ), diff --git a/docs/components/store.md b/docs/components/store.md index dfab1855cc..ce2adb6d6d 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -250,6 +250,18 @@ Flags: The maximum series allowed for a single Series request. The Series call fails if this limit is exceeded. 0 means no limit. + --store.posting-group-max-key-series-ratio=100 + Mark posting group as lazy if it fetches more + keys than R * max series the query should + fetch. With R set to 100, a posting group which + fetches 100K keys will be marked as lazy if + the current query only fetches 1000 series. + thanos_bucket_store_lazy_expanded_posting_groups_total + shows lazy expanded postings groups with + reasons and you can tune this config + accordingly. This config is only valid if lazy + expanded posting is enabled. 0 disables the + limit. --sync-block-duration=15m Repeat interval for syncing the blocks between local and remote view. --tracing.config= diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index d9940221ff..1c434f503f 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -151,6 +151,7 @@ type bucketStoreMetrics struct { emptyPostingCount *prometheus.CounterVec lazyExpandedPostingsCount prometheus.Counter + lazyExpandedPostingGroupsByReason *prometheus.CounterVec lazyExpandedPostingSizeBytes prometheus.Counter lazyExpandedPostingSeriesOverfetchedSizeBytes prometheus.Counter @@ -345,6 +346,11 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { Help: "Total number of times when lazy expanded posting optimization applies.", }) + m.lazyExpandedPostingGroupsByReason = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "thanos_bucket_store_lazy_expanded_posting_groups_total", + Help: "Total number of posting groups that are marked as lazy and corresponding reason", + }, []string{"reason"}) + m.lazyExpandedPostingSizeBytes = promauto.With(reg).NewCounter(prometheus.CounterOpts{ Name: "thanos_bucket_store_lazy_expanded_posting_size_bytes_total", Help: "Total number of lazy posting group size in bytes.", @@ -419,7 +425,8 @@ type BucketStore struct { enableChunkHashCalculation bool - enabledLazyExpandedPostings bool + enabledLazyExpandedPostings bool + postingGroupMaxKeySeriesRatio float64 sortingStrategy sortingStrategy @@ -552,6 +559,13 @@ func WithLazyExpandedPostings(enabled bool) BucketStoreOption { } } +// WithPostingGroupMaxKeySeriesRatio configures a threshold to mark a posting group as lazy if it has more add keys. +func WithPostingGroupMaxKeySeriesRatio(postingGroupMaxKeySeriesRatio float64) BucketStoreOption { + return func(s *BucketStore) { + s.postingGroupMaxKeySeriesRatio = postingGroupMaxKeySeriesRatio + } +} + // WithDontResort disables series resorting in Store Gateway. func WithDontResort(true bool) BucketStoreOption { return func(s *BucketStore) { @@ -1002,8 +1016,11 @@ type blockSeriesClient struct { chunksLimiter ChunksLimiter bytesLimiter BytesLimiter - lazyExpandedPostingEnabled bool + lazyExpandedPostingEnabled bool + // Mark posting group as lazy if it adds too many keys. 0 to disable. + postingGroupMaxKeySeriesRatio float64 lazyExpandedPostingsCount prometheus.Counter + lazyExpandedPostingGroupByReason *prometheus.CounterVec lazyExpandedPostingSizeBytes prometheus.Counter lazyExpandedPostingSeriesOverfetchedSizeBytes prometheus.Counter @@ -1046,7 +1063,9 @@ func newBlockSeriesClient( chunkFetchDurationSum *prometheus.HistogramVec, extLsetToRemove map[string]struct{}, lazyExpandedPostingEnabled bool, + postingGroupMaxKeySeriesRatio float64, lazyExpandedPostingsCount prometheus.Counter, + lazyExpandedPostingByReason *prometheus.CounterVec, lazyExpandedPostingSizeBytes prometheus.Counter, lazyExpandedPostingSeriesOverfetchedSizeBytes prometheus.Counter, tenant string, @@ -1081,7 +1100,9 @@ func newBlockSeriesClient( chunkFetchDurationSum: chunkFetchDurationSum, lazyExpandedPostingEnabled: lazyExpandedPostingEnabled, + postingGroupMaxKeySeriesRatio: postingGroupMaxKeySeriesRatio, lazyExpandedPostingsCount: lazyExpandedPostingsCount, + lazyExpandedPostingGroupByReason: lazyExpandedPostingByReason, lazyExpandedPostingSizeBytes: lazyExpandedPostingSizeBytes, lazyExpandedPostingSeriesOverfetchedSizeBytes: lazyExpandedPostingSeriesOverfetchedSizeBytes, @@ -1133,7 +1154,7 @@ func (b *blockSeriesClient) ExpandPostings( matchers sortedMatchers, seriesLimiter SeriesLimiter, ) error { - ps, err := b.indexr.ExpandedPostings(b.ctx, matchers, b.bytesLimiter, b.lazyExpandedPostingEnabled, b.lazyExpandedPostingSizeBytes, b.tenant) + ps, err := b.indexr.ExpandedPostings(b.ctx, matchers, b.bytesLimiter, b.lazyExpandedPostingEnabled, b.postingGroupMaxKeySeriesRatio, b.lazyExpandedPostingSizeBytes, b.lazyExpandedPostingGroupByReason, b.tenant) if err != nil { return errors.Wrap(err, "expanded matching posting") } @@ -1566,7 +1587,9 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, seriesSrv storepb.Store s.metrics.chunkFetchDurationSum, extLsetToRemove, s.enabledLazyExpandedPostings, + s.postingGroupMaxKeySeriesRatio, s.metrics.lazyExpandedPostingsCount, + s.metrics.lazyExpandedPostingGroupsByReason, s.metrics.lazyExpandedPostingSizeBytes, s.metrics.lazyExpandedPostingSeriesOverfetchedSizeBytes, tenant, @@ -1880,7 +1903,9 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq nil, extLsetToRemove, s.enabledLazyExpandedPostings, + s.postingGroupMaxKeySeriesRatio, s.metrics.lazyExpandedPostingsCount, + s.metrics.lazyExpandedPostingGroupsByReason, s.metrics.lazyExpandedPostingSizeBytes, s.metrics.lazyExpandedPostingSeriesOverfetchedSizeBytes, tenant, @@ -2106,7 +2131,9 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR nil, nil, s.enabledLazyExpandedPostings, + s.postingGroupMaxKeySeriesRatio, s.metrics.lazyExpandedPostingsCount, + s.metrics.lazyExpandedPostingGroupsByReason, s.metrics.lazyExpandedPostingSizeBytes, s.metrics.lazyExpandedPostingSeriesOverfetchedSizeBytes, tenant, @@ -2563,7 +2590,16 @@ func (r *bucketIndexReader) reset(size int) { // Reminder: A posting is a reference (represented as a uint64) to a series reference, which in turn points to the first // chunk where the series contains the matching label-value pair for a given block of data. Postings can be fetched by // single label name=value. -func (r *bucketIndexReader) ExpandedPostings(ctx context.Context, ms sortedMatchers, bytesLimiter BytesLimiter, lazyExpandedPostingEnabled bool, lazyExpandedPostingSizeBytes prometheus.Counter, tenant string) (*lazyExpandedPostings, error) { +func (r *bucketIndexReader) ExpandedPostings( + ctx context.Context, + ms sortedMatchers, + bytesLimiter BytesLimiter, + lazyExpandedPostingEnabled bool, + postingGroupMaxKeySeriesRatio float64, + lazyExpandedPostingSizeBytes prometheus.Counter, + lazyExpandedPostingGroupsByReason *prometheus.CounterVec, + tenant string, +) (*lazyExpandedPostings, error) { // Shortcut the case of `len(postingGroups) == 0`. It will only happen when no // matchers specified, and we don't need to fetch expanded postings from cache. if len(ms) == 0 { @@ -2615,7 +2651,7 @@ func (r *bucketIndexReader) ExpandedPostings(ctx context.Context, ms sortedMatch postingGroups = append(postingGroups, newPostingGroup(true, name, []string{value}, nil)) } - ps, err := fetchLazyExpandedPostings(ctx, postingGroups, r, bytesLimiter, addAllPostings, lazyExpandedPostingEnabled, lazyExpandedPostingSizeBytes, tenant) + ps, err := fetchLazyExpandedPostings(ctx, postingGroups, r, bytesLimiter, addAllPostings, lazyExpandedPostingEnabled, postingGroupMaxKeySeriesRatio, lazyExpandedPostingSizeBytes, lazyExpandedPostingGroupsByReason, tenant) if err != nil { return nil, errors.Wrap(err, "fetch and expand postings") } @@ -2661,13 +2697,14 @@ func ExpandPostingsWithContext(ctx context.Context, p index.Postings) ([]storage // If addAll is not set: Merge of postings for "addKeys" labels minus postings for removeKeys labels // This computation happens in ExpandedPostings. type postingGroup struct { - addAll bool - name string - matchers []*labels.Matcher - addKeys []string - removeKeys []string - cardinality int64 - lazy bool + addAll bool + name string + matchers []*labels.Matcher + addKeys []string + removeKeys []string + cardinality int64 + existentKeys int + lazy bool } func newPostingGroup(addAll bool, name string, addKeys, removeKeys []string) *postingGroup { diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index e8dffd093b..df4d1e189c 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -1288,7 +1288,9 @@ func benchmarkExpandedPostings( {`uniq=~"9|random-shuffled-values|1"`, []*labels.Matcher{iRegexBigValueSet}, bigValueSetSize}, } - dummyCounter := promauto.NewCounter(prometheus.CounterOpts{Name: "test"}) + reg := prometheus.NewRegistry() + dummyCounter := promauto.With(reg).NewCounter(prometheus.CounterOpts{Name: "test"}) + dummyCounterVec := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{Name: "test_counter_vec"}, []string{"reason"}) for _, c := range cases { t.Run(c.name, func(t testutil.TB) { b := &bucketBlock{ @@ -1304,7 +1306,7 @@ func benchmarkExpandedPostings( t.ResetTimer() for i := 0; i < t.N(); i++ { - p, err := indexr.ExpandedPostings(context.Background(), newSortedMatchers(c.matchers), NewBytesLimiterFactory(0)(nil), false, dummyCounter, tenancy.DefaultTenant) + p, err := indexr.ExpandedPostings(context.Background(), newSortedMatchers(c.matchers), NewBytesLimiterFactory(0)(nil), false, 0, dummyCounter, dummyCounterVec, tenancy.DefaultTenant) testutil.Ok(t, err) testutil.Equals(t, c.expectedLen, len(p.postings)) } @@ -1340,8 +1342,10 @@ func TestExpandedPostingsEmptyPostings(t *testing.T) { // Match nothing. matcher2 := labels.MustNewMatcher(labels.MatchRegexp, "i", "500.*") ctx := context.Background() - dummyCounter := promauto.With(prometheus.NewRegistry()).NewCounter(prometheus.CounterOpts{Name: "test"}) - ps, err := indexr.ExpandedPostings(ctx, newSortedMatchers([]*labels.Matcher{matcher1, matcher2}), NewBytesLimiterFactory(0)(nil), false, dummyCounter, tenancy.DefaultTenant) + reg := prometheus.NewRegistry() + dummyCounter := promauto.With(reg).NewCounter(prometheus.CounterOpts{Name: "test"}) + dummyCounterVec := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{Name: "test_counter_vec"}, []string{"reason"}) + ps, err := indexr.ExpandedPostings(ctx, newSortedMatchers([]*labels.Matcher{matcher1, matcher2}), NewBytesLimiterFactory(0)(nil), false, 0, dummyCounter, dummyCounterVec, tenancy.DefaultTenant) testutil.Ok(t, err) testutil.Equals(t, ps, (*lazyExpandedPostings)(nil)) // Make sure even if a matcher doesn't match any postings, we still cache empty expanded postings. @@ -1378,8 +1382,10 @@ func TestLazyExpandedPostingsEmptyPostings(t *testing.T) { matcher2 := labels.MustNewMatcher(labels.MatchRegexp, "n", "1_.*") matcher3 := labels.MustNewMatcher(labels.MatchRegexp, "i", ".+") ctx := context.Background() - dummyCounter := promauto.With(prometheus.NewRegistry()).NewCounter(prometheus.CounterOpts{Name: "test"}) - ps, err := indexr.ExpandedPostings(ctx, newSortedMatchers([]*labels.Matcher{matcher1, matcher2, matcher3}), NewBytesLimiterFactory(0)(nil), true, dummyCounter, tenancy.DefaultTenant) + reg := prometheus.NewRegistry() + dummyCounter := promauto.With(reg).NewCounter(prometheus.CounterOpts{Name: "test"}) + dummyCounterVec := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{Name: "test_counter_vec"}, []string{"reason"}) + ps, err := indexr.ExpandedPostings(ctx, newSortedMatchers([]*labels.Matcher{matcher1, matcher2, matcher3}), NewBytesLimiterFactory(0)(nil), true, 0, dummyCounter, dummyCounterVec, tenancy.DefaultTenant) testutil.Ok(t, err) // We expect emptyLazyPostings rather than lazy postings with 0 length but with matchers. testutil.Equals(t, ps, emptyLazyPostings) @@ -2872,7 +2878,9 @@ func benchmarkBlockSeriesWithConcurrency(b *testing.B, concurrency int, blockMet wg := sync.WaitGroup{} wg.Add(concurrency) - dummyCounter := promauto.NewCounter(prometheus.CounterOpts{Name: "test"}) + reg := prometheus.NewRegistry() + dummyCounter := promauto.With(reg).NewCounter(prometheus.CounterOpts{Name: "test"}) + dummyCounterVec := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{Name: "test_counter_vec"}, []string{"reason"}) for w := 0; w < concurrency; w++ { go func() { defer wg.Done() @@ -2917,7 +2925,9 @@ func benchmarkBlockSeriesWithConcurrency(b *testing.B, concurrency int, blockMet dummyHistogram, nil, false, + 0, dummyCounter, + dummyCounterVec, dummyCounter, dummyCounter, tenancy.DefaultTenant, @@ -3551,7 +3561,9 @@ func TestExpandedPostingsRace(t *testing.T) { l := sync.Mutex{} previousRefs := make(map[int][]storage.SeriesRef) - dummyCounter := promauto.With(prometheus.NewRegistry()).NewCounter(prometheus.CounterOpts{Name: "test"}) + reg := prometheus.NewRegistry() + dummyCounter := promauto.With(reg).NewCounter(prometheus.CounterOpts{Name: "test"}) + dummyCounterVec := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{Name: "test_counter_vec"}, []string{"reason"}) for { if tm.Err() != nil { @@ -3573,7 +3585,7 @@ func TestExpandedPostingsRace(t *testing.T) { wg.Add(1) go func(i int, bb *bucketBlock) { - refs, err := bb.indexReader(logger).ExpandedPostings(context.Background(), m, NewBytesLimiterFactory(0)(nil), false, dummyCounter, tenancy.DefaultTenant) + refs, err := bb.indexReader(logger).ExpandedPostings(context.Background(), m, NewBytesLimiterFactory(0)(nil), false, 0, dummyCounter, dummyCounterVec, tenancy.DefaultTenant) testutil.Ok(t, err) defer wg.Done() diff --git a/pkg/store/lazy_postings.go b/pkg/store/lazy_postings.go index f8363ab477..81b977f5d3 100644 --- a/pkg/store/lazy_postings.go +++ b/pkg/store/lazy_postings.go @@ -39,7 +39,15 @@ func (p *lazyExpandedPostings) lazyExpanded() bool { return p != nil && len(p.matchers) > 0 } -func optimizePostingsFetchByDownloadedBytes(r *bucketIndexReader, postingGroups []*postingGroup, seriesMaxSize int64, seriesMatchRatio float64, lazyExpandedPostingSizeBytes prometheus.Counter) ([]*postingGroup, bool, error) { +func optimizePostingsFetchByDownloadedBytes( + r *bucketIndexReader, + postingGroups []*postingGroup, + seriesMaxSize int64, + seriesMatchRatio float64, + postingGroupMaxKeySeriesRatio float64, + lazyExpandedPostingSizeBytes prometheus.Counter, + lazyExpandedPostingGroupsByReason *prometheus.CounterVec, +) ([]*postingGroup, bool, error) { if len(postingGroups) <= 1 { return postingGroups, false, nil } @@ -55,6 +63,7 @@ func optimizePostingsFetchByDownloadedBytes(r *bucketIndexReader, postingGroups return nil, false, errors.Wrapf(err, "postings offsets for %s", pg.name) } + existentKeys := 0 for _, rng := range rngs { if rng == indexheader.NotFoundRange { continue @@ -63,14 +72,16 @@ func optimizePostingsFetchByDownloadedBytes(r *bucketIndexReader, postingGroups level.Error(r.logger).Log("msg", "invalid index range, fallback to non lazy posting optimization") return postingGroups, false, nil } + existentKeys++ // Each range starts from the #entries field which is 4 bytes. // Need to subtract it when calculating number of postings. // https://github.com/prometheus/prometheus/blob/v2.46.0/tsdb/docs/format/index.md. pg.cardinality += (rng.End - rng.Start - 4) / 4 } + pg.existentKeys = existentKeys // If the posting group adds keys, 0 cardinality means the posting doesn't exist. // If the posting group removes keys, no posting ranges found is fine as it is a noop. - if len(pg.addKeys) > 0 && pg.cardinality == 0 { + if len(pg.addKeys) > 0 && pg.existentKeys == 0 { return nil, true, nil } } @@ -142,6 +153,7 @@ func optimizePostingsFetchByDownloadedBytes(r *bucketIndexReader, postingGroups // Assume only seriesMatchRatio postings will be matched every posting group. seriesMatched := postingGroups[i].cardinality - int64(math.Ceil(float64(negativeCardinalities)*seriesMatchRatio)) + maxSeriesMatched := seriesMatched i++ // Start from next posting group as we always need to fetch at least one posting group with add keys. for i < len(postingGroups) { pg := postingGroups[i] @@ -165,6 +177,13 @@ func optimizePostingsFetchByDownloadedBytes(r *bucketIndexReader, postingGroups seriesMatched -= underfetchedSeries underfetchedSeriesSize = underfetchedSeries * seriesMaxSize } else { + // Only mark posting group as lazy due to too many keys when those keys are known to be existent. + if postingGroupMaxKeySeriesRatio > 0 && maxSeriesMatched > 0 && + float64(pg.existentKeys)/float64(maxSeriesMatched) > postingGroupMaxKeySeriesRatio { + markPostingGroupLazy(pg, "keys_limit", lazyExpandedPostingSizeBytes, lazyExpandedPostingGroupsByReason) + i++ + continue + } underfetchedSeriesSize = seriesMaxSize * int64(math.Ceil(float64(seriesMatched)*(1-seriesMatchRatio))) seriesMatched = int64(math.Ceil(float64(seriesMatched) * seriesMatchRatio)) } @@ -176,13 +195,18 @@ func optimizePostingsFetchByDownloadedBytes(r *bucketIndexReader, postingGroups i++ } for i < len(postingGroups) { - postingGroups[i].lazy = true - lazyExpandedPostingSizeBytes.Add(float64(4 * postingGroups[i].cardinality)) + markPostingGroupLazy(postingGroups[i], "postings_size", lazyExpandedPostingSizeBytes, lazyExpandedPostingGroupsByReason) i++ } return postingGroups, false, nil } +func markPostingGroupLazy(pg *postingGroup, reason string, lazyExpandedPostingSizeBytes prometheus.Counter, lazyExpandedPostingGroupsByReason *prometheus.CounterVec) { + pg.lazy = true + lazyExpandedPostingSizeBytes.Add(float64(4 * pg.cardinality)) + lazyExpandedPostingGroupsByReason.WithLabelValues(reason).Inc() +} + func fetchLazyExpandedPostings( ctx context.Context, postingGroups []*postingGroup, @@ -190,7 +214,9 @@ func fetchLazyExpandedPostings( bytesLimiter BytesLimiter, addAllPostings bool, lazyExpandedPostingEnabled bool, + postingGroupMaxKeySeriesRatio float64, lazyExpandedPostingSizeBytes prometheus.Counter, + lazyExpandedPostingGroupsByReason *prometheus.CounterVec, tenant string, ) (*lazyExpandedPostings, error) { var ( @@ -212,7 +238,9 @@ func fetchLazyExpandedPostings( postingGroups, int64(r.block.estimatedMaxSeriesSize), 0.5, // TODO(yeya24): Expose this as a flag. + postingGroupMaxKeySeriesRatio, lazyExpandedPostingSizeBytes, + lazyExpandedPostingGroupsByReason, ) if err != nil { return nil, err @@ -243,27 +271,25 @@ func keysToFetchFromPostingGroups(postingGroups []*postingGroup) ([]labels.Label for i < len(postingGroups) { pg := postingGroups[i] if pg.lazy { - break + if len(lazyMatchers) == 0 { + lazyMatchers = make([]*labels.Matcher, 0) + } + lazyMatchers = append(lazyMatchers, postingGroups[i].matchers...) + } else { + // Postings returned by fetchPostings will be in the same order as keys + // so it's important that we iterate them in the same order later. + // We don't have any other way of pairing keys and fetched postings. + for _, key := range pg.addKeys { + keys = append(keys, labels.Label{Name: pg.name, Value: key}) + } + for _, key := range pg.removeKeys { + keys = append(keys, labels.Label{Name: pg.name, Value: key}) + } } - // Postings returned by fetchPostings will be in the same order as keys - // so it's important that we iterate them in the same order later. - // We don't have any other way of pairing keys and fetched postings. - for _, key := range pg.addKeys { - keys = append(keys, labels.Label{Name: pg.name, Value: key}) - } - for _, key := range pg.removeKeys { - keys = append(keys, labels.Label{Name: pg.name, Value: key}) - } i++ } - if i < len(postingGroups) { - lazyMatchers = make([]*labels.Matcher, 0) - for i < len(postingGroups) { - lazyMatchers = append(lazyMatchers, postingGroups[i].matchers...) - i++ - } - } + return keys, lazyMatchers } diff --git a/pkg/store/lazy_postings_test.go b/pkg/store/lazy_postings_test.go index 06157affe0..3eac705871 100644 --- a/pkg/store/lazy_postings_test.go +++ b/pkg/store/lazy_postings_test.go @@ -206,6 +206,38 @@ func TestKeysToFetchFromPostingGroups(t *testing.T) { labels.MustNewMatcher(labels.MatchRegexp, "job", "prometheus.*"), }, }, + { + name: "multiple non lazy and lazy posting groups with lazy posting groups in the middle", + pgs: []*postingGroup{ + { + name: "test", + addKeys: []string{"foo", "bar"}, + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")}, + }, + { + name: "cluster", + addKeys: []string{"bar"}, + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "cluster", "bar")}, + lazy: true, + }, + { + name: "env", + addKeys: []string{"beta", "gamma", "prod"}, + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "env", "beta|gamma|prod")}, + lazy: true, + }, + { + name: "job", + addKeys: []string{"prometheus"}, + matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "job", "prometheus.*")}, + }, + }, + expectedLabels: []labels.Label{{Name: "test", Value: "foo"}, {Name: "test", Value: "bar"}, {Name: "job", Value: "prometheus"}}, + expectedMatchers: []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchEqual, "cluster", "bar"), + labels.MustNewMatcher(labels.MatchRegexp, "env", "beta|gamma|prod"), + }, + }, } { t.Run(tc.name, func(t *testing.T) { keys, matchers := keysToFetchFromPostingGroups(tc.pgs) @@ -276,15 +308,16 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { }, } for _, tc := range []struct { - name string - inputPostings map[string]map[string]index.Range - inputError error - postingGroups []*postingGroup - seriesMaxSize int64 - seriesMatchRatio float64 - expectedPostingGroups []*postingGroup - expectedEmptyPosting bool - expectedError string + name string + inputPostings map[string]map[string]index.Range + inputError error + postingGroups []*postingGroup + seriesMaxSize int64 + seriesMatchRatio float64 + postingGroupMaxKeySeriesRatio float64 + expectedPostingGroups []*postingGroup + expectedEmptyPosting bool + expectedError string }{ { name: "empty posting group", @@ -353,7 +386,7 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { }, expectedPostingGroups: []*postingGroup{ {name: "bar", removeKeys: []string{"foo"}, cardinality: 0, addAll: true}, - {name: "foo", addKeys: []string{"bar"}, cardinality: 1}, + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, }, }, { @@ -385,7 +418,7 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { }, expectedPostingGroups: []*postingGroup{ {name: "bar", removeKeys: []string{"foo"}, cardinality: 0, addAll: true}, - {name: "foo", addKeys: []string{"bar"}, cardinality: 1}, + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, }, }, { @@ -401,8 +434,8 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { {name: "bar", addKeys: []string{"foo", "buz"}}, }, expectedPostingGroups: []*postingGroup{ - {name: "bar", addKeys: []string{"foo", "buz"}, cardinality: 1}, - {name: "foo", addKeys: []string{"bar"}, cardinality: 1}, + {name: "bar", addKeys: []string{"foo", "buz"}, cardinality: 1, existentKeys: 1}, + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, }, }, { @@ -418,8 +451,97 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { {name: "bar", addKeys: []string{"foo"}}, }, expectedPostingGroups: []*postingGroup{ - {name: "bar", addKeys: []string{"foo"}, cardinality: 1}, - {name: "foo", addKeys: []string{"bar"}, cardinality: 1}, + {name: "bar", addKeys: []string{"foo"}, cardinality: 1, existentKeys: 1}, + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + }, + }, + { + name: "two posting groups with add keys, posting group not marked as lazy due to some add keys don't exist", + inputPostings: map[string]map[string]index.Range{ + "foo": {"bar": index.Range{End: 8}}, + "bar": {"foo": index.Range{Start: 8, End: 16}}, + }, + seriesMaxSize: 1000, + seriesMatchRatio: 0.5, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}}, + {name: "bar", addKeys: []string{"bar", "baz", "foo"}}, + }, + postingGroupMaxKeySeriesRatio: 2, + expectedPostingGroups: []*postingGroup{ + {name: "bar", addKeys: []string{"bar", "baz", "foo"}, cardinality: 1, existentKeys: 1}, + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + }, + }, + { + name: "two posting groups with add keys, first posting group not marked as lazy even though exceeding 2 keys due to we always mark first posting group as non lazy", + inputPostings: map[string]map[string]index.Range{ + "foo": {"bar": index.Range{End: 108}}, + "bar": {"foo": index.Range{Start: 108, End: 116}, "bar": index.Range{Start: 116, End: 124}, "baz": index.Range{Start: 124, End: 132}}, + }, + seriesMaxSize: 1000, + seriesMatchRatio: 0.5, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}}, + {name: "bar", addKeys: []string{"bar", "baz", "foo"}}, + }, + postingGroupMaxKeySeriesRatio: 2, + expectedPostingGroups: []*postingGroup{ + {name: "bar", addKeys: []string{"bar", "baz", "foo"}, cardinality: 3, existentKeys: 3}, + {name: "foo", addKeys: []string{"bar"}, cardinality: 26, existentKeys: 1}, + }, + }, + { + name: "two posting groups with add keys, one posting group with too many keys not marked as lazy due to postingGroupMaxKeySeriesRatio not set", + inputPostings: map[string]map[string]index.Range{ + "foo": {"bar": index.Range{End: 8}}, + "bar": {"foo": index.Range{Start: 8, End: 16}, "bar": index.Range{Start: 16, End: 24}, "baz": index.Range{Start: 24, End: 32}}, + }, + seriesMaxSize: 1000, + seriesMatchRatio: 0.5, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}}, + {name: "bar", addKeys: []string{"bar", "baz", "foo"}}, + }, + postingGroupMaxKeySeriesRatio: 0, + expectedPostingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + {name: "bar", addKeys: []string{"bar", "baz", "foo"}, cardinality: 3, existentKeys: 3}, + }, + }, + { + name: "two posting groups with add keys, one posting group marked as lazy due to exceeding postingGroupMaxKeySeriesRatio", + inputPostings: map[string]map[string]index.Range{ + "foo": {"bar": index.Range{End: 8}}, + "bar": {"foo": index.Range{Start: 8, End: 16}, "bar": index.Range{Start: 16, End: 24}, "baz": index.Range{Start: 24, End: 32}}, + }, + seriesMaxSize: 1000, + seriesMatchRatio: 0.5, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}}, + {name: "bar", addKeys: []string{"bar", "baz", "foo"}}, + }, + postingGroupMaxKeySeriesRatio: 2, + expectedPostingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + {name: "bar", addKeys: []string{"bar", "baz", "foo"}, cardinality: 3, existentKeys: 3, lazy: true}, + }, + }, + { + name: "two posting groups with remove keys, minAddKeysToMarkLazy won't be applied", + inputPostings: map[string]map[string]index.Range{ + "foo": {"bar": index.Range{End: 8}}, + "bar": {"foo": index.Range{Start: 8, End: 16}, "baz": index.Range{Start: 16, End: 24}}, + }, + seriesMaxSize: 1000, + seriesMatchRatio: 0.5, + postingGroups: []*postingGroup{ + {addAll: true, name: "foo", removeKeys: []string{"bar"}}, + {addAll: true, name: "bar", removeKeys: []string{"baz", "foo"}}, + }, + expectedPostingGroups: []*postingGroup{ + {addAll: true, name: "foo", removeKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + {addAll: true, name: "bar", removeKeys: []string{"baz", "foo"}, cardinality: 2, existentKeys: 2}, }, }, { @@ -437,8 +559,8 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { {addAll: true, name: "bar", removeKeys: []string{"foo"}}, }, expectedPostingGroups: []*postingGroup{ - {addAll: true, name: "bar", removeKeys: []string{"foo"}, cardinality: 1}, - {addAll: true, name: "foo", removeKeys: []string{"bar"}, cardinality: 1}, + {addAll: true, name: "bar", removeKeys: []string{"foo"}, cardinality: 1, existentKeys: 1}, + {addAll: true, name: "foo", removeKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, }, }, { @@ -454,8 +576,8 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { {name: "bar", addKeys: []string{"foo"}}, }, expectedPostingGroups: []*postingGroup{ - {addAll: true, name: "foo", removeKeys: []string{"bar"}, cardinality: 1}, - {name: "bar", addKeys: []string{"foo"}, cardinality: 250000}, + {addAll: true, name: "foo", removeKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + {name: "bar", addKeys: []string{"foo"}, cardinality: 250000, existentKeys: 1}, }, }, { @@ -471,8 +593,8 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { {name: "bar", addKeys: []string{"foo"}}, }, expectedPostingGroups: []*postingGroup{ - {name: "bar", addKeys: []string{"foo"}, cardinality: 1}, - {name: "foo", addKeys: []string{"bar"}, cardinality: 1, lazy: true}, + {name: "bar", addKeys: []string{"foo"}, cardinality: 1, existentKeys: 1}, + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1, lazy: true}, }, }, { @@ -488,8 +610,8 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { {name: "bar", addKeys: []string{"foo"}}, }, expectedPostingGroups: []*postingGroup{ - {name: "foo", addKeys: []string{"bar"}, cardinality: 1}, - {name: "bar", addKeys: []string{"foo"}, cardinality: 250000, lazy: true}, + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + {name: "bar", addKeys: []string{"foo"}, cardinality: 250000, existentKeys: 1, lazy: true}, }, }, { @@ -507,9 +629,51 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { {name: "cluster", addKeys: []string{"us"}}, }, expectedPostingGroups: []*postingGroup{ - {name: "cluster", addKeys: []string{"us"}, cardinality: 1}, - {name: "foo", addKeys: []string{"bar"}, cardinality: 1}, - {name: "bar", addKeys: []string{"foo"}, cardinality: 250000, lazy: true}, + {name: "cluster", addKeys: []string{"us"}, cardinality: 1, existentKeys: 1}, + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + {name: "bar", addKeys: []string{"foo"}, cardinality: 250000, existentKeys: 1, lazy: true}, + }, + }, + { + name: "three posting groups with add keys, middle posting group marked as lazy due to too many add keys", + inputPostings: map[string]map[string]index.Range{ + "foo": {"bar": index.Range{End: 8}}, + "bar": {"bar": index.Range{Start: 8, End: 16}, "baz": index.Range{Start: 16, End: 24}, "foo": index.Range{Start: 24, End: 32}}, + "cluster": {"us": index.Range{Start: 32, End: 108}}, + }, + seriesMaxSize: 1000, + seriesMatchRatio: 0.5, + postingGroupMaxKeySeriesRatio: 2, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}}, + {name: "bar", addKeys: []string{"bar", "baz", "foo"}}, + {name: "cluster", addKeys: []string{"us"}}, + }, + expectedPostingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + {name: "bar", addKeys: []string{"bar", "baz", "foo"}, cardinality: 3, existentKeys: 3, lazy: true}, + {name: "cluster", addKeys: []string{"us"}, cardinality: 18, existentKeys: 1}, + }, + }, + { + name: "three posting groups with add keys, bar not marked as lazy even though too many add keys due to first positive posting group sorted by cardinality", + inputPostings: map[string]map[string]index.Range{ + "foo": {"bar": index.Range{End: 8}}, + "bar": {"bar": index.Range{Start: 8, End: 16}, "baz": index.Range{Start: 16, End: 24}, "foo": index.Range{Start: 24, End: 32}}, + "cluster": {"us": index.Range{Start: 32, End: 108}}, + }, + seriesMaxSize: 1000, + seriesMatchRatio: 0.5, + postingGroupMaxKeySeriesRatio: 2, + postingGroups: []*postingGroup{ + {addAll: true, name: "foo", removeKeys: []string{"bar"}}, + {name: "bar", addKeys: []string{"bar", "baz", "foo"}}, + {name: "cluster", addKeys: []string{"us"}}, + }, + expectedPostingGroups: []*postingGroup{ + {addAll: true, name: "foo", removeKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + {name: "bar", addKeys: []string{"bar", "baz", "foo"}, cardinality: 3, existentKeys: 3}, + {name: "cluster", addKeys: []string{"us"}, cardinality: 18, existentKeys: 1}, }, }, { @@ -527,9 +691,9 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { {name: "cluster", addKeys: []string{"us"}}, }, expectedPostingGroups: []*postingGroup{ - {name: "cluster", addKeys: []string{"us"}, cardinality: 1}, - {addAll: true, name: "foo", removeKeys: []string{"bar"}, cardinality: 1}, - {addAll: true, name: "bar", removeKeys: []string{"foo"}, cardinality: 250000, lazy: true}, + {name: "cluster", addKeys: []string{"us"}, cardinality: 1, existentKeys: 1}, + {addAll: true, name: "foo", removeKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + {addAll: true, name: "bar", removeKeys: []string{"foo"}, cardinality: 250000, existentKeys: 1, lazy: true}, }, }, { @@ -549,10 +713,10 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { {name: "cluster", addKeys: []string{"us"}}, }, expectedPostingGroups: []*postingGroup{ - {addAll: true, name: "foo", removeKeys: []string{"bar"}, cardinality: 1}, - {name: "bar", addKeys: []string{"foo"}, cardinality: 500}, - {name: "baz", addKeys: []string{"foo"}, cardinality: 501}, - {name: "cluster", addKeys: []string{"us"}, cardinality: 250000, lazy: true}, + {addAll: true, name: "foo", removeKeys: []string{"bar"}, cardinality: 1, existentKeys: 1}, + {name: "bar", addKeys: []string{"foo"}, cardinality: 500, existentKeys: 1}, + {name: "baz", addKeys: []string{"foo"}, cardinality: 501, existentKeys: 1}, + {name: "cluster", addKeys: []string{"us"}, cardinality: 250000, existentKeys: 1, lazy: true}, }, }, } { @@ -563,7 +727,8 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { testutil.Ok(t, err) ir := newBucketIndexReader(block, logger) dummyCounter := promauto.With(registry).NewCounter(prometheus.CounterOpts{Name: "test"}) - pgs, emptyPosting, err := optimizePostingsFetchByDownloadedBytes(ir, tc.postingGroups, tc.seriesMaxSize, tc.seriesMatchRatio, dummyCounter) + dummyCounterVec := promauto.With(registry).NewCounterVec(prometheus.CounterOpts{Name: "test_counter_vec"}, []string{"reason"}) + pgs, emptyPosting, err := optimizePostingsFetchByDownloadedBytes(ir, tc.postingGroups, tc.seriesMaxSize, tc.seriesMatchRatio, tc.postingGroupMaxKeySeriesRatio, dummyCounter, dummyCounterVec) if err != nil { testutil.Equals(t, tc.expectedError, err.Error()) return From b3645c80170e5e445191d80631b278fefd9abd6f Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Tue, 10 Dec 2024 11:22:07 +0100 Subject: [PATCH 20/21] sidecar: fix limit mintime (#7970) Signed-off-by: Michael Hoffmann --- CHANGELOG.md | 2 ++ cmd/thanos/sidecar.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bf3378b5a..e883362e15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Fixed +- [#7970](https://github.com/thanos-io/thanos/pull/7970) Sidecar: Respect min-time setting. + ### Added - [#7907](https://github.com/thanos-io/thanos/pull/7907) Receive: Add `--receive.grpc-service-config` flag to configure gRPC service config for the receivers. diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 127584ea94..531ffb8ab4 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -505,7 +505,7 @@ func (s *promMetadata) UpdateTimestamps(ctx context.Context) error { return err } - s.mint = min(s.limitMinTime.PrometheusTimestamp(), mint) + s.mint = max(s.limitMinTime.PrometheusTimestamp(), mint) s.maxt = math.MaxInt64 return nil From 0ea6bac096ce5216b084bfb5fa75d131d984e272 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Tue, 10 Dec 2024 15:43:02 -0800 Subject: [PATCH 21/21] store gateway: fix merge fetched postings with lazy postings (#7979) Signed-off-by: Ben Ye --- pkg/store/lazy_postings.go | 24 +++++---- pkg/store/lazy_postings_test.go | 93 +++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/pkg/store/lazy_postings.go b/pkg/store/lazy_postings.go index 81b977f5d3..ef7ae5d00a 100644 --- a/pkg/store/lazy_postings.go +++ b/pkg/store/lazy_postings.go @@ -305,6 +305,18 @@ func fetchAndExpandPostingGroups(ctx context.Context, r *bucketIndexReader, post return nil, nil, errors.Wrap(err, "get postings") } + result := mergeFetchedPostings(ctx, fetchedPostings, postingGroups) + if err := ctx.Err(); err != nil { + return nil, nil, err + } + ps, err := ExpandPostingsWithContext(ctx, result) + if err != nil { + return nil, nil, errors.Wrap(err, "expand") + } + return ps, lazyMatchers, nil +} + +func mergeFetchedPostings(ctx context.Context, fetchedPostings []index.Postings, postingGroups []*postingGroup) index.Postings { // Get "add" and "remove" postings from groups. We iterate over postingGroups and their keys // again, and this is exactly the same order as before (when building the groups), so we can simply // use one incrementing index to fetch postings from returned slice. @@ -313,7 +325,7 @@ func fetchAndExpandPostingGroups(ctx context.Context, r *bucketIndexReader, post var groupAdds, groupRemovals []index.Postings for _, g := range postingGroups { if g.lazy { - break + continue } // We cannot add empty set to groupAdds, since they are intersected. if len(g.addKeys) > 0 { @@ -333,13 +345,5 @@ func fetchAndExpandPostingGroups(ctx context.Context, r *bucketIndexReader, post } result := index.Without(index.Intersect(groupAdds...), index.Merge(ctx, groupRemovals...)) - - if err := ctx.Err(); err != nil { - return nil, nil, err - } - ps, err := ExpandPostingsWithContext(ctx, result) - if err != nil { - return nil, nil, errors.Wrap(err, "expand") - } - return ps, lazyMatchers, nil + return result } diff --git a/pkg/store/lazy_postings_test.go b/pkg/store/lazy_postings_test.go index 3eac705871..cb52dac412 100644 --- a/pkg/store/lazy_postings_test.go +++ b/pkg/store/lazy_postings_test.go @@ -16,8 +16,10 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" promtest "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/index" + "github.com/stretchr/testify/require" "github.com/thanos-io/objstore/providers/filesystem" "github.com/thanos-io/thanos/pkg/block/indexheader" @@ -745,3 +747,94 @@ func TestOptimizePostingsFetchByDownloadedBytes(t *testing.T) { }) } } + +func TestMergeFetchedPostings(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name string + fetchedPostings []index.Postings + postingGroups []*postingGroup + expectedSeriesRefs []storage.SeriesRef + }{ + { + name: "empty fetched postings and posting groups", + }, + { + name: "single posting group with 1 add key", + fetchedPostings: []index.Postings{index.NewListPostings([]storage.SeriesRef{1, 2, 3, 4, 5})}, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}}, + }, + expectedSeriesRefs: []storage.SeriesRef{1, 2, 3, 4, 5}, + }, + { + name: "single posting group with multiple add keys, merge", + fetchedPostings: []index.Postings{ + index.NewListPostings([]storage.SeriesRef{1, 2, 3, 4, 5}), + index.NewListPostings([]storage.SeriesRef{6, 7, 8, 9}), + }, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar", "baz"}}, + }, + expectedSeriesRefs: []storage.SeriesRef{1, 2, 3, 4, 5, 6, 7, 8, 9}, + }, + { + name: "multiple posting groups with add key, intersect", + fetchedPostings: []index.Postings{ + index.NewListPostings([]storage.SeriesRef{1, 2, 3, 4, 5}), + index.NewListPostings([]storage.SeriesRef{1, 2, 4}), + }, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}}, + {name: "bar", addKeys: []string{"foo"}}, + }, + expectedSeriesRefs: []storage.SeriesRef{1, 2, 4}, + }, + { + name: "posting group with remove keys", + fetchedPostings: []index.Postings{ + index.NewListPostings([]storage.SeriesRef{1, 2, 3, 4, 5}), + index.NewListPostings([]storage.SeriesRef{1, 2, 4}), + }, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}}, + {name: "bar", removeKeys: []string{"foo"}, addAll: true}, + }, + expectedSeriesRefs: []storage.SeriesRef{3, 5}, + }, + { + name: "multiple posting groups with add key and ignore lazy posting groups", + fetchedPostings: []index.Postings{ + index.NewListPostings([]storage.SeriesRef{1, 2, 3, 4, 5}), + }, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}}, + {name: "bar", addKeys: []string{"foo"}, lazy: true}, + {name: "baz", addKeys: []string{"foo"}, lazy: true}, + {name: "job", addKeys: []string{"foo"}, lazy: true}, + }, + expectedSeriesRefs: []storage.SeriesRef{1, 2, 3, 4, 5}, + }, + { + name: "multiple posting groups with add key and non consecutive lazy posting groups", + fetchedPostings: []index.Postings{ + index.NewListPostings([]storage.SeriesRef{1, 2, 3, 4, 5}), + index.NewListPostings([]storage.SeriesRef{1, 2, 4}), + }, + postingGroups: []*postingGroup{ + {name: "foo", addKeys: []string{"bar"}}, + {name: "bar", addKeys: []string{"foo"}, lazy: true}, + {name: "baz", addKeys: []string{"foo"}}, + {name: "job", addKeys: []string{"foo"}, lazy: true}, + }, + expectedSeriesRefs: []storage.SeriesRef{1, 2, 4}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + p := mergeFetchedPostings(ctx, tc.fetchedPostings, tc.postingGroups) + res, err := index.ExpandPostings(p) + require.NoError(t, err) + require.Equal(t, tc.expectedSeriesRefs, res) + }) + } +}