processor: delete label in order

Order matters in Prometheus so we need to delete the tenant label and
keep the original sorted order.

processor.go

@@ -325,6 +325,10 @@ func (p *processor) dispatch(clientIP net.Addr, reqID uuid.UUID, m map[string]*p
 	return
 }
 
+func removeOrdered(slice []prompb.Label, s int) []prompb.Label {
+	return append(slice[:s], slice[s+1:]...)
+}
+
 func (p *processor) processTimeseries(ts *prompb.TimeSeries) (tenant string, err error) {
 	idx := 0
 	for i, l := range ts.Labels {
@@ -343,9 +347,10 @@ func (p *processor) processTimeseries(ts *prompb.TimeSeries) (tenant string, err
 	}
 
 	if p.cfg.Tenant.LabelRemove {
-		l := len(ts.Labels)
-		ts.Labels[idx] = ts.Labels[l-1]
-		ts.Labels = ts.Labels[:l-1]
+		// Order is important. See:
+		// https://github.com/thanos-io/thanos/issues/6452
+		// https://github.com/prometheus/prometheus/issues/11505
+		ts.Labels = removeOrdered(ts.Labels, idx)
 	}
 
 	return

processor_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	fh "github.com/valyala/fasthttp"
 	fhu "github.com/valyala/fasthttp/fasthttputil"
@@ -452,3 +453,34 @@ func Benchmark_marshal(b *testing.B) {
 		_, _ = p.unmarshal(buf)
 	}
 }
+
+func TestRemoveOrdered(t *testing.T) {
+	l := []prompb.Label{
+		{
+			Name:  "aaa",
+			Value: "bbb",
+		},
+	}
+
+	l = removeOrdered(l, 0)
+	require.Equal(t, []prompb.Label{}, l)
+
+	l = []prompb.Label{
+		{
+			Name:  "aaa",
+			Value: "bbb",
+		},
+		{
+			Name:  "ccc",
+			Value: "ddd",
+		},
+	}
+	l = removeOrdered(l, 0)
+	require.Equal(t, []prompb.Label{
+		{
+			Name:  "ccc",
+			Value: "ddd",
+		},
+	}, l)
+
+}