Merge branch 'main' into v1v-patch-1

.go-version

@@ -1 +1 @@
-1.21.4
+1.21.5

.golangci.yml

@@ -3,7 +3,7 @@ run:
   # timeout for analysis, e.g. 30s, 5m, default is 1m
   timeout: 1m
   build-tags: integration
-  go: "1.21.4"
+  go: "1.21.5"
 
 issues:
   # Maximum count of issues with the same text.

.mergify.yml

@@ -194,3 +194,16 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.12 branch
+    conditions:
+      - merged
+      - label=backport-v8.12.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.12"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

dev-tools/integration/.env

@@ -1,4 +1,4 @@
-ELASTICSEARCH_VERSION=8.12.0-9d443b17-SNAPSHOT
+ELASTICSEARCH_VERSION=8.13.0-SNAPSHOT
 ELASTICSEARCH_USERNAME=elastic
 ELASTICSEARCH_PASSWORD=changeme
 TEST_ELASTICSEARCH_HOSTS=localhost:9200

internal/pkg/api/auth.go

@@ -170,7 +170,7 @@ func authAgent(r *http.Request, id *string, bulker bulk.Bulk, c cache.Cache) (*m
 
 		// Update the cache to mark the api key id associated with this agent as not enabled
 		c.SetAPIKey(*key, false)
-		return nil, ErrAgentInactive
+		return agent, ErrAgentInactive
 	}
 
 	return agent, nil

internal/pkg/api/handleAck.go

@@ -567,62 +567,16 @@ func cleanRoles(roles json.RawMessage) (json.RawMessage, int, error) {
 }
 
 func (ack *AckT) invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, toRetireAPIKeyIDs []model.ToRetireAPIKeyIdsItems, skip string) {
-	ids := make([]string, 0, len(toRetireAPIKeyIDs))
-	remoteIds := make(map[string][]string)
-	for _, k := range toRetireAPIKeyIDs {
-		if k.ID == skip || k.ID == "" {
-			continue
-		}
-		if k.Output != "" {
-			if remoteIds[k.Output] == nil {
-				remoteIds[k.Output] = make([]string, 0)
-			}
-			remoteIds[k.Output] = append(remoteIds[k.Output], k.ID)
-		} else {
-			ids = append(ids, k.ID)
-		}
-	}
-	if len(ids) > 0 {
-		zlog.Info().Strs("fleet.policy.apiKeyIDsToRetire", ids).Msg("Invalidate old API keys")
-		if err := ack.bulk.APIKeyInvalidate(ctx, ids...); err != nil {
-			zlog.Info().Err(err).Strs("ids", ids).Msg("Failed to invalidate API keys")
-		}
-	}
-	// using remote es bulker to invalidate api key
-	for outputName, outputIds := range remoteIds {
-		outputBulk := ack.bulk.GetBulker(outputName)
-
-		if outputBulk == nil {
-			// read output config from .fleet-policies, not filtering by policy id as agent could be reassigned
-			policy, err := dl.QueryOutputFromPolicy(ctx, ack.bulk, outputName)
-			if err != nil || policy == nil {
-				zlog.Warn().Str("outputName", outputName).Any("ids", outputIds).Msg("Output policy not found, API keys will be orphaned")
-			} else {
-				outputBulk, _, err = ack.bulk.CreateAndGetBulker(ctx, zlog, outputName, policy.Data.Outputs)
-				if err != nil {
-					zlog.Warn().Str("outputName", outputName).Any("ids", outputIds).Msg("Failed to recreate output bulker, API keys will be orphaned")
-				}
-			}
-		}
-		if outputBulk != nil {
-			if err := outputBulk.APIKeyInvalidate(ctx, outputIds...); err != nil {
-				zlog.Info().Err(err).Strs("ids", outputIds).Str("outputName", outputName).Msg("Failed to invalidate API keys")
-			}
-		}
-	}
+	invalidateAPIKeys(ctx, zlog, ack.bulk, toRetireAPIKeyIDs, skip)
 }
 
 func (ack *AckT) handleUnenroll(ctx context.Context, zlog zerolog.Logger, agent *model.Agent) error {
 	span, ctx := apm.StartSpan(ctx, "ackUnenroll", "process")
 	defer span.End()
-	apiKeys := agent.APIKeyIDs()
-	if len(apiKeys) > 0 {
-		zlog = zlog.With().Strs(LogAPIKeyID, apiKeys).Logger()
 
-		if err := ack.bulk.APIKeyInvalidate(ctx, apiKeys...); err != nil {
-			return fmt.Errorf("handleUnenroll invalidate apikey: %w", err)
-		}
-	}
+	apiKeys := agent.APIKeyIDs()
+	zlog.Info().Any("fleet.policy.apiKeyIDsToRetire", apiKeys).Msg("handleUnenroll invalidate API keys")
+	ack.invalidateAPIKeys(ctx, zlog, apiKeys, "")
 
 	now := time.Now().UTC().Format(time.RFC3339)
 	doc := bulk.UpdateFields{
@@ -740,3 +694,49 @@ func makeUpdatePolicyBody(policyID string, newRev, coordIdx int64) []byte {
 
 	return buf.Bytes()
 }
+
+func invalidateAPIKeys(ctx context.Context, zlog zerolog.Logger, bulk bulk.Bulk, toRetireAPIKeyIDs []model.ToRetireAPIKeyIdsItems, skip string) {
+	ids := make([]string, 0, len(toRetireAPIKeyIDs))
+	remoteIds := make(map[string][]string)
+	for _, k := range toRetireAPIKeyIDs {
+		if k.ID == skip || k.ID == "" {
+			continue
+		}
+		if k.Output != "" {
+			if remoteIds[k.Output] == nil {
+				remoteIds[k.Output] = make([]string, 0)
+			}
+			remoteIds[k.Output] = append(remoteIds[k.Output], k.ID)
+		} else {
+			ids = append(ids, k.ID)
+		}
+	}
+	if len(ids) > 0 {
+		zlog.Info().Strs("fleet.policy.apiKeyIDsToRetire", ids).Msg("Invalidate old API keys")
+		if err := bulk.APIKeyInvalidate(ctx, ids...); err != nil {
+			zlog.Info().Err(err).Strs("ids", ids).Msg("Failed to invalidate API keys")
+		}
+	}
+	// using remote es bulker to invalidate api key
+	for outputName, outputIds := range remoteIds {
+		outputBulk := bulk.GetBulker(outputName)
+
+		if outputBulk == nil {
+			// read output config from .fleet-policies, not filtering by policy id as agent could be reassigned
+			policy, err := dl.QueryOutputFromPolicy(ctx, bulk, outputName)
+			if err != nil || policy == nil {
+				zlog.Warn().Str("outputName", outputName).Any("ids", outputIds).Msg("Output policy not found, API keys will be orphaned")
+			} else {
+				outputBulk, _, err = bulk.CreateAndGetBulker(ctx, zlog, outputName, policy.Data.Outputs)
+				if err != nil {
+					zlog.Warn().Str("outputName", outputName).Any("ids", outputIds).Msg("Failed to recreate output bulker, API keys will be orphaned")
+				}
+			}
+		}
+		if outputBulk != nil {
+			if err := outputBulk.APIKeyInvalidate(ctx, outputIds...); err != nil {
+				zlog.Info().Err(err).Strs("ids", outputIds).Str("outputName", outputName).Msg("Failed to invalidate API keys")
+			}
+		}
+	}
+}

internal/pkg/api/handleCheckin.go

@@ -119,6 +119,11 @@ func (ct *CheckinT) handleCheckin(zlog zerolog.Logger, w http.ResponseWriter, r
 
 	agent, err := authAgent(r, &id, ct.bulker, ct.cache)
 	if err != nil {
+		// invalidate remote API keys of force unenrolled agents
+		if errors.Is(err, ErrAgentInactive) && agent != nil {
+			ctx := zlog.WithContext(r.Context())
+			invalidateAPIKeysOfInactiveAgent(ctx, zlog, ct.bulker, agent)
+		}
 		return err
 	}
 
@@ -136,6 +141,18 @@ func (ct *CheckinT) handleCheckin(zlog zerolog.Logger, w http.ResponseWriter, r
 	return ct.ProcessRequest(zlog, w, r, start, agent, newVer)
 }
 
+func invalidateAPIKeysOfInactiveAgent(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agent *model.Agent) {
+	remoteAPIKeys := make([]model.ToRetireAPIKeyIdsItems, 0)
+	apiKeys := agent.APIKeyIDs()
+	for _, key := range apiKeys {
+		if key.Output != "" {
+			remoteAPIKeys = append(remoteAPIKeys, key)
+		}
+	}
+	zlog.Info().Any("fleet.policy.apiKeyIDsToRetire", remoteAPIKeys).Msg("handleCheckin invalidate remote API keys")
+	invalidateAPIKeys(ctx, zlog, bulker, remoteAPIKeys, "")
+}
+
 // validatedCheckin is a struct to wrap all the things that validateRequest returns.
 type validatedCheckin struct {
 	req     *CheckinRequest

internal/pkg/model/ext.go

@@ -44,22 +44,34 @@ func (a *Agent) CheckDifferentVersion(ver string) string {
 
 // APIKeyIDs returns all the API keys, the valid, in-use as well as the one
 // marked to be retired.
-func (a *Agent) APIKeyIDs() []string {
+func (a *Agent) APIKeyIDs() []ToRetireAPIKeyIdsItems {
 	if a == nil {
 		return nil
 	}
-	keys := make([]string, 0, len(a.Outputs)+1)
+	keys := make([]ToRetireAPIKeyIdsItems, 0, len(a.Outputs)+1)
 	if a.AccessAPIKeyID != "" {
-		keys = append(keys, a.AccessAPIKeyID)
+		keys = append(keys, ToRetireAPIKeyIdsItems{
+			ID:        a.AccessAPIKeyID,
+			Output:    "",
+			RetiredAt: "",
+		})
 	}
 
-	for _, output := range a.Outputs {
+	for outputName, output := range a.Outputs {
 		if output.APIKeyID != "" {
-			keys = append(keys, output.APIKeyID)
+			name := ""
+			if outputName != "default" {
+				name = outputName
+			}
+			keys = append(keys, ToRetireAPIKeyIdsItems{
+				ID:        output.APIKeyID,
+				Output:    name,
+				RetiredAt: "",
+			})
 		}
 		for _, key := range output.ToRetireAPIKeyIds {
 			if key.ID != "" {
-				keys = append(keys, key.ID)
+				keys = append(keys, key)
 			}
 		}
 	}

internal/pkg/model/ext_test.go

@@ -88,7 +88,7 @@ func TestAgentAPIKeyIDs(t *testing.T) {
 	tcs := []struct {
 		name  string
 		agent Agent
-		want  []string
+		want  []ToRetireAPIKeyIdsItems
 	}{
 		{
 			name: "no API key marked to be retired",
@@ -99,7 +99,9 @@ func TestAgentAPIKeyIDs(t *testing.T) {
 					"p2": {APIKeyID: "p2_api_key_id"},
 				},
 			},
-			want: []string{"access_api_key_id", "p1_api_key_id", "p2_api_key_id"},
+			want: []ToRetireAPIKeyIdsItems{{ID: "access_api_key_id", Output: "", RetiredAt: ""},
+				{ID: "p1_api_key_id", Output: "p1", RetiredAt: ""},
+				{ID: "p2_api_key_id", Output: "p2", RetiredAt: ""}},
 		},
 		{
 			name: "with API key marked to be retired",
@@ -109,18 +111,22 @@ func TestAgentAPIKeyIDs(t *testing.T) {
 					"p1": {
 						APIKeyID: "p1_api_key_id",
 						ToRetireAPIKeyIds: []ToRetireAPIKeyIdsItems{{
-							ID: "p1_to_retire_key",
+							ID:     "p1_to_retire_key",
+							Output: "remote",
 						}}},
 					"p2": {
 						APIKeyID: "p2_api_key_id",
 						ToRetireAPIKeyIds: []ToRetireAPIKeyIdsItems{{
-							ID: "p2_to_retire_key",
+							ID:     "p2_to_retire_key",
+							Output: "remote",
 						}}},
 				},
 			},
-			want: []string{
-				"access_api_key_id", "p1_api_key_id", "p2_api_key_id",
-				"p1_to_retire_key", "p2_to_retire_key"},
+			want: []ToRetireAPIKeyIdsItems{{ID: "access_api_key_id", Output: "", RetiredAt: ""},
+				{ID: "p1_api_key_id", Output: "p1", RetiredAt: ""},
+				{ID: "p2_api_key_id", Output: "p2", RetiredAt: ""},
+				{ID: "p1_to_retire_key", Output: "remote", RetiredAt: ""},
+				{ID: "p2_to_retire_key", Output: "remote", RetiredAt: ""}},
 		},
 		{
 			name: "API key empty",
@@ -130,7 +136,7 @@ func TestAgentAPIKeyIDs(t *testing.T) {
 					"p1": {APIKeyID: ""},
 				},
 			},
-			want: []string{"access_api_key_id"},
+			want: []ToRetireAPIKeyIdsItems{{ID: "access_api_key_id", Output: "", RetiredAt: ""}},
 		},
 		{
 			name: "retired API key empty",
@@ -144,8 +150,8 @@ func TestAgentAPIKeyIDs(t *testing.T) {
 						}}},
 				},
 			},
-			want: []string{
-				"access_api_key_id", "p1_api_key_id"},
+			want: []ToRetireAPIKeyIdsItems{{ID: "access_api_key_id", Output: "", RetiredAt: ""},
+				{ID: "p1_api_key_id", Output: "p1", RetiredAt: ""}},
 		},
 	}