Igor api key validation

e2e/_suites/fleet/features/fleet_mode_agent.feature

@@ -7,58 +7,56 @@ Background: Setting up kibana instance with the default profile
 
 @install
 @skip:windows
-Scenario Outline: Deploying the agent
+Scenario: Deploying the agent
   Given an agent is deployed to Fleet with "tar" installer
   When the "elastic-agent" process is in the "started" state on the host
   Then the agent is listed in Fleet as "online"
     And system package dashboards are listed in Fleet
 
-# @enroll
-# @skip:windows
-# Scenario Outline: Deploying the agent with enroll and then run on rpm and deb
-#   Given an agent is deployed to Fleet
-#   When the "elastic-agent" process is in the "started" state on the host
-#   Then the agent is listed in Fleet as "online"
-#     And system package dashboards are listed in Fleet
-
-# @centos
-# Examples: Centos
-# | os     |
-# | centos |
-
-# @debian
-# Examples: Debian
-# | os     |
-# | debian |
+@enroll
+@skip:windows
+Scenario: Deploying the agent with enroll and then run on rpm and deb
+  Given an agent is deployed to Fleet
+  When the "elastic-agent" process is in the "started" state on the host
+  Then the agent is listed in Fleet as "online"
+    And system package dashboards are listed in Fleet
 
 # @upgrade-agent
 @nightly
 @skip:windows
-Scenario Outline: Upgrading the installed agent
+Scenario: Upgrading the installed agent
   Given an agent "stale" is deployed to Fleet with "tar" installer
     And certs are installed
     And the "elastic-agent" process is "restarted" on the host
   When agent is upgraded to version "latest"
   Then agent is in version "latest"
 
-
 @restart-agent
 @skip:windows
-Scenario Outline: Restarting the installed agent
+Scenario: Restarting the installed agent
   Given an agent is deployed to Fleet with "tar" installer
   When the "elastic-agent" process is "restarted" on the host
   Then the agent is listed in Fleet as "online"
 
-@unenroll
+@un-enroll-with-revoke
 @skip:windows
-Scenario Outline: Un-enrolling the agent deactivates the agent
+Scenario: Un-enrolling with revoke agent deactivates the agent
   Given an agent is deployed to Fleet with "tar" installer
   When the agent is un-enrolled
   Then the agent is listed in Fleet as "inactive"
+     And the agent Api key invalidated "true"
+
+@un-enroll-without-revoke
+@skip:windows
+Scenario: Un-enrolling without revoke deactivates the agent
+  Given an agent is deployed to Fleet with "tar" installer
+  When the agent is un-enrolled without revoke
+  Then the agent is listed in Fleet as "inactive"
+     And the agent Api key invalidated "true"
 
 @reenroll
 @skip:windows
-Scenario Outline: Re-enrolling the agent activates the agent in Fleet
+Scenario: Re-enrolling agent activates the agent in Fleet
   Given an agent is deployed to Fleet with "tar" installer
     And the agent is un-enrolled
     And the "elastic-agent" process is "stopped" on the host
@@ -67,14 +65,14 @@ Scenario Outline: Re-enrolling the agent activates the agent in Fleet
   Then the agent is listed in Fleet as "online"
 
 @revoke-token
-Scenario Outline: Revoking the enrollment token for the agent
+Scenario: Revoking the enrollment token for agent
   Given an agent is deployed to Fleet with "tar" installer
   When the enrollment token is revoked
   Then an attempt to enroll a new agent fails
 
 @uninstall-host
 @skip:windows
-Scenario Outline: Un-installing the installed agent
+Scenario: Un-installing the installed agent
   Given an agent is deployed to Fleet with "tar" installer
   When the "elastic-agent" process is "uninstalled" on the host
   Then the file system Agent folder is empty

e2e/_suites/fleet/fleet.go

@@ -11,6 +11,10 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
+
+	//"strconv"
+
 	"strings"
 	"time"
 
@@ -62,6 +66,7 @@ type FleetTestSuite struct {
 	// instrumentation
 	currentContext context.Context
 	DefaultAPIKey  string
+	AgentId        string
 }
 
 // afterScenario destroys the state created by a scenario
@@ -105,7 +110,7 @@ func (fts *FleetTestSuite) afterScenario() {
 			_ = fts.deployer.Logs(fts.currentContext, agentService)
 		}
 
-		err := fts.unenrollHostname()
+		err := fts.unenrollHostname(true)
 		if err != nil {
 			manifest, _ := fts.deployer.Inspect(fts.currentContext, agentService)
 			log.WithFields(log.Fields{
@@ -248,8 +253,10 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
 	s.Step(`^the host is restarted$`, fts.theHostIsRestarted)
 	s.Step(`^system package dashboards are listed in Fleet$`, fts.systemPackageDashboardsAreListedInFleet)
 	s.Step(`^the agent is un-enrolled$`, fts.theAgentIsUnenrolled)
+	s.Step(`^the agent is un-enrolled without revoke$`, fts.theAgentIsUnenrolledWithoutRevoke)
 	s.Step(`^the agent is re-enrolled on the host$`, fts.theAgentIsReenrolledOnTheHost)
 	s.Step(`^the enrollment token is revoked$`, fts.theEnrollmentTokenIsRevoked)
+	s.Step(`^the agent Api key invalidated "(([^"]*))"$`, fts.theAgentApiKeyIsInvalidated)
 	s.Step(`^an attempt to enroll a new agent fails$`, fts.anAttemptToEnrollANewAgentFails)
 	s.Step(`^the "([^"]*)" process is "([^"]*)" on the host$`, fts.processStateChangedOnTheHost)
 	s.Step(`^the file system Agent folder is empty$`, fts.theFileSystemAgentFolderIsEmpty)
@@ -779,7 +786,6 @@ func theAgentIsListedInFleetWithStatus(ctx context.Context, desiredStatus string
 			retryCount++
 			return err
 		}
-
 		if agentID == "" {
 			// the agent is not listed in Fleet
 			if desiredStatus == "offline" || desiredStatus == "inactive" {
@@ -939,7 +945,11 @@ func (fts *FleetTestSuite) systemPackageDashboardsAreListedInFleet() error {
 }
 
 func (fts *FleetTestSuite) theAgentIsUnenrolled() error {
-	return fts.unenrollHostname()
+	return fts.unenrollHostname(true)
+}
+
+func (fts *FleetTestSuite) theAgentIsUnenrolledWithoutRevoke() error {
+	return fts.unenrollHostname(false)
 }
 
 func (fts *FleetTestSuite) theAgentIsReenrolledOnTheHost() error {
@@ -1326,7 +1336,7 @@ func (fts *FleetTestSuite) anAttemptToEnrollANewAgentFails() error {
 }
 
 // unenrollHostname deletes the statuses for an existing agent, filtering by hostname
-func (fts *FleetTestSuite) unenrollHostname() error {
+func (fts *FleetTestSuite) unenrollHostname(revoke bool) error {
 	span, _ := apm.StartSpanOptions(fts.currentContext, "Unenrolling hostname", "elastic-agent.hostname.unenroll", apm.SpanOptions{
 		Parent: apm.SpanFromContext(fts.currentContext).TraceContext(),
 	})
@@ -1347,7 +1357,8 @@ func (fts *FleetTestSuite) unenrollHostname() error {
 				"hostname": manifest.Hostname,
 			}).Debug("Un-enrolling agent in Fleet")
 
-			err := fts.kibanaClient.UnEnrollAgent(fts.currentContext, agent.LocalMetadata.Host.HostName)
+			fts.AgentId = agent.ID
+			err := fts.kibanaClient.UnEnrollAgent(fts.currentContext, agent.LocalMetadata.Host.HostName, revoke)
 			if err != nil {
 				return err
 			}
@@ -1684,3 +1695,46 @@ func (fts *FleetTestSuite) theMetricsInTheDataStream(name string, set string) er
 
 	return nil
 }
+
+func (fts *FleetTestSuite) theAgentApiKeyIsInvalidated(invalidated string) error {
+	maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute
+	exp := utils.GetExponentialBackOff(maxTimeout)
+	retryCount := 1
+
+	invalidatedBool, _ := strconv.ParseBool(invalidated)
+	apiKeyInvalidated := func() error {
+		body, err := elasticsearch.GetSecurityApiKey()
+		if err != nil {
+			log.WithFields(log.Fields{
+				"Error": err,
+			}).Error("Could not return Security Api Keys")
+			return err
+		}
+
+		for _, item := range body.APIKeys {
+			if item.Metadata.AgentID == fts.AgentId {
+				if item.Invalidated == invalidatedBool {
+					log.WithFields(log.Fields{
+						"agentId": fts.AgentId,
+						"Type":    item.Metadata.Type,
+					}).Info("The agent Api key invalidated: ", item.Invalidated)
+				} else {
+					log.WithFields(log.Fields{
+						"agentId": fts.AgentId,
+						"Type":    item.Metadata.Type,
+					}).Error("The agent Api key invalidated: ", item.Invalidated)
+					return errors.New("The agent Api key invalidated is should be: " + invalidated)
+				}
+			}
+		}
+		retryCount++
+
+		return nil
+	}
+
+	err := backoff.Retry(apiKeyInvalidated, exp)
+	if err != nil {
+		return err
+	}
+	return nil
+}

internal/elasticsearch/client.go

@@ -34,6 +34,27 @@ type Query struct {
 	ServiceVersion string
 }
 
+type APIKey struct {
+	APIKeys []APIKeys `json:"api_keys"`
+}
+type Metadata struct {
+	PolicyID  string `json:"policy_id, omitempty"`
+	AgentID   string `json:"agent_id, omitempty"`
+	ManagedBy string `json:"managed_by"`
+	Managed   bool   `json:"managed"`
+	Type      string `json:"type"`
+}
+
+type APIKeys struct {
+	ID          string   `json:"id"`
+	Name        string   `json:"name"`
+	Creation    int64    `json:"creation"`
+	Invalidated bool     `json:"invalidated"`
+	Username    string   `json:"username"`
+	Realm       string   `json:"realm"`
+	Metadata    Metadata `json:"metadata,omitempty"`
+}
+
 // SearchResult wraps a search result
 type SearchResult map[string]interface{}
 
@@ -386,3 +407,34 @@ func WaitForNumberOfHits(ctx context.Context, indexName string, query map[string
 	err := backoff.Retry(numberOfHits, exp)
 	return result, err
 }
+
+// GetSecurityApiKey waits for the elasticsearch SecurityApiKey to return the list of Api Keys.
+func GetSecurityApiKey() (APIKey, error) {
+	esEndpoint := GetElasticSearchEndpoint()
+	data := APIKey{}
+
+	r := curl.HTTPRequest{
+		URL:               fmt.Sprintf("%s://%s:%d/_security/api_key?", esEndpoint.Scheme, esEndpoint.Host, esEndpoint.Port),
+		BasicAuthPassword: shell.GetEnv("ELASTICSEARCH_PASSWORD", "changeme"),
+		BasicAuthUser:     "elastic",
+	}
+
+	response, err := curl.Get(r)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":          err,
+			"statusEndpoint": r.URL,
+		}).Warn("The Elasticsearch Console SecurityApiKey API is not available yet")
+
+		return APIKey{}, err
+	}
+
+	log.WithFields(log.Fields{
+		"statusEndpoint": r.URL,
+		"Response":       response,
+	}).Trace("The Elasticsearch  API is available")
+
+	json.Unmarshal([]byte(response), &data)
+
+	return data, err
+}

internal/kibana/agents.go

@@ -78,6 +78,7 @@ func (c *Client) GetAgentIDByHostname(ctx context.Context, hostname string) (str
 		"agentId":  agent.ID,
 		"hostname": hostname,
 	}).Trace("Agent Id found")
+
 	return agent.ID, nil
 }
 
@@ -245,7 +246,8 @@ func (c *Client) ListAgents(ctx context.Context) ([]Agent, error) {
 }
 
 // UnEnrollAgent unenrolls agent from fleet
-func (c *Client) UnEnrollAgent(ctx context.Context, hostname string) error {
+func (c *Client) UnEnrollAgent(ctx context.Context, hostname string, revoke bool) error {
+	var reqBody = `{}`
 	span, _ := apm.StartSpanOptions(ctx, "UnEnrolling Elastic Agent by hostname", "fleet.agent.un-enroll", apm.SpanOptions{
 		Parent: apm.SpanFromContext(ctx).TraceContext(),
 	})
@@ -256,7 +258,12 @@ func (c *Client) UnEnrollAgent(ctx context.Context, hostname string) error {
 		return err
 	}
 
-	reqBody := `{"revoke": true}`
+	if revoke == true {
+		reqBody = `{"revoke": true}`
+	}
+	log.WithFields(log.Fields{
+		"body": reqBody,
+	}).Info("Sending un-enrollment request")
 	statusCode, respBody, _ := c.post(ctx, fmt.Sprintf("%s/agents/%s/unenroll", FleetAPI, agentID), []byte(reqBody))
 	if statusCode != 200 {
 		return fmt.Errorf("could not unenroll agent; API status code = %d, response body = %s", statusCode, respBody)