[DO NOT MERGE] Debug reproduce access denied

.buildkite/integration.pipeline.yml

@@ -5,40 +5,40 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
 
 steps:
-  - label: "Serverless integration test"
-    key: "serverless-integration-tests"
-    concurrency_group: elastic-agent-extended-testing/serverless-integration
-    concurrency: 8
-    env:
-      # we run each step in a different data center to spread the load
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
+#  - label: "Serverless integration test"
+#    key: "serverless-integration-tests"
+#    concurrency_group: elastic-agent-extended-testing/serverless-integration
+#    concurrency: 8
+#    env:
+#      # we run each step in a different data center to spread the load
+#      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
+#    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+#    artifact_paths:
+#      - "build/TEST-**"
+#      - "build/diagnostics/*"
+#    agents:
+#      provider: "gcp"
+#      machineType: "n1-standard-8"
+#    notify:
+#      - github_commit_status:
+#          context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
 
-  - label: "Extended runtime leak tests"
-    key: "extended-integration-tests"
-    concurrency_group: elastic-agent-extended-testing/leak-tests
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
+#  - label: "Extended runtime leak tests"
+#    key: "extended-integration-tests"
+#    concurrency_group: elastic-agent-extended-testing/leak-tests
+#    concurrency: 8
+#    env:
+#      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
+#    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks"
+#    artifact_paths:
+#      - "build/TEST-**"
+#      - "build/diagnostics/*"
+#    agents:
+#      provider: "gcp"
+#      machineType: "n1-standard-8"
+#    notify:
+#      - github_commit_status:
+#          context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
 
   - label: "Integration tests"
     key: "integration-tests"
@@ -57,20 +57,20 @@ steps:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Integration tests"
 
-  - label: "Serverless Beats Tests"
-    key: "serverless-beats-integration-tests"
-    concurrency_group: elastic-agent-extended-testing/beats-integration
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: ".buildkite/scripts/steps/beats_tests.sh"
-    # if: "build.env('CRON') == 'yes'"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    retry:
-      manual:
-        allowed: true
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
+#  - label: "Serverless Beats Tests"
+#    key: "serverless-beats-integration-tests"
+#    concurrency_group: elastic-agent-extended-testing/beats-integration
+#    concurrency: 8
+#    env:
+#      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
+#    command: ".buildkite/scripts/steps/beats_tests.sh"
+#    # if: "build.env('CRON') == 'yes'"
+#    agents:
+#      provider: "gcp"
+#      machineType: "n1-standard-8"
+#    retry:
+#      manual:
+#        allowed: true
+#    notify:
+#      - github_commit_status:
+#          context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"

.buildkite/scripts/steps/integration_tests.sh

@@ -4,8 +4,8 @@ set -euo pipefail
 source .buildkite/scripts/common.sh
 
 STACK_PROVISIONER="${1:-"stateful"}"
-MAGE_TARGET="${2:-"integration:test"}"
-MAGE_SUBTARGET="${3:-""}"
+MAGE_TARGET="${2:-"integration:single"}"
+MAGE_SUBTARGET="${3:-"TestProxyURL"}"
 
 
 # Override the agent package version using a string with format <major>.<minor>.<patch>
@@ -23,7 +23,7 @@ AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true
 
 # Run integration tests
 set +e
-AGENT_VERSION="${OVERRIDE_TEST_AGENT_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true  STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET
+TEST_PLATFORMS="windows/amd64/2022" TEST_RUN_UNTIL_FAILURE=true KEEP_AGENT_INSTALLED=true AGENT_VERSION="${OVERRIDE_TEST_AGENT_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=false  STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET
 TESTS_EXIT_STATUS=$?
 set -e
 

internal/pkg/agent/storage/encrypted_disk_storage_debug_notwindows.go

@@ -0,0 +1,18 @@
+//go:build !windows
+
+package storage
+
+import (
+	"os"
+)
+
+type loggerFunc func(fmtString string, args ...any)
+
+func DumpFilesystemInfo(path string, logF loggerFunc) {
+	stat, staterr := os.Stat(path)
+	if staterr != nil {
+		logF("Error stat()ing %s: %s", path, staterr)
+	} else {
+		logF("%s stat:\n%+v\n", path, stat)
+	}
+}

internal/pkg/agent/storage/encrypted_disk_storage_debug_windows.go

@@ -0,0 +1,64 @@
+//go:build windows
+
+package storage
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/sys/windows"
+)
+
+type loggerFunc func(fmtString string, args ...any)
+
+func printFileInfo(info os.FileInfo) string {
+	buf := new(strings.Builder)
+	encoder := json.NewEncoder(buf)
+	encoder.SetIndent("", "  ")
+	err := encoder.Encode(info)
+	if err != nil {
+		return fmt.Sprintf("error encoding FileInfo: %s", err)
+	}
+
+	return buf.String()
+}
+
+func printWindowsFileInfo(path string, logF loggerFunc) {
+
+	dumpSecurityDescriptor(path, logF)
+	dumpSecurityDescriptor(filepath.Dir(path), logF)
+}
+
+func dumpSecurityDescriptor(path string, logF loggerFunc) {
+	fileSD, err := windows.GetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.GROUP_SECURITY_INFORMATION)
+	if err != nil {
+		panic(fmt.Errorf("getting security descriptor for %s: %w", path, err))
+	}
+
+	logF("%s security descriptor: %s", path, fileSD)
+
+	ownerSid, _, err := fileSD.Owner()
+	if err != nil {
+		panic(fmt.Errorf("getting owner from security descriptor %s: %w", fileSD, err))
+	}
+
+	account, domain, accType, err := ownerSid.LookupAccount("")
+	if err != nil {
+		panic(fmt.Errorf("looking up account for %s: %w", ownerSid, err))
+	}
+
+	logF("owner for %q: %s\\%s account type %x", path, domain, account, accType)
+}
+
+func DumpFilesystemInfo(path string, logF loggerFunc) {
+	stat, staterr := os.Stat(path)
+	if staterr != nil {
+		logF("Error stat()ing %s: %s\n", path, staterr)
+	} else {
+		logF("%s stat:\n%s", path, printFileInfo(stat))
+		printWindowsFileInfo(path, logF)
+	}
+}

internal/pkg/agent/storage/encrypted_disk_store.go

@@ -12,8 +12,10 @@ import (
 	"io/fs"
 	"os"
 	"runtime"
+	"time"
 
 	"github.com/elastic/elastic-agent-libs/file"
+	"github.com/elastic/elastic-agent/pkg/core/logger"
 
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/secret"
@@ -117,6 +119,19 @@ func (d *EncryptedDiskStore) ensureKey(ctx context.Context) error {
 // Save will write the encrypted storage to disk.
 // Specifically it will write to a .tmp file then rotate the file to the target name to ensure that an error does not corrupt the previously written file.
 func (d *EncryptedDiskStore) Save(in io.Reader) error {
+
+	l, logErr := logger.New("encrypted-disk-store-debug", true)
+	if logErr != nil {
+		fmt.Fprintf(os.Stderr, "error instantiating debug logger: %s", logErr)
+	}
+
+	if d.target == paths.AgentConfigFile() {
+		l.Infof("Save of %s started at %s\n", paths.AgentConfigFile(), time.Now())
+		defer func() {
+			l.Infof("Save of %s finished at %s\n", paths.AgentConfigFile(), time.Now())
+		}()
+	}
+
 	// Ensure has agent key
 	err := d.ensureKey(d.ctx)
 	if err != nil {
@@ -182,7 +197,11 @@ func (d *EncryptedDiskStore) Save(in io.Reader) error {
 			errors.M(errors.MetaKeyPath, tmpFile))
 	}
 
+	DumpFilesystemInfo(d.target, l.Infof)
 	if err := file.SafeFileRotate(d.target, tmpFile); err != nil {
+
+		DumpFilesystemInfo(d.target, l.Infof)
+
 		return errors.New(err,
 			fmt.Sprintf("could not replace target file %s", d.target),
 			errors.TypeFilesystem,
@@ -206,6 +225,13 @@ func (d *EncryptedDiskStore) Load() (rc io.ReadCloser, err error) {
 			errors.M(errors.MetaKeyPath, d.target))
 	}
 
+	if d.target == paths.AgentConfigFile() {
+		fmt.Fprintf(os.Stderr, "Load of %s started at %s", paths.AgentConfigFile(), time.Now())
+		defer func() {
+			fmt.Fprintf(os.Stderr, "Load of %s finished at %s", paths.AgentConfigFile(), time.Now())
+		}()
+	}
+
 	// Close fd if there is an error upon return
 	defer func() {
 		if err != nil && fd != nil {

testing/integration/proxy_url_test.go

@@ -100,7 +100,7 @@ func TestProxyURL(t *testing.T) {
 				ackToken := "ackToken-AckTokenTestNoProxyInThePolicy"
 				mockFleet.checkinWithAcker.AddCheckin(
 					ackToken,
-					0,
+					30*time.Second,
 					action,
 				)
 
@@ -137,7 +137,7 @@ func TestProxyURL(t *testing.T) {
 				require.NoError(t, err, "could not generate action with policy")
 				mockFleet.checkinWithAcker.AddCheckin(
 					ackToken,
-					0,
+					30*time.Second,
 					action,
 				)
 
@@ -179,7 +179,7 @@ func TestProxyURL(t *testing.T) {
 				ackToken := "AckToken-TestValidProxyInThePolicy"
 				mockFleet.checkinWithAcker.AddCheckin(
 					ackToken,
-					0,
+					30*time.Second,
 					action,
 				)
 
@@ -227,7 +227,7 @@ func TestProxyURL(t *testing.T) {
 				ackToken := "AckToken-TestValidProxyInThePolicy"
 				mockFleet.checkinWithAcker.AddCheckin(
 					ackToken,
-					0,
+					30*time.Second,
 					action,
 				)
 				return map[string]*proxytest.Proxy{"proxyFleetPolicy": proxyFleetPolicy}, installArgs{insecure: true, enrollmentURL: mockFleet.fleetServer.LocalhostURL}
@@ -276,7 +276,7 @@ func TestProxyURL(t *testing.T) {
 				ackToken := "AckToken-TestRemoveProxyFromThePolicy"
 				mockFleet.checkinWithAcker.AddCheckin(
 					ackToken,
-					0,
+					30*time.Second,
 					action,
 				)
 
@@ -318,7 +318,7 @@ func TestProxyURL(t *testing.T) {
 				ackToken := "AckToken-TestRemovedProxyFromThePolicy"
 				mockFleet.checkinWithAcker.AddCheckin(
 					ackToken,
-					0,
+					30*time.Second,
 					action,
 				)
 
@@ -385,7 +385,7 @@ func TestProxyURL(t *testing.T) {
 				ackToken := "AckToken-TestValidProxyInThePolicy"
 				mockFleet.checkinWithAcker.AddCheckin(
 					ackToken,
-					0,
+					30*time.Second,
 					action,
 				)
 				return map[string]*proxytest.Proxy{"proxyFleetPolicy": proxyFleetPolicy}, installArgs{insecure: true, enrollmentURL: mockFleet.fleetServer.LocalhostURL}
@@ -486,7 +486,7 @@ func TestProxyURL(t *testing.T) {
 				ackToken := "AckToken-TestValidProxyInThePolicy"
 				mockFleet.checkinWithAcker.AddCheckin(
 					ackToken,
-					0,
+					30*time.Second,
 					action,
 				)
 				return map[string]*proxytest.Proxy{"proxyFleetPolicy": proxyFleetPolicy}, installArgs{insecure: true, enrollmentURL: mockFleet.fleetServer.LocalhostURL}
@@ -626,7 +626,7 @@ func TestProxyURL(t *testing.T) {
 				ackToken := "AckToken-TestValidProxyInThePolicy"
 				mockFleet.checkinWithAcker.AddCheckin(
 					ackToken,
-					0,
+					30*time.Second,
 					action,
 				)
 				return map[string]*proxytest.Proxy{"enroll": proxyEnroll, "proxyFleetPolicy": proxyFleetPolicy},
@@ -710,11 +710,11 @@ func TestProxyURL(t *testing.T) {
 			require.NoError(t, err, "SetupTest: fixture.Prepare failed")
 
 			privileged := false
-			if runtime.GOOS == "windows" {
-				// On windows installing + enrolling mode leads to access denied error when updating fleet.enc (regardless of privileged/unprivileged)
-				// See https://github.com/elastic/elastic-agent/issues/4913
-				t.Skip("Skipped on windows until https://github.com/elastic/elastic-agent/issues/4913 is resolved")
-			}
+			//if runtime.GOOS == "windows" {
+			//	// On windows installing + enrolling mode leads to access denied error when updating fleet.enc (regardless of privileged/unprivileged)
+			//	// See https://github.com/elastic/elastic-agent/issues/4913
+			//	t.Skip("Skipped on windows until https://github.com/elastic/elastic-agent/issues/4913 is resolved")
+			//}
 
 			out, err := fixture.Install(
 				ctx,