Adding upgrade test for unhealthy upgraded Agent (#3274)

* [WIP] Adding upgrade test for unhealthy upgraded Agent

* Fixing up logging

* WIP: fixing some issues with fake package creation

* Use fast watcher config

* [WIP] Different approach to making upgraded Agent unhealthy

* WIP: Adding TODO

* Flesh out rest of the test

* Adding logging

* Make invalid input try to run in upgraded Agent version

* Removing OS restriction

* Tuning status check frequency

* Use artifacts API

* Clarify comment

(cherry picked from commit 5531ad3)

pkg/testing/fixture_install.go

@@ -87,6 +87,7 @@ func (i InstallOpts) toCmdArgs() []string {
 //   - the combined output of stdout and stderr
 //   - an error if any.
 func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ...process.CmdOption) ([]byte, error) {
+	f.t.Logf("[test %s] Inside fixture install function", f.t.Name())
 	installArgs := []string{"install"}
 	if installOpts != nil {
 		installArgs = append(installArgs, installOpts.toCmdArgs()...)
@@ -110,6 +111,7 @@ func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ..
 	f.setClient(c)
 
 	f.t.Cleanup(func() {
+		f.t.Logf("[test %s] Inside fixture cleanup function", f.t.Name())
 		if !f.installed {
 			f.t.Logf("skipping uninstall; agent not installed (fixture.installed is false)")
 			// not installed; no need to clean up or collect diagnostics

testing/integration/upgrade_test.go

@@ -1003,8 +1003,6 @@ func TestStandaloneUpgradeFailsStatus(t *testing.T) {
 		Sudo:    true, // requires Agent installation
 	})
 
-	t.Skip("Affected by https://github.com/elastic/elastic-agent/issues/3371, watcher left running at end of test")
-
 	upgradeFromVersion, err := version.ParseVersion(define.Version())
 	require.NoError(t, err)