Remove PGP signature verification skip for DEV builds (#3590)

* remove PGP signature verification skip for DEV builds
* create pgptest package to sign and give the public key to verify the signature
* fix tests that relied on skipping the PGP verification
* add PGP/GPG how-to on docs
* add test for VerifySHA512HashWithCleanup

(cherry picked from commit e43be2a)

# Conflicts:
#	internal/pkg/agent/application/upgrade/artifact/download/fs/verifier.go
#	internal/pkg/agent/application/upgrade/artifact/download/http/verifier.go

docs/pgp-sign-verify-artifact.md

@@ -0,0 +1,176 @@
+# Signing Elastic Agent artifacts
+
+This doc covers generating a key, exporting the public key, signing a file and verifying it using GPG as well as pure Go.
+
+Full GPG docs: https://www.gnupg.org/documentation/manuals/gnupg/OpenPGP-Key-Management.html
+
+
+## Go
+
+```go
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/crypto/openpgp"
+	"golang.org/x/crypto/openpgp/armor"
+)
+
+func main() {
+	dir, err := os.MkdirTemp(os.TempDir(), "pgp-")
+	NoError(err, "could not create directory to save the files to")
+
+	key := filepath.Join(dir, "key")
+	keyPub := filepath.Join(dir, "key.pub")
+	asc := filepath.Join(dir, "plaindata.asc")
+
+	fmt.Printf("Writing files to %q\n", dir)
+
+	data := []byte("some data")
+	plaindata := filepath.Join(dir, "plaindata")
+	err = os.WriteFile(plaindata, data, 0o600)
+	NoError(err, "could not write plain data file")
+
+	fmt.Printf("wrote %q\n", plaindata)
+
+	// Create files
+	fKeyPub, err := os.OpenFile(
+		keyPub,
+		os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	NoError(err, "could not create %q file", keyPub)
+	defer func() {
+		if err := fKeyPub.Close(); err != nil {
+			fmt.Printf("failed closing %q\n", fKeyPub.Name())
+		}
+		fmt.Printf("wrote %q\n", fKeyPub.Name())
+	}()
+
+	fKey, err := os.OpenFile(
+		key,
+		os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	NoError(err, "could not create %q file", key)
+	defer func() {
+		if err := fKey.Close(); err != nil {
+			fmt.Printf("failed closing %q\n", fKey.Name())
+		}
+		fmt.Printf("wrote %q\n", fKey.Name())
+	}()
+
+	fasc, err := os.OpenFile(
+		asc,
+		os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	NoError(err, "could not create %q file", asc)
+	defer func() {
+		if err := fasc.Close(); err != nil {
+			fmt.Printf("failed closing %q\n", fasc.Name())
+		}
+		fmt.Printf("wrote %q\n", fasc.Name())
+	}()
+
+	// Generate PGP key
+	entity, err := openpgp.NewEntity("someKeyName", "", "", nil)
+
+	// Create an ASCII armored encoder to serialize the private key
+	wPubKey, err := armor.Encode(fKeyPub, openpgp.PublicKeyType, nil)
+	NoError(err, "could not create PGP ASCII Armor encoder for public key")
+	defer func() {
+		err := wPubKey.Close()
+		if err != nil {
+			fmt.Println("failed closing private key writer")
+		}
+	}()
+
+	// Writes the public key to the io.Writer passed to armor.Encode.
+	// Use entity.SerializePrivate if you need the private key.
+	err = entity.Serialize(wPubKey)
+	NoError(err, "could not serialize the public key")
+
+	// Create an ASCII armored encoder to serialize the private key
+	wPrivKey, err := armor.Encode(fKey, openpgp.PrivateKeyType, nil)
+	NoError(err, "could not create PGP ASCII Armor encoder for private key")
+	defer func() {
+		err := wPrivKey.Close()
+		if err != nil {
+			fmt.Println("failed closing private key writer")
+		}
+	}()
+
+	// Writes the private key to the io.Writer passed to armor.Encode.
+	// Use entity.SerializePrivate if you need the private key.
+	err = entity.SerializePrivate(wPrivKey, nil)
+	NoError(err, "could not serialize the private key")
+
+	// Sign data and write the detached signature to fasc
+	err = openpgp.ArmoredDetachSign(fasc, entity, bytes.NewReader(data), nil)
+	NoError(err, "failed signing date")
+}
+
+func NoError(err error, msg string, args ...any) {
+	if err != nil {
+		panic(fmt.Sprintf(msg+": %v", append(args, err)))
+	}
+}
+```
+
+## GPG
+### Generate a key
+
+```shell
+gpg --no-default-keyring --keyring ./some-file-to-be-the-key-ring --quick-generate-key atest  rsa2048 default none
+```
+Where:
+ - `--no-default-keyring`: do not use your keyring
+ - `--keyring ./some-file-to-be-the-key-ring`: keyring to use, as the file do not exist, it'll create it
+ - `--quick-generate-key`: quick generate the key
+ - `atest`: user-id, a.k.a the key identifier
+ - `rsa2048`: algorithm to use
+ - `default`: "usage" for the key. Just use default
+ - `none`: key expiration
+
+
+### Export the public key
+```shell
+gpg --no-default-keyring --keyring ./some-file-to-be-the-key-ring --armor --output public-key.pgp --export atest
+```
+Where:
+- `--no-default-keyring`: do not use your keyring
+ - `--keyring ./some-file-to-be-the-key-ring`: the keyring to use, created in the previous step
+ - `--armor`: create ASCII armoured output. Otherwise, it's a binary format
+ - `--output public-key.pgp`: the output file
+ - `--export`: export the public key
+ - `atest`: the key identifier
+
+### Sing the file
+```shell
+gpg --no-default-keyring --keyring ./some-file-to-be-the-key-ring -a -o elastic-agent-8.0.0-darwin-x86_64.tar.gz.asc --detach-sign elastic-agent-8.0.0-darwin-x86_64.tar.gz
+```
+
+Where:
+ - `-a -o`: --armored, --output
+ - `elastic-agent-8.0.0-darwin-x86_64.tar.gz.asc`: the output file
+ - `--detach-sign`: generate a separated file for signature
+ - `elastic-agent-8.0.0-darwin-x86_64.tar.gz`: the file to sign
+
+
+
+### Verify the file
+
+#### Import the public key
+```shell
+gpg --no-default-keyring --keyring ./new-keyring --import public-key.pgp
+```
+Where:
+ - `--import`: import a key
+ - `public-key.pgp`: the key to import
+
+#### Verify the signature using the imported key
+```shell
+gpg --no-default-keyring --keyring ./new-keyring --verify elastic-agent-8.0.0-darwin-x86_64.tar.gz.asc
+```
+Where:
+ - `--verify`: verify a signature
+ - `elastic-agent-8.0.0-darwin-x86_64.tar.gz.asc`: the detached signature file. It'll assume the file to be verified is `elastic-agent-8.0.0-darwin-x86_64.tar.gz`

internal/pkg/agent/application/coordinator/coordinator.go

@@ -108,7 +108,7 @@ type RuntimeManager interface {
 	// it performs diagnostics for all current units.
 	PerformDiagnostics(context.Context, ...runtime.ComponentUnitDiagnosticRequest) []runtime.ComponentUnitDiagnostic
 
-	//PerformComponentDiagnostics executes the diagnostic action for the provided components. If no components are provided,
+	// PerformComponentDiagnostics executes the diagnostic action for the provided components. If no components are provided,
 	// then it performs the diagnostics for all current units.
 	PerformComponentDiagnostics(ctx context.Context, additionalMetrics []cproto.AdditionalDiagnosticRequest, req ...component.Component) ([]runtime.ComponentDiagnostic, error)
 }
@@ -415,7 +415,7 @@ func (c *Coordinator) ReExec(callback reexec.ShutdownCallbackFn, argOverrides ..
 // Upgrade runs the upgrade process.
 // Called from external goroutines.
 func (c *Coordinator) Upgrade(ctx context.Context, version string, sourceURI string, action *fleetapi.ActionUpgrade, skipVerifyOverride bool, skipDefaultPgp bool, pgpBytes ...string) error {
-	// early check outside of upgrader before overridding the state
+	// early check outside of upgrader before overriding the state
 	if !c.upgradeMgr.Upgradeable() {
 		return ErrNotUpgradable
 	}

internal/pkg/agent/application/upgrade/artifact/download/fs/downloader.go

@@ -52,32 +52,34 @@ func (e *Downloader) Download(ctx context.Context, a artifact.Artifact, version
 	}()
 
 	// download from source to dest
-	path, err := e.download(e.config.OS(), a, version)
+	path, err := e.download(e.config.OS(), a, version, "")
 	downloadedFiles = append(downloadedFiles, path)
 	if err != nil {
 		return "", err
 	}
 
-	hashPath, err := e.downloadHash(e.config.OS(), a, version)
+	hashPath, err := e.download(e.config.OS(), a, version, ".sha512")
 	downloadedFiles = append(downloadedFiles, hashPath)
 	return path, err
 }
 
-func (e *Downloader) download(operatingSystem string, a artifact.Artifact, version string) (string, error) {
-	filename, err := artifact.GetArtifactName(a, version, operatingSystem, e.config.Arch())
-	if err != nil {
-		return "", errors.New(err, "generating package name failed")
-	}
-
-	fullPath, err := artifact.GetArtifactPath(a, version, operatingSystem, e.config.Arch(), e.config.TargetDirectory)
+// DownloadAsc downloads the package .asc file from configured source.
+// It returns absolute path to the downloaded file and a no-nil error if any occurs.
+func (e *Downloader) DownloadAsc(_ context.Context, a artifact.Artifact, version string) (string, error) {
+	path, err := e.download(e.config.OS(), a, version, ".asc")
 	if err != nil {
-		return "", errors.New(err, "generating package path failed")
+		os.Remove(path)
+		return "", err
 	}
 
-	return e.downloadFile(filename, fullPath)
+	return path, nil
 }
 
-func (e *Downloader) downloadHash(operatingSystem string, a artifact.Artifact, version string) (string, error) {
+func (e *Downloader) download(
+	operatingSystem string,
+	a artifact.Artifact,
+	version,
+	extension string) (string, error) {
 	filename, err := artifact.GetArtifactName(a, version, operatingSystem, e.config.Arch())
 	if err != nil {
 		return "", errors.New(err, "generating package name failed")
@@ -88,8 +90,10 @@ func (e *Downloader) downloadHash(operatingSystem string, a artifact.Artifact, v
 		return "", errors.New(err, "generating package path failed")
 	}
 
-	filename = filename + ".sha512"
-	fullPath = fullPath + ".sha512"
+	if extension != "" {
+		filename += extension
+		fullPath += extension
+	}
 
 	return e.downloadFile(filename, fullPath)
 }

internal/pkg/agent/application/upgrade/artifact/download/fs/testdata/drop/elastic-agent-8.0.0-darwin-x86_64.tar.gz.asc

@@ -0,0 +1,14 @@
+-----BEGIN PGP SIGNATURE-----
+
+iQGzBAABCgAdFiEE81a455Doc5DWexOcF4e6ez4rqzAFAmUn7TsACgkQF4e6ez4r
+qzDcIgwArpuXDex9aisWFWkXjCfjhJdrTTXr3wv8W68NeFsAaazLlvsWPxdol1db
+FeKFL+P/P/PhlTvdkZw9xMyXoVRWQXJ2p2jVjV0Wq2SCtbbjdrGjQ4OrchgE9FW7
+onWxqV8RjzPyaMwpDWWtHKgxhQeLP5yXhWm6RXHvBLZ5mqbTCuIq2Q4sijEd6IFD
+9JoAA276tqyKGOsPZ1QzaPUFF69B9QLcWasEuNFf5ytMVFfTcMl6/HYDPO7ErhJx
+E1hnKGIc5rrMghL0LzaVLGYZUtnQwru02ZA0omXzEv1uYgqmZl75g9qHk2Cu2V5W
+0qbg9OtUKOkJ1sODvsVv8O40rVazdZTgL2ifNLi2wFwR3syMdHCih2aKMcPDPzt3
+Q4q0zvsxuR9PGsv5+8zze74iC3oZSvF8h36XGjJuyjEFORUpcWNGDmhsC6l7ql5W
+rEbIPZ19j3r1M4yHG/ptBmrwRnQz9RKFnwTO9ME/5eBVumPLUD5kAcYXjvAFYQI5
+qEc7okL5
+=+nvi
+-----END PGP SIGNATURE-----

internal/pkg/agent/application/upgrade/artifact/download/fs/testdata/drop/beat-8.0.0-darwin-x86_64.tar.gz.sha512 → internal/pkg/agent/application/upgrade/artifact/download/fs/testdata/drop/elastic-agent-8.0.0-darwin-x86_64.tar.gz.sha512

@@ -1 +1 @@
-9af9aa016f3349aa248034629e4336ca2f4d31317bfb8c9a23a9d924c18969cf43ad93727e784da010a272690b2b5ce4c4ded3a5d2039e4408e93e1e18d113db  beat-8.0.0-darwin-x86_64.tar.gz
+9af9aa016f3349aa248034629e4336ca2f4d31317bfb8c9a23a9d924c18969cf43ad93727e784da010a272690b2b5ce4c4ded3a5d2039e4408e93e1e18d113db  elastic-agent-8.0.0-darwin-x86_64.tar.gz

internal/pkg/agent/application/upgrade/artifact/download/fs/testdata/drop/public-key.pgp

@@ -0,0 +1,40 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQGNBGUn7JEBDADH0iBdohpZIQY7QyBz9Hl68b7fq0zoFcB4HTDDMQD1ouDQfPwg
+Frpr/ViNNHsye1QfrUWXN8FQfKztqHtUHeM8ggdSvhYYGaDtVSuVakoNNz3Z3+kD
+YhwH0byZrE2MiCKExtgQYWBIDd1TeCMSOgYcQPAXPqQBwX0G1xRAY3s+eazCjaSU
+aNJtlNuAx36jEBa+X73sTh+Y/OtCSN9s75SVYu5xJ+kXkpcHNvsMJmDCZ0zsKrxT
+TMvXSU9qcczj8+wAee/1E77eR01ttrf67IjVReuVZ0OhxucVxJHOp7x9jfeGsjjn
+6uhFT0KV+VOaaRlI9wZ4AOMmAX5nroNYP/GC+SKiOvKV79+r3jyxbChqd5nWdSBN
+mO9okB72nUpGmL1NosW926MMTauR9/nP1uWB66d/pHYRop7sAbAZ7u8COoRS1wd+
+V6dtb3QUwR9LsfKd1xQfrTFVKZ4i703MN1qkq/6TqLhpwlt0+K4WN7LtkkeFivyx
+N0RLiVDzZP289ssAEQEAAbQFYXRlc3SJAc4EEwEKADgWIQTzVrjnkOhzkNZ7E5wX
+h7p7PiurMAUCZSfskQIbAwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRAXh7p7
+PiurMFkbDAC0QLqwq4dZGjOqcNjj02DOM1eQcC8XUSy0w8X6gX/69wFHGM34zl4+
+IO7H6ujvkBxMHmeEU3nNsLH+WsN6Hc8JBRQZSqjySgL2am+K6XYMcP7h7VGnFR0r
+5IKbGn9zCR7xkVrkvW0T48U0fJ00X3v+GWcxcBQIu58sMmKrmzliPCDhmQ94yum8
+n8Yc1tB3DazAQEDGxtfP8/yc93sWKZ4qKPBMZUsjSSzC8a7zei/J9vJccRy/JJEl
+/mNIQx7FxObrCSSa3wXc4AEbWdq4HNZkahLvnOs4EhNR9ihWg7TtMVyBesV/rdgj
+5cgHU3erir1nSOHmrHqLydeWH4vHW4R6BYuJd6NXhsISMHO8Oerlceqmt7aex3wJ
+09ULyareJ3QMc+HWcjxxYbSLU6j5ZgCqcPz17V88W7SkXnzbPaoVAxMCf+M3a0Ib
+r+Yw6CrvWRj2+bmW8Ars6fND90nX4ZS82VnMc27kFqNYdkAE9kdlZ+L8OU70nWmT
+Clh2FhjhHKe5AY0EZSfskQEMANT+4NWxDtyExZEIvwUdegcetF3hbdHlXMmMnuPU
+vJwPhXhXJtzyX5VKRp3WCUO2TOGMKaHaNPi4XCS4QMzBEEft8C7X896QPGqanGdV
+oZ9Oc/mXNZfuOk62hP6Ifn38VIyxAcpQ11ypKJ5wFSwSvkPIdaXm1125oGIFQg+W
+51GSNz8PBuP5GavLs3L1Wp2VupJ9pOrolxGRP+t41u6rNewaktSO6eLY0o0j/FMY
+Anujnj68sS92e7TnQcaAEUsplYLrZlZI1Ly0W2QakvOUIkDq5DSsNYKypTM1rZ7s
+VYENPjHdhATsHoW1LxirBKHuoi8aANSjsofdggnxtu+sp8mk/+oZpyR78yA/+hIA
+/t/wEVgVXETTB0Y8o6n8+/U/uBHEjYGa8JJEcMbNJesQAusBXYt90N8URKHRWEcR
+L9IH3V4rmssDqgE7voHYvNKFru/socsI3WPmDnPKFWGRd7rqzlkBoqbrPiD/tRIC
+cwDqz5hm3vKqOkHqvsGqzNVp4wARAQABiQG2BBgBCgAgFiEE81a455Doc5DWexOc
+F4e6ez4rqzAFAmUn7JECGwwACgkQF4e6ez4rqzA23gv/UZTQw13urB8Hf6s5FJyz
+z5dCWT1RMW1ig7MuCe/MzRCk29zDc16y5fOo0aLzYMWsQXBrBTAXj6hx2/MYHXg0
+mUXzxrnUqM5H/b1hbx52NdwD1eR1prQIX39ifPzw+FTirD98qx04479En/561PQW
+lbWXtm1/JoaSpGIYP2gWNgb3HfHShEGPxFH39vxmP6XVz99BL+3zaHehcCUP8fbC
+Kabo/qbtNC/nZEBUVVMxEj2O9eEq9otk8K8fBzoCOQ4K0Idn+BnQ0O67x4jemunD
+JX6BGBo0WYxJNarK2sJw5+CVRK472va8U6Y+6yGyv5qu68eOZZXvkrCbDpysSIf7
+YjwhmaZuerd4oBvRKJHbbHoqgde8sviSjm6cdU+ZSHILvwEaBLwW3pTgBJAupQcV
+4Ws7fo7/6R2YWws8c4sseGqLC+XxCXk+SvrvyA02ZBY+0L6IFD6Cb8BT0uMMrLIP
+YcZ1xK3gfrp4PCg2OFj46WER5ufHP1r0zvufY7chA9tP
+=Jwiw
+-----END PGP PUBLIC KEY BLOCK-----