Add logic for beats User-Agent

NOTICE.txt

@@ -15964,6 +15964,29 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/LI
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/elastic/mock-es
+Version: v0.0.0-20240610195132-8fc87dc3f130
+Licence type (autodetected): Apache-2.0
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/LICENSE:
+
+Copyright 2024 Elasticsearch B.V.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/tk-btf
 Version: v0.1.0
@@ -18527,11 +18550,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/google/uuid
-Version: v1.3.1
+Version: v1.6.0
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/google/uuid@v1.3.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/google/uuid@v1.6.0/LICENSE:
 
 Copyright (c) 2009,2014 Google Inc. All rights reserved.
 
@@ -48010,6 +48033,36 @@ Contents of probable licence file $GOMODCACHE/github.com/matttproud/golang_proto
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/mileusna/useragent
+Version: v1.3.4
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/mileusna/[email protected]/LICENSE.md:
+
+MIT License
+
+Copyright (c) 2017 Miloš Mileusnić
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 --------------------------------------------------------------------------------
 Dependency : github.com/minio/asm2plan9s
 Version: v0.0.0-20200509001527-cdd76441f9d8

go.mod

@@ -1,6 +1,6 @@
 module github.com/elastic/beats/v7
 
-go 1.21.0
+go 1.21.9
 
 toolchain go1.21.10
 
@@ -102,7 +102,7 @@ require (
 	github.com/google/flatbuffers v23.5.26+incompatible
 	github.com/google/go-cmp v0.6.0
 	github.com/google/gopacket v1.1.19
-	github.com/google/uuid v1.3.1
+	github.com/google/uuid v1.6.0
 	github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75
 	github.com/h2non/filetype v1.1.1
 	github.com/hashicorp/go-multierror v1.1.1
@@ -210,6 +210,7 @@ require (
 	github.com/elastic/elastic-agent-system-metrics v0.10.2
 	github.com/elastic/go-elasticsearch/v8 v8.14.0
 	github.com/elastic/mito v1.12.2
+	github.com/elastic/mock-es v0.0.0-20240610195132-8fc87dc3f130
 	github.com/elastic/tk-btf v0.1.0
 	github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640
 	github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15
@@ -349,6 +350,7 @@ require (
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mileusna/useragent v1.3.4 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect

go.sum

@@ -594,6 +594,8 @@ github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/u
 github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/elastic/mito v1.12.2 h1:DQCYWg4U7rwNP45Ed2cHtAY94HEK1rfVUbNUT/8cYXk=
 github.com/elastic/mito v1.12.2/go.mod h1:J+wCf4HccW2YoSFmZMGu+d06gN+WmnIlj5ehBqine74=
+github.com/elastic/mock-es v0.0.0-20240610195132-8fc87dc3f130 h1:MSTfdpBTKMrrXUg0ucjFFQAgqxWa5kOkYvikqvlmUYM=
+github.com/elastic/mock-es v0.0.0-20240610195132-8fc87dc3f130/go.mod h1:mVdKBYYwt30xRFjCegbcURHh+3LWOCkQM33fgWbUiRI=
 github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA=
 github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0=
 github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3 h1:ChPwRVv1RR4a0cxoGjKcyWjTEpxYfm5gydMIzo32cAw=
@@ -992,8 +994,9 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.2.4 h1:uGy6JWR/uMIILU8wbf+OkstIrNiMjGpEIyhx8f6W7s4=
 github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -1310,6 +1313,8 @@ github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7
 github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY=
 github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/mileusna/useragent v0.0.0-20190129205925-3e331f0949a5/go.mod h1:JWhYAp2EXqUtsxTKdeGlY8Wp44M7VxThC9FEoNGi2IE=
+github.com/mileusna/useragent v1.3.4 h1:MiuRRuvGjEie1+yZHO88UBYg8YBC/ddF6T7F56i3PCk=
+github.com/mileusna/useragent v1.3.4/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc=
 github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
 github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
 github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=

libbeat/beat/info.go

@@ -25,17 +25,18 @@ import (
 
 // Info stores a beats instance meta data.
 type Info struct {
-	Beat            string    // The actual beat's name
-	IndexPrefix     string    // The beat's index prefix in Elasticsearch.
-	Version         string    // The beat version. Defaults to the libbeat version when an implementation does not set a version
-	ElasticLicensed bool      // Whether the beat is licensed under and Elastic License
-	Name            string    // configured beat name
-	Hostname        string    // hostname
-	FQDN            string    // FQDN
-	ID              uuid.UUID // ID assigned to beat machine
-	EphemeralID     uuid.UUID // ID assigned to beat process invocation (PID)
-	FirstStart      time.Time // The time of the first start of the Beat.
-	StartTime       time.Time // The time of last start of the Beat. Updated when the Beat is started or restarted.
+	Beat             string    // The actual beat's name
+	IndexPrefix      string    // The beat's index prefix in Elasticsearch.
+	Version          string    // The beat version. Defaults to the libbeat version when an implementation does not set a version
+	ElasticLicensed  bool      // Whether the beat is licensed under and Elastic License
+	Name             string    // configured beat name
+	Hostname         string    // hostname
+	FQDN             string    // FQDN
+	ID               uuid.UUID // ID assigned to beat machine
+	EphemeralID      uuid.UUID // ID assigned to beat process invocation (PID)
+	FirstStart       time.Time // The time of the first start of the Beat.
+	StartTime        time.Time // The time of last start of the Beat. Updated when the Beat is started or restarted.
+	UserAgentPostfix string    // A string postfix that's postfixed to the existing user-agent string used by the elasticsearch output
 
 	// Monitoring-related fields
 	Monitoring struct {

libbeat/cmd/instance/beat.go

@@ -68,6 +68,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/publisher/processing"
 	"github.com/elastic/beats/v7/libbeat/publisher/queue/diskqueue"
 	"github.com/elastic/beats/v7/libbeat/version"
+	"github.com/elastic/elastic-agent-client/v7/pkg/proto"
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/file"
 	"github.com/elastic/elastic-agent-libs/filewatcher"
@@ -864,6 +865,19 @@ func (b *Beat) configure(settings Settings) error {
 		version.SetPackageVersion(b.Info.Version)
 	}
 
+	// if we're in fleet mode, construct a custom user-agent
+	if fleetmode.Enabled() {
+		userAgent := ""
+		info := b.Manager.AgentInfo()
+		if info.ManagedMode == proto.AgentManagedMode_MANAGED {
+			userAgent = "Agent-Managed"
+		} else if info.ManagedMode == proto.AgentManagedMode_STANDALONE {
+			userAgent = "Agent-Standalone"
+		}
+		b.Info.UserAgentPostfix = userAgent
+	}
+	logp.L().Infof("got user agent in beat: %s", b.Info.UserAgentPostfix)
+
 	if err := b.Manager.CheckRawConfig(b.RawConfig); err != nil {
 		return err
 	}

libbeat/esleg/eslegclient/connection.go

@@ -86,6 +86,11 @@ type ConnectionSettings struct {
 	IdleConnTimeout time.Duration
 
 	Transport httpcommon.HTTPTransportSettings
+
+	// UserAgentPostfix can be used to report the agent running mode
+	// to ES via the User Agent string. If running under Agent (fleetmode.Enabled() == true)
+	// then this string will be appended to the user agent.
+	UserAgentPostfix string
 }
 
 type ESPingData struct {
@@ -135,6 +140,10 @@ func NewConnection(s ConnectionSettings) (*Connection, error) {
 	if s.Beatname == "" {
 		s.Beatname = "Libbeat"
 	}
+	if s.UserAgentPostfix != "" {
+		s.Beatname = fmt.Sprintf("%s-%s", s.Beatname, s.UserAgentPostfix)
+	}
+
 	userAgent := useragent.UserAgent(s.Beatname, version.GetDefaultVersion(), version.Commit(), version.BuildTime().String())
 
 	// Default the product origin header to beats if it wasn't already set.

libbeat/esleg/eslegclient/connection_test.go

@@ -23,11 +23,17 @@ import (
 	"context"
 	"encoding/base64"
 	"net/http"
+	"net/http/httptest"
+	"strings"
 	"testing"
+	"time"
 
+	"github.com/google/uuid"
+	"github.com/rcrowley/go-metrics"
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/libbeat/common/productorigin"
+	mock "github.com/elastic/mock-es/pkg/api"
 )
 
 func TestAPIKeyEncoding(t *testing.T) {
@@ -104,9 +110,41 @@ func TestHeaders(t *testing.T) {
 		require.NoError(t, err)
 
 		require.Equal(t, req.Header, http.Header(td.expected))
+
 	}
 }
 
+func TestUserAgentHeader(t *testing.T) {
+	metricsReg := metrics.NewRegistry()
+
+	mux := http.NewServeMux()
+	mockHandler := mock.NewAPIHandler(uuid.New(), "", metricsReg, time.Now().Add(time.Hour), 0, 0, 0, 0, 0)
+	mux.Handle("/", mockHandler)
+	server := httptest.NewServer(mux)
+	defer server.Close()
+
+	conn, err := NewConnection(ConnectionSettings{
+		URL:              server.URL,
+		Beatname:         "testbeat",
+		UserAgentPostfix: "Agent",
+	})
+	require.NoError(t, err)
+
+	err = conn.Connect()
+	require.NoError(t, err)
+
+	// search mock metrics for our user-agent string
+	found := false
+	for key := range metricsReg.GetAll() {
+		if strings.Contains(key, "testbeat-Agent") {
+			found = true
+		}
+	}
+
+	require.True(t, found, "did not find user-agent in %#v", metricsReg.GetAll())
+
+}
+
 func BenchmarkExecHTTPRequest(b *testing.B) {
 	for _, td := range []struct {
 		input    map[string]string

libbeat/outputs/elasticsearch/elasticsearch.go

@@ -121,6 +121,7 @@ func makeES(
 				EscapeHTML:       esConfig.EscapeHTML,
 				Transport:        esConfig.Transport,
 				IdleConnTimeout:  esConfig.Transport.IdleConnTimeout,
+				UserAgentPostfix: beatInfo.UserAgentPostfix,
 			},
 			indexSelector:    indexSelector,
 			pipelineSelector: pipelineSelector,