metricbeat/module/postgresql: Add PostgreSQL Replication metricset

CHANGELOG.next.asciidoc

@@ -231,6 +231,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Improve documentation for ActiveMQ module {issue}35113[35113] {pull}35558[35558]
 - Fix EC2 host.cpu.usage {pull}35717[35717]
 - Resolve statsd module's prematurely halting of metrics parsing upon encountering an invalid packet. {pull}35075[35075]
+- Add support for PostgreSQL replication metrics {pull}35562[35562]
 
 *Osquerybeat*
 

libbeat/common/schema/mapstrstr/mapstrstr.go

@@ -129,6 +129,27 @@ func Int(key string, opts ...schema.SchemaOption) schema.Conv {
 	return schema.SetOptions(schema.Conv{Key: key, Func: toInt}, opts)
 }
 
+// toUint converts value to uint. In case of error, returns 0
+func toUint(key string, data map[string]interface{}) (interface{}, error) {
+	str, err := getString(key, data)
+	if err != nil {
+		return false, err
+	}
+
+	value, err := strconv.ParseUint(str, 10, 64)
+	if err != nil {
+		msg := fmt.Sprintf("error converting param to uint: `%s`", str)
+		return 0, schema.NewWrongFormatError(key, msg)
+	}
+
+	return value, nil
+}
+
+// Uint creates a Conv object for parsing integers
+func Uint(key string, opts ...schema.SchemaOption) schema.Conv {
+	return schema.SetOptions(schema.Conv{Key: key, Func: toUint}, opts)
+}
+
 // toStr converts value to str. In case of error, returns ""
 func toStr(key string, data map[string]interface{}) (interface{}, error) {
 	return getString(key, data)

metricbeat/docs/fields.asciidoc

@@ -56324,6 +56324,203 @@ type: date
 
 --
 
+[float]
+=== replication
+
+One row per replication, showing database replication statistics. Collected by querying pg_stat_replication.
+
+
+
+*`postgresql.replication.write_lag`*::
++
+--
+Elapsed time during committed WALs from primary to the replica.
+
+
+type: long
+
+--
+
+*`postgresql.replication.state`*::
++
+--
+Current WAL sender state.
+
+
+type: keyword
+
+--
+
+*`postgresql.replication.sync_priority`*::
++
+--
+Priority of this standby server for being chosen as the synchronous standby.
+
+
+type: long
+
+--
+
+*`postgresql.replication.backend_start`*::
++
+--
+Time when this process was started, i.e., when the client connected to this WAL sender.
+
+
+type: long
+
+--
+
+*`postgresql.replication.backend_xmin`*::
++
+--
+This standby's xmin horizon reported by hot_standby_feedback.
+
+
+type: long
+
+--
+
+*`postgresql.replication.replay_lag`*::
++
+--
+Elapsed time during committed WALs from primary to the replica. Fully committed to replica node.
+
+
+type: long
+
+--
+
+*`postgresql.replication.flush_lag`*::
++
+--
+The elapsed time during committed WALs from primary to the replica. WALs have already been flushed but have not yet been applied.
+
+
+type: long
+
+--
+
+*`postgresql.replication.pid`*::
++
+--
+Process ID of a WAL sender process.
+
+
+type: long
+
+--
+
+*`postgresql.replication.client.address`*::
++
+--
+IP address of the client connected to this WAL sender. If this field is null, it indicates that the client is connected via a Unix socket on the server machine.
+
+
+type: long
+
+--
+
+*`postgresql.replication.client.port`*::
++
+--
+TCP port number that the client is using for communication with this WAL sender, or -1 if a Unix socket is used.
+
+
+type: long
+
+--
+
+*`postgresql.replication.client.hostname`*::
++
+--
+Host name of the connected client, as reported by a reverse DNS lookup of client_addr. This field will only be non-null for IP connections, and only when log_hostname is enabled.
+
+
+type: long
+
+--
+
+*`postgresql.replication.replay_lsn`*::
++
+--
+Last transaction log position replayed into the database on this standby server.
+
+
+type: long
+
+--
+
+*`postgresql.replication.user.name`*::
++
+--
+Name of the user logged into this WAL sender process.
+
+
+type: keyword
+
+--
+
+*`postgresql.replication.user.id`*::
++
+--
+OID of the user logged into this WAL sender process.
+
+
+type: long
+
+--
+
+*`postgresql.replication.sync_state`*::
++
+--
+Synchronous state of this standby server.
+
+
+type: keyword
+
+--
+
+*`postgresql.replication.flush_lsn`*::
++
+--
+Last transaction log position flushed to disk by this standby server.
+
+
+type: long
+
+--
+
+*`postgresql.replication.application_name`*::
++
+--
+Name of the application that is connected to this WAL sender.
+
+
+type: keyword
+
+--
+
+*`postgresql.replication.write_lsn`*::
++
+--
+Last transaction log position written to disk by this standby server.
+
+
+type: long
+
+--
+
+*`postgresql.replication.sent_lsn`*::
++
+--
+Last transaction log position sent on this connection.
+
+
+type: long
+
+--
+
 [float]
 === statement
 

metricbeat/docs/modules/postgresql.asciidoc

@@ -96,6 +96,9 @@ metricbeat.modules:
     # Stats about every PostgreSQL process
     - activity
 
+    # Stats about every PostgreSQL replication process
+    - replication
+
     # Stats about every statement executed in the server. It requires the
     # `pg_stats_statement` library to be configured in the server.
     #- statement
@@ -126,6 +129,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-postgresql-database,database>>
 
+* <<metricbeat-metricset-postgresql-replication,replication>>
+
 * <<metricbeat-metricset-postgresql-statement,statement>>
 
 include::postgresql/activity.asciidoc[]
@@ -134,6 +139,8 @@ include::postgresql/bgwriter.asciidoc[]
 
 include::postgresql/database.asciidoc[]
 
+include::postgresql/replication.asciidoc[]
+
 include::postgresql/statement.asciidoc[]
 
 :edit_url!:

metricbeat/docs/modules/postgresql/replication.asciidoc

@@ -0,0 +1,29 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/postgresql/replication/_meta/docs.asciidoc
+
+
+[[metricbeat-metricset-postgresql-replication]]
+=== PostgreSQL replication metricset
+
+beta[]
+
+include::../../../module/postgresql/replication/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+:edit_url:
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-postgresql,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/postgresql/replication/_meta/data.json[]
+----
+:edit_url!:

metricbeat/docs/modules_list.asciidoc

@@ -247,9 +247,10 @@ This file is generated! See scripts/mage/docs_collector.go
 .2+| .2+|  |<<metricbeat-metricset-php_fpm-pool,pool>>   
 |<<metricbeat-metricset-php_fpm-process,process>>   
 |<<metricbeat-module-postgresql,PostgreSQL>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.4+| .4+|  |<<metricbeat-metricset-postgresql-activity,activity>>   
+.5+| .5+|  |<<metricbeat-metricset-postgresql-activity,activity>>   
 |<<metricbeat-metricset-postgresql-bgwriter,bgwriter>>   
 |<<metricbeat-metricset-postgresql-database,database>>   
+|<<metricbeat-metricset-postgresql-replication,replication>> beta[]  
 |<<metricbeat-metricset-postgresql-statement,statement>>   
 |<<metricbeat-module-prometheus,Prometheus>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .3+| .3+|  |<<metricbeat-metricset-prometheus-collector,collector>>   

metricbeat/metricbeat.reference.yml

@@ -814,6 +814,9 @@ metricbeat.modules:
     # Stats about every PostgreSQL process
     - activity
 
+    # Stats about every PostgreSQL replication process
+    - replication
+
     # Stats about every statement executed in the server. It requires the
     # `pg_stats_statement` library to be configured in the server.
     #- statement

metricbeat/module/postgresql/_meta/config.reference.yml

@@ -10,6 +10,9 @@
     # Stats about every PostgreSQL process
     - activity
 
+    # Stats about every PostgreSQL replication process
+    - replication
+
     # Stats about every statement executed in the server. It requires the
     # `pg_stats_statement` library to be configured in the server.
     #- statement

metricbeat/module/postgresql/replication/_meta/data.json

@@ -0,0 +1,42 @@
+{
+    "@timestamp": "2023-05-24T12:22:59.233Z",
+    "event": {
+        "dataset": "postgresql.replication",
+        "duration": 115000,
+        "module": "postgresql"
+    },
+    "metricset": {
+        "name": "replication",
+        "period": 10000
+    },
+    "postgresql": {
+        "replication": {
+            "write_lag": "0",
+            "state": "streaming",
+            "sync_priority": 0,
+            "backend_start": "2023-05-23T17:01:53.697Z",
+            "replay_lag": "0",
+            "flush_lag": "0",
+            "pid": 2516780,
+            "client": {
+                "address": "192.168.128.2",
+                "port": 46594,
+                "hostname": ""
+            },
+            "replay_lsn": "A0/60003558",
+            "user": {
+                "name": "replicator",
+                "id": 16442
+            },
+            "sync_state": "async",
+            "flush_lsn": "A0/60003558",
+            "application_name": "walreceiver",
+            "write_lsn": "A0/60003558",
+            "sent_lsn": "A0/60003558"
+        }
+    },
+    "service": {
+        "address": "postgres://localhost:5432/postgres?connect_timeout=10",
+        "type": "postgresql"
+    }
+}