diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c63a2234573d..de2eab34db4b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -120,6 +120,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add context with timeout in AWS API calls {pull}35425[35425] - Fix EC2 host.cpu.usage {pull}35717[35717] - Resolve statsd module's prematurely halting of metrics parsing upon encountering an invalid packet. {pull}35075[35075] +- Add support for PostgreSQL replication metrics {pull}35562[35562] - Fix the gap in fetching forecast API metrics at the end of each month for Azure billing module {pull}36142[36142] - Add option in SQL module to execute queries for all dbs. {pull}35688[35688] - Fix Azure Monitor empty metricnamespace. {pull}36295[36295] @@ -129,6 +130,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add missing 'TransactionType' dimension for Azure Storage Account. {pull}36413[36413] - Add log error when statsd server fails to start {pull}36477[36477] + *Osquerybeat* diff --git a/libbeat/common/schema/mapstrstr/mapstrstr.go b/libbeat/common/schema/mapstrstr/mapstrstr.go index 8bde7d6f221c..b81f6b4ea841 100644 --- a/libbeat/common/schema/mapstrstr/mapstrstr.go +++ b/libbeat/common/schema/mapstrstr/mapstrstr.go @@ -129,6 +129,27 @@ func Int(key string, opts ...schema.SchemaOption) schema.Conv { return schema.SetOptions(schema.Conv{Key: key, Func: toInt}, opts) } +// toUint converts value to uint. In case of error, returns 0 +func toUint(key string, data map[string]interface{}) (interface{}, error) { + str, err := getString(key, data) + if err != nil { + return false, err + } + + value, err := strconv.ParseUint(str, 10, 64) + if err != nil { + msg := fmt.Sprintf("error converting param to uint: `%s`", str) + return 0, schema.NewWrongFormatError(key, msg) + } + + return value, nil +} + +// Uint creates a Conv object for parsing integers +func Uint(key string, opts ...schema.SchemaOption) schema.Conv { + return schema.SetOptions(schema.Conv{Key: key, Func: toUint}, opts) +} + // toStr converts value to str. In case of error, returns "" func toStr(key string, data map[string]interface{}) (interface{}, error) { return getString(key, data) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 6e7e895a56d4..2f72307e1350 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -56324,6 +56324,203 @@ type: date -- +[float] +=== replication + +One row per replication, showing database replication statistics. Collected by querying pg_stat_replication. + + + +*`postgresql.replication.write_lag`*:: ++ +-- +Elapsed time during committed WALs from primary to the replica. + + +type: long + +-- + +*`postgresql.replication.state`*:: ++ +-- +Current WAL sender state. + + +type: keyword + +-- + +*`postgresql.replication.sync_priority`*:: ++ +-- +Priority of this standby server for being chosen as the synchronous standby. + + +type: long + +-- + +*`postgresql.replication.backend_start`*:: ++ +-- +Time when this process was started, i.e., when the client connected to this WAL sender. + + +type: long + +-- + +*`postgresql.replication.backend_xmin`*:: ++ +-- +This standby's xmin horizon reported by hot_standby_feedback. + + +type: long + +-- + +*`postgresql.replication.replay_lag`*:: ++ +-- +Elapsed time during committed WALs from primary to the replica. Fully committed to replica node. + + +type: long + +-- + +*`postgresql.replication.flush_lag`*:: ++ +-- +The elapsed time during committed WALs from primary to the replica. WALs have already been flushed but have not yet been applied. + + +type: long + +-- + +*`postgresql.replication.pid`*:: ++ +-- +Process ID of a WAL sender process. + + +type: long + +-- + +*`postgresql.replication.client.address`*:: ++ +-- +IP address of the client connected to this WAL sender. If this field is null, it indicates that the client is connected via a Unix socket on the server machine. + + +type: long + +-- + +*`postgresql.replication.client.port`*:: ++ +-- +TCP port number that the client is using for communication with this WAL sender, or -1 if a Unix socket is used. + + +type: long + +-- + +*`postgresql.replication.client.hostname`*:: ++ +-- +Host name of the connected client, as reported by a reverse DNS lookup of client_addr. This field will only be non-null for IP connections, and only when log_hostname is enabled. + + +type: long + +-- + +*`postgresql.replication.replay_lsn`*:: ++ +-- +Last transaction log position replayed into the database on this standby server. + + +type: long + +-- + +*`postgresql.replication.user.name`*:: ++ +-- +Name of the user logged into this WAL sender process. + + +type: keyword + +-- + +*`postgresql.replication.user.id`*:: ++ +-- +OID of the user logged into this WAL sender process. + + +type: long + +-- + +*`postgresql.replication.sync_state`*:: ++ +-- +Synchronous state of this standby server. + + +type: keyword + +-- + +*`postgresql.replication.flush_lsn`*:: ++ +-- +Last transaction log position flushed to disk by this standby server. + + +type: long + +-- + +*`postgresql.replication.application_name`*:: ++ +-- +Name of the application that is connected to this WAL sender. + + +type: keyword + +-- + +*`postgresql.replication.write_lsn`*:: ++ +-- +Last transaction log position written to disk by this standby server. + + +type: long + +-- + +*`postgresql.replication.sent_lsn`*:: ++ +-- +Last transaction log position sent on this connection. + + +type: long + +-- + [float] === statement diff --git a/metricbeat/docs/modules/postgresql.asciidoc b/metricbeat/docs/modules/postgresql.asciidoc index b09ade4bd127..fcc35873d070 100644 --- a/metricbeat/docs/modules/postgresql.asciidoc +++ b/metricbeat/docs/modules/postgresql.asciidoc @@ -96,6 +96,9 @@ metricbeat.modules: # Stats about every PostgreSQL process - activity + # Stats about every PostgreSQL replication process + - replication + # Stats about every statement executed in the server. It requires the # `pg_stats_statement` library to be configured in the server. #- statement @@ -126,6 +129,8 @@ The following metricsets are available: * <> +* <> + * <> include::postgresql/activity.asciidoc[] @@ -134,6 +139,8 @@ include::postgresql/bgwriter.asciidoc[] include::postgresql/database.asciidoc[] +include::postgresql/replication.asciidoc[] + include::postgresql/statement.asciidoc[] :edit_url!: diff --git a/metricbeat/docs/modules/postgresql/replication.asciidoc b/metricbeat/docs/modules/postgresql/replication.asciidoc new file mode 100644 index 000000000000..0cda7f433b14 --- /dev/null +++ b/metricbeat/docs/modules/postgresql/replication.asciidoc @@ -0,0 +1,29 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/postgresql/replication/_meta/docs.asciidoc + + +[[metricbeat-metricset-postgresql-replication]] +=== PostgreSQL replication metricset + +beta[] + +include::../../../module/postgresql/replication/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/postgresql/replication/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 2a77f4d38cd3..79ddbeb03712 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -247,9 +247,10 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.4+| .4+| |<> +.5+| .5+| |<> |<> |<> +|<> beta[] |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .3+| .3+| |<> diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 434d2d7fc72c..88f6d0a06106 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -142,6 +142,7 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/postgresql/activity" _ "github.com/elastic/beats/v7/metricbeat/module/postgresql/bgwriter" _ "github.com/elastic/beats/v7/metricbeat/module/postgresql/database" + _ "github.com/elastic/beats/v7/metricbeat/module/postgresql/replication" _ "github.com/elastic/beats/v7/metricbeat/module/postgresql/statement" _ "github.com/elastic/beats/v7/metricbeat/module/prometheus" _ "github.com/elastic/beats/v7/metricbeat/module/prometheus/collector" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index eabdcc8e918e..cc30fbb74d24 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -815,6 +815,9 @@ metricbeat.modules: # Stats about every PostgreSQL process - activity + # Stats about every PostgreSQL replication process + - replication + # Stats about every statement executed in the server. It requires the # `pg_stats_statement` library to be configured in the server. #- statement diff --git a/metricbeat/module/postgresql/_meta/config.reference.yml b/metricbeat/module/postgresql/_meta/config.reference.yml index 3b4ed4579d11..8891d1d1cf8a 100644 --- a/metricbeat/module/postgresql/_meta/config.reference.yml +++ b/metricbeat/module/postgresql/_meta/config.reference.yml @@ -10,6 +10,9 @@ # Stats about every PostgreSQL process - activity + # Stats about every PostgreSQL replication process + - replication + # Stats about every statement executed in the server. It requires the # `pg_stats_statement` library to be configured in the server. #- statement diff --git a/metricbeat/module/postgresql/fields.go b/metricbeat/module/postgresql/fields.go index cd679d7edc24..630e3c950e31 100644 --- a/metricbeat/module/postgresql/fields.go +++ b/metricbeat/module/postgresql/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPostgresql returns asset data. // This is the base64 encoded zlib format compressed contents of module/postgresql. func AssetPostgresql() string { - return "eJzUWk+P47oNv8+nIN7l7RbZoL3OoUCxr0AX6O7bh92ix4CR6ViILHklOZn00xeU5D+xncwkY8+2Oc0kFvkTKZE/kv4Aezo9QmWc31lyP9QDgJde0SP88jV++e2Pf/7yAJCRE1ZWXhr9CH99AAD4TN5K4UAYpUh4yiC3poRuHTiyB7Ju/QDgCmP9Rhidy90j5KgcPQBYUoSOHmGHDwC5JJW5xyD8A2gsaQCNP/5U8fPW1FX6ZgIaf3o4yoh0nX7r6+nrQuHlQfpT+8OUtisa+fO7JsiMqEvSHiqyyQZQWSPIuRUb4ij1DqTOjS2RZbAZkO3nDfiCQNTWkvZnchtsYHLwBfqewFoUgA6cR0+AOmvWw4+a7GkNH1v/bE9nMsPvjKXabXj1plGy7j127qLmMzRh34wZetyio7WR2dkDjTmV0bvBD1csGqz66be4cWqlgy+kgy2KPekMJB9DreM2vVlfB8b/TiLb0+lo7BD1M+C+YEkzoKtms9bXeDSgMVqHZFpz7ciul/AVCwZldjvKQOpwul+EZcI/N/jgDq1YVUqKcBk3r1PekxTv6cD3LwAjlCTt15hllpy7Dcqnr5DWNYCitDsxFMb52+3xD+N8kNNiaJVHuSuOV5YqY2NUAgRLnCkIfvvyDZQx+7rixfHxDW/pKk6WNNPx/f7xK7A40HW5JRud2DOkdFA7Dpq5sSBMWda68fdR+iLYdiQ02XoFxsKHv4DMAeFfWj6BM2JPSShd8EVavOEd3biXU9X5ICWFxu2cH53cKgqWcoCWAGtvDijqugSFtRYF2VX/y6Oxe7KrkR5ldlKgYpe2h78TMPVrkgQVWlSKVPsFw+N0q4fhCOBopedHkiNam4qCxL4yUodfnUfr62oFR1SWBMkDf3tkwqEzsiFBHlFFYeuRkr8/edJOGu2gxBNY2knnySZ8LvoYs0zyNlC1IT4Y8br/ArLJU5qhv9WzsiQ4FqTjXU5kAI6RB/C1WoFc03rVPDQZCEZi+blIWKa34i1qxyzB6DfYzq/toe3p7e9xGmSgNYvBa2+SOkVSRlHhue2N5UseKZl0oM0QSmJ01HOQQufHsqb3GCRvRIF6N81kXrvJCJ1xBFhR0wUwR5RejuJsxLE1RhHqG6HYmth+IxLVWj6pBKMBQRmxv2Km23R/TEfOHIhDUzLEkEd10fOAqo7hs2PDI6EAf0r+foTvBfX3RE8k6rAXTIR9crXM1HhtYwVORQiajt0lL0sc5va+KJC6f6lGkiXbtffACra1v3SS+dO55oYNDVDAO9wGSvCe8UjX3R8nS6nQMndJ6yZBnAGmJ0GVB6PbFBjEcWEW9sff9JUL5BQ8KRc1kLVmIl3wTnJ0vkJfQF7rRpRSVx3NSz6crZkWnUmHW0XZ0B4td+JLYlHsm9JNkuPfm3Vxn8+RveClG28oPfnhpfjVQcnMj7NuV31+6oXBFC/DolBBjuRydewGUbYzXCNSA99M4wuur1m4W4H03eIxQelCa+BzHNei2GsxbUOHYRn+rGH+jdJDWBcZsIxH71oQew7APcSviLSOXRTBcIg4FlIUQzgjEC1t2UWO9JpuyDePXjovhQPcmtq3yiPFS5SuzfeubXP0uxaRbg/d2vQsGpjjWHN756Jjkm7tREFZrUYR4d664kssJ0wOreS+vnguCzwQbIk0VGRzY8tLx7OP1NKPmpxfAGkreSakXpbk1sFf63JYJUW4uTJ445X7bjwqwNLUOsYkJjIJpIsYXRVKtxj0OXRylOb6sgU3kprOJB+9Y0GWIJcqlUm8Ac88yXCg3a9YcCmVko6E0dmFMmBsCHfS4v/ZDoy/sEbL/8S2wg3G2NZ5Ttate0aZ/fQmHa27stryLvp+uI5tgrjOj2p7mg6KL8C2yWulZgcYzuaFQO28qSrKACEAYHM6gRq2FNgTyPH5KTDrXRgDJepTu42re0xJavlzIS0JzsehEXWVKQ2gbXK+Agt5oIUSTOhNQ1hAegfmqCEoD1wT3mljS1RqyOLggh8L1JkKXjaOAkPoKr9Ga2aYS0Zd072Y91dthEoZgUukpcaBrYbLxZ/bWHI0ax+A6WNDpFwsKhLFOXJ4DFQzKB1TqmY68BpK9bsmsOYYZkuNvG6q1Hzz4SizPrbzKVA7+ZlkVBMob6dSP2/2E5ovfw6E1xVoKWNvmNqKS/25nzwNWgGVlT/dAjjchI3JN0nkArkzCe5VK+MJQjtPe65X6NbClOUoOcwQKXs62lq3Z/UzohoxXAwXZ3itUYpt8HMRMwq+sHipv7VVRuy5AsD5oyyzuaQAWMEI7VVIxRLeDnmxjytE3DxkNlQM8tSQ25gjQKAoODxOtbrRh7kT8xMMjS3QxFQX7QnehZ0arVigUHVGDgrZNY66lwtGgs81s1heYCqyGJoY7uQ8lb+6QKTTf/Hp91ctyrsPnr5UMtwx6+J8FmsCFtwkkIgsmXh76oLB8ASsphpxLyD/vQ1dLQZftSOW/FY7subIt9DXVi9RhpsjX8EovUnfodP3ohsZwOXkRbEUtiT8TmhSO7KLtC8YWyP9TnB1lS3CYQO2JPxOaBkpWgxaEn47NGF0rqRYoJ5vcAjUgjgvZjUxF2k1xuGsJWEOZE8N3pHEZ2gLlZWxaE/r0OiYP4k18lMjRVh69gzA30Z1PowEoSUQptZh/mhph5ZLvPAeyLGITYbzJZz2RlIbOO9ovVtz4rRxzsY1oyuk3r1fhRH6uYIwvjS7DSvYTNkNwJG/3OzujL49+dmMPmyIhUzQa94NLTh2weWzwy65wwXPeDEwkeSCJeycEWYhCc7PFhvJkJGPZcLL4sX/TpU+KtfbeeFr6/WzF0LjGIr/Ci+qTdfxE2+HnkmV+mDSqzTNC6FBbvc6qKhqqB3u4iuhPlyFwLhueB+0G5i+bq7yli8Ydg2sYBWLkbBPTKLf/H3V/psittYwnPqezWPns1eHpfXnNb2enm4cPPwRB6vDdediBSq1QDZtm9XRtO2kw9ZXjcscZ9Yso6+z9QtHL1mcr2W4obNPgzpY/SKmqflqrbux+nMAS6lnhPdZalnW5awA8WlOgPg0O0DCiya8/dx9JtRzonM+y+gwH76vpqpVTFHOo87QZpDRQXZZq9d96CN94QgxQi+pNPa0jo3TGbtOw+uTOrOhhRC7NbEflMZ3z5r4HOeMDbsXAA09rvuAZtJ6OVuN+QKsSeGdcBO5fzu4gynuS+EqI1AteFqD/Fcf1ohywbM6hnnPUY0wlz2pY6R3HtQIdtlzOgZ75zHlMnNJ/7P8V7s/gFzWoCOc0/b8bwAAAP//1pQyXQ==" + return "eJzUW02P4zbSvs+vKOQyMy88xrvXPiwQJFnsAJOkg5lFjkaZKllEU6RCUu1Wfv2iSOrDkuz2h9zJ+tRti8WHVcXiU8XSJ3ii5gEq4/zOkvtDvQPw0it6gO8e45dff/vy3TuAjJywsvLS6Af45zsAgJ/JWykcCKMUCU8Z5NaU0I8DR/aZrFu/A3CFsX4jjM7l7gFyVI7eAVhShI4eYIfvAHJJKnMPQfgn0FjSCBp/fFPx89bUVfpmBhp/BjjKiHSdfhvOM5wLhZfP0jfdD3OznZiRP79qgsyIuiTtoSKbdACVNYKcW7Ei9lLvQOrc2BJZBqsBWX/egC8IRG0taX8gt8UGJgdfoB8IrEUB6MB59ASos3Y8/FGTbdbwQ2efbXMgM/zOWKrdhkdv2knWg8cOTdR+xiocqjFDj1t0tDYyO3igVacyejf64YRGg1Y//xgXTp108IV0sEXxRDoDyW6odVymN+vTwPjfWWRP1OyNHaN+BdwvWNIC6KrFtPUYXQNapfVI5meuHdn1PWzFgkGZ3Y4ykDp491lYZuxzgQ2umBWrSkkRNuPmtskHkuI+Hdn+DDBCSdJ+jVlmybnLoHx+hDSuBRSlXYmhMM5fro9/G+eDnA5DN3mUu+J4ZakyNkYlQLDEJwXBj798BWXMU13x4Pj4hpd0EidLWsh9v/3wCCwOdF1uyUYjDhQpHdSOg2ZuLAhTlrVu7b2Xvgi6nQhNul6BsfDpHyBzQPiPli/gjHiiJJSO2CIN3vCKLlxLU/U2SIdCa3Y+H53cKgqacoCWAGtvnlHUdQkKay0Ksqvhl3tjn8iuJvMos5MCFZu0c/5ewNyvSRJUaFEpUt0XDI+PWz0ORwB7Kz0/kgzR6VQUJJ4qI3X41Xm0vq5WsEdlSZB85m/3TDh0RjYckHtUUdh6MslPL560k0Y7KLEBSzvpPNmEz0UbY5ZJXgaqLsQHJZ62X0A266UZ+kstK0uCfUE67uVEBmAfeQBvqxXINa1X7UOzgWAilp+LhGV+Kd6idswSjH6D5bzvnHYw73CN8yADrbkbvG4nqSaSMooTHureWN7kkZJJB9qMoSRGRwMDKXR+Kmt+jUHyRhSod/NM5tZFRuiMI8CKMx0Bs0fp5STORhxbYxShvhCKrYn1NyFRnebTlGA0ICgjnk6o6bK5f0guZ56JQ1NSxJhH9dHzGVUdw2fPhidCAf4v2fsBvhU0XBO9kKjDWjAR9tnRMlPTsa0W+ChC0LTvN3lZ4vhsH4oCqYebaiJZsl4HD6xgW/tjnsyf3jQXLGiEAj7gNlCCj4xHun7/OFlKhZa5Sxo3C+IAML0IqjwY3R2BQRwnZmF9/M1wcoF8BM/KRQ1krZk5LnglOTpfoS8gr3UrSqmThuYhnw7GzIvOpMOtomysj4478SaxKJ7a1E2S49/bcXGdr5G9YKULdyi9+PGmeO+gZObHp26ffX4ehMEUL8OgkEFO5HJ27EZRtldcK1ID70zjC86vWbhbgfT94ClB6UNr4HMc16LYUzFtQ8/jNPxVxfyO0kMYFxmwjK53Koi9BuAa4ldEWscmimA4ROwLKYoxnAmIjrbsIke6pRry1aOXzkvhALem9t3kkeIlSted964rcwyrFpFuj83a1ixamNNYc3nlomeSbu1EQVmtJhHh2rzil5hOmBw6ycP5ol8W+EywJdJQkc2NLY+55xCppT9qcv4OSDvJCyH1siS3DvZal+MsKcLNlcELt9w341EBlqbWMSYxkUkgXcToqpC6xaDPoZOjNOeXHbiJ1OST7Hr7gixBLlVKk3gBnnmS4UD7tGLBpVRKOhJGZ0fSgKkiXKPF/7IeGH9hjZZ/xrLCBcrY1nlO1q0HSlnce9Mcnbmy2vIqhnY4jW2GuC6PatvMB8UzsG3yWqnFAQbfPBKonTdVRRkgBACsTidQw5YCewI59Z8Cs8GGMVCibrplnFxjOqTu7xfSkuDzOBSiTjKlEbRNzlvgThbooAQVetMSFpDegdlrCJMHrgkftLElKjVmcXDEjgXqTAUrG0eBIfSZXztrZphLxrnmazEfT+oIlTIC73EstQbsZjie/LmNJUeL1gGYPrZEysWkIlGcPYfHQDXDpFNK1d4O3EKpftUE1uzD3VIrr79Var/5tJfZENvhLVB38zPLqGZQXk6l/rq7n1B8+f9AeF2BljK2hqmtOFaf+4tvg1ZAZeWbSwCHnbAx+SaJvMPZmQQPspXpDUJ3n/ZardCthSnLyeGwQKQczNHlugOtHxDViOFouDjAa41SrIO/FjGj4A2Lx+pbW2XEE2cAuHyUZTaXJgCeYIL2JKTiHtYO5+IQV4i4eTjZUDHIpiW38YwAgaLg8DhX6kYf7p2Yn2AobIEmprpoG/gQVmq0YoFC1Rk5KGRfOOqbCyaCD2dmsTzAVGQxFDFc4zyV710g0um/+PTHkxrl1QdLH0sZrrjr4vMs5gQsuD1AIrKk4m3TB4OxB6zmCnFnkP/Bgk4mgzetiCW/1Yqs2fMu9LXV90jDzZ63YJTeHt+h0nfWjgzgcvKiuBe2JPxKaFI7sncpXzC2VvqV4OoquwuHDdiS8CuhZaTobtCS8MuhCaNzJcUd8vkWh0AtiM/FrCbmIt2M8XLWkjDPZJsW70TiK7SFyspYtM06FDqWP8Ra+amQIiy96gPw/STPh4kgtATC1DrcP1raoeUUL/SB7ItYZDgcwsfeRGoL5wOtd2s+OG28Z+Oc0RVS7z6uwhX64QTh+tLsNjzBZk5vAI788WJ3r/Rt4xdT+rggFk6CQfFurMGpCY77DpvkChO8YsXARJIJ7qHnjDALh+DybLGVDBn5mCacFy/+Pln6JF0fNKwslbEPRE6T9oMOmety94GE2auQLflzM/jAzDYK5+/1L/eVnxRWjrNHNk9bi21TMvj9+y8u9g9XVpa8H1IzbFrR+ff7N+Tu7d3/799/gdQuFGY4Mnmjxaay0tjDnuFblPSYxHV3q86jzrZN20acGwtbimVs40gDxlJtW4o3dTfk2makK0n3zc1IcXCv+tP4X8oJUb8a/kDP7x2wZCiMlX+Gzuy+SbEwfpMe2+RE2fG0nH0Wm7/p3uHPv2oVKjqtBG/aZ0Gb7Ii/56p2xYKL+lYQ0MILCyNCAaWtB4RCSoDOVqxTeYXT8oZ8/DX07B4rC92rLRuHUSZtmSv6ga8Fc0Gj8LEtGcTkg26O0B9UKxWaMaTO+CAiN9NBO2Wd3aTPEke9sUYP2hKhRFFIfSz3+Bt1A09Edd3BA4Ve1xB8ujn72qXe0LU9kXW8iztG2+gue6lUrLFteUfqT3p6nwlBs58fWzTSaBeZcRgYDhRmxK0yQqeVDs1Pp2OzW+r4+MJUcthIpswOKuNkerWHp+vfRBhcChg9d8Rf9lrEQhcV869MnBug3vj9kXNhBX62MEP8esizBp2h51gxnaBv5HrtoZc6M2KnwZlIX3khZiG3O+9lmdcoYUpU3kiro+afi7TqOAy+FVAXuphTjOmj5/RWumuLvTXJPXjvMXZb8l9hF89fV8+8BHkgVepnk/yjfe8xyO3fehRVDbXDXXzz0YeKT7hYuOC1x74v+Lb2wbeMg32fRtCKxciSZhqu3/y1zOELEbbWMG5uPmg7Xk5fPZbOnqfm9fRyYX/db7F/eDzuUKxApe5QNO56sqJqu4Y+W59UrjX7ZYup+vSl1BHXSxrnbRl26OJNjz2s4V1de7VZa913j78GsJR6QXg/Sy3LulwUIL4sCRBfFgdIeFSFl/vdz4R6SXTOZxk9L4fv0VS16kq2OkObQUbPsj+1BpfsQ6RndspG6CWVxjbr2B+0YHPFePukBqRwUx6bEmLbQ+pSfVXFhzgX7Es5A2ho5bgOaCatl4tdpZ6BNU14JdzEQd8O7qhZ+Vy4yghUd/TWIP9mZ40o7+irU5jXuGqEeV9PnSK90lEj2Pv66RTslW7qqazuaX+Wf7P5A8j7KnSCc16f/w0AAP//9q5frw==" } diff --git a/metricbeat/module/postgresql/replication/_meta/data.json b/metricbeat/module/postgresql/replication/_meta/data.json new file mode 100644 index 000000000000..c1d29d778cac --- /dev/null +++ b/metricbeat/module/postgresql/replication/_meta/data.json @@ -0,0 +1,42 @@ +{ + "@timestamp": "2023-05-24T12:22:59.233Z", + "event": { + "dataset": "postgresql.replication", + "duration": 115000, + "module": "postgresql" + }, + "metricset": { + "name": "replication", + "period": 10000 + }, + "postgresql": { + "replication": { + "write_lag": "0", + "state": "streaming", + "sync_priority": 0, + "backend_start": "2023-05-23T17:01:53.697Z", + "replay_lag": "0", + "flush_lag": "0", + "pid": 2516780, + "client": { + "address": "192.168.128.2", + "port": 46594, + "hostname": "" + }, + "replay_lsn": "A0/60003558", + "user": { + "name": "replicator", + "id": 16442 + }, + "sync_state": "async", + "flush_lsn": "A0/60003558", + "application_name": "walreceiver", + "write_lsn": "A0/60003558", + "sent_lsn": "A0/60003558" + } + }, + "service": { + "address": "postgres://localhost:5432/postgres?connect_timeout=10", + "type": "postgresql" + } +} diff --git a/metricbeat/module/postgresql/replication/_meta/data_shared.json b/metricbeat/module/postgresql/replication/_meta/data_shared.json new file mode 100644 index 000000000000..a5273171c017 --- /dev/null +++ b/metricbeat/module/postgresql/replication/_meta/data_shared.json @@ -0,0 +1,42 @@ +{ + "@timestamp": "2023-05-24T12:22:59.233Z", + "event": { + "dataset": "postgresql.replication", + "duration": 115000, + "module": "postgresql" + }, + "metricset": { + "name": "replication", + "period": 10000 + }, + "postgresql": { + "replication": { + "write_lag": "0", + "state": "", + "sync_priority": 0, + "backend_start": "", + "replay_lag": "0", + "flush_lag": "0", + "pid": 0, + "client": { + "address": "", + "port": 0, + "hostname": "" + }, + "replay_lsn": "", + "user": { + "name": "", + "id": 0 + }, + "sync_state": "", + "flush_lsn": "", + "application_name": "", + "write_lsn": "", + "sent_lsn": "" + } + }, + "service": { + "address": "postgres://localhost:5432/postgres?connect_timeout=10", + "type": "postgresql" + } +} diff --git a/metricbeat/module/postgresql/replication/_meta/docs.asciidoc b/metricbeat/module/postgresql/replication/_meta/docs.asciidoc new file mode 100644 index 000000000000..b5377bf6548b --- /dev/null +++ b/metricbeat/module/postgresql/replication/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `replication` metricset of the PostgreSQL module. diff --git a/metricbeat/module/postgresql/replication/_meta/fields.yml b/metricbeat/module/postgresql/replication/_meta/fields.yml new file mode 100644 index 000000000000..103e2a357ba3 --- /dev/null +++ b/metricbeat/module/postgresql/replication/_meta/fields.yml @@ -0,0 +1,90 @@ +- name: replication + type: group + description: > + One row per replication, showing database replication statistics. Collected by querying + pg_stat_replication. + release: beta + fields: + - name: write_lag + type: long + description: > + Elapsed time during committed WALs from primary to the replica. + - name: state + type: keyword + description: > + Current WAL sender state. + - name: sync_priority + type: long + description: > + Priority of this standby server for being chosen as the synchronous standby. + - name: backend_start + type: long + description: > + Time when this process was started, i.e., when the client connected to this WAL sender. + - name: backend_xmin + type: long + description: > + This standby's xmin horizon reported by hot_standby_feedback. + - name: replay_lag + type: long + description: > + Elapsed time during committed WALs from primary to the replica. + Fully committed to replica node. + - name: flush_lag + type: long + description: > + The elapsed time during committed WALs from primary to the replica. + WALs have already been flushed but have not yet been applied. + - name: pid + type: long + description: > + Process ID of a WAL sender process. + - name: client.address + type: long + description: > + IP address of the client connected to this WAL sender. + If this field is null, it indicates that the client is + connected via a Unix socket on the server machine. + - name: client.port + type: long + description: > + TCP port number that the client is using for communication + with this WAL sender, or -1 if a Unix socket is used. + - name: client.hostname + type: long + description: > + Host name of the connected client, as reported by a reverse + DNS lookup of client_addr. This field will only be non-null + for IP connections, and only when log_hostname is enabled. + - name: replay_lsn + type: long + description: > + Last transaction log position replayed into the database on this standby server. + - name: user.name + type: keyword + description: > + Name of the user logged into this WAL sender process. + - name: user.id + type: long + description: > + OID of the user logged into this WAL sender process. + - name: sync_state + type: keyword + description: > + Synchronous state of this standby server. + - name: flush_lsn + type: long + description: > + Last transaction log position flushed to disk by this standby server. + - name: application_name + type: keyword + description: > + Name of the application that is connected to this WAL sender. + - name: write_lsn + type: long + description: > + Last transaction log position written to disk by this standby server. + - name: sent_lsn + type: long + description: > + Last transaction log position sent on this connection. diff --git a/metricbeat/module/postgresql/replication/data.go b/metricbeat/module/postgresql/replication/data.go new file mode 100644 index 000000000000..1c2ccb345cad --- /dev/null +++ b/metricbeat/module/postgresql/replication/data.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package replication + +import ( + "time" + + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" +) + +// Based on https://www.postgresql.org/docs/9.5/monitoring-stats.html#PG-STAT-REPLICATION-VIEW +var schema = s.Schema{ + "pid": c.Int("pid"), + "user": s.Object{ + "id": c.Int("usesysid", s.Optional), + "name": c.Str("usename"), + }, + "application_name": c.Str("application_name"), + "client": s.Object{ + "address": c.Str("client_addr", s.Optional), + "hostname": c.Str("client_hostname", s.Optional), + "port": c.Int("client_port", s.Optional), + }, + "backend_start": c.Time(time.RFC3339Nano, "backend_start"), + "backend_xmin": c.Uint("backend_xmin", s.Optional), + "state": c.Str("state"), + "sent_lsn": c.Str("sent_lsn"), + "write_lsn": c.Str("write_lsn"), + "flush_lsn": c.Str("flush_lsn"), + "replay_lsn": c.Str("replay_lsn"), + "write_lag": c.Uint("write_lag", s.Optional), + "flush_lag": c.Uint("flush_lag", s.Optional), + "replay_lag": c.Uint("replay_lag", s.Optional), + "sync_priority": c.Int("sync_priority", s.Optional), + "sync_state": c.Str("sync_state"), +} diff --git a/metricbeat/module/postgresql/replication/replication.go b/metricbeat/module/postgresql/replication/replication.go new file mode 100644 index 000000000000..6c9365debbb2 --- /dev/null +++ b/metricbeat/module/postgresql/replication/replication.go @@ -0,0 +1,72 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package replication + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + + // Register postgresql as the sql driver + _ "github.com/lib/pq" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("postgresql", "replication", New, + mb.WithHostParser(postgresql.ParseURL), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet. +type MetricSet struct { + *postgresql.MetricSet +} + +// New create a new instance of the postgresql replication MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := postgresql.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} + +// Fetch method implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + ctx := context.Background() + results, err := m.QueryStats(ctx, "SELECT EXTRACT(EPOCH FROM write_lag)::INT AS write_lag, EXTRACT(EPOCH FROM flush_lag)::INT AS flush_lag, EXTRACT(EPOCH FROM replay_lag)::INT AS replay_lag, pid, usesysid, usename, application_name, client_addr, client_hostname, client_port, backend_start, backend_xmin, state, sent_lsn, write_lsn, replay_lsn, sync_priority, sync_state FROM pg_stat_replication;") + if err != nil { + return fmt.Errorf("error in QueryStats: %w", err) + } + + for _, result := range results { + data, _ := schema.Apply(result) + reporter.Event(mb.Event{ + MetricSetFields: data, + }) + } + + return nil +} diff --git a/metricbeat/module/postgresql/replication/replication_integration_test.go b/metricbeat/module/postgresql/replication/replication_integration_test.go new file mode 100644 index 000000000000..aafcde204e45 --- /dev/null +++ b/metricbeat/module/postgresql/replication/replication_integration_test.go @@ -0,0 +1,98 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration +// +build integration + +package replication + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/tests/compose" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + "github.com/elastic/elastic-agent-libs/mapstr" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetch(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d: %v", len(errs), errs) + } + require.NotEmpty(t, events) + event := events[0].MetricSetFields + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) + + // Check event fields + db_oid := event["oid"].(int64) + assert.True(t, db_oid >= 0) + assert.Contains(t, event, "name") + _, ok := event["name"].(string) + assert.True(t, ok) + + rows := event["rows"].(mapstr.M) + assert.Contains(t, rows, "returned") + assert.Contains(t, rows, "fetched") + assert.Contains(t, rows, "inserted") + assert.Contains(t, rows, "updated") + assert.Contains(t, rows, "deleted") +} + +func TestData(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + + getOid := func(event mapstr.M) int { + oid, err := event.GetValue("postgresql.database.oid") + require.NoError(t, err) + + switch oid := oid.(type) { + case int: + return oid + case int64: + return int(oid) + } + t.Log(event) + t.Fatalf("no numeric oid in event: %v (%T)", oid, oid) + return 0 + } + + f := mbtest.NewFetcher(t, getConfig(service.Host())) + f.WriteEventsCond(t, "", func(event mapstr.M) bool { + return getOid(event) != 0 + }) + f.WriteEventsCond(t, "./_meta/data_shared.json", func(event mapstr.M) bool { + return getOid(event) == 0 + }) +} + +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "postgresql", + "metricsets": []string{"replication"}, + "hosts": []string{postgresql.GetDSN(host)}, + "username": postgresql.GetEnvUsername(), + "password": postgresql.GetEnvPassword(), + } +} diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index fa15aca7fb6f..974d689fa8e4 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1249,6 +1249,9 @@ metricbeat.modules: # Stats about every PostgreSQL process - activity + # Stats about every PostgreSQL replication process + - replication + # Stats about every statement executed in the server. It requires the # `pg_stats_statement` library to be configured in the server. #- statement