From c5afb46b15de8a8edbc4fcfe040d4eb6cb090a00 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 31 Jan 2024 11:21:56 +0200 Subject: [PATCH 1/4] Auto-set 5s on-demand heartbeat if --enable_heartbeat is disabled Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/flags/endtoend/vtcombo.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- go/vt/vttablet/tabletserver/tabletenv/config.go | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index c648b2fe6ef..957c102462a 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -169,7 +169,7 @@ Flags: --healthcheck_timeout duration the health check timeout period (default 1m0s) --heartbeat_enable If true, vttablet records (if master) or checks (if replica) the current time of a replication heartbeat in the sidecar database's heartbeat table. The result is used to inform the serving state of the vttablet via healthchecks. --heartbeat_interval duration How frequently to read and write replication heartbeat. (default 1s) - --heartbeat_on_demand_duration duration If non-zero, heartbeats are only written upon consumer request, and only run for up to given duration following the request. Frequent requests can keep the heartbeat running consistently; when requests are infrequent heartbeat may completely stop between requests + --heartbeat_on_demand_duration duration If non-zero, heartbeats are only written upon consumer request, and only run for up to given duration following the request. Frequent requests can keep the heartbeat running consistently. Automatically set to 5s when --heartbeat_enable is unset. -h, --help help for vtcombo --hot_row_protection_concurrent_transactions int Number of concurrent transactions let through to the txpool/MySQL for the same hot row. Should be > 1 to have enough 'ready' transactions in MySQL and benefit from a pipelining effect. (default 5) --hot_row_protection_max_global_queue_size int Global queue limit across all row (ranges). Useful to prevent that the queue can grow unbounded. (default 1000) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index c7969a79d4a..99066ddd3da 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -196,7 +196,7 @@ Flags: --health_check_interval duration Interval between health checks (default 20s) --heartbeat_enable If true, vttablet records (if master) or checks (if replica) the current time of a replication heartbeat in the sidecar database's heartbeat table. The result is used to inform the serving state of the vttablet via healthchecks. --heartbeat_interval duration How frequently to read and write replication heartbeat. (default 1s) - --heartbeat_on_demand_duration duration If non-zero, heartbeats are only written upon consumer request, and only run for up to given duration following the request. Frequent requests can keep the heartbeat running consistently; when requests are infrequent heartbeat may completely stop between requests + --heartbeat_on_demand_duration duration If non-zero, heartbeats are only written upon consumer request, and only run for up to given duration following the request. Frequent requests can keep the heartbeat running consistently. Automatically set to 5s when --heartbeat_enable is unset. -h, --help help for vttablet --hot_row_protection_concurrent_transactions int Number of concurrent transactions let through to the txpool/MySQL for the same hot row. Should be > 1 to have enough 'ready' transactions in MySQL and benefit from a pipelining effect. (default 5) --hot_row_protection_max_global_queue_size int Global queue limit across all row (ranges). Useful to prevent that the queue can grow unbounded. (default 1000) diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 25352aba91b..829fcf2ebc7 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -183,7 +183,7 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) { fs.BoolVar(&enableHeartbeat, "heartbeat_enable", false, "If true, vttablet records (if master) or checks (if replica) the current time of a replication heartbeat in the sidecar database's heartbeat table. The result is used to inform the serving state of the vttablet via healthchecks.") fs.DurationVar(&heartbeatInterval, "heartbeat_interval", 1*time.Second, "How frequently to read and write replication heartbeat.") - fs.DurationVar(&heartbeatOnDemandDuration, "heartbeat_on_demand_duration", 0, "If non-zero, heartbeats are only written upon consumer request, and only run for up to given duration following the request. Frequent requests can keep the heartbeat running consistently; when requests are infrequent heartbeat may completely stop between requests") + fs.DurationVar(&heartbeatOnDemandDuration, "heartbeat_on_demand_duration", 0, "If non-zero, heartbeats are only written upon consumer request, and only run for up to given duration following the request. Frequent requests can keep the heartbeat running consistently. Automatically set to 5s when --heartbeat_enable is unset.") fs.BoolVar(¤tConfig.EnforceStrictTransTables, "enforce_strict_trans_tables", defaultConfig.EnforceStrictTransTables, "If true, vttablet requires MySQL to run with STRICT_TRANS_TABLES or STRICT_ALL_TABLES on. It is recommended to not turn this flag off. Otherwise MySQL may alter your supplied values before saving them to the database.") flagutil.DualFormatBoolVar(fs, &enableConsolidator, "enable_consolidator", true, "This option enables the query consolidator.") @@ -251,6 +251,9 @@ func Init() { if heartbeatOnDemandDuration < 0 { heartbeatOnDemandDuration = 0 } + if !enableHeartbeat { + heartbeatOnDemandDuration = 5 * time.Second + } currentConfig.ReplicationTracker.HeartbeatInterval = heartbeatInterval currentConfig.ReplicationTracker.HeartbeatOnDemand = heartbeatOnDemandDuration From f45b23cbe09b59946cd4e71372235837d0073fde Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 31 Jan 2024 12:29:35 +0200 Subject: [PATCH 2/4] only when heartbeatOnDemandDuration isn't already set Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/tabletenv/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 829fcf2ebc7..8deba62db2f 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -251,7 +251,7 @@ func Init() { if heartbeatOnDemandDuration < 0 { heartbeatOnDemandDuration = 0 } - if !enableHeartbeat { + if heartbeatOnDemandDuration == 0 && !enableHeartbeat { heartbeatOnDemandDuration = 5 * time.Second } currentConfig.ReplicationTracker.HeartbeatInterval = heartbeatInterval From b0b94dcc620b60f95c38b4d7d4248ff21db42994 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 31 Jan 2024 12:33:37 +0200 Subject: [PATCH 3/4] comment Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/tabletenv/config.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 8deba62db2f..decc9feaab6 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -252,6 +252,14 @@ func Init() { heartbeatOnDemandDuration = 0 } if heartbeatOnDemandDuration == 0 && !enableHeartbeat { + // Neither heartbeat-related flag was set. We want to always have some heartbeat capability, + // because the tablet throttler depends on the availability of a heartbeat. + // To that effect, we choose to set a minimal and relaxed heartbeat setup: a 5s on-demand lease. + // Ad on-demand heartbeats go, this has no impact when the throttler is disabled, and no impact + // when the throttler is enabled but otherwise not engaged with some consumer. Heartbeats are + // only leased and generated when the throttler is requested by some consumer (e.g. vreplication) + // to provide throttling advice. We keep the lease to a very low 5s, so that heartbeats are only + // generated while an active consumer is requesting them, and only up to 5s afterwards. heartbeatOnDemandDuration = 5 * time.Second } currentConfig.ReplicationTracker.HeartbeatInterval = heartbeatInterval From 02297853deb0598d526023cae323e66ddceafbaa Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 31 Jan 2024 14:23:17 +0200 Subject: [PATCH 4/4] unit test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../tabletserver/tabletenv/config_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/go/vt/vttablet/tabletserver/tabletenv/config_test.go b/go/vt/vttablet/tabletserver/tabletenv/config_test.go index 1cf1559ba6e..202d664f16e 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config_test.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config_test.go @@ -214,6 +214,7 @@ func TestFlags(t *testing.T) { want.Healthcheck.UnhealthyThreshold = 2 * time.Hour want.ReplicationTracker.HeartbeatInterval = time.Second want.ReplicationTracker.Mode = Disable + want.ReplicationTracker.HeartbeatOnDemand = 5 * time.Second assert.Equal(t, want.DB, currentConfig.DB) assert.Equal(t, want, currentConfig) @@ -266,21 +267,36 @@ func TestFlags(t *testing.T) { assert.Equal(t, want, currentConfig) enableHeartbeat = true + heartbeatOnDemandDuration = 0 heartbeatInterval = 1 * time.Second currentConfig.ReplicationTracker.Mode = "" currentConfig.ReplicationTracker.HeartbeatInterval = 0 Init() want.ReplicationTracker.Mode = Heartbeat want.ReplicationTracker.HeartbeatInterval = time.Second + want.ReplicationTracker.HeartbeatOnDemand = 0 assert.Equal(t, want, currentConfig) enableHeartbeat = false + heartbeatOnDemandDuration = 0 heartbeatInterval = 1 * time.Second currentConfig.ReplicationTracker.Mode = "" currentConfig.ReplicationTracker.HeartbeatInterval = 0 Init() want.ReplicationTracker.Mode = Disable want.ReplicationTracker.HeartbeatInterval = time.Second + want.ReplicationTracker.HeartbeatOnDemand = 5 * time.Second + assert.Equal(t, want, currentConfig) + + enableHeartbeat = false + heartbeatOnDemandDuration = 7 * time.Second + heartbeatInterval = 1 * time.Second + currentConfig.ReplicationTracker.Mode = "" + currentConfig.ReplicationTracker.HeartbeatInterval = 0 + Init() + want.ReplicationTracker.Mode = Disable + want.ReplicationTracker.HeartbeatInterval = time.Second + want.ReplicationTracker.HeartbeatOnDemand = 7 * time.Second assert.Equal(t, want, currentConfig) enableReplicationReporter = true