Consul client registers itself at the cluster, but it says no servers available

[EugenKon]

From here you can see, that Consul server sees the client_node, but client

Reproduction Steps

Deploy two consul servers within one cluster
Terminate client instance
Run new one

Consul info for both Client and Server

Client info

$ consul info
agent:
	check_monitors = 0
	check_ttls = 0
	checks = 2
	services = 3
build:
	prerelease =
	revision = 9f62fb41
	version = 1.19.1
	version_metadata =
consul:
	acl = enabled
	known_servers = 2
	server = false
runtime:
	arch = amd64
	cpu_count = 4
	goroutines = 56
	max_procs = 4
	os = linux
	version = go1.22.5
serf_lan:
	coordinate_resets = 0
	encrypted = false
	event_queue = 0
	event_time = 3
	failed = 0
	health_score = 0
	intent_queue = 0
	left = 0
	member_time = 106
	members = 4
	query_queue = 0
	query_time = 1

# https://developer.hashicorp.com/consul/docs/agent#configuring-consul-agents
datacenter = "xxx"

data_dir = "/data/consul"
server = false                              # Agent mode
# bootstrap_expect = SERVER_COUNT
# https://developer.hashicorp.com/nomad/docs/configuration/server_join#amazon-ec2
retry_join = ["provider=aws tag_key=ConsulAutoJoin tag_value=nomad-auto-join"]                 # The default value comes from `aws/variables.tf` file

# DNS
# https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_domain
domain = "xxx.internal"
# https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html
recursors = [ "169.254.169.253" ]
client_addr = "0.0.0.0"

# addresses = { }
bind_addr = "0.0.0.0"
advertise_addr = "172.31.31.36"

acl {
    enabled = true
    default_policy = "deny"
    down_policy = "extend-cache"
    # https://developer.hashicorp.com/consul/docs/agent/config/config-files#acl_tokens_default
    tokens {
      default = "xxxxxx"
    }
}

# Logging
# https://developer.hashicorp.com/consul/commands/monitor
enable_debug = false
log_level = "warn"
enable_syslog = true
log_rotate_max_files = 30
log_rotate_duration = "24h"

ui_config {
  enabled = false
}

# server_name =
tls {
  defaults {
    tls_min_version = "TLSv1_3"
  }
}

ports {
  dns = 8600
  grpc = 8502
}

# Service Mesh related
# https://developer.hashicorp.com/consul/docs/connect/configuration#agent-configuration
connect {
  enabled = true
}

# https://developer.hashicorp.com/consul/docs/services/usage/define-services#define-multiple-services-in-a-single-file
services {
  id      = "dns"
  name    = "dns"
  tags    = ["primary"]
  address = "localhost"
  port    = 8600

  # https://developer.hashicorp.com/consul/docs/services/usage/checks
  check {
    id       = "dns"
    name     = "Consul DNS TCP on port 8600"
    tcp      = "localhost:8600"
    interval = "10s"
    timeout  = "1s"
  }
}

services {
  name    = "consul"
}```

</details>

<details>
  <summary>Server info</summary>

consul info
Error querying agent: Unexpected response code: 403 (Permission denied: anonymous token lacks permission 'agent:read' on "nomad-control-plane-0-36ca". The anonymous token is used implicitly when a request does not specify a token.)```

# https://developer.hashicorp.com/consul/docs/agent#configuring-consul-agents
datacenter = "xxx"

data_dir = "/data/consul"
server = true                               # Agent mode
bootstrap_expect = 1             # Ignored in client mode
# https://developer.hashicorp.com/nomad/docs/configuration/server_join#amazon-ec2
retry_join = ["provider=aws tag_key=ConsulAutoJoin tag_value=nomad-auto-join"]                 # The default value comes from `aws/variables.tf` file

# DNS
# https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_domain
domain = "xxx.internal"
# https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html
recursors = [ "169.254.169.253" ]
client_addr = "0.0.0.0"

# addresses = { }
bind_addr = "0.0.0.0"
advertise_addr = "172.31.202.17"

acl {
    enabled = true
    default_policy = "deny"
    down_policy = "extend-cache"
}

# Logging
# https://developer.hashicorp.com/consul/commands/monitor
enable_debug = false
log_level = "warn"
enable_syslog = true
log_rotate_max_files = 30
log_rotate_duration = "24h"

ui_config {
  enabled = true
}

# server_name =
tls {
  defaults {
    tls_min_version = "TLSv1_3"
  }
}

ports {
  dns = 8600
  grpc = 8502
}

# Service Mesh related
# https://developer.hashicorp.com/consul/docs/connect/configuration#agent-configuration
connect {
  enabled = true
}

# https://developer.hashicorp.com/consul/docs/services/usage/define-services#define-multiple-services-in-a-single-file
services {
  id      = "dns"
  name    = "dns"
  tags    = ["primary"]
  address = "localhost"
  port    = 8600

  # https://developer.hashicorp.com/consul/docs/services/usage/checks
  check {
    id       = "dns"
    name     = "Consul DNS TCP on port 8600"
    tcp      = "localhost:8600"
    interval = "10s"
    timeout  = "1s"
  }
}

services {
  name    = "consul"
}

Operating system and Environment details

uname -a
Linux nomad-control-plane-0-36ca 6.8.0-1016-aws #17-Ubuntu SMP Mon Sep  2 13:48:07 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux

Log Fragments

By incident it looks like we have two consul servers and they conflict with each other in some way.

Still it is not clear why client node says that there is no servers available?

Expected

Client consul should report that there is two server available.