Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Report own_peer address in HealthCheck method #48

Merged
merged 1 commit into from
Jan 10, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions config.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,10 @@ type Config struct {
// using multi data center support.
DataCenter string

// (Optional) This is the AdvertiseAddr of the instance. This value will be used to
// find the peer that is owned by this instance
AdvertiseAddr string

// (Optional) A Logger which implements the declared logger interface (typically *logrus.Entry)
Logger FieldLogger

Expand Down
1 change: 1 addition & 0 deletions daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ func (s *Daemon) Start(ctx context.Context) error {
Workers: s.conf.Workers,
InstanceID: s.conf.InstanceID,
EventChannel: s.conf.EventChannel,
AdvertiseAddr: s.conf.AdvertiseAddress,
}

s.V1Server, err = NewV1Instance(s.instanceConf)
Expand Down
20 changes: 18 additions & 2 deletions gubernator.go
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,7 @@ func (s *V1Instance) HealthCheck(ctx context.Context, r *HealthCheckReq) (health
span := trace.SpanFromContext(ctx)

var errs []string
ownPeerAddress := ""

s.peerMutex.RLock()
defer s.peerMutex.RUnlock()
Expand All @@ -554,6 +555,10 @@ func (s *V1Instance) HealthCheck(ctx context.Context, r *HealthCheckReq) (health
span.RecordError(err)
errs = append(errs, err.Error())
}

if ownPeerAddress == "" && peer.Info().GRPCAddress == s.conf.AdvertiseAddr {
ownPeerAddress = peer.Info().GRPCAddress
}
}

// Do the same for region peers
Expand All @@ -564,18 +569,29 @@ func (s *V1Instance) HealthCheck(ctx context.Context, r *HealthCheckReq) (health
span.RecordError(err)
errs = append(errs, err.Error())
}

if ownPeerAddress == "" && peer.Info().GRPCAddress == s.conf.AdvertiseAddr &&
peer.Info().DataCenter == s.conf.DataCenter {
ownPeerAddress = peer.Info().GRPCAddress
}
}

health = &HealthCheckResp{
PeerCount: int32(len(localPeers) + len(regionPeers)),
Status: Healthy,
PeerCount: int32(len(localPeers) + len(regionPeers)),
Status: Healthy,
AdvertiseAddress: ownPeerAddress,
}

if len(errs) != 0 {
health.Status = UnHealthy
health.Message = strings.Join(errs, "|")
}

if health.AdvertiseAddress == "" {
health.Status = UnHealthy
health.Message = strings.Join(errs, "this instance is not found in the peer list")
}

span.SetAttributes(
attribute.Int64("health.peerCount", int64(health.PeerCount)),
attribute.String("health.status", health.Status),
Expand Down
99 changes: 60 additions & 39 deletions gubernator.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 12 additions & 0 deletions gubernator.proto
Original file line number Diff line number Diff line change
Expand Up @@ -210,4 +210,16 @@ message HealthCheckResp {
string message = 2;
// The number of peers we know about
int32 peer_count = 3;

// advertise_address is the address advertised to other peers and
// is verified as owned by this instance. If advertise_address is
// empty, this indicates the instance is unable to find it's advertised
// address in the list of peers provided to SetPeers(). This is likely
// due to an incorrect GUBER_ADVERTISE_ADDRESS, or the discovery method
// used to call SetPeers() is missing our instance address in the peer
// list provided.
//
// If advertise_address is empty, this gubernator instance is considered
// unhealthy.
string advertise_address = 4;
}
5 changes: 3 additions & 2 deletions tls_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,7 @@ func TestTLSClusterWithClientAuthentication(t *testing.T) {
func TestHTTPSClientAuth(t *testing.T) {
conf := gubernator.DaemonConfig{
GRPCListenAddress: "127.0.0.1:9695",
AdvertiseAddress: "127.0.0.1:9695",
HTTPListenAddress: "127.0.0.1:9685",
HTTPStatusListenAddress: "127.0.0.1:9686",
TLS: &gubernator.TLSConfig{
Expand Down Expand Up @@ -332,7 +333,7 @@ func TestHTTPSClientAuth(t *testing.T) {
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
require.NoError(t, err)
assert.Equal(t, `{"status":"healthy","message":"","peer_count":1}`, strings.ReplaceAll(string(b), " ", ""))
assert.Equal(t, `{"status":"healthy","message":"","peer_count":1,"advertise_address":"127.0.0.1:9695"}`, strings.ReplaceAll(string(b), " ", ""))

// Verify we get an error when we try to access existing HTTPListenAddress without cert
//nolint:bodyclose // Expect error, no body to close.
Expand All @@ -345,7 +346,7 @@ func TestHTTPSClientAuth(t *testing.T) {
defer resp2.Body.Close()
b, err = io.ReadAll(resp2.Body)
require.NoError(t, err)
assert.Equal(t, `{"status":"healthy","message":"","peer_count":1}`, strings.ReplaceAll(string(b), " ", ""))
assert.Equal(t, `{"status":"healthy","message":"","peer_count":1,"advertise_address":"127.0.0.1:9695"}`, strings.ReplaceAll(string(b), " ", ""))

}

Expand Down
Loading