Skip to content

Commit

Permalink
[s3inbox] run healthchecks in main thread, on same port
Browse files Browse the repository at this point in the history
  • Loading branch information
MalinAhlberg committed Sep 3, 2024
1 parent 986c441 commit 0ee0e2b
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 74 deletions.
104 changes: 35 additions & 69 deletions sda/cmd/s3inbox/healthchecks.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,90 +5,58 @@ import (
"database/sql"
"fmt"
"net/http"
"strconv"
"time"

"github.com/heptiolabs/healthcheck"
"github.com/neicnordic/sensitive-data-archive/internal/broker"
"github.com/neicnordic/sensitive-data-archive/internal/config"
)

// HealthCheck registers and endpoint for healthchecking the service
type HealthCheck struct {
port int
DB *sql.DB
s3URL string
brokerURL string
tlsConfig *tls.Config
serverCert string
serverKey string
DB *sql.DB
broker *broker.AMQPBroker
s3URL string
tlsConfig *tls.Config
}

// NewHealthCheck creates a new healthchecker. It needs to know where to find
// the backend S3 storage and the Message Broker so it can report readiness.
func NewHealthCheck(port int, db *sql.DB, conf *config.Config, tlsConfig *tls.Config) *HealthCheck {
func NewHealthCheck(db *sql.DB, conf *config.Config, messenger *broker.AMQPBroker, tlsConfig *tls.Config) *HealthCheck {
s3URL := conf.Inbox.S3.URL
if conf.Inbox.S3.Port != 0 {
s3URL = fmt.Sprintf("%s:%d", s3URL, conf.Inbox.S3.Port)
}
if conf.Inbox.S3.Readypath != "" {
s3URL += conf.Inbox.S3.Readypath
}

brokerURL := fmt.Sprintf("%s:%d", conf.Broker.Host, conf.Broker.Port)

serverCert := conf.Server.Cert
serverKey := conf.Server.Key

return &HealthCheck{port, db, s3URL, brokerURL, tlsConfig, serverCert, serverKey}
return &HealthCheck{db, messenger, s3URL, tlsConfig}
}

// RunHealthChecks should be run as a go routine in the main app. It registers
// the healthcheck handler on the port specified in when creating a new
// healthcheck.
func (h *HealthCheck) RunHealthChecks() {
health := healthcheck.NewHandler()

health.AddLivenessCheck("goroutine-threshold", healthcheck.GoroutineCountCheck(100))

health.AddReadinessCheck("S3-backend-http", h.httpsGetCheck(h.s3URL, 5000*time.Millisecond))

health.AddReadinessCheck("broker-tcp", healthcheck.TCPDialCheck(h.brokerURL, 5000*time.Millisecond))

health.AddReadinessCheck("database", healthcheck.DatabasePingCheck(h.DB, 1*time.Second))
func (hc *HealthCheck) checkHealth(w http.ResponseWriter, r *http.Request) {

mux := http.NewServeMux()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodHead {
// readyEndpoint does not accept method head
r.Method = http.MethodGet
health.ReadyEndpoint(w, r)
}
})
mux.HandleFunc("/health", health.ReadyEndpoint)

addr := ":" + strconv.Itoa(h.port)
server := &http.Server{
Addr: addr,
Handler: mux,
ReadTimeout: 5 * time.Second,
WriteTimeout: 5 * time.Second,
IdleTimeout: 30 * time.Second,
ReadHeaderTimeout: 3 * time.Second,
// Check that the mq channel is open
if hc.broker.Channel.IsClosed() {
w.WriteHeader(http.StatusServiceUnavailable)
}
// Ping database
err := hc.DB.Ping()
if err != nil {
w.WriteHeader(http.StatusServiceUnavailable)
}
if h.serverCert != "" && h.serverKey != "" {
if err := server.ListenAndServeTLS(h.serverCert, h.serverKey); err != nil {
panic(err)
}
} else {
if err := server.ListenAndServe(); err != nil {
panic(err)
}

// Check that s3 backend responds
s3URL := hc.s3URL
err = hc.httpsGetCheck(s3URL, 5000*time.Millisecond)
if err != nil {
w.WriteHeader(http.StatusServiceUnavailable)
}
w.WriteHeader(http.StatusOK)
}

func (h *HealthCheck) httpsGetCheck(url string, timeout time.Duration) healthcheck.Check {
// httpsGetCheck sends a request to the S3 backend and makes sure it is healthy
func (hc *HealthCheck) httpsGetCheck(url string, timeout time.Duration) error {
cfg := &tls.Config{MinVersion: tls.VersionTLS12}
cfg.RootCAs = h.tlsConfig.RootCAs
cfg.RootCAs = hc.tlsConfig.RootCAs
tr := &http.Transport{TLSClientConfig: cfg}
client := http.Client{
Transport: tr,
Expand All @@ -99,16 +67,14 @@ func (h *HealthCheck) httpsGetCheck(url string, timeout time.Duration) healthche
},
}

return func() error {
resp, e := client.Get(url)
if e != nil {
return e
}
_ = resp.Body.Close() // ignoring error
if resp.StatusCode != 200 {
return fmt.Errorf("returned status %d", resp.StatusCode)
}

return nil
resp, e := client.Get(url)
if e != nil {
return e
}
_ = resp.Body.Close() // ignoring error
if resp.StatusCode != 200 {
return fmt.Errorf("returned status %d", resp.StatusCode)
}

return nil
}
10 changes: 5 additions & 5 deletions sda/cmd/s3inbox/s3inbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,12 @@ func main() {
log.Panicf("Error while getting key %s: %v", Conf.Server.Jwtpubkeypath, err)
}
}
mux := http.NewServeMux()
proxy := NewProxy(Conf.Inbox.S3, auth, messenger, sdaDB, tlsProxy)

http.Handle("/", proxy)

hc := NewHealthCheck(8001, sdaDB.DB, Conf, tlsProxy)
go hc.RunHealthChecks()
hc := NewHealthCheck(sdaDB.DB, Conf, messenger, tlsProxy)
mux.HandleFunc("HEAD /", hc.checkHealth)
mux.HandleFunc("/health", hc.checkHealth)
mux.Handle("/", proxy)

server := &http.Server{
Addr: ":8000",
Expand Down

0 comments on commit 0ee0e2b

Please sign in to comment.