Adding Health endpoint (#836)

* Adding Health endpoint

* pr comments + 503 if not healthy

* refactored admin server and api + health endpoint tests

* fix health condition

* fix admin routing

* added comments + changed from ChainSync to ChainBootstrapStatus

* Adding healthcheck for solo mode

* adding solo + tests

* fix log_level handler funcs

* refactor health package + add p2p count

* remove solo methods

* moving health service to api pkg

* added defaults + api health query

* pr comments

* pr comments

* pr comments

* Update cmd/thor/main.go

api/admin/admin.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or <https://www.gnu.org/licenses/lgpl-3.0.html>
+
+package admin
+
+import (
+	"log/slog"
+	"net/http"
+
+	"github.com/gorilla/handlers"
+	"github.com/gorilla/mux"
+	healthAPI "github.com/vechain/thor/v2/api/admin/health"
+	"github.com/vechain/thor/v2/api/admin/loglevel"
+)
+
+func New(logLevel *slog.LevelVar, health *healthAPI.Health) http.HandlerFunc {
+	router := mux.NewRouter()
+	subRouter := router.PathPrefix("/admin").Subrouter()
+
+	loglevel.New(logLevel).Mount(subRouter, "/loglevel")
+	healthAPI.NewAPI(health).Mount(subRouter, "/health")
+
+	handler := handlers.CompressHandler(router)
+
+	return handler.ServeHTTP
+}

api/admin/health/health.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or <https://www.gnu.org/licenses/lgpl-3.0.html>
+
+package health
+
+import (
+	"time"
+
+	"github.com/vechain/thor/v2/chain"
+	"github.com/vechain/thor/v2/comm"
+	"github.com/vechain/thor/v2/thor"
+)
+
+type BlockIngestion struct {
+	ID        *thor.Bytes32 `json:"id"`
+	Timestamp *time.Time    `json:"timestamp"`
+}
+
+type Status struct {
+	Healthy              bool       `json:"healthy"`
+	BestBlockTime        *time.Time `json:"bestBlockTime"`
+	PeerCount            int        `json:"peerCount"`
+	IsNetworkProgressing bool       `json:"isNetworkProgressing"`
+}
+
+type Health struct {
+	repo *chain.Repository
+	p2p  *comm.Communicator
+}
+
+const (
+	defaultBlockTolerance = time.Duration(2*thor.BlockInterval) * time.Second // 2 blocks tolerance
+	defaultMinPeerCount   = 2
+)
+
+func New(repo *chain.Repository, p2p *comm.Communicator) *Health {
+	return &Health{
+		repo: repo,
+		p2p:  p2p,
+	}
+}
+
+// isNetworkProgressing checks if the network is producing new blocks within the allowed interval.
+func (h *Health) isNetworkProgressing(now time.Time, bestBlockTimestamp time.Time, blockTolerance time.Duration) bool {
+	return now.Sub(bestBlockTimestamp) <= blockTolerance
+}
+
+// isNodeConnectedP2P checks if the node is connected to peers
+func (h *Health) isNodeConnectedP2P(peerCount int, minPeerCount int) bool {
+	return peerCount >= minPeerCount
+}
+
+func (h *Health) Status(blockTolerance time.Duration, minPeerCount int) (*Status, error) {
+	// Fetch the best block details
+	bestBlock := h.repo.BestBlockSummary()
+	bestBlockTimestamp := time.Unix(int64(bestBlock.Header.Timestamp()), 0)
+
+	// Fetch the current connected peers
+	var connectedPeerCount int
+	if h.p2p == nil {
+		connectedPeerCount = minPeerCount // ignore peers in solo mode
+	} else {
+		connectedPeerCount = h.p2p.PeerCount()
+	}
+
+	now := time.Now()
+
+	// Perform the checks
+	networkProgressing := h.isNetworkProgressing(now, bestBlockTimestamp, blockTolerance)
+	nodeConnected := h.isNodeConnectedP2P(connectedPeerCount, minPeerCount)
+
+	// Calculate overall health status
+	healthy := networkProgressing && nodeConnected
+
+	// Return the current status
+	return &Status{
+		Healthy:              healthy,
+		BestBlockTime:        &bestBlockTimestamp,
+		IsNetworkProgressing: networkProgressing,
+		PeerCount:            connectedPeerCount,
+	}, nil
+}

api/admin/health/health_api.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or <https://www.gnu.org/licenses/lgpl-3.0.html>
+
+package health
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/vechain/thor/v2/api/utils"
+)
+
+type API struct {
+	healthStatus *Health
+}
+
+func NewAPI(healthStatus *Health) *API {
+	return &API{
+		healthStatus: healthStatus,
+	}
+}
+
+func (h *API) handleGetHealth(w http.ResponseWriter, r *http.Request) error {
+	// Parse query parameters
+	query := r.URL.Query()
+
+	// Default to constants if query parameters are not provided
+	blockTolerance := defaultBlockTolerance
+	minPeerCount := defaultMinPeerCount
+
+	// Override with query parameters if they exist
+	if queryBlockTolerance := query.Get("blockTolerance"); queryBlockTolerance != "" {
+		if parsed, err := time.ParseDuration(queryBlockTolerance); err == nil {
+			blockTolerance = parsed
+		}
+	}
+
+	if queryMinPeerCount := query.Get("minPeerCount"); queryMinPeerCount != "" {
+		if parsed, err := strconv.Atoi(queryMinPeerCount); err == nil {
+			minPeerCount = parsed
+		}
+	}
+
+	acc, err := h.healthStatus.Status(blockTolerance, minPeerCount)
+	if err != nil {
+		return err
+	}
+
+	if !acc.Healthy {
+		w.WriteHeader(http.StatusServiceUnavailable) // Set the status to 503
+	} else {
+		w.WriteHeader(http.StatusOK) // Set the status to 200
+	}
+	return utils.WriteJSON(w, acc)
+}
+
+func (h *API) Mount(root *mux.Router, pathPrefix string) {
+	sub := root.PathPrefix(pathPrefix).Subrouter()
+
+	sub.Path("").
+		Methods(http.MethodGet).
+		Name("health").
+		HandlerFunc(utils.WrapHandlerFunc(h.handleGetHealth))
+}

api/admin/health/health_api_test.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or <https://www.gnu.org/licenses/lgpl-3.0.html>
+
+package health
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gorilla/mux"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/vechain/thor/v2/comm"
+	"github.com/vechain/thor/v2/test/testchain"
+	"github.com/vechain/thor/v2/txpool"
+)
+
+var ts *httptest.Server
+
+func TestHealth(t *testing.T) {
+	initAPIServer(t)
+
+	var healthStatus Status
+	respBody, statusCode := httpGet(t, ts.URL+"/health")
+	require.NoError(t, json.Unmarshal(respBody, &healthStatus))
+	assert.False(t, healthStatus.Healthy)
+	assert.Equal(t, http.StatusServiceUnavailable, statusCode)
+}
+
+func initAPIServer(t *testing.T) {
+	thorChain, err := testchain.NewIntegrationTestChain()
+	require.NoError(t, err)
+
+	router := mux.NewRouter()
+	NewAPI(
+		New(thorChain.Repo(), comm.New(thorChain.Repo(), txpool.New(thorChain.Repo(), nil, txpool.Options{}))),
+	).Mount(router, "/health")
+
+	ts = httptest.NewServer(router)
+}
+
+func httpGet(t *testing.T, url string) ([]byte, int) {
+	res, err := http.Get(url) //#nosec G107
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer res.Body.Close()
+
+	r, err := io.ReadAll(res.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return r, res.StatusCode
+}

api/admin/health/health_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or <https://www.gnu.org/licenses/lgpl-3.0.html>
+
+package health
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHealth_isNetworkProgressing(t *testing.T) {
+	h := &Health{}
+
+	now := time.Now()
+
+	tests := []struct {
+		name                string
+		bestBlockTimestamp  time.Time
+		expectedProgressing bool
+	}{
+		{
+			name:                "Progressing - block within timeBetweenBlocks",
+			bestBlockTimestamp:  now.Add(-5 * time.Second),
+			expectedProgressing: true,
+		},
+		{
+			name:                "Not Progressing - block outside timeBetweenBlocks",
+			bestBlockTimestamp:  now.Add(-25 * time.Second),
+			expectedProgressing: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			isProgressing := h.isNetworkProgressing(now, tt.bestBlockTimestamp, defaultBlockTolerance)
+			assert.Equal(t, tt.expectedProgressing, isProgressing, "isNetworkProgressing result mismatch")
+		})
+	}
+}
+
+func TestHealth_isNodeConnectedP2P(t *testing.T) {
+	h := &Health{}
+
+	tests := []struct {
+		name              string
+		peerCount         int
+		expectedConnected bool
+	}{
+		{
+			name:              "Connected - more than one peer",
+			peerCount:         3,
+			expectedConnected: true,
+		},
+		{
+			name:              "Not Connected - one or fewer peers",
+			peerCount:         1,
+			expectedConnected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			isConnected := h.isNodeConnectedP2P(tt.peerCount, defaultMinPeerCount)
+			assert.Equal(t, tt.expectedConnected, isConnected, "isNodeConnectedP2P result mismatch")
+		})
+	}
+}