PWX-28654 Add quorum member flag to the StorageNode SDK object

This field is populated from cluster database's NonQuorumMember field. Which in turn is calculated in QuorumMember() listener function impl in PX. It looks at the storage devices in the metadata store. Signed-off-by: Piyush Nimbalkar <[email protected]>
libopenstorage · Oct 25, 2023 · 913b4f2 · 913b4f2
1 parent 416f35f
commit 913b4f2
Show file tree

Hide file tree

Showing 8 changed files with 3,987 additions and 3,941 deletions.
diff --git a/SDK_CHANGELOG.md b/SDK_CHANGELOG.md
@@ -2,10 +2,16 @@
 
 ## Releases
 
+### v0.169.0 - (10/24/2023)
+
+* Add quorum member flag to StorageNode
+
 ### v0.168.0 - (09/14/2023)
+
 * Add new field cpu cores to api Node
 
 ### v0.167.0 - (09/14/2023)
+
 * Add new field verbose to fastpath configuration
 
 ### v0.166.0 - (09/01/2023)

diff --git a/api/api.go b/api/api.go
@@ -9,9 +9,10 @@ import (
 	"time"
 
 	"github.com/golang/protobuf/ptypes"
-	"github.com/libopenstorage/openstorage/pkg/auth"
 	"github.com/mohae/deepcopy"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+
+	"github.com/libopenstorage/openstorage/pkg/auth"
 )
 
 // Strings for VolumeSpec
@@ -281,6 +282,8 @@ type Node struct {
 	SecurityStatus StorageNode_SecurityStatus
 	// SchedulerTopology topology information of the node in scheduler context
 	SchedulerTopology *SchedulerTopology
+	// Flag indicating whether the node is a quorum member or not
+	NonQuorumMember bool
 }
 
 // FluentDConfig describes ip and port of a fluentdhost.
@@ -1030,6 +1033,7 @@ func (s *Node) ToStorageNode() *StorageNode {
 		HWType:            s.HWType,
 		SecurityStatus:    s.SecurityStatus,
 		SchedulerTopology: s.SchedulerTopology,
+		NonQuorumMember:   s.NonQuorumMember,
 	}
 
 	node.Disks = make(map[string]*StorageResource)

diff --git a/api/api.pb.go b/api/api.pb.go
diff --git a/api/api.proto b/api/api.proto
@@ -1622,6 +1622,11 @@ message StorageNode {
   SecurityStatus security_status = 19;
   // Topology information of the node in scheduler context
   SchedulerTopology scheduler_topology = 20;
+  // Flag indicating whether the node is a quorum member or not. Using inverse value
+  // to handle intialization and upgrades. Node will always be counted as a quorum member
+  // when initializing until it reaches a point where we can definitely determine whether
+  // it is a quorum member or not.
+  bool non_quorum_member = 22;
 }
 
 // StorageCluster represents the state and information about the cluster
@@ -5494,7 +5499,7 @@ message SdkVersion {
     // SDK version major value of this specification
     Major = 0;
     // SDK version minor value of this specification
-    Minor = 168;
+    Minor = 169;
     // SDK version patch value of this specification
     Patch = 0;
   }

diff --git a/api/server/sdk/api/api.swagger.json b/api/server/sdk/api/api.swagger.json
diff --git a/api/server/sdk/node_test.go b/api/server/sdk/node_test.go
@@ -22,10 +22,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/libopenstorage/openstorage/api"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+
+	"github.com/libopenstorage/openstorage/api"
 )
 
 func TestSdkNodeEnumerateNoNodes(t *testing.T) {
@@ -153,6 +154,7 @@ func TestSdkNodeEnumerateWithFilters(t *testing.T) {
 						"foo": "bar",
 					},
 				},
+				NonQuorumMember: true,
 			},
 		},
 	}
@@ -170,6 +172,7 @@ func TestSdkNodeEnumerateWithFilters(t *testing.T) {
 				"foo": "bar",
 			},
 		},
+		NonQuorumMember: true,
 	}
 
 	s.MockCluster().EXPECT().Enumerate().Return(cluster, nil).Times(1)
@@ -240,7 +243,8 @@ func TestSdkNodeInspect(t *testing.T) {
 		NodeLabels: map[string]string{
 			"hello": "world",
 		},
-		HWType: api.HardwareType_VirtualMachine,
+		HWType:          api.HardwareType_VirtualMachine,
+		NonQuorumMember: true,
 	}
 	s.MockCluster().EXPECT().Inspect(nodeid).Return(node, nil).Times(1)
 
@@ -265,6 +269,7 @@ func TestSdkNodeInspect(t *testing.T) {
 	assert.Equal(t, rn.GetAvgLoad(), int64(node.Avgload))
 	assert.Equal(t, rn.GetStatus(), node.Status)
 	assert.Equal(t, rn.GetHWType(), node.HWType)
+	assert.Equal(t, node.NonQuorumMember, rn.NonQuorumMember)
 
 	// Check Disk
 	assert.Len(t, rn.GetDisks(), 2)
@@ -354,6 +359,7 @@ func TestSdkNodeInspectCurrent(t *testing.T) {
 				"foo": "bar",
 			},
 		},
+		NonQuorumMember: true,
 	}
 
 	cluster := api.Cluster{
@@ -385,6 +391,7 @@ func TestSdkNodeInspectCurrent(t *testing.T) {
 	assert.Equal(t, rn.GetAvgLoad(), int64(node.Avgload))
 	assert.Equal(t, rn.GetStatus(), node.Status)
 	assert.Equal(t, rn.GetHWType(), node.HWType)
+	assert.Equal(t, node.NonQuorumMember, rn.NonQuorumMember)
 
 	// Check Disk
 	assert.Len(t, rn.GetDisks(), 1)

diff --git a/cluster/cluster.go b/cluster/cluster.go
@@ -5,19 +5,19 @@ import (
 	"errors"
 	"time"
 
-	"github.com/libopenstorage/openstorage/pkg/diags"
+	"github.com/portworx/kvdb"
 
 	"github.com/libopenstorage/gossip/types"
 	"github.com/libopenstorage/openstorage/api"
 	"github.com/libopenstorage/openstorage/objectstore"
 	"github.com/libopenstorage/openstorage/osdconfig"
 	"github.com/libopenstorage/openstorage/pkg/auth"
 	"github.com/libopenstorage/openstorage/pkg/clusterdomain"
+	"github.com/libopenstorage/openstorage/pkg/diags"
 	"github.com/libopenstorage/openstorage/pkg/job"
 	"github.com/libopenstorage/openstorage/pkg/nodedrain"
 	sched "github.com/libopenstorage/openstorage/schedpolicy"
 	"github.com/libopenstorage/openstorage/secrets"
-	"github.com/portworx/kvdb"
 )
 
 const (

diff --git a/cluster/manager/manager.go b/cluster/manager/manager.go
@@ -19,6 +19,9 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/portworx/kvdb"
+	"github.com/sirupsen/logrus"
+
 	"github.com/libopenstorage/gossip"
 	"github.com/libopenstorage/gossip/types"
 	"github.com/libopenstorage/openstorage/api"
@@ -36,8 +39,6 @@ import (
 	sched "github.com/libopenstorage/openstorage/schedpolicy"
 	"github.com/libopenstorage/openstorage/secrets"
 	"github.com/libopenstorage/systemutils"
-	"github.com/portworx/kvdb"
-	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -287,6 +288,7 @@ func (c *ClusterManager) getNodeEntry(nodeID string, clustDBRef *cluster.Cluster
 			n.NodeLabels = v.NodeLabels
 			n.HWType = v.HWType
 			n.SecurityStatus = v.SecurityStatus
+			n.NonQuorumMember = v.NonQuorumMember
 		} else {
 			logrus.Warnf("Could not query NodeID %v", nodeID)
 			// Node entry won't be refreshed form DB, will use the "offline" original
@@ -746,6 +748,7 @@ func (c *ClusterManager) joinCluster(
 	selfNodeEntry.NonQuorumMember =
 		selfNodeEntry.Status == api.Status_STATUS_DECOMMISSION ||
 			!c.quorumMember()
+	c.selfNode.NonQuorumMember = selfNodeEntry.NonQuorumMember
 	if selfNodeEntry.NonQuorumMember != prevNonQuorumMemberState {
 		if !selfNodeEntry.NonQuorumMember {
 			logrus.Infof("This node now participates in quorum decisions")
@@ -1251,6 +1254,7 @@ func (c *ClusterManager) initListeners(
 	selfNodeEntry.NonQuorumMember =
 		selfNodeEntry.Status == api.Status_STATUS_DECOMMISSION ||
 			!c.quorumMember()
+	c.selfNode.NonQuorumMember = selfNodeEntry.NonQuorumMember
 	if !selfNodeEntry.NonQuorumMember {
 		logrus.Infof("This node participates in quorum decisions")
 	} else {
@@ -1638,6 +1642,7 @@ func (c *ClusterManager) nodes(clusterDB *cluster.ClusterInfo) []*api.Node {
 			node.Hostname = n.Hostname
 			node.NodeLabels = n.NodeLabels
 			node.SecurityStatus = n.SecurityStatus
+			node.NonQuorumMember = n.NonQuorumMember
 		}
 		nodes = append(nodes, &node)
 	}