DAOS-16209 pool: return mem file size on pool query

src/control/common/proto/logging.go

@@ -92,7 +92,7 @@ func Debug(msg proto.Message) string {
 		fmt.Fprintf(&bld, "ranks:%s ", ranks.String())
 		fmt.Fprint(&bld, "tiers:")
 		for i, b := range m.TierBytes {
-			fmt.Fprintf(&bld, "%d: %d ", i, b)
+			fmt.Fprintf(&bld, "%d: %s (%d)", i, humanize.Bytes(b), b)
 			if len(m.TierRatio) > i+1 {
 				fmt.Fprintf(&bld, "(%.02f%%) ", m.TierRatio[i])
 			}
@@ -112,9 +112,10 @@ func Debug(msg proto.Message) string {
 		fmt.Fprintf(&bld, "tgt_ranks:%s ", ranks.String())
 		fmt.Fprint(&bld, "tiers:")
 		for i, b := range m.TierBytes {
-			fmt.Fprintf(&bld, "%d:%d ", i, b)
+			fmt.Fprintf(&bld, "%d: %s (%d)", i, humanize.Bytes(b), b)
 		}
-		fmt.Fprintf(&bld, "meta-file-size:%s", humanize.Bytes(m.MemFileBytes))
+		fmt.Fprintf(&bld, "meta-file-size: %s (%d)", humanize.Bytes(m.MemFileBytes),
+			m.MemFileBytes)
 	case *mgmtpb.PoolEvictReq:
 		fmt.Fprintf(&bld, "%T pool:%s", m, m.Id)
 		if len(m.Handles) > 0 {

src/control/server/mgmt_pool.go

@@ -482,6 +482,14 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq
 		return nil, errors.Wrap(err, "unmarshal PoolCreate response")
 	}
 
+	// Zero mem_file_bytes in non-MD-on-SSD mode.
+	if !svc.harness.Instances()[0].GetStorage().BdevRoleMetaConfigured() {
+		resp.MemFileBytes = 0
+	} else {
+		svc.log.Tracef("%T mem_file_bytes: %s (%d)", resp,
+			humanize.Bytes(resp.MemFileBytes), resp.MemFileBytes)
+	}
+
 	if resp.GetStatus() != 0 {
 		if err := svc.sysdb.RemovePoolService(ctx, ps.PoolUUID); err != nil {
 			return nil, err
@@ -960,11 +968,12 @@ func (svc *mgmtSvc) PoolQuery(ctx context.Context, req *mgmtpb.PoolQueryReq) (*m
 	// Preserve compatibility with pre-2.6 callers.
 	resp.Leader = resp.SvcLdr
 
-	// TODO DAOS-16209: After VOS query API is updated, zero-value mem_file_bytes will be
-	//                  returned in non-MD-on-SSD mode and this hack can be removed.
-	storage := svc.harness.Instances()[0].GetStorage()
-	if !storage.ControlMetadataPathConfigured() {
+	// Zero mem_file_bytes in non-MD-on-SSD mode.
+	if !svc.harness.Instances()[0].GetStorage().BdevRoleMetaConfigured() {
 		resp.MemFileBytes = 0
+	} else {
+		svc.log.Tracef("%T mem_file_bytes: %s (%d)", resp,
+			humanize.Bytes(resp.MemFileBytes), resp.MemFileBytes)
 	}
 
 	return resp, nil
@@ -986,13 +995,17 @@ func (svc *mgmtSvc) PoolQueryTarget(ctx context.Context, req *mgmtpb.PoolQueryTa
 		return nil, errors.Wrap(err, "unmarshal PoolQueryTarget response")
 	}
 
-	// TODO DAOS-16209: After VOS query API is updated, zero-value mem_file_bytes will be
-	//                  returned in non-MD-on-SSD mode and this hack can be removed.
-	storage := svc.harness.Instances()[0].GetStorage()
-	if !storage.ControlMetadataPathConfigured() {
+	// Zero mem_file_bytes in non-MD-on-SSD mode.
+	if !svc.harness.Instances()[0].GetStorage().BdevRoleMetaConfigured() {
 		for _, tgtInfo := range resp.Infos {
 			tgtInfo.MemFileBytes = 0
 		}
+	} else {
+		for _, tgtInfo := range resp.Infos {
+			svc.log.Tracef("%T mem_file_bytes: %s (%d)", resp,
+				humanize.Bytes(tgtInfo.MemFileBytes), tgtInfo.MemFileBytes)
+			break
+		}
 	}
 
 	return resp, nil

src/control/server/mgmt_pool_test.go

@@ -24,7 +24,6 @@ import (
 	mgmtpb "github.com/daos-stack/daos/src/control/common/proto/mgmt"
 	"github.com/daos-stack/daos/src/control/common/test"
 	"github.com/daos-stack/daos/src/control/drpc"
-	"github.com/daos-stack/daos/src/control/events"
 	"github.com/daos-stack/daos/src/control/lib/daos"
 	"github.com/daos-stack/daos/src/control/lib/ranklist"
 	"github.com/daos-stack/daos/src/control/logging"
@@ -504,6 +503,16 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 				TgtRanks:  []uint32{0, 1},
 			},
 		},
+		"create with memory file ratio; mdonssd not enabled": {
+			targetCount: 8,
+			req: &mgmtpb.PoolCreateReq{
+				Uuid:       test.MockUUID(1),
+				TierBytes:  []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
+				MemRatio:   storage.DefaultMemoryFileRatio,
+				Properties: testPoolLabelProp(),
+			},
+			expErr: errors.New("MD-on-SSD has not been enabled"),
+		},
 		"successful creation with memory file ratio": {
 			targetCount:    8,
 			mdonssdEnabled: true,
@@ -524,6 +533,23 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 				TgtRanks:     []uint32{0, 1},
 			},
 		},
+		"successful creation with memory file bytes in resp; mdonssd not enabled": {
+			targetCount: 8,
+			req: &mgmtpb.PoolCreateReq{
+				Uuid:       test.MockUUID(1),
+				TierBytes:  []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
+				Properties: testPoolLabelProp(),
+			},
+			drpcRet: &mgmtpb.PoolCreateResp{
+				TierBytes:    []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
+				MemFileBytes: 100 * humanize.GiByte,
+				TgtRanks:     []uint32{0, 1},
+			},
+			expResp: &mgmtpb.PoolCreateResp{
+				TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte},
+				TgtRanks:  []uint32{0, 1},
+			},
+		},
 		"successful creation minimum size": {
 			targetCount: 8,
 			req: &mgmtpb.PoolCreateReq{
@@ -629,8 +655,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 			buf.Reset()
 			defer test.ShowBufferOnFailure(t, buf)
 
-			ctx := test.Context(t)
-
 			if tc.mgmtSvc == nil {
 				tier := storage.NewTierConfig().
 					WithStorageClass("nvme").
@@ -639,30 +663,11 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) {
 					tier.WithBdevDeviceRoles(7)
 				}
 				engineCfg := engine.MockConfig().
-					WithTargetCount(tc.targetCount).
+					WithTargetCount(16).
 					WithStorage(tier)
-				r := engine.NewTestRunner(nil, engineCfg)
-				if _, err := r.Start(ctx); err != nil {
-					t.Fatal(err)
-				}
-
 				mp := storage.NewProvider(log, 0, &engineCfg.Storage,
 					nil, nil, nil, nil)
-				srv := NewEngineInstance(log, mp, nil, r, nil)
-				srv.setDrpcSocket("/dontcare")
-				srv.ready.SetTrue()
-
-				harness := NewEngineHarness(log)
-				if err := harness.AddInstance(srv); err != nil {
-					panic(err)
-				}
-				harness.started.SetTrue()
-
-				db := raft.MockDatabase(t, log)
-				ms := system.MockMembership(t, log, db, mockTCPResolver)
-				tc.mgmtSvc = newMgmtSvc(harness, ms, db, nil,
-					events.NewPubSub(ctx, log))
-				tc.mgmtSvc.startAsyncLoops(ctx)
+				tc.mgmtSvc = newTestMgmtSvcWithProvider(t, log, mp)
 			}
 
 			numMembers := tc.memberCount
@@ -2315,11 +2320,12 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) {
 	}
 
 	for name, tc := range map[string]struct {
-		mgmtSvc       *mgmtSvc
-		setupMockDrpc func(_ *mgmtSvc, _ error)
-		req           *mgmtpb.PoolQueryReq
-		expResp       *mgmtpb.PoolQueryResp
-		expErr        error
+		mdonssdEnabled bool
+		mgmtSvc        *mgmtSvc
+		setupMockDrpc  func(_ *mgmtSvc, _ error)
+		req            *mgmtpb.PoolQueryReq
+		expResp        *mgmtpb.PoolQueryResp
+		expErr         error
 	}{
 		"nil request": {
 			expErr: errors.New("nil request"),
@@ -2369,15 +2375,16 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) {
 				Uuid:  mockUUID,
 			},
 		},
-		"successful query (includes pre-2.6 Leader field)": {
+		"successful query (includes pre-2.6 Leader field); mdonssd not enabled": {
 			req: &mgmtpb.PoolQueryReq{
 				Id: mockUUID,
 			},
 			setupMockDrpc: func(svc *mgmtSvc, err error) {
 				resp := &mgmtpb.PoolQueryResp{
-					State:  mgmtpb.PoolServiceState_Ready,
-					Uuid:   mockUUID,
-					SvcLdr: 42,
+					State:        mgmtpb.PoolServiceState_Ready,
+					Uuid:         mockUUID,
+					SvcLdr:       42,
+					MemFileBytes: humanize.GiByte,
 				}
 				setupMockDrpcClient(svc, resp, nil)
 			},
@@ -2388,19 +2395,50 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) {
 				Leader: 42,
 			},
 		},
+		"successful query; mdonssd enabled": {
+			mdonssdEnabled: true,
+			req: &mgmtpb.PoolQueryReq{
+				Id: mockUUID,
+			},
+			setupMockDrpc: func(svc *mgmtSvc, err error) {
+				resp := &mgmtpb.PoolQueryResp{
+					State:        mgmtpb.PoolServiceState_Ready,
+					Uuid:         mockUUID,
+					MemFileBytes: humanize.GiByte,
+				}
+				setupMockDrpcClient(svc, resp, nil)
+			},
+			expResp: &mgmtpb.PoolQueryResp{
+				State:        mgmtpb.PoolServiceState_Ready,
+				Uuid:         mockUUID,
+				MemFileBytes: humanize.GiByte,
+			},
+		},
 	} {
 		t.Run(name, func(t *testing.T) {
 			buf.Reset()
 			defer test.ShowBufferOnFailure(t, buf)
 
 			if tc.mgmtSvc == nil {
-				tc.mgmtSvc = newTestMgmtSvc(t, log)
+				tier := storage.NewTierConfig().
+					WithStorageClass("nvme").
+					WithBdevDeviceList("foo", "bar")
+				if tc.mdonssdEnabled {
+					tier.WithBdevDeviceRoles(7)
+				}
+				engineCfg := engine.MockConfig().
+					WithTargetCount(16).
+					WithStorage(tier)
+				mp := storage.NewProvider(log, 0, &engineCfg.Storage,
+					nil, nil, nil, nil)
+				tc.mgmtSvc = newTestMgmtSvcWithProvider(t, log, mp)
 			}
 			addTestPools(t, tc.mgmtSvc.sysdb, mockUUID)
 
 			if tc.setupMockDrpc == nil {
 				tc.setupMockDrpc = func(svc *mgmtSvc, err error) {
-					setupSvcDrpcClient(svc, 0, getMockDrpcClient(tc.expResp, tc.expErr))
+					setupSvcDrpcClient(svc, 0,
+						getMockDrpcClient(tc.expResp, tc.expErr))
 				}
 			}
 			tc.setupMockDrpc(tc.mgmtSvc, tc.expErr)

src/control/server/util_test.go

@@ -234,15 +234,13 @@ func mockTCPResolver(netString string, address string) (*net.TCPAddr, error) {
 	return &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 10001}, nil
 }
 
-// newTestMgmtSvc creates a mgmtSvc that contains an EngineInstance
-// properly set up as an MS.
-func newTestMgmtSvc(t *testing.T, log logging.Logger) *mgmtSvc {
+// newTestMgmtSvcWithProvider creates a mgmtSvc that contains an EngineInstance
+// properly set up as an MS using an input storage provider.
+func newTestMgmtSvcWithProvider(t *testing.T, log logging.Logger, provider *storage.Provider) *mgmtSvc {
 	harness := NewEngineHarness(log)
-	provider := storage.MockProvider(log, 0, nil, nil, nil, nil, nil)
-
-	srv := newTestEngine(log, true, provider)
 
-	if err := harness.AddInstance(srv); err != nil {
+	e := newTestEngine(log, true, provider)
+	if err := harness.AddInstance(e); err != nil {
 		t.Fatal(err)
 	}
 	harness.started.SetTrue()
@@ -254,9 +252,17 @@ func newTestMgmtSvc(t *testing.T, log logging.Logger) *mgmtSvc {
 	svc.batchInterval = 100 * time.Microsecond // Speed up tests
 	svc.startAsyncLoops(ctx)
 	svc.startLeaderLoops(ctx)
+
 	return svc
 }
 
+// newTestMgmtSvc creates a mgmtSvc that contains an EngineInstance
+// properly set up as an MS.
+func newTestMgmtSvc(t *testing.T, log logging.Logger) *mgmtSvc {
+	return newTestMgmtSvcWithProvider(t, log,
+		storage.MockProvider(log, 0, nil, nil, nil, nil, nil))
+}
+
 // newTestMgmtSvcMulti creates a mgmtSvc that contains the requested
 // number of EngineInstances. If requested, the first instance is
 // configured as an access point.

src/include/daos_pool.h

@@ -70,6 +70,16 @@ struct daos_space {
 	uint64_t		s_total[DAOS_MEDIA_MAX];
 	/** Free space in bytes */
 	uint64_t		s_free[DAOS_MEDIA_MAX];
+	/** Memory file size (in bytes) for md-on-ssd phase2 pool */
+	uint64_t                s_total_mem;
+};
+
+/** Pool target space usage information v6 */
+struct daos_space_v6 {
+	/** Total space in bytes */
+	uint64_t s_total[DAOS_MEDIA_MAX];
+	/** Free space in bytes */
+	uint64_t s_free[DAOS_MEDIA_MAX];
 };
 
 /** Target information */
@@ -100,6 +110,22 @@ struct daos_pool_space {
 	uint32_t		ps_padding;
 };
 
+/** Pool space usage information v6 */
+struct daos_pool_space_v6 {
+	/** Aggregated space for all live targets */
+	struct daos_space_v6 ps_space;
+	/** Min target free space in bytes */
+	uint64_t             ps_free_min[DAOS_MEDIA_MAX];
+	/** Max target free space in bytes */
+	uint64_t             ps_free_max[DAOS_MEDIA_MAX];
+	/** Average target free space in bytes */
+	uint64_t             ps_free_mean[DAOS_MEDIA_MAX];
+	/** Target(VOS) count */
+	uint32_t             ps_ntargets;
+	/** padding - not used */
+	uint32_t             ps_padding;
+};
+
 enum daos_rebuild_state_t {
 	DRS_IN_PROGRESS		= 0,
 	DRS_NOT_STARTED		= 1,

src/mgmt/srv_drpc.c

@@ -412,6 +412,7 @@ static int pool_create_fill_resp(Mgmt__PoolCreateResp *resp, uuid_t uuid, d_rank
 	for (index = 0; index < DAOS_MEDIA_MAX; ++index) {
 		D_ASSERT(pool_info.pi_space.ps_space.s_total[index] % resp->n_tgt_ranks == 0);
 	}
+	D_ASSERT(pool_info.pi_space.ps_space.s_total_mem % resp->n_tgt_ranks == 0);
 	D_ALLOC_ARRAY(resp->tier_bytes, DAOS_MEDIA_MAX);
 	if (resp->tier_bytes == NULL) {
 		rc = -DER_NOMEM;
@@ -422,6 +423,7 @@ static int pool_create_fill_resp(Mgmt__PoolCreateResp *resp, uuid_t uuid, d_rank
 		resp->tier_bytes[index] =
 			pool_info.pi_space.ps_space.s_total[index] / resp->n_tgt_ranks;
 	}
+	resp->mem_file_bytes = pool_info.pi_space.ps_space.s_total_mem / resp->n_tgt_ranks;
 
 out:
 	d_rank_list_free(enabled_ranks);
@@ -519,14 +521,6 @@ ds_mgmt_drpc_pool_create(Drpc__Call *drpc_req, Drpc__Response *drpc_resp)
 	rc = pool_create_fill_resp(&resp, pool_uuid, svc);
 	d_rank_list_free(svc);
 
-	/**
-	 * TODO DAOS-16209: Populate per-rank VOS-file sizes. For now just calculate here based on
-	 *                  the supplied input values but really should be returned from
-	 *                  ds_mgmt_pool_query() through the VOS query API and set in
-	 *                  pool_create_fill_resp(). Return zero for non-MD-on-SSD mode.
-	 */
-	resp.mem_file_bytes = req->tier_bytes[DAOS_MEDIA_SCM] * req->mem_ratio;
-
 out:
 	resp.status = rc;
 	len = mgmt__pool_create_resp__get_packed_size(&resp);
@@ -1866,12 +1860,7 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp)
 	pool_rebuild_status_from_info(&rebuild, &pool_info.pi_rebuild_st);
 	resp.rebuild = &rebuild;
 
-	/**
-	 * TODO DAOS-16209: Populate VOS-file sizes in response. For now just return the meta-blob
-	 *                  size until VOS query API is updated. When updated, zero-value should
-	 *                  be returned in non-MD-on-SSD mode.
-	 */
-	resp.mem_file_bytes = scm.total;
+	resp.mem_file_bytes = pool_info.pi_space.ps_space.s_total_mem;
 
 error:
 	resp.status = rc;
@@ -1983,6 +1972,7 @@ ds_mgmt_drpc_pool_query_targets(Drpc__Call *drpc_req, Drpc__Response *drpc_resp)
 			resp.infos[i]->space[j]->free = infos[i].ta_space.s_free[j];
 			resp.infos[i]->space[j]->media_type = j;
 		}
+		resp.infos[i]->mem_file_bytes = infos[i].ta_space.s_total_mem;
 	}
 
 out_infos:

src/mgmt/tests/mocks.c

@@ -373,6 +373,7 @@ mock_ds_mgmt_pool_query_targets_gen_infos(uint32_t n_infos)
 		infos[i].ta_space.s_free[DAOS_MEDIA_SCM] = 800000000 + i;
 		infos[i].ta_space.s_total[DAOS_MEDIA_NVME] = 9000000000;
 		infos[i].ta_space.s_free[DAOS_MEDIA_NVME] = 600000000 + i;
+		infos[i].ta_space.s_total_mem              = 2000000000;
 	}
 	ds_mgmt_pool_query_targets_info_out = infos;
 }