DAOS-16823 container: Add metrics_enabled property

Add a boolean container property that can be used to
toggle client-side metrics collection and reporting
behavior on a per-container basis. By default, containers
will not participate in metrics collection, and must
be opted-in via the property. The initial use case for
this property is to enable DFS-level metrics in
POSIX containers.

Skip-nlt: true
Features: container
Required-githooks: true
Signed-off-by: Michael MacDonald <[email protected]>

docs/user/container.md

@@ -507,6 +507,12 @@ corruption proactively. (see data_integrity.md for more details). This can be
 disabled per container using the `DAOS_PROP_CO_SCRUBBER_DISABLED` container
 property.
 
+### Per-Container Metrics Collection
+
+If enabled, per-container metrics (currently only POSIX containers are supported)
+will be collected on the client. Set the `DAOS_PROP_CO_METRICS_ENABLED` container
+property to enable this behavior.
+
 ### Deduplication (Preview)
 
 Data deduplication (dedup) is a process that allows to eliminate duplicated

src/client/dfs/mnt.c

@@ -570,7 +570,8 @@ dfs_mount_int(daos_handle_t poh, daos_handle_t coh, int flags, daos_epoch_t epoc
 	int                        amode, omode;
 	int                        rc;
 	int                        i;
-	uint32_t  props[] = {DAOS_PROP_CO_LAYOUT_TYPE, DAOS_PROP_CO_ROOTS, DAOS_PROP_CO_REDUN_FAC};
+	uint32_t  props[]   = {DAOS_PROP_CO_LAYOUT_TYPE, DAOS_PROP_CO_ROOTS, DAOS_PROP_CO_REDUN_FAC,
+			       DAOS_PROP_CO_METRICS_ENABLED};
 	const int num_props = ARRAY_SIZE(props);
 
 	if (_dfs == NULL)

src/client/serialize/daos_serialize.c

@@ -298,6 +298,8 @@ prop_to_str(uint32_t type)
 		return "DAOS_PROP_CO_ROOTS";
 	case DAOS_PROP_CO_SCRUBBER_DISABLED:
 		return "DAOS_PROP_CO_SCRUBBER_DISABLED";
+	case DAOS_PROP_CO_METRICS_ENABLED:
+		return "DAOS_PROP_CO_METRICS_ENABLED";
 	default:
 		return "PROPERTY NOT SUPPORTED";
 	}
@@ -362,7 +364,8 @@ daos_cont_serialize_props(hid_t file_id, daos_prop_t *prop_query)
 			   type == DAOS_PROP_CO_PERF_DOMAIN ||
 			   type == DAOS_PROP_CO_GLOBAL_VERSION ||
 			   type == DAOS_PROP_CO_ALLOCED_OID ||
-			   type == DAOS_PROP_CO_SCRUBBER_DISABLED) {
+			   type == DAOS_PROP_CO_SCRUBBER_DISABLED ||
+			   type == DAOS_PROP_CO_METRICS_ENABLED) {
 			entry = &prop_query->dpp_entries[i];
 			rc = serialize_uint(file_id, entry->dpe_val,
 					    prop_str);
@@ -1070,6 +1073,16 @@ deserialize_props(daos_handle_t poh, hid_t file_id, daos_prop_t **_prop, uint64_
 			D_GOTO(out, rc);
 		prop_num++;
 	}
+	if (H5Aexists(file_id, "DAOS_PROP_CO_METRICS_ENABLED") > 0) {
+		type = DAOS_PROP_CO_METRICS_ENABLED;
+		prop->dpp_entries[prop_num].dpe_type = type;
+		entry = &prop->dpp_entries[prop_num];
+		rc = deserialize_uint(file_id, &entry->dpe_val,
+				      "DAOS_PROP_CO_METRICS_ENABLED");
+		if (rc != 0)
+			D_GOTO(out, rc);
+		prop_num++;
+	}
 	/* deserialize_label stays false if property doesn't exist above */
 	if (deserialize_label) {
 		prop->dpp_entries[prop_num].dpe_type = DAOS_PROP_CO_LABEL;

src/common/prop.c

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2019-2023 Intel Corporation.
+ * (C) Copyright 2019-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -444,6 +444,13 @@ daos_prop_valid(daos_prop_t *prop, bool pool, bool input)
 		case DAOS_PROP_CO_SCRUBBER_DISABLED:
 			/* Placeholder */
 			break;
+		case DAOS_PROP_CO_METRICS_ENABLED:
+			val = prop->dpp_entries[i].dpe_val;
+			if (val != 0 && val != 1) {
+				D_ERROR("invalid metrics disable value: " DF_U64 "\n", val);
+				return false;
+			}
+			break;
 		case DAOS_PROP_CO_CSUM_SERVER_VERIFY:
 			val = prop->dpp_entries[i].dpe_val;
 			if (val != DAOS_PROP_CO_CSUM_SV_OFF &&
@@ -1085,6 +1092,9 @@ parse_entry(char *str, struct daos_prop_entry *entry)
 	} else if (strcmp(name, DAOS_PROP_ENTRY_PERF_DOMAIN) == 0) {
 		entry->dpe_type = DAOS_PROP_CO_PERF_DOMAIN;
 		entry->dpe_val = strtoull(val, NULL, 0);
+	} else if (strcmp(name, DAOS_PROP_ENTRY_METRICS_ENABLED) == 0) {
+		entry->dpe_type = DAOS_PROP_CO_METRICS_ENABLED;
+		entry->dpe_val  = strtoull(val, NULL, 0);
 	} else if (strcmp(name, DAOS_PROP_ENTRY_LAYOUT_TYPE) == 0 ||
 		   strcmp(name, DAOS_PROP_ENTRY_LAYOUT_VER) == 0 ||
 		   strcmp(name, DAOS_PROP_ENTRY_REDUN_LVL) == 0 ||

src/container/cli.c

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -1387,6 +1387,9 @@ cont_query_bits(daos_prop_t *prop)
 		case DAOS_PROP_CO_SCRUBBER_DISABLED:
 			bits |= DAOS_CO_QUERY_PROP_SCRUB_DIS;
 			break;
+		case DAOS_PROP_CO_METRICS_ENABLED:
+			bits |= DAOS_CO_QUERY_PROP_METRICS_ENA;
+			break;
 		case DAOS_PROP_CO_OBJ_VERSION:
 			bits |= DAOS_CO_QUERY_PROP_OBJ_VERSION;
 			break;

src/container/container_iv.c

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2019-2023 Intel Corporation.
+ * (C) Copyright 2019-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -392,6 +392,10 @@ cont_iv_prop_l2g(daos_prop_t *prop, struct cont_iv_prop *iv_prop)
 			iv_prop->cip_scrubbing_disabled = prop_entry->dpe_val;
 			bits |= DAOS_CO_QUERY_PROP_SCRUB_DIS;
 			break;
+		case DAOS_PROP_CO_METRICS_ENABLED:
+			iv_prop->cip_metrics_enabled = prop_entry->dpe_val;
+			bits |= DAOS_CO_QUERY_PROP_METRICS_ENA;
+			break;
 		default:
 			D_ASSERTF(0, "bad dpe_type %d\n", prop_entry->dpe_type);
 			break;
@@ -1410,6 +1414,11 @@ cont_iv_prop_g2l(struct cont_iv_prop *iv_prop, daos_prop_t **prop_out)
 		prop_entry->dpe_val = iv_prop->cip_scrubbing_disabled;
 		prop_entry->dpe_type = DAOS_PROP_CO_SCRUBBER_DISABLED;
 	}
+	if (bits & DAOS_CO_QUERY_PROP_METRICS_ENA) {
+		prop_entry           = &prop->dpp_entries[i++];
+		prop_entry->dpe_val  = iv_prop->cip_metrics_enabled;
+		prop_entry->dpe_type = DAOS_PROP_CO_METRICS_ENABLED;
+	}
 out:
 	if (rc)
 		daos_prop_free(prop);

src/container/rpc.h

@@ -426,8 +426,9 @@ CRT_RPC_DECLARE(cont_close_v8, DAOS_ISEQ_CONT_CLOSE_V8, DAOS_OSEQ_CONT_CLOSE)
 #define DAOS_CO_QUERY_PROP_SCRUB_DIS		(1ULL << 23)
 #define DAOS_CO_QUERY_PROP_OBJ_VERSION		(1ULL << 24)
 #define DAOS_CO_QUERY_PROP_PERF_DOMAIN		(1ULL << 25)
+#define DAOS_CO_QUERY_PROP_METRICS_ENA  	(1ULL << 26)
 
-#define DAOS_CO_QUERY_PROP_BITS_NR		(26)
+#define DAOS_CO_QUERY_PROP_BITS_NR		(27)
 #define DAOS_CO_QUERY_PROP_ALL					\
 	((1ULL << DAOS_CO_QUERY_PROP_BITS_NR) - 1)
 

src/container/srv_container.c

@@ -667,6 +667,7 @@ cont_create_prop_prepare(struct ds_pool_hdl *pool_hdl,
 		case DAOS_PROP_CO_RP_PDA:
 		case DAOS_PROP_CO_PERF_DOMAIN:
 		case DAOS_PROP_CO_SCRUBBER_DISABLED:
+		case DAOS_PROP_CO_METRICS_ENABLED:
 			entry_def->dpe_val = entry->dpe_val;
 			break;
 		case DAOS_PROP_CO_REDUN_FAC:
@@ -991,6 +992,12 @@ cont_prop_write(struct rdb_tx *tx, const rdb_path_t *kvs, daos_prop_t *prop,
 			if (rc)
 				return rc;
 			break;
+		case DAOS_PROP_CO_METRICS_ENABLED:
+			d_iov_set(&value, &entry->dpe_val, sizeof(entry->dpe_val));
+			rc = rdb_tx_update(tx, kvs, &ds_cont_prop_metrics_enabled, &value);
+			if (rc)
+				return rc;
+			break;
 		default:
 			D_ERROR("bad dpe_type %d.\n", entry->dpe_type);
 			return -DER_INVAL;
@@ -2895,6 +2902,18 @@ cont_prop_read(struct rdb_tx *tx, struct cont *cont, uint64_t bits,
 		prop->dpp_entries[idx].dpe_val = val;
 		idx++;
 	}
+	if (bits & DAOS_CO_QUERY_PROP_METRICS_ENA) {
+		d_iov_set(&value, &val, sizeof(val));
+		rc = rdb_tx_lookup(tx, &cont->c_prop, &ds_cont_prop_metrics_enabled, &value);
+		if (rc == -DER_NONEXIST)
+			val = 0;
+		else if (rc != 0)
+			D_GOTO(out, rc);
+		D_ASSERT(idx < nr);
+		prop->dpp_entries[idx].dpe_type = DAOS_PROP_CO_METRICS_ENABLED;
+		prop->dpp_entries[idx].dpe_val  = val;
+		idx++;
+	}
 	if (bits & DAOS_CO_QUERY_PROP_REDUN_FAC) {
 		d_iov_set(&value, &val, sizeof(val));
 		rc = rdb_tx_lookup(tx, &cont->c_prop, &ds_cont_prop_redun_fac,
@@ -3459,6 +3478,7 @@ cont_query(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont,
 			case DAOS_PROP_CO_PERF_DOMAIN:
 			case DAOS_PROP_CO_GLOBAL_VERSION:
 			case DAOS_PROP_CO_SCRUBBER_DISABLED:
+			case DAOS_PROP_CO_METRICS_ENABLED:
 			case DAOS_PROP_CO_OBJ_VERSION:
 				if (entry->dpe_val != iv_entry->dpe_val) {
 					D_ERROR("type %d mismatch "DF_U64" - "
@@ -4795,6 +4815,19 @@ upgrade_cont_cb(daos_handle_t ih, d_iov_t *key, d_iov_t *val, void *varg)
 		upgraded = true;
 	}
 
+	rc = rdb_tx_lookup(ap->tx, &cont->c_prop, &ds_cont_prop_metrics_enabled, &value);
+	if (rc && rc != -DER_NONEXIST)
+		goto out;
+	if (rc == -DER_NONEXIST) {
+		rc = rdb_tx_update(ap->tx, &cont->c_prop, &ds_cont_prop_metrics_enabled, &value);
+		if (rc) {
+			DL_ERROR(rc, "failed to upgrade container metrics disabled prop: " DF_CONTF,
+				 DP_CONT(ap->pool_uuid, cont_uuid));
+			goto out;
+		}
+		upgraded = true;
+	}
+
 	/* Initialize number of open handles to zero */
 	d_iov_set(&value, &nhandles, sizeof(nhandles));
 	rc = rdb_tx_lookup(ap->tx, &cont->c_prop, &ds_cont_prop_nhandles, &value);

src/container/srv_internal.h

@@ -136,6 +136,7 @@ struct cont_iv_prop {
 	uint64_t	cip_csum_chunk_size;
 	uint64_t	cip_csum_server_verify;
 	uint64_t	cip_scrubbing_disabled;
+	uint64_t	cip_metrics_enabled;
 	uint64_t	cip_dedup;
 	uint64_t	cip_dedup_size;
 	uint64_t	cip_alloced_oid;

src/container/srv_layout.c

@@ -54,6 +54,7 @@ RDB_STRING_KEY(ds_cont_prop_, scrubber_disabled);
 RDB_STRING_KEY(ds_cont_prop_, co_md_times);
 RDB_STRING_KEY(ds_cont_prop_, cont_obj_version);
 RDB_STRING_KEY(ds_cont_prop_, nhandles);
+RDB_STRING_KEY(ds_cont_prop_, metrics_enabled);
 
 /* dummy value for container roots, avoid malloc on demand */
 static struct daos_prop_co_roots dummy_roots;
@@ -205,6 +206,9 @@ struct daos_prop_entry cont_prop_entries_default[CONT_PROP_NUM] = {
 	}, {
 		.dpe_type	= DAOS_PROP_CO_PERF_DOMAIN,
 		.dpe_val	= 0, /* inherit from pool by default */
+	}, {
+		.dpe_type	= DAOS_PROP_CO_METRICS_ENABLED,
+		.dpe_val	= 0, /* disabled by default */
 	}
 };
 

src/container/srv_layout.h

@@ -1,5 +1,5 @@
 /*
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -137,6 +137,7 @@ extern d_iov_t ds_cont_prop_co_md_times;	/* co_md_times */
 extern d_iov_t ds_cont_prop_cont_obj_version;	/* uint32_t */
 extern d_iov_t ds_cont_prop_nhandles;		/* uint32_t */
 extern d_iov_t ds_cont_prop_oit_oids;		/* snapshot OIT OID KVS */
+extern d_iov_t ds_cont_prop_metrics_enabled;	/* uint64_t */
 /* Please read the IMPORTANT notes above before adding new keys. */
 
 struct co_md_times {

src/control/cmd/daos/property.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2021-2023 Intel Corporation.
+// (C) Copyright 2021-2024 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -658,6 +658,36 @@ var propHdlrs = propHdlrMap{
 		},
 		true,
 	},
+	C.DAOS_PROP_ENTRY_METRICS_ENABLED: {
+		C.DAOS_PROP_CO_METRICS_ENABLED,
+		"Container Metrics Enabled",
+		func(_ *propHdlr, e *C.struct_daos_prop_entry, v string) error {
+			boolVal, err := strconv.ParseBool(v)
+			if err != nil {
+				return propError("invalid bool value %q", v)
+			}
+			var value uint64
+			if boolVal {
+				value = 1
+			}
+
+			C.set_dpe_val(e, C.uint64_t(value))
+			return nil
+		},
+		nil,
+		func(e *C.struct_daos_prop_entry, name string) string {
+			if e == nil {
+				return propNotFound(name)
+			}
+			var boolVal bool
+			if C.get_dpe_val(e) != 0 {
+				boolVal = true
+			}
+
+			return fmt.Sprintf("%t", boolVal)
+		},
+		false,
+	},
 }
 
 var contDeprProps = map[string]string{
@@ -1037,13 +1067,13 @@ type SetPropertiesFlag struct {
 }
 
 func (f *SetPropertiesFlag) Complete(match string) []flags.Completion {
-	f.SettableKeys("label", "status")
+	f.SettableKeys("label", "status", "metrics_enabled")
 
 	return f.PropertiesFlag.Complete(match)
 }
 
 func (f *SetPropertiesFlag) UnmarshalFlag(fv string) error {
-	f.SettableKeys("label", "status")
+	f.SettableKeys("label", "status", "metrics_enabled")
 
 	if err := f.PropertiesFlag.UnmarshalFlag(fv); err != nil {
 		return err

src/control/cmd/daos/property_test.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2021-2022 Intel Corporation.
+// (C) Copyright 2021-2024 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -17,6 +17,42 @@ import (
 	"github.com/daos-stack/daos/src/control/lib/daos"
 )
 
+func TestProperty_MetricsEnabled(t *testing.T) {
+	for input, tc := range map[string]struct {
+		expValStr string
+		expErr    error
+	}{
+		// NB: Not testing all permutations because we don't need to test
+		// strconv.ParseBool()... Just enough to verify our code.
+		"true": {
+			expValStr: "true",
+		},
+		"1": {
+			expValStr: "true",
+		},
+		"false": {
+			expValStr: "false",
+		},
+		"0": {
+			expValStr: "false",
+		},
+		"b0rken": {
+			expErr: errors.New("invalid bool value"),
+		},
+	} {
+		t.Run(input, func(t *testing.T) {
+			propEntry := newTestPropEntry()
+			gotErr := propHdlrs[daos.PropEntryMetricsEnabled].nameHdlr(nil, propEntry, input)
+			test.CmpErr(t, tc.expErr, gotErr)
+			if tc.expErr != nil {
+				return
+			}
+			valStr := propHdlrs[daos.PropEntryMetricsEnabled].toString(propEntry, daos.PropEntryMetricsEnabled)
+			test.AssertEqual(t, valStr, tc.expValStr, "Invalid Metrics Enabled Value")
+		})
+	}
+}
+
 func TestProperty_EcCellSize(t *testing.T) {
 	for name, tc := range map[string]struct {
 		SizeStr    string

src/control/lib/daos/pool_cont_prop.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2019-2023 Intel Corporation.
+// (C) Copyright 2019-2024 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -72,6 +72,8 @@ const (
 	PropEntryServerChecksum = C.DAOS_PROP_ENTRY_SRV_CKSUM
 	// PropEntryStatus is the status property.
 	PropEntryStatus = C.DAOS_PROP_ENTRY_STATUS
+	// PropEntryMetricsEnabled is the metrics enabled property.
+	PropEntryMetricsEnabled = C.DAOS_PROP_ENTRY_METRICS_ENABLED
 )
 
 const (

src/include/daos/cont_props.h

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2020-2023 Intel Corporation.
+ * (C) Copyright 2020-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -33,6 +33,7 @@
 #define DAOS_PROP_ENTRY_GLOBAL_VERSION	"global_version"
 #define DAOS_PROP_ENTRY_OBJ_VERSION	"obj_version"
 #define DAOS_PROP_ENTRY_PERF_DOMAIN	"perf_domain"
+#define DAOS_PROP_ENTRY_METRICS_ENABLED "metrics_enabled"
 
 /** DAOS deprecated property entry names keeped for backward compatibility */
 #define DAOS_PROP_ENTRY_REDUN_FAC_OLD	"rf"