range_tree: Add zfs_recover_rt parameter and extra debug info

There are production cases where unexpected range tree segment
adding/removal leads to panic. The root cause investigation requires
more debug info about the range tree and the segments in question when
it happens. In addition, the zfs_recover_rt parameter allows converting
such panics into warnings with a potential space leak as a trade-off.

Signed-off-by: Igor Ostapenko <igor.ostapenko@klarasystems.com>

Author: ihoro

include/sys/range_tree.h

@@ -48,6 +48,13 @@ typedef enum zfs_range_seg_type {
 	ZFS_RANGE_SEG_NUM_TYPES,
 } zfs_range_seg_type_t;
 
+typedef enum zfs_range_tree_usecase {
+	ZFS_RANGE_TREE_UC_UNKNOWN,
+	ZFS_RANGE_TREE_UC_ALLOCATED_SPACE,
+	ZFS_RANGE_TREE_UC_FREE_SPACE,
+	ZFS_RANGE_TREE_UC_NUM_CASES,
+} zfs_range_tree_usecase_t;
+
 /*
  * Note: the range_tree may not be accessed concurrently; consumers
  * must provide external locking if required.
@@ -66,6 +73,8 @@ typedef struct zfs_range_tree {
 	const zfs_range_tree_ops_t *rt_ops;
 	void		*rt_arg;
 	uint64_t	rt_gap;		/* allowable inter-segment gap */
+	zfs_range_tree_usecase_t	rt_usecase;
+	char		*rt_instance;	/* use case details for debug */
 
 	/*
 	 * The rt_histogram maintains a histogram of ranges. Each bucket,
@@ -280,6 +289,9 @@ zfs_range_tree_t *zfs_range_tree_create_gap(const zfs_range_tree_ops_t *ops,
     uint64_t gap);
 zfs_range_tree_t *zfs_range_tree_create(const zfs_range_tree_ops_t *ops,
     zfs_range_seg_type_t type, void *arg, uint64_t start, uint64_t shift);
+zfs_range_tree_t *zfs_range_tree_create_usecase(const zfs_range_tree_ops_t *ops,
+    zfs_range_seg_type_t type, void *arg, uint64_t start, uint64_t shift,
+    zfs_range_tree_usecase_t usecase, char *instance);
 void zfs_range_tree_destroy(zfs_range_tree_t *rt);
 boolean_t zfs_range_tree_contains(zfs_range_tree_t *rt, uint64_t start,
     uint64_t size);

man/man4/zfs.4

@@ -1987,6 +1987,12 @@ Set to attempt to recover from fatal errors.
 This should only be used as a last resort,
 as it typically results in leaked space, or worse.
 .
+.It Sy zfs_recover_rt Ns = Ns Sy 0 Ns | Ns 1 Pq int
+Set to attempt to recover from fatal errors while adding or removing
+unexpected segments to a range tree.
+This should only be used as a last resort,
+as it typically results in leaked space.
+.
 .It Sy zfs_removal_ignore_errors Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Ignore hard I/O errors during device removal.
 When set, if a device encounters a hard I/O error during the removal process

module/zfs/dnode.c

@@ -2435,8 +2435,10 @@ dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
 	{
 		int txgoff = tx->tx_txg & TXG_MASK;
 		if (dn->dn_free_ranges[txgoff] == NULL) {
-			dn->dn_free_ranges[txgoff] = zfs_range_tree_create(NULL,
-			    ZFS_RANGE_SEG64, NULL, 0, 0);
+			dn->dn_free_ranges[txgoff] =
+			    zfs_range_tree_create_usecase(
+			    NULL, ZFS_RANGE_SEG64, NULL, 0, 0,
+			    ZFS_RANGE_TREE_UC_FREE_SPACE, "dn_free_ranges");
 		}
 		zfs_range_tree_clear(dn->dn_free_ranges[txgoff], blkid, nblks);
 		zfs_range_tree_add(dn->dn_free_ranges[txgoff], blkid, nblks);

module/zfs/metaslab.c

@@ -368,6 +368,22 @@ static metaslab_stats_t metaslab_stats = {
 #define	METASLABSTAT_BUMP(stat) \
 	atomic_inc_64(&metaslab_stats.stat.value.ui64);
 
+static inline char *
+metaslab_range_tree_instance(metaslab_group_t *mg, metaslab_t *ms,
+    char *rt_name)
+{
+	const size_t len = 4 * ZFS_MAX_DATASET_NAME_LEN;
+	char *buf = kmem_zalloc(len, KM_SLEEP);
+
+	snprintf(buf, len, "{spa=%s vdev_guid=%llu ms_id=%llu %s}",
+	    mg->mg_vd->vdev_spa->spa_name,
+	    mg->mg_vd->vdev_guid,
+	    ms->ms_id,
+	    rt_name);
+
+	return (buf);
+}
+
 
 static kstat_t *metaslab_ksp;
 
@@ -2753,30 +2769,42 @@ metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object,
 	zfs_range_seg_type_t type =
 	    metaslab_calculate_range_tree_type(vd, ms, &start, &shift);
 
-	ms->ms_allocatable = zfs_range_tree_create(NULL, type, NULL, start,
-	    shift);
+	ms->ms_allocatable = zfs_range_tree_create_usecase(NULL, type, NULL,
+	    start, shift, ZFS_RANGE_TREE_UC_FREE_SPACE,
+	    metaslab_range_tree_instance(mg, ms, "ms_allocatable"));
 	for (int t = 0; t < TXG_SIZE; t++) {
-		ms->ms_allocating[t] = zfs_range_tree_create(NULL, type,
-		    NULL, start, shift);
-	}
-	ms->ms_freeing = zfs_range_tree_create(NULL, type, NULL, start, shift);
-	ms->ms_freed = zfs_range_tree_create(NULL, type, NULL, start, shift);
+		ms->ms_allocating[t] = zfs_range_tree_create_usecase(NULL, type,
+		    NULL, start, shift, ZFS_RANGE_TREE_UC_ALLOCATED_SPACE,
+		    metaslab_range_tree_instance(mg, ms, "ms_allocating"));
+	}
+	ms->ms_freeing = zfs_range_tree_create_usecase(NULL, type, NULL, start,
+	    shift, ZFS_RANGE_TREE_UC_FREE_SPACE,
+	    metaslab_range_tree_instance(mg, ms, "ms_freeing"));
+	ms->ms_freed = zfs_range_tree_create_usecase(NULL, type, NULL, start,
+	    shift, ZFS_RANGE_TREE_UC_FREE_SPACE,
+	    metaslab_range_tree_instance(mg, ms, "ms_freed"));
 	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
-		ms->ms_defer[t] = zfs_range_tree_create(NULL, type, NULL,
-		    start, shift);
+		ms->ms_defer[t] = zfs_range_tree_create_usecase(NULL, type,
+		    NULL, start, shift, ZFS_RANGE_TREE_UC_FREE_SPACE,
+		    metaslab_range_tree_instance(mg, ms, "ms_defer"));
 	}
-	ms->ms_checkpointing =
-	    zfs_range_tree_create(NULL, type, NULL, start, shift);
-	ms->ms_unflushed_allocs =
-	    zfs_range_tree_create(NULL, type, NULL, start, shift);
+	ms->ms_checkpointing = zfs_range_tree_create_usecase(NULL, type, NULL,
+	    start, shift, ZFS_RANGE_TREE_UC_FREE_SPACE,
+	    metaslab_range_tree_instance(mg, ms, "ms_checkpointing"));
+	ms->ms_unflushed_allocs = zfs_range_tree_create_usecase(NULL, type,
+	    NULL, start, shift, ZFS_RANGE_TREE_UC_ALLOCATED_SPACE,
+	    metaslab_range_tree_instance(mg, ms, "ms_unflushed_allocs"));
 
 	metaslab_rt_arg_t *mrap = kmem_zalloc(sizeof (*mrap), KM_SLEEP);
 	mrap->mra_bt = &ms->ms_unflushed_frees_by_size;
 	mrap->mra_floor_shift = metaslab_by_size_min_shift;
-	ms->ms_unflushed_frees = zfs_range_tree_create(&metaslab_rt_ops,
-	    type, mrap, start, shift);
+	ms->ms_unflushed_frees = zfs_range_tree_create_usecase(&metaslab_rt_ops,
+	    type, mrap, start, shift, ZFS_RANGE_TREE_UC_FREE_SPACE,
+	    metaslab_range_tree_instance(mg, ms, "ms_unflushed_frees"));
 
-	ms->ms_trim = zfs_range_tree_create(NULL, type, NULL, start, shift);
+	ms->ms_trim = zfs_range_tree_create_usecase(NULL, type, NULL, start,
+	    shift, ZFS_RANGE_TREE_UC_FREE_SPACE,
+	    metaslab_range_tree_instance(mg, ms, "ms_trim"));
 
 	metaslab_group_add(mg, ms);
 	metaslab_set_fragmentation(ms, B_FALSE);
@@ -3750,7 +3778,9 @@ metaslab_condense(metaslab_t *msp, dmu_tx_t *tx)
 	type = metaslab_calculate_range_tree_type(msp->ms_group->mg_vd, msp,
 	    &start, &shift);
 
-	condense_tree = zfs_range_tree_create(NULL, type, NULL, start, shift);
+	condense_tree = zfs_range_tree_create_usecase(NULL, type, NULL, start,
+	    shift, ZFS_RANGE_TREE_UC_FREE_SPACE,
+	    metaslab_range_tree_instance(msp->ms_group, msp, "condense_tree"));
 
 	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
 		zfs_range_tree_walk(msp->ms_defer[t],