Add asize to psize logic

Signed-off-by: Paul Dagnelie <paul.dagnelie@klarasystems.com>

Author: Paul Dagnelie

include/sys/metaslab.h

@@ -88,7 +88,7 @@ int metaslab_alloc(spa_t *, metaslab_class_t *, uint64_t,
     int);
 int metaslab_alloc_range(spa_t *, metaslab_class_t *, uint64_t, uint64_t,
     blkptr_t *, int, uint64_t, blkptr_t *, int, zio_alloc_list_t *, zio_t *,
-    int);
+    int, uint64_t *);
 int metaslab_alloc_dva(spa_t *, metaslab_class_t *, uint64_t,
     dva_t *, int, dva_t *, uint64_t, int, zio_alloc_list_t *, int);
 void metaslab_free(spa_t *, const blkptr_t *, uint64_t, boolean_t);

include/sys/vdev.h

@@ -132,6 +132,8 @@ extern void vdev_space_update(vdev_t *vd,
 
 extern int64_t vdev_deflated_space(vdev_t *vd, int64_t space);
 
+extern uint64_t vdev_asize_to_psize_txg(vdev_t *vd, uint64_t asize,
+    uint64_t txg);
 extern uint64_t vdev_psize_to_asize_txg(vdev_t *vd, uint64_t psize,
     uint64_t txg);
 extern uint64_t vdev_psize_to_asize(vdev_t *vd, uint64_t psize);

include/sys/vdev_draid.h

@@ -94,7 +94,7 @@ extern int vdev_draid_generate_perms(const draid_map_t *, uint8_t **);
  */
 extern boolean_t vdev_draid_readable(vdev_t *, uint64_t);
 extern boolean_t vdev_draid_missing(vdev_t *, uint64_t, uint64_t, uint64_t);
-extern uint64_t vdev_draid_asize_to_psize(vdev_t *, uint64_t);
+extern uint64_t vdev_draid_asize_to_psize(vdev_t *, uint64_t, uint64_t);
 extern void vdev_draid_map_alloc_empty(zio_t *, struct raidz_row *);
 extern int vdev_draid_map_verify_empty(zio_t *, struct raidz_row *);
 extern nvlist_t *vdev_draid_read_config_spare(vdev_t *);

include/sys/vdev_impl.h

@@ -106,7 +106,8 @@ typedef const struct vdev_ops {
 	vdev_fini_func_t		*vdev_op_fini;
 	vdev_open_func_t		*vdev_op_open;
 	vdev_close_func_t		*vdev_op_close;
-	vdev_asize_func_t		*vdev_op_asize;
+	vdev_asize_func_t		*vdev_op_psize_to_asize;
+	vdev_asize_func_t		*vdev_op_asize_to_psize;
 	vdev_min_asize_func_t		*vdev_op_min_asize;
 	vdev_min_alloc_func_t		*vdev_op_min_alloc;
 	vdev_io_start_func_t		*vdev_op_io_start;
@@ -618,6 +619,7 @@ extern vdev_ops_t vdev_indirect_ops;
  */
 extern void vdev_default_xlate(vdev_t *vd, const zfs_range_seg64_t *logical_rs,
     zfs_range_seg64_t *physical_rs, zfs_range_seg64_t *remain_rs);
+extern uint64_t vdev_default_psize(vdev_t *vd, uint64_t asize, uint64_t txg);
 extern uint64_t vdev_default_asize(vdev_t *vd, uint64_t psize, uint64_t txg);
 extern uint64_t vdev_default_min_asize(vdev_t *vd);
 extern uint64_t vdev_get_min_asize(vdev_t *vd);

lib/libzfs/libzfs_dataset.c

@@ -5409,7 +5409,7 @@ zfs_get_holds(zfs_handle_t *zhp, nvlist_t **nvl)
  * +-------+-------+-------+-------+-------+
  *
  * Above, notice that the 4k block required one sector for parity and another
- * for data.  vdev_raidz_asize() will return 8k and as such the pool's allocated
+ * for data.  vdev_raidz_psize_to_asize() will return 8k and as such the pool's allocated
  * and free properties will be adjusted by 8k.  The dataset will not be charged
  * 8k.  Rather, it will be charged a value that is scaled according to the
  * overhead of the 128k block on the same vdev.  This 8k allocation will be
@@ -5461,7 +5461,7 @@ zfs_get_holds(zfs_handle_t *zhp, nvlist_t **nvl)
  * not necessarily equal to "blksize", due to RAIDZ deflation.
  */
 static uint64_t
-vdev_raidz_asize(uint64_t ndisks, uint64_t nparity, uint64_t ashift,
+vdev_raidz_psize_to_asize(uint64_t ndisks, uint64_t nparity, uint64_t ashift,
     uint64_t blksize)
 {
 	uint64_t asize, ndata;
@@ -5481,7 +5481,7 @@ vdev_raidz_asize(uint64_t ndisks, uint64_t nparity, uint64_t ashift,
  * size.
  */
 static uint64_t
-vdev_draid_asize(uint64_t ndisks, uint64_t nparity, uint64_t ashift,
+vdev_draid_psize_to_asize(uint64_t ndisks, uint64_t nparity, uint64_t ashift,
     uint64_t blksize)
 {
 	ASSERT3U(ndisks, >, nparity);
@@ -5541,11 +5541,11 @@ volsize_from_vdevs(zpool_handle_t *zhp, uint64_t nblocks, uint64_t blksize)
 				continue;
 
 			/* allocation size for the "typical" 128k block */
-			tsize = vdev_raidz_asize(ndisks, nparity, ashift,
+			tsize = vdev_raidz_psize_to_asize(ndisks, nparity, ashift,
 			    SPA_OLD_MAXBLOCKSIZE);
 
 			/* allocation size for the blksize block */
-			asize = vdev_raidz_asize(ndisks, nparity, ashift,
+			asize = vdev_raidz_psize_to_asize(ndisks, nparity, ashift,
 			    blksize);
 		} else {
 			uint64_t ndata;
@@ -5555,11 +5555,11 @@ volsize_from_vdevs(zpool_handle_t *zhp, uint64_t nblocks, uint64_t blksize)
 				continue;
 
 			/* allocation size for the "typical" 128k block */
-			tsize = vdev_draid_asize(ndata + nparity, nparity,
+			tsize = vdev_draid_psize_to_asize(ndata + nparity, nparity,
 			    ashift, SPA_OLD_MAXBLOCKSIZE);
 
 			/* allocation size for the blksize block */
-			asize = vdev_draid_asize(ndata + nparity, nparity,
+			asize = vdev_draid_psize_to_asize(ndata + nparity, nparity,
 			    ashift, blksize);
 		}
 

module/os/freebsd/zfs/vdev_geom.c

@@ -1275,7 +1275,8 @@ vdev_ops_t vdev_disk_ops = {
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_geom_open,
 	.vdev_op_close = vdev_geom_close,
-	.vdev_op_asize = vdev_default_asize,
+	.vdev_op_psize_to_asize = vdev_default_asize,
+	.vdev_op_asize_to_psize = vdev_default_psize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_geom_io_start,

module/os/linux/zfs/vdev_disk.c

@@ -1553,7 +1553,8 @@ vdev_ops_t vdev_disk_ops = {
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_disk_open,
 	.vdev_op_close = vdev_disk_close,
-	.vdev_op_asize = vdev_default_asize,
+	.vdev_op_asize_to_psize = vdev_default_psize,
+	.vdev_op_psize_to_asize = vdev_default_asize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_disk_io_start,

module/zfs/metaslab.c

@@ -5230,7 +5230,7 @@ metaslab_group_alloc(metaslab_group_t *mg, zio_alloc_list_t *zal,
 static int
 metaslab_alloc_dva_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
     uint64_t max_psize, dva_t *dva, int d, dva_t *hintdva, uint64_t txg,
-    int flags, zio_alloc_list_t *zal, int allocator)
+    int flags, zio_alloc_list_t *zal, int allocator, uint64_t *actual_psize)
 {
 	metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator];
 	metaslab_group_t *mg, *rotor;
@@ -5253,9 +5253,13 @@ metaslab_alloc_dva_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
 		    allocator);
 		return (SET_ERROR(ENOSPC));
 	}
-	if (max_psize > psize && max_psize >= metaslab_force_ganging) {
-		max_psize = (metaslab_force_ganging - psize) / 2 + psize;
+	if (max_psize > psize && max_psize >= metaslab_force_ganging &&
+	    metaslab_force_ganging_pct > 0 &&
+	    (random_in_range(100) < MIN(metaslab_force_ganging_pct, 100))) {
+		max_psize = MAX((psize + max_psize) / 2,
+		    metaslab_force_ganging);
 	}
+	ASSERT3U(psize, <=, max_psize);
 
 	/*
 	 * Start at the rotor and loop through all mgs until we find something.
@@ -5377,6 +5381,8 @@ metaslab_alloc_dva_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
 		uint64_t offset = metaslab_group_alloc(mg, zal, asize,
 		    max_asize, txg, !try_hard, dva, d, allocator, try_hard,
 		    &asize);
+		if (actual_psize)
+			*actual_psize = vdev_asize_to_psize_txg(vd, asize, txg);
 
 		if (offset != -1ULL) {
 			/*
@@ -5471,7 +5477,7 @@ metaslab_alloc_dva(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
     zio_alloc_list_t *zal, int allocator)
 {
 	return (metaslab_alloc_dva_range(spa, mc, psize, psize, dva, d, hintdva,
-	    txg, flags, zal, allocator));
+	    txg, flags, zal, allocator, NULL));
 }
 
 void
@@ -5952,14 +5958,14 @@ metaslab_alloc(spa_t *spa, metaslab_class_t *mc, uint64_t psize, blkptr_t *bp,
     zio_alloc_list_t *zal, zio_t *zio, int allocator)
 {
 	return (metaslab_alloc_range(spa, mc, psize, psize, bp, ndvas, txg,
-	    hintbp, flags, zal, zio, allocator));
+	    hintbp, flags, zal, zio, allocator, NULL));
 }
 
 int
 metaslab_alloc_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
     uint64_t max_psize, blkptr_t *bp, int ndvas, uint64_t txg,
     blkptr_t *hintbp, int flags, zio_alloc_list_t *zal, zio_t *zio,
-    int allocator)
+    int allocator, uint64_t *actual_psize)
 {
 	dva_t *dva = bp->blk_dva;
 	dva_t *hintdva = (hintbp != NULL) ? hintbp->blk_dva : NULL;
@@ -5981,9 +5987,11 @@ metaslab_alloc_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
 	ASSERT(hintbp == NULL || ndvas <= BP_GET_NDVAS(hintbp));
 	ASSERT3P(zal, !=, NULL);
 
+	uint64_t cur_psize = 0, min_psize = UINT64_MAX;
+
 	for (int d = 0; d < ndvas; d++) {
 		error = metaslab_alloc_dva_range(spa, mc, psize, max_psize,
-		    dva, d, hintdva, txg, flags, zal, allocator);
+		    dva, d, hintdva, txg, flags, zal, allocator, &cur_psize);
 		if (error != 0) {
 			for (d--; d >= 0; d--) {
 				metaslab_unalloc_dva(spa, &dva[d], txg);
@@ -6001,10 +6009,14 @@ metaslab_alloc_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
 			 */
 			metaslab_group_alloc_increment(spa,
 			    DVA_GET_VDEV(&dva[d]), zio, flags, allocator);
+			min_psize = MIN(cur_psize, min_psize);
 		}
 	}
 	ASSERT(error == 0);
 	ASSERT(BP_GET_NDVAS(bp) == ndvas);
+	ASSERT3U(min_psize, !=, UINT64_MAX);
+	if (actual_psize)
+		*actual_psize = min_psize;
 
 	spa_config_exit(spa, SCL_ALLOC, FTAG);
 

module/zfs/vdev.c

@@ -322,6 +322,19 @@ vdev_derive_alloc_bias(const char *bias)
 	return (alloc_bias);
 }
 
+uint64_t
+vdev_default_psize(vdev_t *vd, uint64_t asize, uint64_t txg)
+{
+	ASSERT0(asize % (1ULL << vd->vdev_top->vdev_ashift));
+	uint64_t csize, psize = asize;
+	for (int c = 0; c < vd->vdev_children; c++) {
+		csize = vdev_asize_to_psize_txg(vd->vdev_child[c], asize, txg);
+		psize = MIN(psize, csize);
+	}
+
+	return (psize);
+}
+
 /*
  * Default asize function: return the MAX of psize with the asize of
  * all children.  This is what's used by anything other than RAID-Z.
@@ -4114,17 +4127,22 @@ vdev_sync(vdev_t *vd, uint64_t txg)
 	(void) txg_list_add(&spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg));
 	dmu_tx_commit(tx);
 }
+uint64_t
+vdev_asize_to_psize_txg(vdev_t *vd, uint64_t asize, uint64_t txg)
+{
+	return (vd->vdev_ops->vdev_op_asize_to_psize(vd, asize, txg));
+}
 
 /*
  * Return the amount of space that should be (or was) allocated for the given
  * psize (compressed block size) in the given TXG. Note that for expanded
  * RAIDZ vdevs, the size allocated for older BP's may be larger. See
- * vdev_raidz_asize().
+ * vdev_raidz_psize_to_asize().
  */
 uint64_t
 vdev_psize_to_asize_txg(vdev_t *vd, uint64_t psize, uint64_t txg)
 {
-	return (vd->vdev_ops->vdev_op_asize(vd, psize, txg));
+	return (vd->vdev_ops->vdev_op_psize_to_asize(vd, psize, txg));
 }
 
 uint64_t

module/zfs/vdev_draid.c

@@ -577,7 +577,7 @@ vdev_draid_permute_id(vdev_draid_config_t *vdc,
  * i.e. vdev_draid_psize_to_asize().
  */
 static uint64_t
-vdev_draid_asize(vdev_t *vd, uint64_t psize, uint64_t txg)
+vdev_draid_psize_to_asize(vdev_t *vd, uint64_t psize, uint64_t txg)
 {
 	(void) txg;
 	vdev_draid_config_t *vdc = vd->vdev_tsd;
@@ -598,8 +598,9 @@ vdev_draid_asize(vdev_t *vd, uint64_t psize, uint64_t txg)
  * Deflate the asize to the psize, this includes stripping parity.
  */
 uint64_t
-vdev_draid_asize_to_psize(vdev_t *vd, uint64_t asize)
+vdev_draid_asize_to_psize(vdev_t *vd, uint64_t asize, uint64_t txg)
 {
+	(void) txg;
 	vdev_draid_config_t *vdc = vd->vdev_tsd;
 
 	ASSERT0(asize % vdc->vdc_groupwidth);
@@ -961,7 +962,7 @@ vdev_draid_map_alloc_row(zio_t *zio, raidz_row_t **rrp, uint64_t io_offset,
 	vdev_draid_config_t *vdc = vd->vdev_tsd;
 	uint64_t ashift = vd->vdev_top->vdev_ashift;
 	uint64_t io_size = abd_size;
-	uint64_t io_asize = vdev_draid_asize(vd, io_size, 0);
+	uint64_t io_asize = vdev_draid_psize_to_asize(vd, io_size, 0);
 	uint64_t group = vdev_draid_offset_to_group(vd, io_offset);
 	uint64_t start_offset = vdev_draid_group_to_offset(vd, group + 1);
 
@@ -971,7 +972,7 @@ vdev_draid_map_alloc_row(zio_t *zio, raidz_row_t **rrp, uint64_t io_offset,
 	 */
 	if (io_offset + io_asize > start_offset) {
 		io_size = vdev_draid_asize_to_psize(vd,
-		    start_offset - io_offset);
+		    start_offset - io_offset, 0);
 	}
 
 	/*
@@ -1116,7 +1117,7 @@ vdev_draid_map_alloc(zio_t *zio)
 	if (size < abd_size) {
 		vdev_t *vd = zio->io_vd;
 
-		io_offset += vdev_draid_asize(vd, size, 0);
+		io_offset += vdev_draid_psize_to_asize(vd, size, 0);
 		abd_offset += size;
 		abd_size -= size;
 		nrows++;
@@ -1769,7 +1770,7 @@ vdev_draid_need_resilver(vdev_t *vd, const dva_t *dva, size_t psize,
     uint64_t phys_birth)
 {
 	uint64_t offset = DVA_GET_OFFSET(dva);
-	uint64_t asize = vdev_draid_asize(vd, psize, 0);
+	uint64_t asize = vdev_draid_psize_to_asize(vd, psize, 0);
 
 	if (phys_birth == TXG_UNKNOWN) {
 		/*
@@ -1826,7 +1827,7 @@ vdev_draid_io_verify(vdev_t *vd, raidz_row_t *rr, int col)
 	zfs_range_seg64_t logical_rs, physical_rs, remain_rs;
 	logical_rs.rs_start = rr->rr_offset;
 	logical_rs.rs_end = logical_rs.rs_start +
-	    vdev_draid_asize(vd, rr->rr_size, 0);
+	    vdev_draid_psize_to_asize(vd, rr->rr_size, 0);
 
 	raidz_col_t *rc = &rr->rr_col[col];
 	vdev_t *cvd = vd->vdev_child[rc->rc_devidx];
@@ -2310,7 +2311,8 @@ vdev_ops_t vdev_draid_ops = {
 	.vdev_op_fini = vdev_draid_fini,
 	.vdev_op_open = vdev_draid_open,
 	.vdev_op_close = vdev_draid_close,
-	.vdev_op_asize = vdev_draid_asize,
+	.vdev_op_psize_to_asize = vdev_draid_psize_to_asize,
+	.vdev_op_asize_to_psize = vdev_draid_asize_to_psize,
 	.vdev_op_min_asize = vdev_draid_min_asize,
 	.vdev_op_min_alloc = vdev_draid_min_alloc,
 	.vdev_op_io_start = vdev_draid_io_start,
@@ -2800,7 +2802,8 @@ vdev_ops_t vdev_draid_spare_ops = {
 	.vdev_op_fini = vdev_draid_spare_fini,
 	.vdev_op_open = vdev_draid_spare_open,
 	.vdev_op_close = vdev_draid_spare_close,
-	.vdev_op_asize = vdev_default_asize,
+	.vdev_op_psize_to_asize = vdev_default_asize,
+	.vdev_op_asize_to_psize = vdev_default_psize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_draid_spare_io_start,

module/zfs/vdev_file.c

@@ -312,7 +312,8 @@ vdev_ops_t vdev_file_ops = {
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_file_open,
 	.vdev_op_close = vdev_file_close,
-	.vdev_op_asize = vdev_default_asize,
+	.vdev_op_psize_to_asize = vdev_default_asize,
+	.vdev_op_asize_to_psize = vdev_default_psize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_file_io_start,
@@ -342,7 +343,7 @@ vdev_ops_t vdev_disk_ops = {
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_file_open,
 	.vdev_op_close = vdev_file_close,
-	.vdev_op_asize = vdev_default_asize,
+	.vdev_op_psize_to_asize = vdev_default_asize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_file_io_start,

module/zfs/vdev_indirect.c

@@ -1866,7 +1866,8 @@ vdev_ops_t vdev_indirect_ops = {
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_indirect_open,
 	.vdev_op_close = vdev_indirect_close,
-	.vdev_op_asize = vdev_default_asize,
+	.vdev_op_psize_to_asize = vdev_default_asize,
+	.vdev_op_asize_to_psize = vdev_default_psize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_indirect_io_start,

module/zfs/vdev_mirror.c

@@ -971,7 +971,8 @@ vdev_ops_t vdev_mirror_ops = {
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_mirror_open,
 	.vdev_op_close = vdev_mirror_close,
-	.vdev_op_asize = vdev_default_asize,
+	.vdev_op_psize_to_asize = vdev_default_asize,
+	.vdev_op_asize_to_psize = vdev_default_psize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_mirror_io_start,
@@ -996,7 +997,8 @@ vdev_ops_t vdev_replacing_ops = {
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_mirror_open,
 	.vdev_op_close = vdev_mirror_close,
-	.vdev_op_asize = vdev_default_asize,
+	.vdev_op_psize_to_asize = vdev_default_asize,
+	.vdev_op_asize_to_psize = vdev_default_psize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_mirror_io_start,
@@ -1021,7 +1023,8 @@ vdev_ops_t vdev_spare_ops = {
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_mirror_open,
 	.vdev_op_close = vdev_mirror_close,
-	.vdev_op_asize = vdev_default_asize,
+	.vdev_op_psize_to_asize = vdev_default_asize,
+	.vdev_op_asize_to_psize = vdev_default_psize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_mirror_io_start,