aboutsummaryrefslogtreecommitdiff
path: root/sys/contrib/openzfs/module/zfs/zio.c
diff options
context:
space:
mode:
Diffstat (limited to 'sys/contrib/openzfs/module/zfs/zio.c')
-rw-r--r--sys/contrib/openzfs/module/zfs/zio.c113
1 files changed, 62 insertions, 51 deletions
diff --git a/sys/contrib/openzfs/module/zfs/zio.c b/sys/contrib/openzfs/module/zfs/zio.c
index 218aec6093e2..4cf8912d4269 100644
--- a/sys/contrib/openzfs/module/zfs/zio.c
+++ b/sys/contrib/openzfs/module/zfs/zio.c
@@ -339,8 +339,8 @@ zio_fini(void)
}
for (size_t i = 0; i < n; i++) {
- VERIFY3P(zio_buf_cache[i], ==, NULL);
- VERIFY3P(zio_data_buf_cache[i], ==, NULL);
+ VERIFY0P(zio_buf_cache[i]);
+ VERIFY0P(zio_data_buf_cache[i]);
}
if (zio_ksp != NULL) {
@@ -771,7 +771,7 @@ zio_add_child_impl(zio_t *pio, zio_t *cio, boolean_t first)
else
mutex_enter(&cio->io_lock);
- ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0);
+ ASSERT0(pio->io_state[ZIO_WAIT_DONE]);
uint64_t *countp = pio->io_children[cio->io_child_type];
for (int w = 0; w < ZIO_WAIT_TYPES; w++)
@@ -821,7 +821,7 @@ zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait)
boolean_t waiting = B_FALSE;
mutex_enter(&zio->io_lock);
- ASSERT(zio->io_stall == NULL);
+ ASSERT0P(zio->io_stall);
for (int c = 0; c < ZIO_CHILD_TYPES; c++) {
if (!(ZIO_CHILD_BIT_IS_SET(childbits, c)))
continue;
@@ -955,8 +955,8 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
zio_t *zio;
IMPLY(type != ZIO_TYPE_TRIM, psize <= SPA_MAXBLOCKSIZE);
- ASSERT(P2PHASE(psize, SPA_MINBLOCKSIZE) == 0);
- ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0);
+ ASSERT0(P2PHASE(psize, SPA_MINBLOCKSIZE));
+ ASSERT0(P2PHASE(offset, SPA_MINBLOCKSIZE));
ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER));
ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER));
@@ -1451,7 +1451,7 @@ zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp)
metaslab_check_free(spa, bp);
bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp);
} else {
- VERIFY3P(zio_free_sync(NULL, spa, txg, bp, 0), ==, NULL);
+ VERIFY0P(zio_free_sync(NULL, spa, txg, bp, 0));
}
}
@@ -1559,7 +1559,7 @@ zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
{
zio_t *zio;
- ASSERT(vd->vdev_children == 0);
+ ASSERT0(vd->vdev_children);
ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE ||
offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
ASSERT3U(offset + size, <=, vd->vdev_psize);
@@ -1580,7 +1580,7 @@ zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
{
zio_t *zio;
- ASSERT(vd->vdev_children == 0);
+ ASSERT0(vd->vdev_children);
ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE ||
offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
ASSERT3U(offset + size, <=, vd->vdev_psize);
@@ -1747,7 +1747,7 @@ zio_flush(zio_t *pio, vdev_t *vd)
void
zio_shrink(zio_t *zio, uint64_t size)
{
- ASSERT3P(zio->io_executor, ==, NULL);
+ ASSERT0P(zio->io_executor);
ASSERT3U(zio->io_orig_size, ==, zio->io_size);
ASSERT3U(size, <=, zio->io_size);
@@ -1941,7 +1941,7 @@ zio_write_compress(zio_t *zio)
}
ASSERT(zio->io_child_type != ZIO_CHILD_DDT);
- ASSERT(zio->io_bp_override == NULL);
+ ASSERT0P(zio->io_bp_override);
if (!BP_IS_HOLE(bp) && BP_GET_BIRTH(bp) == zio->io_txg) {
/*
@@ -2436,7 +2436,7 @@ __zio_execute(zio_t *zio)
ASSERT(!MUTEX_HELD(&zio->io_lock));
ASSERT(ISP2(stage));
- ASSERT(zio->io_stall == NULL);
+ ASSERT0P(zio->io_stall);
do {
stage <<= 1;
@@ -2509,7 +2509,7 @@ zio_wait(zio_t *zio)
int error;
ASSERT3S(zio->io_stage, ==, ZIO_STAGE_OPEN);
- ASSERT3P(zio->io_executor, ==, NULL);
+ ASSERT0P(zio->io_executor);
zio->io_waiter = curthread;
ASSERT0(zio->io_queued_timestamp);
@@ -2551,7 +2551,7 @@ zio_nowait(zio_t *zio)
if (zio == NULL)
return;
- ASSERT3P(zio->io_executor, ==, NULL);
+ ASSERT0P(zio->io_executor);
if (zio->io_child_type == ZIO_CHILD_LOGICAL &&
list_is_empty(&zio->io_parent_list)) {
@@ -2590,8 +2590,8 @@ zio_reexecute(void *arg)
ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL);
ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN);
- ASSERT(pio->io_gang_leader == NULL);
- ASSERT(pio->io_gang_tree == NULL);
+ ASSERT0P(pio->io_gang_leader);
+ ASSERT0P(pio->io_gang_tree);
mutex_enter(&pio->io_lock);
pio->io_flags = pio->io_orig_flags;
@@ -2689,7 +2689,7 @@ zio_suspend(spa_t *spa, zio_t *zio, zio_suspend_reason_t reason)
ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
ASSERT(zio != spa->spa_suspend_zio_root);
ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
- ASSERT(zio_unique_parent(zio) == NULL);
+ ASSERT0P(zio_unique_parent(zio));
ASSERT(zio->io_stage == ZIO_STAGE_DONE);
zio_add_child(spa->spa_suspend_zio_root, zio);
}
@@ -2908,7 +2908,7 @@ zio_gang_node_alloc(zio_gang_node_t **gnpp, uint64_t gangblocksize)
{
zio_gang_node_t *gn;
- ASSERT(*gnpp == NULL);
+ ASSERT0P(*gnpp);
gn = kmem_zalloc(sizeof (*gn) +
(gbh_nblkptrs(gangblocksize) * sizeof (gn)), KM_SLEEP);
@@ -2925,7 +2925,7 @@ zio_gang_node_free(zio_gang_node_t **gnpp)
zio_gang_node_t *gn = *gnpp;
for (int g = 0; g < gbh_nblkptrs(gn->gn_allocsize); g++)
- ASSERT(gn->gn_child[g] == NULL);
+ ASSERT0P(gn->gn_child[g]);
zio_buf_free(gn->gn_gbh, gn->gn_allocsize);
kmem_free(gn, sizeof (*gn) +
@@ -3362,11 +3362,11 @@ zio_nop_write(zio_t *zio)
zio_prop_t *zp = &zio->io_prop;
ASSERT(BP_IS_HOLE(bp));
- ASSERT(BP_GET_LEVEL(bp) == 0);
+ ASSERT0(BP_GET_LEVEL(bp));
ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
ASSERT(zp->zp_nopwrite);
ASSERT(!zp->zp_dedup);
- ASSERT(zio->io_bp_override == NULL);
+ ASSERT0P(zio->io_bp_override);
ASSERT(IO_IS_ALLOCATING(zio));
/*
@@ -3495,7 +3495,7 @@ zio_ddt_read_start(zio_t *zio)
ddt_univ_phys_t *ddp = dde->dde_phys;
blkptr_t blk;
- ASSERT(zio->io_vsd == NULL);
+ ASSERT0P(zio->io_vsd);
zio->io_vsd = dde;
if (v_self == DDT_PHYS_NONE)
@@ -3560,7 +3560,7 @@ zio_ddt_read_done(zio_t *zio)
zio->io_vsd = NULL;
}
- ASSERT(zio->io_vsd == NULL);
+ ASSERT0P(zio->io_vsd);
return (zio);
}
@@ -4415,7 +4415,7 @@ static void
zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp)
{
ASSERT(BP_GET_BIRTH(bp) == zio->io_txg || BP_IS_HOLE(bp));
- ASSERT(zio->io_bp_override == NULL);
+ ASSERT0P(zio->io_bp_override);
if (!BP_IS_HOLE(bp)) {
metaslab_free(zio->io_spa, bp, BP_GET_BIRTH(bp), B_TRUE);
@@ -4434,12 +4434,15 @@ zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp)
*/
int
zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp,
- uint64_t size, boolean_t *slog)
+ uint64_t min_size, uint64_t max_size, boolean_t *slog,
+ boolean_t allow_larger)
{
int error;
zio_alloc_list_t io_alloc_list;
+ uint64_t alloc_size = 0;
ASSERT(txg > spa_syncing_txg(spa));
+ ASSERT3U(min_size, <=, max_size);
metaslab_trace_init(&io_alloc_list);
@@ -4448,7 +4451,7 @@ zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp,
* Fill in the obvious ones before calling into metaslab_alloc().
*/
BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG);
- BP_SET_PSIZE(new_bp, size);
+ BP_SET_PSIZE(new_bp, max_size);
BP_SET_LEVEL(new_bp, 0);
/*
@@ -4463,43 +4466,51 @@ zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp,
ZIOSTAT_BUMP(ziostat_total_allocations);
/* Try log class (dedicated slog devices) first */
- error = metaslab_alloc(spa, spa_log_class(spa), size, new_bp, 1,
- txg, NULL, flags, &io_alloc_list, allocator, NULL);
+ error = metaslab_alloc_range(spa, spa_log_class(spa), min_size,
+ max_size, new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator,
+ NULL, &alloc_size);
*slog = (error == 0);
/* Try special_embedded_log class (reserved on special vdevs) */
if (error != 0) {
- error = metaslab_alloc(spa, spa_special_embedded_log_class(spa),
- size, new_bp, 1, txg, NULL, flags, &io_alloc_list,
- allocator, NULL);
+ error = metaslab_alloc_range(spa,
+ spa_special_embedded_log_class(spa), min_size, max_size,
+ new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator,
+ NULL, &alloc_size);
}
/* Try special class (general special vdev allocation) */
if (error != 0) {
- error = metaslab_alloc(spa, spa_special_class(spa), size,
- new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator,
- NULL);
+ error = metaslab_alloc_range(spa, spa_special_class(spa),
+ min_size, max_size, new_bp, 1, txg, NULL, flags,
+ &io_alloc_list, allocator, NULL, &alloc_size);
}
/* Try embedded_log class (reserved on normal vdevs) */
if (error != 0) {
- error = metaslab_alloc(spa, spa_embedded_log_class(spa), size,
- new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator,
- NULL);
+ error = metaslab_alloc_range(spa, spa_embedded_log_class(spa),
+ min_size, max_size, new_bp, 1, txg, NULL, flags,
+ &io_alloc_list, allocator, NULL, &alloc_size);
}
/* Finally fall back to normal class */
if (error != 0) {
ZIOSTAT_BUMP(ziostat_alloc_class_fallbacks);
- error = metaslab_alloc(spa, spa_normal_class(spa), size,
- new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator,
- NULL);
+ error = metaslab_alloc_range(spa, spa_normal_class(spa),
+ min_size, max_size, new_bp, 1, txg, NULL, flags,
+ &io_alloc_list, allocator, NULL, &alloc_size);
}
metaslab_trace_fini(&io_alloc_list);
if (error == 0) {
- BP_SET_LSIZE(new_bp, size);
- BP_SET_PSIZE(new_bp, size);
+ if (!allow_larger)
+ alloc_size = MIN(alloc_size, max_size);
+ else if (max_size <= SPA_OLD_MAXBLOCKSIZE)
+ alloc_size = MIN(alloc_size, SPA_OLD_MAXBLOCKSIZE);
+ alloc_size = P2ALIGN_TYPED(alloc_size, ZIL_MIN_BLKSZ, uint64_t);
+
+ BP_SET_LSIZE(new_bp, alloc_size);
+ BP_SET_PSIZE(new_bp, alloc_size);
BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF);
BP_SET_CHECKSUM(new_bp,
spa_version(spa) >= SPA_VERSION_SLIM_ZIL
@@ -4527,8 +4538,8 @@ zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp,
}
} else {
zfs_dbgmsg("%s: zil block allocation failure: "
- "size %llu, error %d", spa_name(spa), (u_longlong_t)size,
- error);
+ "min_size %llu, max_size %llu, error %d", spa_name(spa),
+ (u_longlong_t)min_size, (u_longlong_t)max_size, error);
}
return (error);
@@ -4559,8 +4570,8 @@ zio_vdev_io_start(zio_t *zio)
zio->io_delay = 0;
- ASSERT(zio->io_error == 0);
- ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0);
+ ASSERT0(zio->io_error);
+ ASSERT0(zio->io_child_error[ZIO_CHILD_VDEV]);
if (vd == NULL) {
if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
@@ -4751,7 +4762,7 @@ zio_vdev_io_done(zio_t *zio)
ops->vdev_op_io_done(zio);
if (unexpected_error && vd->vdev_remove_wanted == B_FALSE)
- VERIFY(vdev_probe(vd, zio) == NULL);
+ VERIFY0P(vdev_probe(vd, zio));
return (zio);
}
@@ -4903,7 +4914,7 @@ void
zio_vdev_io_reissue(zio_t *zio)
{
ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);
- ASSERT(zio->io_error == 0);
+ ASSERT0(zio->io_error);
zio->io_stage >>= 1;
}
@@ -4920,7 +4931,7 @@ void
zio_vdev_io_bypass(zio_t *zio)
{
ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START);
- ASSERT(zio->io_error == 0);
+ ASSERT0(zio->io_error);
zio->io_flags |= ZIO_FLAG_IO_BYPASS;
zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1;
@@ -5298,7 +5309,7 @@ zio_ready(zio_t *zio)
ASSERT(IO_IS_ALLOCATING(zio));
ASSERT(BP_GET_BIRTH(bp) == zio->io_txg ||
BP_IS_HOLE(bp) || (zio->io_flags & ZIO_FLAG_NOPWRITE));
- ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0);
+ ASSERT0(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY]);
zio->io_ready(zio);
}
@@ -5448,7 +5459,7 @@ zio_done(zio_t *zio)
for (int c = 0; c < ZIO_CHILD_TYPES; c++)
for (int w = 0; w < ZIO_WAIT_TYPES; w++)
- ASSERT(zio->io_children[c][w] == 0);
+ ASSERT0(zio->io_children[c][w]);
if (zio->io_bp != NULL && !BP_IS_EMBEDDED(zio->io_bp)) {
ASSERT(memcmp(zio->io_bp, &zio->io_bp_copy,
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-22 11:54:53 +0000