aboutsummaryrefslogtreecommitdiff
path: root/sys/contrib/openzfs/module
diff options
context:
space:
mode:
authorMartin Matuska <mm@FreeBSD.org>2021-06-13 02:03:58 +0000
committerMartin Matuska <mm@FreeBSD.org>2021-06-13 02:38:47 +0000
commit3ff01b231dfa83d518854c63e7c9cd1debd1139e (patch)
treed760bee3bdd8c4b94fb9962c9e43577a92c1cd2c /sys/contrib/openzfs/module
parent345077ccd52193591a1bad91fef3583f7129345d (diff)
parentafa7b3484556d3ae610a34582ce5ebd2c3e27bba (diff)
downloadsrc-3ff01b231dfa83d518854c63e7c9cd1debd1139e.tar.gz
src-3ff01b231dfa83d518854c63e7c9cd1debd1139e.zip
zfs: merge openzfs/zfs@afa7b3484 (master) into main
Notable upstream pull request merges: #12149: Multiple man-pages: Move to appropriate section #12158: Re-embed multilist_t storage #12191: Convert non-libzpool libraries to -fvisibility=hidden #12196: Unify manpage makefiles, move pages to better sexions, ... #12210: libzutil: import filtering optimisation #12212: Remove pool io kstats Obtained from: OpenZFS OpenZFS commit: afa7b3484556d3ae610a34582ce5ebd2c3e27bba
Diffstat (limited to 'sys/contrib/openzfs/module')
-rw-r--r--sys/contrib/openzfs/module/os/freebsd/spl/spl_kstat.c62
-rw-r--r--sys/contrib/openzfs/module/os/linux/spl/spl-kstat.c66
-rw-r--r--sys/contrib/openzfs/module/os/linux/zfs/zfs_znode.c15
-rw-r--r--sys/contrib/openzfs/module/zfs/arc.c86
-rw-r--r--sys/contrib/openzfs/module/zfs/dbuf.c20
-rw-r--r--sys/contrib/openzfs/module/zfs/dmu_objset.c30
-rw-r--r--sys/contrib/openzfs/module/zfs/dnode.c2
-rw-r--r--sys/contrib/openzfs/module/zfs/dsl_dataset.c3
-rw-r--r--sys/contrib/openzfs/module/zfs/dsl_pool.c3
-rw-r--r--sys/contrib/openzfs/module/zfs/metaslab.c24
-rw-r--r--sys/contrib/openzfs/module/zfs/multilist.c14
-rw-r--r--sys/contrib/openzfs/module/zfs/spa_stats.c50
-rw-r--r--sys/contrib/openzfs/module/zfs/vdev_queue.c47
13 files changed, 99 insertions, 323 deletions
diff --git a/sys/contrib/openzfs/module/os/freebsd/spl/spl_kstat.c b/sys/contrib/openzfs/module/os/freebsd/spl/spl_kstat.c
index e591921ace1b..059ada235c4a 100644
--- a/sys/contrib/openzfs/module/os/freebsd/spl/spl_kstat.c
+++ b/sys/contrib/openzfs/module/os/freebsd/spl/spl_kstat.c
@@ -508,65 +508,3 @@ kstat_delete(kstat_t *ksp)
kmem_free(ksp->ks_data, ksp->ks_data_size);
free(ksp, M_KSTAT);
}
-
-void
-kstat_waitq_enter(kstat_io_t *kiop)
-{
- hrtime_t new, delta;
- ulong_t wcnt;
-
- new = gethrtime();
- delta = new - kiop->wlastupdate;
- kiop->wlastupdate = new;
- wcnt = kiop->wcnt++;
- if (wcnt != 0) {
- kiop->wlentime += delta * wcnt;
- kiop->wtime += delta;
- }
-}
-
-void
-kstat_waitq_exit(kstat_io_t *kiop)
-{
- hrtime_t new, delta;
- ulong_t wcnt;
-
- new = gethrtime();
- delta = new - kiop->wlastupdate;
- kiop->wlastupdate = new;
- wcnt = kiop->wcnt--;
- ASSERT3S(wcnt, >, 0);
- kiop->wlentime += delta * wcnt;
- kiop->wtime += delta;
-}
-
-void
-kstat_runq_enter(kstat_io_t *kiop)
-{
- hrtime_t new, delta;
- ulong_t rcnt;
-
- new = gethrtime();
- delta = new - kiop->rlastupdate;
- kiop->rlastupdate = new;
- rcnt = kiop->rcnt++;
- if (rcnt != 0) {
- kiop->rlentime += delta * rcnt;
- kiop->rtime += delta;
- }
-}
-
-void
-kstat_runq_exit(kstat_io_t *kiop)
-{
- hrtime_t new, delta;
- ulong_t rcnt;
-
- new = gethrtime();
- delta = new - kiop->rlastupdate;
- kiop->rlastupdate = new;
- rcnt = kiop->rcnt--;
- ASSERT3S(rcnt, >, 0);
- kiop->rlentime += delta * rcnt;
- kiop->rtime += delta;
-}
diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-kstat.c b/sys/contrib/openzfs/module/os/linux/spl/spl-kstat.c
index c7f1aadf784e..0c46708326d8 100644
--- a/sys/contrib/openzfs/module/os/linux/spl/spl-kstat.c
+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-kstat.c
@@ -50,72 +50,6 @@ kstat_resize_raw(kstat_t *ksp)
return (0);
}
-void
-kstat_waitq_enter(kstat_io_t *kiop)
-{
- hrtime_t new, delta;
- ulong_t wcnt;
-
- new = gethrtime();
- delta = new - kiop->wlastupdate;
- kiop->wlastupdate = new;
- wcnt = kiop->wcnt++;
- if (wcnt != 0) {
- kiop->wlentime += delta * wcnt;
- kiop->wtime += delta;
- }
-}
-EXPORT_SYMBOL(kstat_waitq_enter);
-
-void
-kstat_waitq_exit(kstat_io_t *kiop)
-{
- hrtime_t new, delta;
- ulong_t wcnt;
-
- new = gethrtime();
- delta = new - kiop->wlastupdate;
- kiop->wlastupdate = new;
- wcnt = kiop->wcnt--;
- ASSERT((int)wcnt > 0);
- kiop->wlentime += delta * wcnt;
- kiop->wtime += delta;
-}
-EXPORT_SYMBOL(kstat_waitq_exit);
-
-void
-kstat_runq_enter(kstat_io_t *kiop)
-{
- hrtime_t new, delta;
- ulong_t rcnt;
-
- new = gethrtime();
- delta = new - kiop->rlastupdate;
- kiop->rlastupdate = new;
- rcnt = kiop->rcnt++;
- if (rcnt != 0) {
- kiop->rlentime += delta * rcnt;
- kiop->rtime += delta;
- }
-}
-EXPORT_SYMBOL(kstat_runq_enter);
-
-void
-kstat_runq_exit(kstat_io_t *kiop)
-{
- hrtime_t new, delta;
- ulong_t rcnt;
-
- new = gethrtime();
- delta = new - kiop->rlastupdate;
- kiop->rlastupdate = new;
- rcnt = kiop->rcnt--;
- ASSERT((int)rcnt > 0);
- kiop->rlentime += delta * rcnt;
- kiop->rtime += delta;
-}
-EXPORT_SYMBOL(kstat_runq_exit);
-
static int
kstat_seq_show_headers(struct seq_file *f)
{
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode.c
index 6015aea62dca..577927747aef 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode.c
@@ -606,17 +606,24 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz,
* number is already hashed for this super block. This can never
* happen because the inode numbers map 1:1 with the object numbers.
*
- * The one exception is rolling back a mounted file system, but in
- * this case all the active inode are unhashed during the rollback.
+ * Exceptions include rolling back a mounted file system, either
+ * from the zfs rollback or zfs recv command.
+ *
+ * Active inodes are unhashed during the rollback, but since zrele
+ * can happen asynchronously, we can't guarantee they've been
+ * unhashed. This can cause hash collisions in unlinked drain
+ * processing so do not hash unlinked znodes.
*/
- VERIFY3S(insert_inode_locked(ip), ==, 0);
+ if (links > 0)
+ VERIFY3S(insert_inode_locked(ip), ==, 0);
mutex_enter(&zfsvfs->z_znodes_lock);
list_insert_tail(&zfsvfs->z_all_znodes, zp);
zfsvfs->z_nr_znodes++;
mutex_exit(&zfsvfs->z_znodes_lock);
- unlock_new_inode(ip);
+ if (links > 0)
+ unlock_new_inode(ip);
return (zp);
error:
diff --git a/sys/contrib/openzfs/module/zfs/arc.c b/sys/contrib/openzfs/module/zfs/arc.c
index 5526cae378fb..737904f34855 100644
--- a/sys/contrib/openzfs/module/zfs/arc.c
+++ b/sys/contrib/openzfs/module/zfs/arc.c
@@ -2327,7 +2327,7 @@ add_reference(arc_buf_hdr_t *hdr, void *tag)
(state != arc_anon)) {
/* We don't use the L2-only state list. */
if (state != arc_l2c_only) {
- multilist_remove(state->arcs_list[arc_buf_type(hdr)],
+ multilist_remove(&state->arcs_list[arc_buf_type(hdr)],
hdr);
arc_evictable_space_decrement(hdr, state);
}
@@ -2361,7 +2361,7 @@ remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
*/
if (((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
(state != arc_anon)) {
- multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr);
+ multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
arc_evictable_space_increment(hdr, state);
}
@@ -2464,7 +2464,7 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
if (refcnt == 0) {
if (old_state != arc_anon && old_state != arc_l2c_only) {
ASSERT(HDR_HAS_L1HDR(hdr));
- multilist_remove(old_state->arcs_list[buftype], hdr);
+ multilist_remove(&old_state->arcs_list[buftype], hdr);
if (GHOST_STATE(old_state)) {
ASSERT0(bufcnt);
@@ -2481,7 +2481,7 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
* beforehand.
*/
ASSERT(HDR_HAS_L1HDR(hdr));
- multilist_insert(new_state->arcs_list[buftype], hdr);
+ multilist_insert(&new_state->arcs_list[buftype], hdr);
if (GHOST_STATE(new_state)) {
ASSERT0(bufcnt);
@@ -2633,8 +2633,8 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
* L2 headers should never be on the L2 state list since they don't
* have L1 headers allocated.
*/
- ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
- multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
+ ASSERT(multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
+ multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
}
void
@@ -4200,7 +4200,7 @@ arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
arc_buf_contents_t type)
{
uint64_t total_evicted = 0;
- multilist_t *ml = state->arcs_list[type];
+ multilist_t *ml = &state->arcs_list[type];
int num_sublists;
arc_buf_hdr_t **markers;
@@ -4534,8 +4534,8 @@ arc_evict_meta(uint64_t meta_used)
static arc_buf_contents_t
arc_evict_type(arc_state_t *state)
{
- multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA];
- multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA];
+ multilist_t *data_ml = &state->arcs_list[ARC_BUFC_DATA];
+ multilist_t *meta_ml = &state->arcs_list[ARC_BUFC_METADATA];
int data_idx = multilist_get_random_index(data_ml);
int meta_idx = multilist_get_random_index(meta_ml);
multilist_sublist_t *data_mls;
@@ -7455,44 +7455,44 @@ arc_state_init(void)
arc_mfu_ghost = &ARC_mfu_ghost;
arc_l2c_only = &ARC_l2c_only;
- arc_mru->arcs_list[ARC_BUFC_METADATA] =
- multilist_create(sizeof (arc_buf_hdr_t),
+ multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
+ sizeof (arc_buf_hdr_t),
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
arc_state_multilist_index_func);
- arc_mru->arcs_list[ARC_BUFC_DATA] =
- multilist_create(sizeof (arc_buf_hdr_t),
+ multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
+ sizeof (arc_buf_hdr_t),
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
arc_state_multilist_index_func);
- arc_mru_ghost->arcs_list[ARC_BUFC_METADATA] =
- multilist_create(sizeof (arc_buf_hdr_t),
+ multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
+ sizeof (arc_buf_hdr_t),
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
arc_state_multilist_index_func);
- arc_mru_ghost->arcs_list[ARC_BUFC_DATA] =
- multilist_create(sizeof (arc_buf_hdr_t),
+ multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
+ sizeof (arc_buf_hdr_t),
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
arc_state_multilist_index_func);
- arc_mfu->arcs_list[ARC_BUFC_METADATA] =
- multilist_create(sizeof (arc_buf_hdr_t),
+ multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
+ sizeof (arc_buf_hdr_t),
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
arc_state_multilist_index_func);
- arc_mfu->arcs_list[ARC_BUFC_DATA] =
- multilist_create(sizeof (arc_buf_hdr_t),
+ multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
+ sizeof (arc_buf_hdr_t),
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
arc_state_multilist_index_func);
- arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA] =
- multilist_create(sizeof (arc_buf_hdr_t),
+ multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
+ sizeof (arc_buf_hdr_t),
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
arc_state_multilist_index_func);
- arc_mfu_ghost->arcs_list[ARC_BUFC_DATA] =
- multilist_create(sizeof (arc_buf_hdr_t),
+ multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
+ sizeof (arc_buf_hdr_t),
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
arc_state_multilist_index_func);
- arc_l2c_only->arcs_list[ARC_BUFC_METADATA] =
- multilist_create(sizeof (arc_buf_hdr_t),
+ multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
+ sizeof (arc_buf_hdr_t),
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
arc_state_multilist_index_func);
- arc_l2c_only->arcs_list[ARC_BUFC_DATA] =
- multilist_create(sizeof (arc_buf_hdr_t),
+ multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
+ sizeof (arc_buf_hdr_t),
offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
arc_state_multilist_index_func);
@@ -7558,16 +7558,16 @@ arc_state_fini(void)
zfs_refcount_destroy(&arc_mfu_ghost->arcs_size);
zfs_refcount_destroy(&arc_l2c_only->arcs_size);
- multilist_destroy(arc_mru->arcs_list[ARC_BUFC_METADATA]);
- multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
- multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_METADATA]);
- multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
- multilist_destroy(arc_mru->arcs_list[ARC_BUFC_DATA]);
- multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
- multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_DATA]);
- multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
- multilist_destroy(arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
- multilist_destroy(arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
+ multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
+ multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
+ multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
+ multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
+ multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
+ multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
+ multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
+ multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
+ multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
+ multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
aggsum_fini(&arc_meta_used);
aggsum_fini(&arc_size);
@@ -8624,16 +8624,16 @@ l2arc_sublist_lock(int list_num)
switch (list_num) {
case 0:
- ml = arc_mfu->arcs_list[ARC_BUFC_METADATA];
+ ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
break;
case 1:
- ml = arc_mru->arcs_list[ARC_BUFC_METADATA];
+ ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
break;
case 2:
- ml = arc_mfu->arcs_list[ARC_BUFC_DATA];
+ ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
break;
case 3:
- ml = arc_mru->arcs_list[ARC_BUFC_DATA];
+ ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
break;
default:
return (NULL);
diff --git a/sys/contrib/openzfs/module/zfs/dbuf.c b/sys/contrib/openzfs/module/zfs/dbuf.c
index 764383b2d039..9d741545fae3 100644
--- a/sys/contrib/openzfs/module/zfs/dbuf.c
+++ b/sys/contrib/openzfs/module/zfs/dbuf.c
@@ -203,8 +203,8 @@ static boolean_t dbuf_evict_thread_exit;
* by those caches' matching enum values (from dbuf_cached_state_t).
*/
typedef struct dbuf_cache {
- multilist_t *cache;
- zfs_refcount_t size;
+ multilist_t cache;
+ zfs_refcount_t size ____cacheline_aligned;
} dbuf_cache_t;
dbuf_cache_t dbuf_caches[DB_CACHE_MAX];
@@ -667,9 +667,9 @@ dbuf_cache_above_lowater(void)
static void
dbuf_evict_one(void)
{
- int idx = multilist_get_random_index(dbuf_caches[DB_DBUF_CACHE].cache);
+ int idx = multilist_get_random_index(&dbuf_caches[DB_DBUF_CACHE].cache);
multilist_sublist_t *mls = multilist_sublist_lock(
- dbuf_caches[DB_DBUF_CACHE].cache, idx);
+ &dbuf_caches[DB_DBUF_CACHE].cache, idx);
ASSERT(!MUTEX_HELD(&dbuf_evict_lock));
@@ -833,8 +833,8 @@ retry:
dbu_evict_taskq = taskq_create("dbu_evict", 1, defclsyspri, 0, 0, 0);
for (dbuf_cached_state_t dcs = 0; dcs < DB_CACHE_MAX; dcs++) {
- dbuf_caches[dcs].cache =
- multilist_create(sizeof (dmu_buf_impl_t),
+ multilist_create(&dbuf_caches[dcs].cache,
+ sizeof (dmu_buf_impl_t),
offsetof(dmu_buf_impl_t, db_cache_link),
dbuf_cache_multilist_index_func);
zfs_refcount_create(&dbuf_caches[dcs].size);
@@ -901,7 +901,7 @@ dbuf_fini(void)
for (dbuf_cached_state_t dcs = 0; dcs < DB_CACHE_MAX; dcs++) {
zfs_refcount_destroy(&dbuf_caches[dcs].size);
- multilist_destroy(dbuf_caches[dcs].cache);
+ multilist_destroy(&dbuf_caches[dcs].cache);
}
if (dbuf_ksp != NULL) {
@@ -2755,7 +2755,7 @@ dbuf_destroy(dmu_buf_impl_t *db)
ASSERT(db->db_caching_status == DB_DBUF_CACHE ||
db->db_caching_status == DB_DBUF_METADATA_CACHE);
- multilist_remove(dbuf_caches[db->db_caching_status].cache, db);
+ multilist_remove(&dbuf_caches[db->db_caching_status].cache, db);
(void) zfs_refcount_remove_many(
&dbuf_caches[db->db_caching_status].size,
db->db.db_size, db);
@@ -3465,7 +3465,7 @@ dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid,
ASSERT(db->db_caching_status == DB_DBUF_CACHE ||
db->db_caching_status == DB_DBUF_METADATA_CACHE);
- multilist_remove(dbuf_caches[db->db_caching_status].cache, db);
+ multilist_remove(&dbuf_caches[db->db_caching_status].cache, db);
(void) zfs_refcount_remove_many(
&dbuf_caches[db->db_caching_status].size,
db->db.db_size, db);
@@ -3707,7 +3707,7 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag, boolean_t evicting)
DB_DBUF_METADATA_CACHE : DB_DBUF_CACHE;
db->db_caching_status = dcs;
- multilist_insert(dbuf_caches[dcs].cache, db);
+ multilist_insert(&dbuf_caches[dcs].cache, db);
size = zfs_refcount_add_many(
&dbuf_caches[dcs].size,
db->db.db_size, db);
diff --git a/sys/contrib/openzfs/module/zfs/dmu_objset.c b/sys/contrib/openzfs/module/zfs/dmu_objset.c
index bfb4adf262d5..8c244dc4c317 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_objset.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_objset.c
@@ -601,7 +601,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
os->os_zil = zil_alloc(os, &os->os_zil_header);
for (i = 0; i < TXG_SIZE; i++) {
- os->os_dirty_dnodes[i] = multilist_create(sizeof (dnode_t),
+ multilist_create(&os->os_dirty_dnodes[i], sizeof (dnode_t),
offsetof(dnode_t, dn_dirty_link[i]),
dnode_multilist_index_func);
}
@@ -995,9 +995,8 @@ dmu_objset_evict_done(objset_t *os)
mutex_destroy(&os->os_obj_lock);
mutex_destroy(&os->os_user_ptr_lock);
mutex_destroy(&os->os_upgrade_lock);
- for (int i = 0; i < TXG_SIZE; i++) {
- multilist_destroy(os->os_dirty_dnodes[i]);
- }
+ for (int i = 0; i < TXG_SIZE; i++)
+ multilist_destroy(&os->os_dirty_dnodes[i]);
spa_evicting_os_deregister(os->os_spa, os);
kmem_free(os, sizeof (objset_t));
}
@@ -1520,7 +1519,7 @@ dmu_objset_sync_dnodes(multilist_sublist_t *list, dmu_tx_t *tx)
* of why this dnode hold is always needed (even when not
* doing user accounting).
*/
- multilist_t *newlist = dn->dn_objset->os_synced_dnodes;
+ multilist_t *newlist = &dn->dn_objset->os_synced_dnodes;
(void) dnode_add_ref(dn, newlist);
multilist_insert(newlist, dn);
@@ -1689,17 +1688,16 @@ dmu_objset_sync(objset_t *os, zio_t *pio, dmu_tx_t *tx)
* dn_dirty_link[] of this txg. But it may already
* exist because we call dsl_dataset_sync() twice per txg.
*/
- if (os->os_synced_dnodes == NULL) {
- os->os_synced_dnodes =
- multilist_create(sizeof (dnode_t),
+ if (os->os_synced_dnodes.ml_sublists == NULL) {
+ multilist_create(&os->os_synced_dnodes, sizeof (dnode_t),
offsetof(dnode_t, dn_dirty_link[txgoff]),
dnode_multilist_index_func);
} else {
- ASSERT3U(os->os_synced_dnodes->ml_offset, ==,
+ ASSERT3U(os->os_synced_dnodes.ml_offset, ==,
offsetof(dnode_t, dn_dirty_link[txgoff]));
}
- ml = os->os_dirty_dnodes[txgoff];
+ ml = &os->os_dirty_dnodes[txgoff];
num_sublists = multilist_get_num_sublists(ml);
for (int i = 0; i < num_sublists; i++) {
if (multilist_sublist_is_empty_idx(ml, i))
@@ -1738,7 +1736,7 @@ dmu_objset_sync(objset_t *os, zio_t *pio, dmu_tx_t *tx)
boolean_t
dmu_objset_is_dirty(objset_t *os, uint64_t txg)
{
- return (!multilist_is_empty(os->os_dirty_dnodes[txg & TXG_MASK]));
+ return (!multilist_is_empty(&os->os_dirty_dnodes[txg & TXG_MASK]));
}
static file_info_cb_t *file_cbs[DMU_OST_NUMTYPES];
@@ -1949,7 +1947,7 @@ userquota_updates_task(void *arg)
userquota_cache_t cache = { { 0 } };
multilist_sublist_t *list =
- multilist_sublist_lock(os->os_synced_dnodes, uua->uua_sublist_idx);
+ multilist_sublist_lock(&os->os_synced_dnodes, uua->uua_sublist_idx);
ASSERT(multilist_sublist_head(list) == NULL ||
dmu_objset_userused_enabled(os));
@@ -2006,7 +2004,7 @@ userquota_updates_task(void *arg)
mutex_exit(&dn->dn_mtx);
multilist_sublist_remove(list, dn);
- dnode_rele(dn, os->os_synced_dnodes);
+ dnode_rele(dn, &os->os_synced_dnodes);
}
do_userquota_cacheflush(os, &cache, tx);
multilist_sublist_unlock(list);
@@ -2032,12 +2030,12 @@ dnode_rele_task(void *arg)
objset_t *os = uua->uua_os;
multilist_sublist_t *list =
- multilist_sublist_lock(os->os_synced_dnodes, uua->uua_sublist_idx);
+ multilist_sublist_lock(&os->os_synced_dnodes, uua->uua_sublist_idx);
dnode_t *dn;
while ((dn = multilist_sublist_head(list)) != NULL) {
multilist_sublist_remove(list, dn);
- dnode_rele(dn, os->os_synced_dnodes);
+ dnode_rele(dn, &os->os_synced_dnodes);
}
multilist_sublist_unlock(list);
kmem_free(uua, sizeof (*uua));
@@ -2093,7 +2091,7 @@ dmu_objset_sync_done(objset_t *os, dmu_tx_t *tx)
{
boolean_t need_userquota = dmu_objset_do_userquota_updates_prep(os, tx);
- int num_sublists = multilist_get_num_sublists(os->os_synced_dnodes);
+ int num_sublists = multilist_get_num_sublists(&os->os_synced_dnodes);
for (int i = 0; i < num_sublists; i++) {
userquota_updates_arg_t *uua =
kmem_alloc(sizeof (*uua), KM_SLEEP);
diff --git a/sys/contrib/openzfs/module/zfs/dnode.c b/sys/contrib/openzfs/module/zfs/dnode.c
index 0fc788e28fe4..8434e72aa4f8 100644
--- a/sys/contrib/openzfs/module/zfs/dnode.c
+++ b/sys/contrib/openzfs/module/zfs/dnode.c
@@ -1671,7 +1671,7 @@ dnode_setdirty(dnode_t *dn, dmu_tx_t *tx)
*/
dmu_objset_userquota_get_ids(dn, B_TRUE, tx);
- multilist_t *dirtylist = os->os_dirty_dnodes[txg & TXG_MASK];
+ multilist_t *dirtylist = &os->os_dirty_dnodes[txg & TXG_MASK];
multilist_sublist_t *mls = multilist_sublist_lock_obj(dirtylist, dn);
/*
diff --git a/sys/contrib/openzfs/module/zfs/dsl_dataset.c b/sys/contrib/openzfs/module/zfs/dsl_dataset.c
index 6da5faf01edf..9b9bb42287d5 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_dataset.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_dataset.c
@@ -2267,8 +2267,7 @@ dsl_dataset_sync_done(dsl_dataset_t *ds, dmu_tx_t *tx)
dsl_bookmark_sync_done(ds, tx);
- multilist_destroy(os->os_synced_dnodes);
- os->os_synced_dnodes = NULL;
+ multilist_destroy(&os->os_synced_dnodes);
if (os->os_encrypted)
os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_FALSE;
diff --git a/sys/contrib/openzfs/module/zfs/dsl_pool.c b/sys/contrib/openzfs/module/zfs/dsl_pool.c
index c770eafa75d8..e66c136a9e02 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_pool.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_pool.c
@@ -568,8 +568,7 @@ dsl_pool_sync_mos(dsl_pool_t *dp, dmu_tx_t *tx)
VERIFY0(zio_wait(zio));
dmu_objset_sync_done(dp->dp_meta_objset, tx);
taskq_wait(dp->dp_sync_taskq);
- multilist_destroy(dp->dp_meta_objset->os_synced_dnodes);
- dp->dp_meta_objset->os_synced_dnodes = NULL;
+ multilist_destroy(&dp->dp_meta_objset->os_synced_dnodes);
dprintf_bp(&dp->dp_meta_rootbp, "meta objset rootbp is %s", "");
spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp);
diff --git a/sys/contrib/openzfs/module/zfs/metaslab.c b/sys/contrib/openzfs/module/zfs/metaslab.c
index 3b2b79b2f42f..e588765b3382 100644
--- a/sys/contrib/openzfs/module/zfs/metaslab.c
+++ b/sys/contrib/openzfs/module/zfs/metaslab.c
@@ -416,7 +416,7 @@ metaslab_class_create(spa_t *spa, metaslab_ops_t *ops)
mc->mc_spa = spa;
mc->mc_ops = ops;
mutex_init(&mc->mc_lock, NULL, MUTEX_DEFAULT, NULL);
- mc->mc_metaslab_txg_list = multilist_create(sizeof (metaslab_t),
+ multilist_create(&mc->mc_metaslab_txg_list, sizeof (metaslab_t),
offsetof(metaslab_t, ms_class_txg_node), metaslab_idx_func);
for (int i = 0; i < spa->spa_alloc_count; i++) {
metaslab_class_allocator_t *mca = &mc->mc_allocator[i];
@@ -443,7 +443,7 @@ metaslab_class_destroy(metaslab_class_t *mc)
zfs_refcount_destroy(&mca->mca_alloc_slots);
}
mutex_destroy(&mc->mc_lock);
- multilist_destroy(mc->mc_metaslab_txg_list);
+ multilist_destroy(&mc->mc_metaslab_txg_list);
kmem_free(mc, offsetof(metaslab_class_t,
mc_allocator[spa->spa_alloc_count]));
}
@@ -639,7 +639,7 @@ metaslab_class_expandable_space(metaslab_class_t *mc)
void
metaslab_class_evict_old(metaslab_class_t *mc, uint64_t txg)
{
- multilist_t *ml = mc->mc_metaslab_txg_list;
+ multilist_t *ml = &mc->mc_metaslab_txg_list;
for (int i = 0; i < multilist_get_num_sublists(ml); i++) {
multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
metaslab_t *msp = multilist_sublist_head(mls);
@@ -1139,7 +1139,7 @@ metaslab_group_remove(metaslab_group_t *mg, metaslab_t *msp)
metaslab_class_t *mc = msp->ms_group->mg_class;
multilist_sublist_t *mls =
- multilist_sublist_lock_obj(mc->mc_metaslab_txg_list, msp);
+ multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
if (multilist_link_active(&msp->ms_class_txg_node))
multilist_sublist_remove(mls, msp);
multilist_sublist_unlock(mls);
@@ -2175,20 +2175,20 @@ metaslab_potentially_evict(metaslab_class_t *mc)
uint64_t size = spl_kmem_cache_entry_size(zfs_btree_leaf_cache);
int tries = 0;
for (; allmem * zfs_metaslab_mem_limit / 100 < inuse * size &&
- tries < multilist_get_num_sublists(mc->mc_metaslab_txg_list) * 2;
+ tries < multilist_get_num_sublists(&mc->mc_metaslab_txg_list) * 2;
tries++) {
unsigned int idx = multilist_get_random_index(
- mc->mc_metaslab_txg_list);
+ &mc->mc_metaslab_txg_list);
multilist_sublist_t *mls =
- multilist_sublist_lock(mc->mc_metaslab_txg_list, idx);
+ multilist_sublist_lock(&mc->mc_metaslab_txg_list, idx);
metaslab_t *msp = multilist_sublist_head(mls);
multilist_sublist_unlock(mls);
while (msp != NULL && allmem * zfs_metaslab_mem_limit / 100 <
inuse * size) {
VERIFY3P(mls, ==, multilist_sublist_lock(
- mc->mc_metaslab_txg_list, idx));
+ &mc->mc_metaslab_txg_list, idx));
ASSERT3U(idx, ==,
- metaslab_idx_func(mc->mc_metaslab_txg_list, msp));
+ metaslab_idx_func(&mc->mc_metaslab_txg_list, msp));
if (!multilist_link_active(&msp->ms_class_txg_node)) {
multilist_sublist_unlock(mls);
@@ -2535,7 +2535,7 @@ metaslab_unload(metaslab_t *msp)
if (msp->ms_group != NULL) {
metaslab_class_t *mc = msp->ms_group->mg_class;
multilist_sublist_t *mls =
- multilist_sublist_lock_obj(mc->mc_metaslab_txg_list, msp);
+ multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
if (multilist_link_active(&msp->ms_class_txg_node))
multilist_sublist_remove(mls, msp);
multilist_sublist_unlock(mls);
@@ -2600,7 +2600,7 @@ metaslab_set_selected_txg(metaslab_t *msp, uint64_t txg)
ASSERT(MUTEX_HELD(&msp->ms_lock));
metaslab_class_t *mc = msp->ms_group->mg_class;
multilist_sublist_t *mls =
- multilist_sublist_lock_obj(mc->mc_metaslab_txg_list, msp);
+ multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
if (multilist_link_active(&msp->ms_class_txg_node))
multilist_sublist_remove(mls, msp);
msp->ms_selected_txg = txg;
@@ -5682,7 +5682,7 @@ metaslab_claim_concrete(vdev_t *vd, uint64_t offset, uint64_t size,
if (spa_writeable(spa)) { /* don't dirty if we're zdb(8) */
metaslab_class_t *mc = msp->ms_group->mg_class;
multilist_sublist_t *mls =
- multilist_sublist_lock_obj(mc->mc_metaslab_txg_list, msp);
+ multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
if (!multilist_link_active(&msp->ms_class_txg_node)) {
msp->ms_selected_txg = txg;
multilist_sublist_insert_head(mls, msp);
diff --git a/sys/contrib/openzfs/module/zfs/multilist.c b/sys/contrib/openzfs/module/zfs/multilist.c
index 36c0d33bf1f6..eeac73bd7adf 100644
--- a/sys/contrib/openzfs/module/zfs/multilist.c
+++ b/sys/contrib/openzfs/module/zfs/multilist.c
@@ -68,8 +68,8 @@ multilist_d2l(multilist_t *ml, void *obj)
* requirement, but a general rule of thumb in order to garner the
* best multi-threaded performance out of the data structure.
*/
-static multilist_t *
-multilist_create_impl(size_t size, size_t offset,
+static void
+multilist_create_impl(multilist_t *ml, size_t size, size_t offset,
unsigned int num, multilist_sublist_index_func_t *index_func)
{
ASSERT3U(size, >, 0);
@@ -77,7 +77,6 @@ multilist_create_impl(size_t size, size_t offset,
ASSERT3U(num, >, 0);
ASSERT3P(index_func, !=, NULL);
- multilist_t *ml = kmem_alloc(sizeof (*ml), KM_SLEEP);
ml->ml_offset = offset;
ml->ml_num_sublists = num;
ml->ml_index_func = index_func;
@@ -92,7 +91,6 @@ multilist_create_impl(size_t size, size_t offset,
mutex_init(&mls->mls_lock, NULL, MUTEX_NOLOCKDEP, NULL);
list_create(&mls->mls_list, size, offset);
}
- return (ml);
}
/*
@@ -103,8 +101,8 @@ multilist_create_impl(size_t size, size_t offset,
* reserve the RAM necessary to create the extra slots for additional CPUs up
* front, and dynamically adding them is a complex task.
*/
-multilist_t *
-multilist_create(size_t size, size_t offset,
+void
+multilist_create(multilist_t *ml, size_t size, size_t offset,
multilist_sublist_index_func_t *index_func)
{
int num_sublists;
@@ -115,7 +113,7 @@ multilist_create(size_t size, size_t offset,
num_sublists = MAX(boot_ncpus, 4);
}
- return (multilist_create_impl(size, offset, num_sublists, index_func));
+ multilist_create_impl(ml, size, offset, num_sublists, index_func);
}
/*
@@ -141,7 +139,7 @@ multilist_destroy(multilist_t *ml)
ml->ml_num_sublists = 0;
ml->ml_offset = 0;
- kmem_free(ml, sizeof (multilist_t));
+ ml->ml_sublists = NULL;
}
/*
diff --git a/sys/contrib/openzfs/module/zfs/spa_stats.c b/sys/contrib/openzfs/module/zfs/spa_stats.c
index c3eacc14239e..534ac72fee7b 100644
--- a/sys/contrib/openzfs/module/zfs/spa_stats.c
+++ b/sys/contrib/openzfs/module/zfs/spa_stats.c
@@ -550,54 +550,6 @@ spa_tx_assign_add_nsecs(spa_t *spa, uint64_t nsecs)
/*
* ==========================================================================
- * SPA IO History Routines
- * ==========================================================================
- */
-static int
-spa_io_history_update(kstat_t *ksp, int rw)
-{
- if (rw == KSTAT_WRITE)
- memset(ksp->ks_data, 0, ksp->ks_data_size);
-
- return (0);
-}
-
-static void
-spa_io_history_init(spa_t *spa)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.io_history;
- char *name;
- kstat_t *ksp;
-
- mutex_init(&shk->lock, NULL, MUTEX_DEFAULT, NULL);
-
- name = kmem_asprintf("zfs/%s", spa_name(spa));
-
- ksp = kstat_create(name, 0, "io", "disk", KSTAT_TYPE_IO, 1, 0);
- shk->kstat = ksp;
-
- if (ksp) {
- ksp->ks_lock = &shk->lock;
- ksp->ks_private = spa;
- ksp->ks_update = spa_io_history_update;
- kstat_install(ksp);
- }
- kmem_strfree(name);
-}
-
-static void
-spa_io_history_destroy(spa_t *spa)
-{
- spa_history_kstat_t *shk = &spa->spa_stats.io_history;
-
- if (shk->kstat)
- kstat_delete(shk->kstat);
-
- mutex_destroy(&shk->lock);
-}
-
-/*
- * ==========================================================================
* SPA MMP History Routines
* ==========================================================================
*/
@@ -996,7 +948,6 @@ spa_stats_init(spa_t *spa)
spa_read_history_init(spa);
spa_txg_history_init(spa);
spa_tx_assign_init(spa);
- spa_io_history_init(spa);
spa_mmp_history_init(spa);
spa_state_init(spa);
spa_iostats_init(spa);
@@ -1010,7 +961,6 @@ spa_stats_destroy(spa_t *spa)
spa_tx_assign_destroy(spa);
spa_txg_history_destroy(spa);
spa_read_history_destroy(spa);
- spa_io_history_destroy(spa);
spa_mmp_history_destroy(spa);
}
diff --git a/sys/contrib/openzfs/module/zfs/vdev_queue.c b/sys/contrib/openzfs/module/zfs/vdev_queue.c
index 25a4bc69cc23..198861edb816 100644
--- a/sys/contrib/openzfs/module/zfs/vdev_queue.c
+++ b/sys/contrib/openzfs/module/zfs/vdev_queue.c
@@ -35,8 +35,6 @@
#include <sys/dsl_pool.h>
#include <sys/metaslab_impl.h>
#include <sys/spa.h>
-#include <sys/spa_impl.h>
-#include <sys/kstat.h>
#include <sys/abd.h>
/*
@@ -516,35 +514,17 @@ vdev_queue_fini(vdev_t *vd)
static void
vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio)
{
- spa_t *spa = zio->io_spa;
- spa_history_kstat_t *shk = &spa->spa_stats.io_history;
-
ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);
avl_add(vdev_queue_type_tree(vq, zio->io_type), zio);
-
- if (shk->kstat != NULL) {
- mutex_enter(&shk->lock);
- kstat_waitq_enter(shk->kstat->ks_data);
- mutex_exit(&shk->lock);
- }
}
static void
vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio)
{
- spa_t *spa = zio->io_spa;
- spa_history_kstat_t *shk = &spa->spa_stats.io_history;
-
ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);
avl_remove(vdev_queue_type_tree(vq, zio->io_type), zio);
-
- if (shk->kstat != NULL) {
- mutex_enter(&shk->lock);
- kstat_waitq_exit(shk->kstat->ks_data);
- mutex_exit(&shk->lock);
- }
}
static boolean_t
@@ -564,9 +544,6 @@ vdev_queue_is_interactive(zio_priority_t p)
static void
vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio)
{
- spa_t *spa = zio->io_spa;
- spa_history_kstat_t *shk = &spa->spa_stats.io_history;
-
ASSERT(MUTEX_HELD(&vq->vq_lock));
ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
vq->vq_class[zio->io_priority].vqc_active++;
@@ -577,20 +554,11 @@ vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio)
vq->vq_nia_credit--;
}
avl_add(&vq->vq_active_tree, zio);
-
- if (shk->kstat != NULL) {
- mutex_enter(&shk->lock);
- kstat_runq_enter(shk->kstat->ks_data);
- mutex_exit(&shk->lock);
- }
}
static void
vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio)
{
- spa_t *spa = zio->io_spa;
- spa_history_kstat_t *shk = &spa->spa_stats.io_history;
-
ASSERT(MUTEX_HELD(&vq->vq_lock));
ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
vq->vq_class[zio->io_priority].vqc_active--;
@@ -602,21 +570,6 @@ vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio)
} else if (vq->vq_ia_active == 0)
vq->vq_nia_credit++;
avl_remove(&vq->vq_active_tree, zio);
-
- if (shk->kstat != NULL) {
- kstat_io_t *ksio = shk->kstat->ks_data;
-
- mutex_enter(&shk->lock);
- kstat_runq_exit(ksio);
- if (zio->io_type == ZIO_TYPE_READ) {
- ksio->reads++;
- ksio->nread += zio->io_size;
- } else if (zio->io_type == ZIO_TYPE_WRITE) {
- ksio->writes++;
- ksio->nwritten += zio->io_size;
- }
- mutex_exit(&shk->lock);
- }
}
static void