diff options
Diffstat (limited to 'tests/zfs-tests/tests/functional/events')
13 files changed, 825 insertions, 9 deletions
diff --git a/tests/zfs-tests/tests/functional/events/cleanup.ksh b/tests/zfs-tests/tests/functional/events/cleanup.ksh index 72570d9288fd..669b8ae99456 100755 --- a/tests/zfs-tests/tests/functional/events/cleanup.ksh +++ b/tests/zfs-tests/tests/functional/events/cleanup.ksh @@ -7,7 +7,7 @@ # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. +# or https://opensource.org/licenses/CDDL-1.0. # See the License for the specific language governing permissions # and limitations under the License. # @@ -26,8 +26,10 @@ . $STF_SUITE/include/libtest.shlib +zed_stop + zed_cleanup all-debug.sh all-syslog.sh all-dumpfds -zed_stop +zed_events_drain default_cleanup diff --git a/tests/zfs-tests/tests/functional/events/events.cfg b/tests/zfs-tests/tests/functional/events/events.cfg index 27046c1b0c6b..5dbdb3074f31 100644 --- a/tests/zfs-tests/tests/functional/events/events.cfg +++ b/tests/zfs-tests/tests/functional/events/events.cfg @@ -6,7 +6,7 @@ # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. +# or https://opensource.org/licenses/CDDL-1.0. # See the License for the specific language governing permissions # and limitations under the License. # diff --git a/tests/zfs-tests/tests/functional/events/events_001_pos.ksh b/tests/zfs-tests/tests/functional/events/events_001_pos.ksh index 189cf435e88e..b509a44606f0 100755 --- a/tests/zfs-tests/tests/functional/events/events_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/events/events_001_pos.ksh @@ -7,7 +7,7 @@ # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. +# or https://opensource.org/licenses/CDDL-1.0. # See the License for the specific language governing permissions # and limitations under the License. # diff --git a/tests/zfs-tests/tests/functional/events/events_002_pos.ksh b/tests/zfs-tests/tests/functional/events/events_002_pos.ksh index 42088fc0905f..e355b8977378 100755 --- a/tests/zfs-tests/tests/functional/events/events_002_pos.ksh +++ b/tests/zfs-tests/tests/functional/events/events_002_pos.ksh @@ -7,7 +7,7 @@ # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. +# or https://opensource.org/licenses/CDDL-1.0. # See the License for the specific language governing permissions # and limitations under the License. # diff --git a/tests/zfs-tests/tests/functional/events/events_common.kshlib b/tests/zfs-tests/tests/functional/events/events_common.kshlib index 73bf4cf357b5..7a0b032c3b2f 100644 --- a/tests/zfs-tests/tests/functional/events/events_common.kshlib +++ b/tests/zfs-tests/tests/functional/events/events_common.kshlib @@ -6,7 +6,7 @@ # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. +# or https://opensource.org/licenses/CDDL-1.0. # See the License for the specific language governing permissions # and limitations under the License. # diff --git a/tests/zfs-tests/tests/functional/events/setup.ksh b/tests/zfs-tests/tests/functional/events/setup.ksh index 2f81d16b1814..0f058b8fe18c 100755 --- a/tests/zfs-tests/tests/functional/events/setup.ksh +++ b/tests/zfs-tests/tests/functional/events/setup.ksh @@ -7,7 +7,7 @@ # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. +# or https://opensource.org/licenses/CDDL-1.0. # See the License for the specific language governing permissions # and limitations under the License. # diff --git a/tests/zfs-tests/tests/functional/events/zed_cksum_config.ksh b/tests/zfs-tests/tests/functional/events/zed_cksum_config.ksh new file mode 100755 index 000000000000..5aae3d034660 --- /dev/null +++ b/tests/zfs-tests/tests/functional/events/zed_cksum_config.ksh @@ -0,0 +1,158 @@ +#!/bin/ksh -p +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2022, Klara Inc. +# + +# DESCRIPTION: +# Verify that vdev properties, checksum_n and checksum_t, work with ZED. +# +# STRATEGY: +# 1. Create a pool with single vdev +# 2. Set checksum_n/checksum_t to non-default values +# 3. Inject checksum errors +# 4. Verify that ZED degrades vdev +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/events/events_common.kshlib + +verify_runnable "both" + +MOUNTDIR="$TEST_BASE_DIR/checksum_mount" +FILEPATH="$MOUNTDIR/checksum_file" +VDEV="$TEST_BASE_DIR/vdevfile.$$" +POOL="checksum_pool" +FILESIZE="10M" + +function cleanup +{ + log_must zed_stop + + log_must zinject -c all + if poolexists $POOL ; then + destroy_pool $POOL + fi + log_must rm -fd $VDEV $MOUNTDIR +} + +log_onexit cleanup + +log_assert "Test ZED checksum_N and checksum_T configurability" + +function do_setup +{ + log_must zpool create -f -m $MOUNTDIR $POOL $VDEV + log_must zpool events -c + log_must truncate -s 0 $ZED_DEBUG_LOG + log_must zfs set compression=off $POOL + log_must zfs set primarycache=none $POOL + log_must zfs set recordsize=512 $POOL +} + +function do_clean +{ + log_must zinject -c all + log_must zpool destroy $POOL +} + +function must_degrade +{ + log_must wait_vdev_state $POOL $VDEV "DEGRADED" 60 +} + +function mustnot_degrade +{ + log_must file_wait $ZED_DEBUG_LOG 5 + log_must wait_vdev_state $POOL $VDEV "ONLINE" 60 +} + +# Test default settings of ZED: +# checksum_n=10 +# checksum_t=600 +# fire 10 events, should degrade. +function default_degrade +{ + do_setup + + log_must mkfile $FILESIZE $FILEPATH + log_must zinject -a -t data -e checksum -T read -f 100 $FILEPATH + + blk=0 + for _ in {1..10}; do + dd if=$FILEPATH of=/dev/null bs=1 count=1 skip=$blk 2>/dev/null + blk=$((blk+512)) + done + + must_degrade + + do_clean +} + +# Set checksum_t=1 +# fire 10 events over 2.5 seconds, should not degrade. +function checksum_t_no_degrade +{ + do_setup + + log_must zpool set checksum_t=1 $POOL $VDEV + log_must mkfile $FILESIZE $FILEPATH + log_must zinject -a -t data -e checksum -T read -f 100 $FILEPATH + + blk=0 + for _ in {1..10}; do + dd if=$FILEPATH of=/dev/null bs=1 count=1 skip=$blk 2>/dev/null + blk=$((blk+512)) + sleep 0.25 + done + + mustnot_degrade + + do_clean +} + +# Set checksum_n=1 +# fire 1 event, should degrade. +function checksum_n_degrade +{ + do_setup + + log_must zpool set checksum_n=1 $POOL $VDEV + log_must mkfile $FILESIZE $FILEPATH + log_must zinject -a -t data -e checksum -T read -f 100 $FILEPATH + + dd if=$FILEPATH of=/dev/null bs=1 count=1 2>/dev/null + + must_degrade + + do_clean +} + +log_must truncate -s $MINVDEVSIZE $VDEV +log_must mkdir -p $MOUNTDIR + +log_must zed_start +default_degrade +checksum_n_degrade +checksum_t_no_degrade + +log_pass "Test ZED checksum_N and checksum_T configurability" diff --git a/tests/zfs-tests/tests/functional/events/zed_cksum_reported.ksh b/tests/zfs-tests/tests/functional/events/zed_cksum_reported.ksh new file mode 100755 index 000000000000..78acd76c2eeb --- /dev/null +++ b/tests/zfs-tests/tests/functional/events/zed_cksum_reported.ksh @@ -0,0 +1,124 @@ +#!/bin/ksh -p +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2022, Klara Inc. +# +# This software was developed by Rob Wing <rob.wing@klarasystems.com> +# under sponsorship from Seagate Technology LLC and Klara Inc. + +# DESCRIPTION: +# Verify that checksum errors are accurately reported to ZED +# +# STRATEGY: +# 1. Create a mirrored/raidz pool +# 2. Inject checksum error +# 3. Verify checksum error count reported to ZED is not zero +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/events/events_common.kshlib + +verify_runnable "both" + +MOUNTDIR="$TEST_BASE_DIR/checksum_mount" +FILEPATH="$MOUNTDIR/checksum_file" +VDEV="$TEST_BASE_DIR/vdevfile.$$" +VDEV1="$TEST_BASE_DIR/vdevfile1.$$" +POOL="checksum_pool" +FILESIZE="10M" + +function cleanup +{ + log_must zed_stop + + log_must zinject -c all + if poolexists $POOL ; then + destroy_pool $POOL + fi + log_must rm -fd $VDEV $MOUNTDIR +} +log_onexit cleanup + +log_assert "Test reported checksum errors to ZED" + +function setup_pool +{ + type="$1" + + log_must zpool create -f -m $MOUNTDIR $POOL $type $VDEV $VDEV1 + log_must zpool events -c + log_must truncate -s 0 $ZED_DEBUG_LOG + log_must zfs set compression=off $POOL + log_must zfs set primarycache=none $POOL +} + +function do_clean +{ + log_must zinject -c all + log_must zpool destroy $POOL +} + +function do_checksum_error +{ + log_must mkfile $FILESIZE $FILEPATH + log_must zinject -a -t data -e checksum -T read -f 100 $FILEPATH + + dd if=$FILEPATH of=/dev/null bs=1 count=1 2>/dev/null + + log_must file_wait_event $ZED_DEBUG_LOG "ereport.fs.zfs.checksum" 10 + + # checksum error as reported from the vdev. + zpool_cksum=`zpool get -H -o value checksum_errors $POOL $VDEV` + + # first checksum error reported to ZED. + zed_cksum=$(awk '/ZEVENT_CLASS=ereport.fs.zfs.checksum/, \ + /ZEVENT_VDEV_CKSUM_ERRORS=/ { \ + if ($1 ~ "ZEVENT_VDEV_CKSUM_ERRORS") \ + { print $0; exit } }' $ZED_DEBUG_LOG) + + log_must [ $zpool_cksum -gt 0 ] + + log_mustnot [ "$zed_cksum" = "ZEVENT_VDEV_CKSUM_ERRORS=0" ] + + log_must [ "$zed_cksum" = "ZEVENT_VDEV_CKSUM_ERRORS=1" ] +} + +# Set checksum_n=1 +# fire 1 event, should degrade. +function checksum_error +{ + type=$1 + + setup_pool $type + do_checksum_error + do_clean +} + +log_must truncate -s $MINVDEVSIZE $VDEV +log_must truncate -s $MINVDEVSIZE $VDEV1 +log_must mkdir -p $MOUNTDIR + +log_must zed_start +checksum_error mirror +checksum_error raidz + +log_pass "Test reported checksum errors to ZED" diff --git a/tests/zfs-tests/tests/functional/events/zed_fd_spill.ksh b/tests/zfs-tests/tests/functional/events/zed_fd_spill.ksh index d70dd0c7b445..4803f0f3de7c 100755 --- a/tests/zfs-tests/tests/functional/events/zed_fd_spill.ksh +++ b/tests/zfs-tests/tests/functional/events/zed_fd_spill.ksh @@ -7,7 +7,7 @@ # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. +# or https://opensource.org/licenses/CDDL-1.0. # See the License for the specific language governing permissions # and limitations under the License. # diff --git a/tests/zfs-tests/tests/functional/events/zed_io_config.ksh b/tests/zfs-tests/tests/functional/events/zed_io_config.ksh new file mode 100755 index 000000000000..637f979fe8d1 --- /dev/null +++ b/tests/zfs-tests/tests/functional/events/zed_io_config.ksh @@ -0,0 +1,150 @@ +#!/bin/ksh -p +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2022, Klara Inc. +# + +# DESCRIPTION: +# Verify that vdev properties, io_n and io_t, work with ZED. +# +# STRATEGY: +# 1. Create a mirrored pool. +# 3. Set io_n/io_t to non-default values +# 3. Inject io errors +# 4. Verify that ZED degrades vdev + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/events/events_common.kshlib + +verify_runnable "both" + +MOUNTDIR="$TEST_BASE_DIR/io_mount" +FILEPATH="$MOUNTDIR/io_file" +VDEV="$TEST_BASE_DIR/vdevfile.$$" +VDEV1="$TEST_BASE_DIR/vdevfile1.$$" +POOL="io_pool" + +function cleanup +{ + log_must zed_stop + + log_must zinject -c all + if poolexists $POOL ; then + destroy_pool $POOL + fi + log_must rm -fd $VDEV $VDEV1 $MOUNTDIR + log_must set_tunable32 PREFETCH_DISABLE $zfsprefetch +} +log_onexit cleanup + +log_assert "Test ZED io_n and io_t configurability" + +zfsprefetch=$(get_tunable PREFETCH_DISABLE) +log_must set_tunable32 PREFETCH_DISABLE 1 + +function setup_pool +{ + log_must zpool create -f -m $MOUNTDIR $POOL mirror $VDEV $VDEV1 + log_must zpool events -c + log_must truncate -s 0 $ZED_DEBUG_LOG + log_must zfs set compression=off $POOL + log_must zfs set primarycache=none $POOL + log_must zfs set recordsize=512 $POOL +} + +function do_clean +{ + log_must zinject -c all + log_must zpool destroy $POOL +} + +# Test default ZED settings: +# io_n=10 (events) +# io_t=600 (seconds) +# fire 10 events over 2.5 seconds, should degrade. +function default_degrade +{ + setup_pool + + log_must dd if=/dev/urandom of=$FILEPATH bs=1M count=64 + log_must zinject -a -d $VDEV -e io -T read -f 100 $POOL + + blk=0 + for _ in {1..10}; do + dd if=$FILEPATH of=/dev/null bs=1 count=1 skip=$blk 2>/dev/null + blk=$((blk+512)) + sleep 0.25 + done + + log_must wait_vdev_state $POOL $VDEV "FAULTED" 60 + do_clean +} + +# set io_n=1 +# fire 1 event, should degrade +function io_n_degrade +{ + setup_pool + + log_must zpool set io_n=1 $POOL $VDEV + log_must dd if=/dev/urandom of=$FILEPATH bs=1M count=64 + log_must zinject -a -d $VDEV -e io -T read -f 100 $POOL + + dd if=$FILEPATH of=/dev/null bs=1 count=1 2>/dev/null + + log_must wait_vdev_state $POOL $VDEV "FAULTED" 60 + do_clean +} + +# set io_t=1 +# fire 10 events over 2.5 seconds, should not degrade +function io_t_nodegrade +{ + setup_pool + + log_must zpool set io_t=1 $POOL $VDEV + log_must dd if=/dev/urandom of=$FILEPATH bs=1M count=64 + log_must zinject -a -d $VDEV -e io -T read -f 100 $POOL + + blk=0 + for _ in {1..10}; do + dd if=$FILEPATH of=/dev/null bs=1 count=1 skip=$blk 2>/dev/null + blk=$((blk+512)) + sleep 0.25 + done + + log_must file_wait $ZED_DEBUG_LOG 30 + log_must wait_vdev_state $POOL $VDEV "ONLINE" 1 + + do_clean +} + +log_must truncate -s $MINVDEVSIZE $VDEV +log_must truncate -s $MINVDEVSIZE $VDEV1 +log_must mkdir -p $MOUNTDIR + +log_must zed_start +default_degrade +io_n_degrade +io_t_nodegrade + +log_pass "Test ZED io_n and io_t configurability" diff --git a/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh b/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh index 0bef0ef1f96b..766468b5660d 100755 --- a/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh +++ b/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh @@ -7,7 +7,7 @@ # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. +# or https://opensource.org/licenses/CDDL-1.0. # See the License for the specific language governing permissions # and limitations under the License. # diff --git a/tests/zfs-tests/tests/functional/events/zed_slow_io.ksh b/tests/zfs-tests/tests/functional/events/zed_slow_io.ksh new file mode 100755 index 000000000000..d9fabb2c3bc9 --- /dev/null +++ b/tests/zfs-tests/tests/functional/events/zed_slow_io.ksh @@ -0,0 +1,205 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2023, Klara Inc. +# + +# DESCRIPTION: +# Verify that vdev properties, slow_io_n and slow_io_t, work with ZED. +# +# STRATEGY: +# 1. Create a pool with single vdev +# 2. Set slow_io_n/slow_io_t to non-default values +# 3. Inject slow io errors +# 4. Verify that ZED degrades vdev +# + +. $STF_SUITE/include/libtest.shlib + +TESTDIR="$TEST_BASE_DIR/zed_slow_io" +VDEV="$TEST_BASE_DIR/vdevfile.$$" +TESTPOOL="slow_io_pool" +FILEPATH="$TESTDIR/slow_io.testfile" + +OLD_SLOW_IO=$(get_tunable ZIO_SLOW_IO_MS) +OLD_SLOW_IO_EVENTS=$(get_tunable SLOW_IO_EVENTS_PER_SECOND) + +verify_runnable "both" + +function do_setup +{ + log_must truncate -s 1G $VDEV + default_setup_noexit $VDEV + zed_events_drain + log_must zfs set compression=off $TESTPOOL + log_must zfs set primarycache=none $TESTPOOL + log_must zfs set prefetch=none $TESTPOOL + log_must zfs set recordsize=512 $TESTPOOL + for i in {1..10}; do + dd if=/dev/urandom of=${FILEPATH}$i bs=512 count=1 2>/dev/null + done + zpool sync +} + +# intermediate cleanup +function do_clean +{ + log_must zinject -c all + log_must zpool destroy $TESTPOOL + log_must rm -f $VDEV +} + +# final cleanup +function cleanup +{ + log_must zinject -c all + + # if pool still exists then something failed so log additional info + if poolexists $TESTPOOL ; then + log_note "$(zpool status -s $TESTPOOL)" + echo "=================== zed log search ===================" + grep "Diagnosis Engine" $ZEDLET_DIR/zed.log + destroy_pool $TESTPOOL + fi + log_must zed_stop + + log_must rm -f $VDEV + + log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO + log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS +} + +function start_slow_io +{ + zpool sync + log_must set_tunable64 ZIO_SLOW_IO_MS 10 + log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND 1000 + + log_must zinject -d $VDEV -D10:1 -T read $TESTPOOL + zpool sync +} + +function stop_slow_io +{ + log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO + log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS + + log_must zinject -c all +} + +# Test default ZED settings: +# inject 10 events over 2.5 seconds, should not degrade. +function default_degrade +{ + do_setup + + start_slow_io + for i in {1..10}; do + dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null + sleep 0.25 + done + stop_slow_io + log_note "$(zpool status -s $TESTPOOL)" + + # give slow ZED a chance to process the delay events + sleep 18 + log_note "$(zpool status -s $TESTPOOL)" + + degrades=$(grep "zpool_vdev_degrade" $ZEDLET_DIR/zed.log | wc -l) + log_note $degrades vdev degrades in ZED log + [ $degrades -eq "0" ] || \ + log_fail "expecting no degrade events, found $degrades" + + do_clean +} + +# change slow_io_n, slow_io_t to 5 events in 60 seconds +# fire more than 5 events, should degrade +function slow_io_degrade +{ + do_setup + + zpool set slow_io_n=5 $TESTPOOL $VDEV + zpool set slow_io_t=60 $TESTPOOL $VDEV + + start_slow_io + for i in {1..16}; do + dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null + sleep 0.5 + done + stop_slow_io + zpool sync + + # + # wait up to 60 seconds for kernel to produce at least 5 delay events + # + typeset -i i=0 + typeset -i events=0 + while [[ $i -lt 60 ]]; do + events=$(zpool events | grep "ereport\.fs\.zfs.delay" | wc -l) + [[ $events -ge "5" ]] && break + i=$((i+1)) + sleep 1 + done + log_note "$events delay events found" + + if [[ $events -ge "5" ]]; then + log_must wait_vdev_state $TESTPOOL $VDEV "DEGRADED" 10 + fi + + do_clean +} + +# change slow_io_n, slow_io_t to 10 events in 1 second +# inject events spaced 0.5 seconds apart, should not degrade +function slow_io_no_degrade +{ + do_setup + + zpool set slow_io_n=10 $TESTPOOL $VDEV + zpool set slow_io_t=1 $TESTPOOL $VDEV + + start_slow_io + for i in {1..16}; do + dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null + sleep 0.5 + done + stop_slow_io + zpool sync + + log_mustnot wait_vdev_state $TESTPOOL $VDEV "DEGRADED" 45 + + do_clean +} + +log_assert "Test ZED slow io configurability" +log_onexit cleanup + +log_must zed_events_drain +log_must zed_start + +default_degrade +slow_io_degrade +slow_io_no_degrade + +log_pass "Test ZED slow io configurability" diff --git a/tests/zfs-tests/tests/functional/events/zed_slow_io_many_vdevs.ksh b/tests/zfs-tests/tests/functional/events/zed_slow_io_many_vdevs.ksh new file mode 100755 index 000000000000..3357ae2e3510 --- /dev/null +++ b/tests/zfs-tests/tests/functional/events/zed_slow_io_many_vdevs.ksh @@ -0,0 +1,177 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2023, Klara Inc. +# + +# DESCRIPTION: +# Verify that delay events from multiple vdevs doesnt degrade +# +# STRATEGY: +# 1. Create a pool with a 3 disk raidz vdev +# 2. Inject slow io errors +# 3. Verify that ZED detects slow I/Os but doesn't degrade any vdevs +# + +. $STF_SUITE/include/libtest.shlib + +TESTDIR="$TEST_BASE_DIR/zed_slow_io" +VDEV1="$TEST_BASE_DIR/vdevfile1.$$" +VDEV2="$TEST_BASE_DIR/vdevfile2.$$" +VDEV3="$TEST_BASE_DIR/vdevfile3.$$" +VDEV4="$TEST_BASE_DIR/vdevfile4.$$" +VDEVS="$VDEV1 $VDEV2 $VDEV3 $VDEV4" +TESTPOOL="slow_io_pool" +FILEPATH="$TESTDIR/slow_io.testfile" + +OLD_SLOW_IO=$(get_tunable ZIO_SLOW_IO_MS) +OLD_SLOW_IO_EVENTS=$(get_tunable SLOW_IO_EVENTS_PER_SECOND) + +verify_runnable "both" + +function cleanup +{ + log_must zinject -c all + + # if pool still exists then something failed so log additional info + if poolexists $TESTPOOL ; then + log_note "$(zpool status -s $TESTPOOL)" + echo "=================== zed log search ===================" + grep "Diagnosis Engine" $ZEDLET_DIR/zed.log + destroy_pool $TESTPOOL + fi + log_must zed_stop + + log_must rm -f $VDEVS + log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO + log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS +} + +function start_slow_io +{ + for vdev in $VDEVS + do + log_must zpool set slow_io_n=4 $TESTPOOL $vdev + log_must zpool set slow_io_t=60 $TESTPOOL $vdev + done + zpool sync + + log_must set_tunable64 ZIO_SLOW_IO_MS 10 + log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND 1000 + + for vdev in $VDEVS + do + log_must zinject -d $vdev -D10:1 $TESTPOOL + done + zpool sync +} + +function stop_slow_io +{ + log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO + log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS + + log_must zinject -c all +} + +function multiple_slow_vdevs_test +{ + log_must truncate -s 1G $VDEVS + default_raidz_setup_noexit $VDEVS + + log_must zpool events -c + log_must zfs set compression=off $TESTPOOL + log_must zfs set primarycache=none $TESTPOOL + log_must zfs set recordsize=4K $TESTPOOL + + log_must dd if=/dev/urandom of=$FILEPATH bs=1M count=20 + zpool sync + + # + # Read the file with slow io injected on the disks + # This will cause multiple errors on each disk to trip ZED SERD + # + # pool: slow_io_pool + # state: ONLINE + # config: + # + # NAME STATE READ WRITE CKSUM SLOW + # slow_io_pool ONLINE 0 0 0 - + # raidz1-0 ONLINE 0 0 0 - + # /var/tmp/vdevfile1.499278 ONLINE 0 0 0 113 + # /var/tmp/vdevfile2.499278 ONLINE 0 0 0 109 + # /var/tmp/vdevfile3.499278 ONLINE 0 0 0 96 + # /var/tmp/vdevfile4.499278 ONLINE 0 0 0 109 + # + start_slow_io + dd if=$FILEPATH of=/dev/null bs=1M count=20 2>/dev/null + stop_slow_io + + # count events available for processing + typeset -i i=0 + typeset -i events=0 + while [[ $i -lt 60 ]]; do + events=$(zpool events | grep "ereport\.fs\.zfs.delay" | wc -l) + [[ $events -ge "50" ]] && break + i=$((i+1)) + sleep 1 + done + log_note "$events delay events found" + if [[ $events -lt "50" ]]; then + log_note "bailing: not enough events to complete the test" + destroy_pool $TESTPOOL + return + fi + + # + # give slow ZED a chance to process the delay events + # + typeset -i i=0 + typeset -i skips=0 + while [[ $i -lt 75 ]]; do + skips=$(grep "retiring case" \ + $ZEDLET_DIR/zed.log | wc -l) + [[ $skips -gt "0" ]] && break + i=$((i+1)) + sleep 1 + done + + log_note $skips degrade skips in ZED log after $i seconds + [ $skips -gt "0" ] || log_fail "expecting to see skips" + + degrades=$(grep "zpool_vdev_degrade" $ZEDLET_DIR/zed.log | wc -l) + log_note $degrades vdev degrades in ZED log + [ $degrades -eq "0" ] || \ + log_fail "expecting no degrade events, found $degrades" + + destroy_pool $TESTPOOL +} + +log_assert "Test ZED slow io across multiple vdevs" +log_onexit cleanup + +log_must zed_events_drain +log_must zed_start +multiple_slow_vdevs_test + +log_pass "Test ZED slow io across multiple vdevs" |