diff options
Diffstat (limited to 'sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend')
60 files changed, 5865 insertions, 0 deletions
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/Makefile.am b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/Makefile.am new file mode 100644 index 000000000000..cf6c727dfa16 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/Makefile.am @@ -0,0 +1,63 @@ +pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/rsend +dist_pkgdata_SCRIPTS = \ + setup.ksh \ + cleanup.ksh \ + recv_dedup.ksh \ + recv_dedup_encrypted_zvol.ksh \ + rsend_001_pos.ksh \ + rsend_002_pos.ksh \ + rsend_003_pos.ksh \ + rsend_004_pos.ksh \ + rsend_005_pos.ksh \ + rsend_006_pos.ksh \ + rsend_007_pos.ksh \ + rsend_008_pos.ksh \ + rsend_009_pos.ksh \ + rsend_010_pos.ksh \ + rsend_011_pos.ksh \ + rsend_012_pos.ksh \ + rsend_013_pos.ksh \ + rsend_014_pos.ksh \ + rsend_016_neg.ksh \ + rsend_019_pos.ksh \ + rsend_020_pos.ksh \ + rsend_021_pos.ksh \ + rsend_022_pos.ksh \ + rsend_024_pos.ksh \ + send_encrypted_files.ksh \ + send_encrypted_hierarchy.ksh \ + send_encrypted_props.ksh \ + send_encrypted_truncated_files.ksh \ + send-c_embedded_blocks.ksh \ + send-c_incremental.ksh \ + send-c_lz4_disabled.ksh \ + send-c_mixed_compression.ksh \ + send-c_props.ksh \ + send-c_recv_dedup.ksh \ + send-c_recv_lz4_disabled.ksh \ + send-c_resume.ksh \ + send-c_stream_size_estimate.ksh \ + send-c_verify_contents.ksh \ + send-c_verify_ratio.ksh \ + send-c_volume.ksh \ + send-c_zstreamdump.ksh \ + send-cpL_varied_recsize.ksh \ + send-L_toggle.ksh \ + send_freeobjects.ksh \ + send_partial_dataset.ksh \ + send_realloc_dnode_size.ksh \ + send_realloc_files.ksh \ + send_realloc_encrypted_files.ksh \ + send_spill_block.ksh \ + send_holds.ksh \ + send_hole_birth.ksh \ + send_mixed_raw.ksh \ + send-wR_encrypted_zvol.ksh + +dist_pkgdata_DATA = \ + dedup.zsend.bz2 \ + dedup_encrypted_zvol.bz2 \ + dedup_encrypted_zvol.zsend.bz2 \ + fs.tar.gz \ + rsend.cfg \ + rsend.kshlib diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/cleanup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/cleanup.ksh new file mode 100755 index 000000000000..063fe9dd1f61 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/cleanup.ksh @@ -0,0 +1,47 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +verify_runnable "both" + +if is_global_zone ; then + destroy_pool $POOL + destroy_pool $POOL2 + poolexists $POOL3 && destroy_pool $POOL3 +else + cleanup_pool $POOL + cleanup_pool $POOL2 + poolexists $POOL3 && cleanup_pool $POOL3 +fi +log_must rm -rf $BACKDIR $TESTDIR + +log_pass diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..585e14852662 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 Binary files differnew file mode 100644 index 000000000000..73a5742fc302 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..04a6cb53f040 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/fs.tar.gz b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/fs.tar.gz Binary files differnew file mode 100644 index 000000000000..cb6861c155b7 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/fs.tar.gz diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh new file mode 100755 index 000000000000..e6e282a1c6f8 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh @@ -0,0 +1,53 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Verifies that we can receive a dedup send stream by processing it with +# "zstream redup". +# + +verify_runnable "both" + +function cleanup +{ + destroy_dataset $TESTPOOL/recv "-r" + rm -r /$TESTPOOL/tar + rm $sendfile +} +log_onexit cleanup + +log_assert "Verify zfs can receive dedup send streams with 'zstream redup'" + +typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup.zsend.bz2 +typeset sendfile=/$TESTPOOL/dedup.zsend +typeset tarfile=$STF_SUITE/tests/functional/rsend/fs.tar.gz + +log_must eval "bzcat <$sendfile_compressed >$sendfile" +log_must zfs create $TESTPOOL/recv +log_must eval "zstream redup $sendfile | zfs recv -d $TESTPOOL/recv" + +log_must mkdir /$TESTPOOL/tar +log_must tar --directory /$TESTPOOL/tar -xzf $tarfile +log_must diff -r /$TESTPOOL/tar /$TESTPOOL/recv + +log_pass "zfs can receive dedup send streams with 'zstream redup'" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh new file mode 100755 index 000000000000..569fcd893e7d --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh @@ -0,0 +1,60 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Verifies that we can receive a dedup send stream of a zvol by processing it +# with "zstream redup". +# + +verify_runnable "both" + +function cleanup +{ + destroy_dataset $TESTPOOL/recv "-r" + rm $sendfile + rm $volfile + rm $keyfile +} +log_onexit cleanup + +log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams" + +typeset keyfile=/$TESTPOOL/pkey +typeset recvdev=$ZVOL_DEVDIR/$TESTPOOL/recv +typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 +typeset sendfile=/$TESTPOOL/dedup_encrypted_zvol.zsend +typeset volfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.bz2 +typeset volfile=/$TESTPOOL/dedup_encrypted_zvol + +log_must eval "echo 'password' > $keyfile" + +log_must eval "bzcat <$sendfile_compressed >$sendfile" +log_must eval "zstream redup $sendfile | zfs recv $TESTPOOL/recv" + +log_must zfs load-key $TESTPOOL/recv +block_device_wait + +log_must eval "bzcat <$volfile_compressed >$volfile" +log_must diff $volfile $recvdev + +log_pass "zfs can receive raw, recursive, and deduplicated send streams" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend.cfg b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend.cfg new file mode 100644 index 000000000000..8400ecfe35b4 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend.cfg @@ -0,0 +1,39 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +export BACKDIR=${TEST_BASE_DIR%%/}/backdir-rsend + +export DISK1=$(echo $DISKS | awk '{print $1}') +export DISK2=$(echo $DISKS | awk '{print $2}') +export DISK3=$(echo $DISKS | awk '{print $3}') + +export POOL=$TESTPOOL +export POOL2=$TESTPOOL2 +export POOL3=$TESTPOOL3 +export FS=$TESTFS diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend.kshlib new file mode 100644 index 000000000000..c82b0f008e32 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend.kshlib @@ -0,0 +1,850 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2018 by Delphix. All rights reserved. +# Copyright (c) 2020 by Datto Inc. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/include/math.shlib +. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib +. $STF_SUITE/tests/functional/rsend/rsend.cfg + +# +# Set up test model which includes various datasets +# +# @final +# @snapB +# @init +# | +# ______ pclone +# | / +# |@psnap +# || @final +# ||@final @final @snapC +# ||@snapC @snapC @snapB +# ||@snapA @snapB @snapA +# ||@init @init @init +# ||| | | +# $pool -------- $FS ------- fs1 ------- fs2 +# \ \\_____ \ | +# vol vol \____ \ @fsnap +# | | \ \ \ +# @init @vsnap | ------------ fclone +# @snapA @init \ | | +# @final @snapB \ | @init +# @snapC vclone @snapA +# @final | @final +# @init +# @snapC +# @final +# +# $1 pool name +# +function setup_test_model +{ + typeset pool=$1 + + log_must zfs create -p $pool/$FS/fs1/fs2 + + log_must zfs snapshot $pool@psnap + log_must zfs clone $pool@psnap $pool/pclone + + if is_global_zone ; then + log_must zfs create -V 16M $pool/vol + log_must zfs create -V 16M $pool/$FS/vol + block_device_wait + + log_must zfs snapshot $pool/$FS/vol@vsnap + log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone + block_device_wait + fi + + log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap + log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone + log_must zfs snapshot -r $pool@init + + log_must snapshot_tree $pool@snapA + log_must snapshot_tree $pool@snapC + log_must snapshot_tree $pool/pclone@snapB + log_must snapshot_tree $pool/$FS@snapB + log_must snapshot_tree $pool/$FS@snapC + log_must snapshot_tree $pool/$FS/fs1@snapA + log_must snapshot_tree $pool/$FS/fs1@snapB + log_must snapshot_tree $pool/$FS/fs1@snapC + log_must snapshot_tree $pool/$FS/fs1/fclone@snapA + + if is_global_zone ; then + log_must zfs snapshot $pool/vol@snapA + log_must zfs snapshot $pool/$FS/vol@snapB + log_must zfs snapshot $pool/$FS/vol@snapC + log_must zfs snapshot $pool/$FS/vclone@snapC + fi + + log_must zfs snapshot -r $pool@final + + return 0 +} + +# +# Cleanup the BACKDIR and given pool content and all the sub datasets +# +# $1 pool name +# +function cleanup_pool +{ + typeset pool=$1 + log_must rm -rf $BACKDIR/* + + if is_global_zone ; then + log_must_busy zfs destroy -Rf $pool + else + typeset list=$(zfs list -H -r -t all -o name $pool) + for ds in $list ; do + if [[ $ds != $pool ]] ; then + if datasetexists $ds ; then + log_must_busy zfs destroy -Rf $ds + fi + fi + done + fi + + typeset mntpnt=$(get_prop mountpoint $pool) + if ! ismounted $pool ; then + # Make sure mountpoint directory is empty + if [[ -d $mntpnt ]]; then + log_must rm -rf $mntpnt/* + fi + + log_must zfs mount $pool + fi + if [[ -d $mntpnt ]]; then + rm -rf $mntpnt/* + fi + + return 0 +} + +function cleanup_pools +{ + cleanup_pool $POOL2 + destroy_pool $POOL3 +} + +function cmp_md5s { + typeset file1=$1 + typeset file2=$2 + + typeset sum1=$(md5digest $file1) + typeset sum2=$(md5digest $file2) + test "$sum1" = "$sum2" +} + +# +# Detect if the given two filesystems have same sub-datasets +# +# $1 source filesystem +# $2 destination filesystem +# +function cmp_ds_subs +{ + typeset src_fs=$1 + typeset dst_fs=$2 + + zfs list -r -H -t all -o name $src_fs > $BACKDIR/src1 + zfs list -r -H -t all -o name $dst_fs > $BACKDIR/dst1 + + eval sed -e 's:^$src_fs:PREFIX:g' < $BACKDIR/src1 > $BACKDIR/src + eval sed -e 's:^$dst_fs:PREFIX:g' < $BACKDIR/dst1 > $BACKDIR/dst + + diff $BACKDIR/src $BACKDIR/dst + typeset -i ret=$? + + rm -f $BACKDIR/src $BACKDIR/dst $BACKDIR/src1 $BACKDIR/dst1 + + return $ret +} + +# +# Compare all the directories and files in two filesystems +# +# $1 source filesystem +# $2 destination filesystem +# +function cmp_ds_cont +{ + typeset src_fs=$1 + typeset dst_fs=$2 + + typeset srcdir dstdir + srcdir=$(get_prop mountpoint $src_fs) + dstdir=$(get_prop mountpoint $dst_fs) + + diff -r $srcdir $dstdir > /dev/null 2>&1 + return $? +} + +# +# Compare the given two dataset properties +# +# $1 dataset 1 +# $2 dataset 2 +# +function cmp_ds_prop +{ + typeset dtst1=$1 + typeset dtst2=$2 + typeset -a props=("type" "origin" "volblocksize" "acltype" "dnodesize" \ + "atime" "canmount" "checksum" "compression" "copies" "devices" \ + "exec" "quota" "readonly" "recordsize" "reservation" "setuid" \ + "snapdir" "version" "volsize" "xattr" "mountpoint"); + if is_freebsd; then + props+=("jailed") + else + props+=("zoned") + fi + + for prop in $props; + do + zfs get -H -o property,value,source $prop $dtst1 >> \ + $BACKDIR/dtst1 + zfs get -H -o property,value,source $prop $dtst2 >> \ + $BACKDIR/dtst2 + done + + eval sed -e 's:$dtst1:PREFIX:g' < $BACKDIR/dtst1 > $BACKDIR/dtst1 + eval sed -e 's:$dtst2:PREFIX:g' < $BACKDIR/dtst2 > $BACKDIR/dtst2 + + diff $BACKDIR/dtst1 $BACKDIR/dtst2 + typeset -i ret=$? + + rm -f $BACKDIR/dtst1 $BACKDIR/dtst2 + + return $ret + +} + +# +# Random create directories and files +# +# $1 directory +# +function random_tree +{ + typeset dir=$1 + + if [[ -d $dir ]]; then + rm -rf $dir + fi + mkdir -p $dir + typeset -i ret=$? + + typeset -i nl nd nf + ((nl = RANDOM % 6 + 1)) + ((nd = RANDOM % 3 )) + ((nf = RANDOM % 5 )) + mktree -b $dir -l $nl -d $nd -f $nf + ((ret |= $?)) + + return $ret +} + +# +# Put data in filesystem and take snapshot +# +# $1 snapshot name +# +function snapshot_tree +{ + typeset snap=$1 + typeset ds=${snap%%@*} + typeset type=$(get_prop "type" $ds) + + typeset -i ret=0 + if [[ $type == "filesystem" ]]; then + typeset mntpnt=$(get_prop mountpoint $ds) + ((ret |= $?)) + + if ((ret == 0)) ; then + eval random_tree $mntpnt/${snap##$ds} + ((ret |= $?)) + fi + fi + + if ((ret == 0)) ; then + zfs snapshot $snap + ((ret |= $?)) + fi + + return $ret +} + +# +# Destroy the given snapshot and stuff +# +# $1 snapshot +# +function destroy_tree +{ + typeset -i ret=0 + typeset snap + for snap in "$@" ; do + log_must_busy zfs destroy $snap + + typeset ds=${snap%%@*} + typeset type=$(get_prop "type" $ds) + if [[ $type == "filesystem" ]]; then + typeset mntpnt=$(get_prop mountpoint $ds) + if [[ -n $mntpnt ]]; then + rm -rf $mntpnt/$snap + fi + fi + done + + return 0 +} + +# +# Get all the sub-datasets of give dataset with specific suffix +# +# $1 Given dataset +# $2 Suffix +# +function getds_with_suffix +{ + typeset ds=$1 + typeset suffix=$2 + + typeset list=$(zfs list -r -H -t all -o name $ds | grep "$suffix$") + + echo $list +} + +# +# Output inherited properties which is edited for file system +# +function fs_inherit_prop +{ + typeset fs_prop + if is_global_zone ; then + fs_prop=$(zfs inherit 2>&1 | \ + awk '$2=="YES" && $3=="YES" {print $1}') + if ! is_te_enabled ; then + fs_prop=$(echo $fs_prop | grep -v "mlslabel") + fi + else + fs_prop=$(zfs inherit 2>&1 | \ + awk '$2=="YES" && $3=="YES" {print $1}'| + egrep -v "devices|mlslabel|sharenfs|sharesmb|zoned") + fi + + echo $fs_prop +} + +# +# Output inherited properties for volume +# +function vol_inherit_prop +{ + echo "checksum readonly" +} + +# +# Get the destination dataset to compare +# +function get_dst_ds +{ + typeset srcfs=$1 + typeset dstfs=$2 + + # + # If the srcfs is not pool + # + if ! zpool list $srcfs > /dev/null 2>&1 ; then + eval dstfs="$dstfs/${srcfs#*/}" + fi + + echo $dstfs +} + +# +# Make test files +# +# $1 Number of files to create +# $2 Maximum file size +# $3 File ID offset +# $4 File system to create the files on +# +function mk_files +{ + nfiles=$1 + maxsize=$2 + file_id_offset=$3 + fs=$4 + bs=512 + + for ((i=0; i<$nfiles; i=i+1)); do + file_name="/$fs/file-$maxsize-$((i+$file_id_offset))" + file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1)) + + # + # Create an interesting mix of files which contain both + # data blocks and holes for more realistic test coverage. + # Half the files are created as sparse then partially filled, + # the other half is dense then a hole is punched in the file. + # + if [ $((RANDOM % 2)) -eq 0 ]; then + truncate -s $file_size $file_name || \ + log_fail "Failed to create $file_name" + dd if=/dev/urandom of=$file_name \ + bs=$bs count=$(($file_size / 2 / $bs)) \ + seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \ + conv=notrunc >/dev/null 2>&1 || \ + log_fail "Failed to create $file_name" + else + dd if=/dev/urandom of=$file_name \ + bs=$file_size count=1 >/dev/null 2>&1 || \ + log_fail "Failed to create $file_name" + dd if=/dev/zero of=$file_name \ + bs=$bs count=$(($file_size / 2 / $bs)) \ + seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \ + conv=notrunc >/dev/null 2>&1 || \ + log_fail "Failed to create $file_name" + fi + done + echo Created $nfiles files of random sizes up to $maxsize bytes +} + +# +# Remove test files +# +# $1 Number of files to remove +# $2 Maximum file size +# $3 File ID offset +# $4 File system to remove the files from +# +function rm_files +{ + nfiles=$1 + maxsize=$2 + file_id_offset=$3 + fs=$4 + + for ((i=0; i<$nfiles; i=i+1)); do + rm -f /$fs/file-$maxsize-$((i+$file_id_offset)) + done + echo Removed $nfiles files of random sizes up to $maxsize bytes +} + +# +# Simulate a random set of operations which could be reasonably expected +# to occur on an average filesystem. +# +# $1 Number of files to modify +# $2 Maximum file size +# $3 File system to modify the file on +# $4 Enabled xattrs (optional) +# +function churn_files +{ + nfiles=$1 + maxsize=$2 + fs=$3 + xattrs=${4:-1} + + # + # Remove roughly half of the files in order to make it more + # likely that a dnode will be reallocated. + # + for ((i=0; i<$nfiles; i=i+1)); do + file_name="/$fs/file-$i" + + if [[ -e $file_name ]]; then + if [ $((RANDOM % 2)) -eq 0 ]; then + rm $file_name || \ + log_fail "Failed to remove $file_name" + fi + fi + done + + # + # Remount the filesystem to simulate normal usage. This resets + # the last allocated object id allowing for new objects to be + # reallocated in the locations of previously freed objects. + # + log_must zfs unmount $fs + log_must zfs mount $fs + + for i in {0..$nfiles}; do + file_name="/$fs/file-$i" + file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1)) + + # + # When the file exists modify it in one of five ways to + # simulate normal usage: + # - (20%) Remove and set and extended attribute on the file + # - (20%) Overwrite the existing file + # - (20%) Truncate the existing file to a random length + # - (20%) Truncate the existing file to zero length + # - (20%) Remove the file + # + # Otherwise create the missing file. 20% of the created + # files will be small and use embedded block pointers, the + # remainder with have random sizes up to the maximum size. + # Three extended attributes are attached to all of the files. + # + if [[ -e $file_name ]]; then + value=$((RANDOM % 5)) + if [ $value -eq 0 -a $xattrs -ne 0 ]; then + attrname="testattr$((RANDOM % 3))" + attrlen="$(((RANDOM % 1000) + 1))" + attrvalue="$(random_string VALID_NAME_CHAR \ + $attrlen)" + rm_xattr $attrname $file_name || \ + log_fail "Failed to remove $attrname" + set_xattr $attrname "$attrvalue" $file_name || \ + log_fail "Failed to set $attrname" + elif [ $value -eq 1 ]; then + dd if=/dev/urandom of=$file_name \ + bs=$file_size count=1 >/dev/null 2>&1 || \ + log_fail "Failed to overwrite $file_name" + elif [ $value -eq 2 ]; then + truncate -s $file_size $file_name || \ + log_fail "Failed to truncate $file_name" + elif [ $value -eq 3 ]; then + truncate -s 0 $file_name || \ + log_fail "Failed to truncate $file_name" + else + rm $file_name || \ + log_fail "Failed to remove $file_name" + fi + else + if [ $((RANDOM % 5)) -eq 0 ]; then + file_size=$((($RANDOM % 64) + 1)) + fi + + dd if=/dev/urandom of=$file_name \ + bs=$file_size count=1 >/dev/null 2>&1 || \ + log_fail "Failed to create $file_name" + + if [ $xattrs -ne 0 ]; then + for j in {0..2}; do + attrname="testattr$j" + attrlen="$(((RANDOM % 1000) + 1))" + attrvalue="$(random_string \ + VALID_NAME_CHAR $attrlen)" + set_xattr $attrname \ + "$attrvalue" $file_name || \ + log_fail "Failed to set $attrname" + done + fi + fi + done + + return 0 +} + +# +# Mess up a send file's contents +# +# $1 The send file path +# +function mess_send_file +{ + file=$1 + + filesize=$(stat_size $file) + + offset=$(($RANDOM * $RANDOM % $filesize)) + + # The random offset might truncate the send stream to be + # smaller than the DRR_BEGIN record. If this happens, then + # the receiving system won't have enough info to create the + # partial dataset at all. We use zstreamdump to check for + # this and retry in this case. + nr_begins=$(head -c $offset $file | zstreamdump | \ + grep DRR_BEGIN | awk '{ print $5 }') + while [ "$nr_begins" -eq 0 ]; do + offset=$(($RANDOM * $RANDOM % $filesize)) + nr_begins=$(head -c $offset $file | zstreamdump | \ + grep DRR_BEGIN | awk '{ print $5 }') + done + + if (($RANDOM % 7 <= 1)); then + # + # We corrupt 2 bytes to minimize the chance that we + # write the same value that's already there. + # + log_must eval "dd if=/dev/urandom of=$file conv=notrunc " \ + "bs=1 count=2 seek=$offset >/dev/null 2>&1" + else + log_must truncate -s $offset $file + fi +} + +# +# Diff the send/receive filesystems +# +# $1 The sent filesystem +# $2 The received filesystem +# +function file_check +{ + sendfs=$1 + recvfs=$2 + + if [[ -d /$recvfs/.zfs/snapshot/a && -d \ + /$sendfs/.zfs/snapshot/a ]]; then + diff -r /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a + [[ $? -eq 0 ]] || log_fail "Differences found in snap a" + fi + if [[ -d /$recvfs/.zfs/snapshot/b && -d \ + /$sendfs/.zfs/snapshot/b ]]; then + diff -r /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b + [[ $? -eq 0 ]] || log_fail "Differences found in snap b" + fi +} + +# +# Resume test helper +# +# $1 The ZFS send command +# $2 The filesystem where the streams are sent +# $3 The receive filesystem +# +function resume_test +{ + sendcmd=$1 + streamfs=$2 + recvfs=$3 + + stream_num=1 + log_must eval "$sendcmd >/$streamfs/$stream_num" + + for ((i=0; i<2; i=i+1)); do + mess_send_file /$streamfs/$stream_num + log_mustnot zfs recv -suv $recvfs </$streamfs/$stream_num + stream_num=$((stream_num+1)) + + token=$(zfs get -Hp -o value receive_resume_token $recvfs) + log_must eval "zfs send -t $token >/$streamfs/$stream_num" + [[ -f /$streamfs/$stream_num ]] || \ + log_fail "NO FILE /$streamfs/$stream_num" + done + log_must zfs recv -suv $recvfs </$streamfs/$stream_num +} + +function get_resume_token +{ + sendcmd=$1 + streamfs=$2 + recvfs=$3 + + log_must eval "$sendcmd > /$streamfs/1" + mess_send_file /$streamfs/1 + log_mustnot zfs recv -suv $recvfs < /$streamfs/1 2>&1 + token=$(zfs get -Hp -o value receive_resume_token $recvfs) + echo "$token" > /$streamfs/resume_token + + return 0 +} + +# +# Setup filesystems for the resumable send/receive tests +# +# $1 The "send" filesystem +# $2 The "recv" filesystem +# +function test_fs_setup +{ + typeset sendfs=$1 + typeset recvfs=$2 + typeset streamfs=$3 + typeset sendpool=${sendfs%%/*} + typeset recvpool=${recvfs%%/*} + + datasetexists $sendfs && log_must_busy zfs destroy -r $sendpool + datasetexists $recvfs && log_must_busy zfs destroy -r $recvpool + datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs + + if datasetexists $sendfs || zfs create -o compress=lz4 $sendfs; then + mk_files 1000 256 0 $sendfs & + mk_files 1000 131072 0 $sendfs & + mk_files 100 1048576 0 $sendfs & + mk_files 10 10485760 0 $sendfs & + mk_files 1 104857600 0 $sendfs & + log_must wait + log_must zfs snapshot $sendfs@a + + rm_files 200 256 0 $sendfs & + rm_files 200 131072 0 $sendfs & + rm_files 20 1048576 0 $sendfs & + rm_files 2 10485760 0 $sendfs & + log_must wait + + mk_files 400 256 0 $sendfs & + mk_files 400 131072 0 $sendfs & + mk_files 40 1048576 0 $sendfs & + mk_files 4 10485760 0 $sendfs & + log_must wait + + log_must zfs snapshot $sendfs@b + log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend" + log_must eval "zfs send -v -i @a $sendfs@b " \ + ">/$sendpool/incremental.zsend" + fi + + log_must zfs create -o compress=lz4 $streamfs +} + +# +# Check to see if the specified features are set in a send stream. +# The values for these features are found in include/sys/zfs_ioctl.h +# +# $1 The stream file +# $2-$n The flags expected in the stream +# +function stream_has_features +{ + typeset file=$1 + shift + + [[ -f $file ]] || log_fail "Couldn't find file: $file" + typeset flags=$(cat $file | zstreamdump | \ + awk '/features =/ {features = $3} END {print features}') + typeset -A feature + feature[dedup]="1" + feature[dedupprops]="2" + feature[sa_spill]="4" + feature[embed_data]="10000" + feature[lz4]="20000" + feature[mooch_byteswap]="40000" + feature[large_blocks]="80000" + feature[resuming]="100000" + feature[redacted]="200000" + feature[compressed]="400000" + + typeset flag known derived=0 + for flag in "$@"; do + known=${feature[$flag]} + [[ -z $known ]] && log_fail "Unknown feature: $flag" + + derived=$(printf "%x" $((0x${flags} & 0x${feature[$flag]}))) + [[ $derived = $known ]] || return 1 + done + + return 0 +} + +# +# Given a send stream, verify that the size of the stream matches what's +# expected based on the source or target dataset. If the stream is an +# incremental stream, subtract the size of the source snapshot before +# comparing. This function does not currently handle incremental streams +# that remove data. +# +# $1 The zstreamdump output file +# $2 The dataset to compare against +# This can be a source of a send or recv target (fs, not snapshot) +# $3 The percentage below which verification is deemed a failure +# $4 The source snapshot of an incremental send +# + +function verify_stream_size +{ + typeset stream=$1 + typeset ds=$2 + typeset percent=${3:-90} + typeset inc_src=$4 + + [[ -f $stream ]] || log_fail "No such file: $stream" + datasetexists $ds || log_fail "No such dataset: $ds" + + typeset stream_size=$(cat $stream | zstreamdump | sed -n \ + 's/ Total payload size = \(.*\) (0x.*)/\1/p') + + typeset inc_size=0 + if [[ -n $inc_src ]]; then + inc_size=$(get_prop lrefer $inc_src) + if stream_has_features $stream compressed; then + inc_size=$(get_prop refer $inc_src) + fi + fi + + if stream_has_features $stream compressed; then + ds_size=$(get_prop refer $ds) + else + ds_size=$(get_prop lrefer $ds) + fi + ds_size=$((ds_size - inc_size)) + + within_percent $stream_size $ds_size $percent || log_fail \ + "$stream_size $ds_size differed by too much" +} + +# Cleanup function for tests involving resumable send +function resume_cleanup +{ + typeset sendfs=$1 + typeset streamfs=$2 + typeset sendpool=${sendfs%%/*} + + datasetexists $sendfs && log_must_busy zfs destroy -r $sendfs + datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs + cleanup_pool $POOL2 + rm -f /$sendpool/initial.zsend /$sendpool/incremental.zsend +} + +# Randomly set the property to one of the enumerated values. +function rand_set_prop +{ + typeset dtst=$1 + typeset prop=$2 + shift 2 + typeset value=$(random_get $@) + + log_must eval "zfs set $prop='$value' $dtst" +} + +# Generate a recursive checksum of a filesystem which includes the file +# contents and any associated extended attributes. +function recursive_cksum +{ + case "$(uname)" in + FreeBSD) + find $1 -type f -exec sh -c 'sha256 -q {}; lsextattr -q \ + system {} | sha256 -q; lsextattr -q user {} | sha256 -q' \ + \; | sort | sha256 -q + ;; + *) + find $1 -type f -exec sh -c 'sha256sum {}; getfattr \ + --absolute-names --only-values -d {} | sha256sum' \; | \ + sort -k 2 | awk '{ print $1 }' | sha256sum | \ + awk '{ print $1 }' + ;; + esac +} diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_001_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_001_pos.ksh new file mode 100755 index 000000000000..3864d8a7d1bd --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_001_pos.ksh @@ -0,0 +1,73 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# zfs send -R send replication stream up to the named snap. +# +# STRATEGY: +# 1. Back up all the data from POOL/FS +# 2. Verify all the datasets and data can be recovered in POOL2 +# 3. Back up all the data from root filesystem POOL2 +# 4. Verify all the data can be recovered, too +# + +verify_runnable "both" + +log_assert "zfs send -R send replication stream up to the named snap." +log_onexit cleanup_pool $POOL2 + +# +# Verify the entire pool and sub-ds can be backup and restored. +# +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R" + +dstds=$(get_dst_ds $POOL $POOL2) +log_must cmp_ds_subs $POOL $dstds +log_must cmp_ds_cont $POOL $dstds + +# Cleanup POOL2 +log_must cleanup_pool $POOL2 + +# +# Verify all the filesystem and sub-fs can be backup and restored. +# +log_must eval "zfs send -R $POOL/$FS@final > $BACKDIR/fs-final-R" +log_must eval "zfs receive -d $POOL2 < $BACKDIR/fs-final-R" + +dstds=$(get_dst_ds $POOL/$FS $POOL2) +log_must cmp_ds_subs $POOL/$FS $dstds +log_must cmp_ds_cont $POOL/$FS $dstds + +log_pass "zfs send -R send replication stream up to the named snap." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_002_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_002_pos.ksh new file mode 100755 index 000000000000..f2c1e03937e2 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_002_pos.ksh @@ -0,0 +1,93 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# zfs send -I sends all incrementals from fs@init to fs@final. +# +# STRATEGY: +# 1. Create several snapshots in pool2 +# 2. Send -I @snapA @final +# 3. Destroy all the snapshot except @snapA +# 4. Make sure all the snapshots and content are recovered +# + +verify_runnable "both" + +log_assert "zfs send -I sends all incrementals from fs@init to fs@final." +log_onexit cleanup_pool $POOL2 + +# +# Duplicate POOL2 +# +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-R" + +if is_global_zone ; then + # + # Verify send -I will backup all the incrementals in pool + # + log_must eval "zfs send -I $POOL2@init $POOL2@final > " \ + "$BACKDIR/pool-init-final-I" + log_must destroy_tree $POOL2@final $POOL2@snapC $POOL2@snapA + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-init-final-I" + log_must cmp_ds_subs $POOL $POOL2 + log_must cmp_ds_cont $POOL $POOL2 +fi + +dstds=$(get_dst_ds $POOL $POOL2) + +# +# Verify send -I will backup all the incrementals in filesystem +# +log_must eval "zfs send -I @init $dstds/$FS@final > $BACKDIR/fs-init-final-I" +log_must destroy_tree $dstds/$FS@final $dstds/$FS@snapC $dstds/$FS@snapB +log_must eval "zfs receive -d -F $dstds < $BACKDIR/fs-init-final-I" +log_must cmp_ds_subs $POOL $dstds +log_must cmp_ds_cont $POOL $dstds + +if is_global_zone ; then + # + # Verify send -I will backup all the incrementals in volume + # + dataset=$POOL2/$FS/vol + log_must eval "zfs send -I @vsnap $dataset@final > " \ + "$BACKDIR/vol-vsnap-final-I" + log_must destroy_tree $dataset@final $dataset@snapC \ + $dataset@snapB $dataset@init + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/vol-vsnap-final-I" + log_must cmp_ds_subs $POOL $POOL2 + log_must cmp_ds_cont $POOL $POOL2 +fi + +log_pass "zfs send -I sends all incrementals from fs@init to fs@final." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh new file mode 100755 index 000000000000..2ace6737b14e --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh @@ -0,0 +1,95 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# zfs send -I dataset@init to clone@snap can create a clone +# +# STRATEGY: +# 1. Setup test model +# 2. send -I pool@init to clone@snap +# 3. Verify the clone and snapshot can be recovered via receive +# 4. Verify the similar operating in filesystem and volume +# + +verify_runnable "both" + +log_assert "zfs send -I send all incrementals from dataset@init to clone@snap" +log_onexit cleanup_pool $POOL2 + +# +# Duplicate POOL2 +# +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-R" + +if is_global_zone ; then + # + # Verify send -I backup all incrementals from pool + # + log_must eval "zfs send -I $POOL2@psnap $POOL2/pclone@final > " \ + "$BACKDIR/pool-clone-I" + log_must_busy zfs destroy -rf $POOL2/pclone + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-clone-I" + log_must cmp_ds_subs $POOL $POOL2 + log_must cmp_ds_cont $POOL $POOL2 +fi + +dstds=$(get_dst_ds $POOL $POOL2) + +# +# Verify send -I backup all incrementals from filesystem +# +ds=$dstds/$FS/fs1 +log_must eval "zfs send -I $ds/fs2@fsnap $ds/fclone@final > " \ + "$BACKDIR/fs-clone-I" +log_must_busy zfs destroy -rf $ds/fclone +log_must eval "zfs receive -F $ds/fclone < $BACKDIR/fs-clone-I" + +log_must cmp_ds_subs $POOL $dstds +log_must cmp_ds_cont $POOL $dstds + +if is_global_zone ; then + # + # Verify send -I backup all incrementals from volume + # + ds=$POOL2/$FS + log_must eval "zfs send -I $ds/vol@vsnap $ds/vclone@final > " \ + "$BACKDIR/vol-clone-I" + log_must_busy zfs destroy -rf $ds/vclone + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/vol-clone-I" + log_must cmp_ds_subs $POOL $POOL2 + log_must cmp_ds_cont $POOL $POOL2 +fi + +log_pass "zfs send -I send all incrementals from dataset@init to clone@snap" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_004_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_004_pos.ksh new file mode 100755 index 000000000000..c0b36b2329e7 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_004_pos.ksh @@ -0,0 +1,120 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# zfs send -R -i send incremental from fs@init to fs@final. +# +# STRATEGY: +# 1. Create a set of snapshots and fill with data. +# 2. Create sub filesystems. +# 3. Create final snapshot +# 4. Verify zfs send -R -i will backup all the datasets which has +# snapshot suffix @final +# + +verify_runnable "both" + +log_assert "zfs send -R -i send incremental from fs@init to fs@final." +log_onexit cleanup_pool $POOL2 + +# +# Duplicate POOL2 for testing +# +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R" + +if is_global_zone ; then + # + # Testing send -R -i backup from pool + # + srclist=$(getds_with_suffix $POOL2 @final) + interlist="$srclist $(getds_with_suffix $POOL2 @snapC)" + interlist="$interlist $(getds_with_suffix $POOL2 @snapB)" + interlist="$interlist $(getds_with_suffix $POOL2 @snapA)" + + log_must eval "zfs send -R -i @init $POOL2@final > " \ + "$BACKDIR/pool-init-final-iR" + log_must destroy_tree $interlist + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-init-final-iR" + + # Get current datasets with suffix @final + dstlist=$(getds_with_suffix $POOL2 @final) + if [[ $srclist != $dstlist ]]; then + log_fail "Unexpected: srclist($srclist) != dstlist($dstlist)" + fi + log_must cmp_ds_cont $POOL $POOL2 +fi + +dstds=$(get_dst_ds $POOL $POOL2) +# +# Testing send -R -i backup from filesystem +# +log_must eval "zfs send -R -i @init $dstds/$FS@final > " \ + "$BACKDIR/fs-init-final-iR" + +srclist=$(getds_with_suffix $dstds/$FS @final) +interlist="$srclist $(getds_with_suffix $dstds/$FS @snapC)" +interlist="$interlist $(getds_with_suffix $dstds/$FS @snapB)" +interlist="$interlist $(getds_with_suffix $dstds/$FS @snapA)" +log_must destroy_tree $interlist +if is_global_zone ; then + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/fs-init-final-iR" +else + zfs receive -F -d $dstds/$FS < $BACKDIR/fs-init-final-iR +fi + +dstlist=$(getds_with_suffix $dstds/$FS @final) +if [[ $srclist != $dstlist ]]; then + log_fail "Unexpected: srclist($srclist) != dstlist($dstlist)" +fi +log_must cmp_ds_cont $POOL $POOL2 + +if is_global_zone ; then + # + # Testing send -R -i backup from volume + # + srclist=$(getds_with_suffix $POOL2/$FS/vol @final) + log_must eval "zfs send -R -i @init $POOL2/$FS/vol@final > " \ + "$BACKDIR/vol-init-final-iR" + log_must destroy_tree $srclist + log_must eval "zfs receive -d $POOL2 < $BACKDIR/vol-init-final-iR" + + dstlist=$(getds_with_suffix $POOL2/$FS/vol @final) + if [[ $srclist != $dstlist ]]; then + log_fail "Unexpected: srclist($srclist) != dstlist($dstlist)" + fi + log_must cmp_ds_cont $POOL $POOL2 +fi + +log_pass "zfs send -R -i send incremental from fs@init to fs@final." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_005_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_005_pos.ksh new file mode 100755 index 000000000000..4d156690e38a --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_005_pos.ksh @@ -0,0 +1,104 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# zfs send -R -I send all the incremental between fs@init with fs@final +# +# STRATEGY: +# 1. Setup test model +# 2. Send -R -I @init @final on pool +# 3. Destroy all the snapshots which is later than @init +# 4. Verify receive can restore all the snapshots and data +# 5. Do the same test on filesystem and volume +# + +verify_runnable "both" + +log_assert "zfs send -R -I send all the incremental between @init with @final" +log_onexit cleanup_pool $POOL2 + +# +# Duplicate POOL2 for testing +# +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R" + +if is_global_zone ; then + # + # Testing send -R -I from pool + # + log_must eval "zfs send -R -I @init $POOL2@final > " \ + "$BACKDIR/pool-init-final-IR" + list=$(getds_with_suffix $POOL2 @snapA) + list="$list $(getds_with_suffix $POOL2 @snapB)" + list="$list $(getds_with_suffix $POOL2 @snapC)" + list="$list $(getds_with_suffix $POOL2 @final)" + log_must destroy_tree $list + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-init-final-IR" + log_must cmp_ds_cont $POOL $POOL2 +fi + +dstds=$(get_dst_ds $POOL $POOL2) +# +# Testing send -R -I from filesystem +# +log_must eval "zfs send -R -I @init $dstds/$FS@final > " \ + "$BACKDIR/fs-init-final-IR" +list=$(getds_with_suffix $dstds/$FS @snapA) +list="$list $(getds_with_suffix $dstds/$FS @snapB)" +list="$list $(getds_with_suffix $dstds/$FS @snapC)" +list="$list $(getds_with_suffix $dstds/$FS @final)" +log_must destroy_tree $list +if is_global_zone ; then + log_must eval "zfs receive -d -F $dstds < $BACKDIR/fs-init-final-IR" +else + zfs receive -d -F $dstds < $BACKDIR/fs-init-final-IR +fi +log_must cmp_ds_subs $POOL $dstds +log_must cmp_ds_cont $POOL $dstds + +if is_global_zone ; then + # + # Testing send -I -R for volume + # + vol=$POOL2/$FS/vol + log_must eval "zfs send -R -I @init $vol@final > " \ + "$BACKDIR/vol-init-final-IR" + log_must destroy_tree $vol@snapB $vol@snapC $vol@final + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/vol-init-final-IR" + log_must cmp_ds_subs $POOL $POOL2 + log_must cmp_ds_cont $POOL $POOL2 +fi + +log_pass "zfs send -R -I send all the incremental between @init with @final" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_006_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_006_pos.ksh new file mode 100755 index 000000000000..8c598f4ec29c --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_006_pos.ksh @@ -0,0 +1,82 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Rename snapshot name will not change the dependent order. +# +# STRATEGY: +# 1. Set up a set of datasets. +# 2. Rename part of snapshots. +# 3. Send -R all the POOL +# 4. Verify snapshot name will not change the dependent order. +# + +verify_runnable "both" + +# Source Target +# +set -A snaps "$POOL@init" "$POOL@snap0" \ + "$POOL@snapA" "$POOL@snap1" \ + "$POOL@snapC" "$POOL@snap2" \ + "$POOL@final" "$POOL@init" + +function cleanup +{ + log_must cleanup_pool $POOL + log_must cleanup_pool $POOL2 + + log_must setup_test_model $POOL +} + +log_assert "Rename snapshot name will not change the dependent order." +log_onexit cleanup + +typeset -i i=0 +while ((i < ${#snaps[@]})); do + log_must zfs rename -r ${snaps[$i]} ${snaps[((i+1))]} + + ((i += 2)) +done + +# +# Duplicate POOL2 for testing +# +log_must eval "zfs send -R $POOL@init > $BACKDIR/pool-final-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R" + +dstds=$(get_dst_ds $POOL $POOL2) +log_must cmp_ds_subs $POOL $dstds +log_must cmp_ds_cont $POOL $dstds + +log_pass "Rename snapshot name will not change the dependent order." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_007_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_007_pos.ksh new file mode 100755 index 000000000000..29e9f1859579 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_007_pos.ksh @@ -0,0 +1,97 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Rename parent filesystem name will not change the dependent order. +# +# STRATEGY: +# 1. Separately rename pool clone, filesystem and volume name. +# 2. Send -R all the POOL +# 3. Verify renamed dataset will not change the snapshot dependent order. +# + +verify_runnable "both" + +set -A dtst \ + "$POOL/pclone" "$POOL/$FS/pclone" \ + "$POOL/$FS/fs1/fs2" "$POOL/fs2" +if is_global_zone ; then + typeset -i n=${#dtst[@]} + dtst[((n))]="$POOL/vol"; dtst[((n+1))]="$POOL/$FS/fs1/vol" +fi + +function cleanup +{ + log_must cleanup_pool $POOL + log_must cleanup_pool $POOL2 + + log_must setup_test_model $POOL +} + +log_assert "Rename parent filesystem name will not change the dependent order." +log_onexit cleanup + +typeset -i i=0 +while ((i < ${#dtst[@]})); do + log_must zfs rename ${dtst[$i]} ${dtst[((i+1))]} + + ((i += 2)) +done + +# +# Verify zfs send -R should succeed +# +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R" +dstds=$(get_dst_ds $POOL $POOL2) +log_must cmp_ds_subs $POOL $dstds + +# +# Verify zfs send -R -I should succeed +# +log_must eval "zfs send -R -I @init $dstds@final > " \ + "$BACKDIR/pool-init-final-IR" +list=$(getds_with_suffix $dstds @snapA) +list="$list $(getds_with_suffix $dstds @snapB)" +list="$list $(getds_with_suffix $dstds @snapC)" +list="$list $(getds_with_suffix $dstds @final)" +log_must destroy_tree $list +if is_global_zone ; then + log_must eval "zfs receive -d -F $dstds < $BACKDIR/pool-init-final-IR" +else + zfs receive -d -F $dstds < $BACKDIR/pool-init-final-IR +fi +log_must cmp_ds_subs $POOL $dstds + +log_pass "Rename parent filesystem name will not change the dependent order." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_008_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_008_pos.ksh new file mode 100755 index 000000000000..531478760457 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_008_pos.ksh @@ -0,0 +1,131 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Changes made by 'zfs promote' can be properly received. +# +# STRATEGY: +# 1. Separately promote pool clone, filesystem clone and volume clone. +# 2. Recursively backup all the POOL and restore in POOL2 +# 3. Verify all the datasets and properties were properly received. +# + +verify_runnable "both" + +# See issue: https://github.com/zfsonlinux/zfs/issues/6066 +log_unsupported "Occasionally hangs" + +# Origin Clone +# +set -A dtst "$POOL" "$POOL/pclone" \ + "$POOL/$FS/fs1/fs2" "$POOL/$FS/fs1/fclone" +if is_global_zone ; then + typeset -i n=${#dtst[@]} + dtst[((n))]="$POOL/$FS/vol"; dtst[((n+1))]="$POOL/$FS/vclone" +fi + +function cleanup +{ + typeset origin + typeset -i i=0 + while ((i < ${#dtst[@]})); do + origin=$(get_prop origin ${dtst[$i]}) + + if [[ $origin != "-" ]]; then + log_must zfs promote ${dtst[$i]} + fi + + ((i += 2)) + done + + origin=$(get_prop origin $POOL2) + if [[ $origin != "-" ]]; then + log_must zfs promote $POOL2 + fi + log_must cleanup_pool $POOL2 +} + +log_assert "Changes made by 'zfs promote' can be properly received." +log_onexit cleanup + +typeset -i i=0 +while ((i < ${#dtst[@]})); do + log_must zfs promote ${dtst[((i+1))]} + + ((i += 2)) +done + +# +# Verify zfs send -R should succeed +# +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R" + +dstds=$(get_dst_ds $POOL $POOL2) +# +# Define all the POOL/POOL2 datasets pair +# +set -A pair "$POOL" "$dstds" \ + "$POOL/$FS" "$dstds/$FS" \ + "$POOL/$FS/fs1" "$dstds/$FS/fs1" \ + "$POOL/$FS/fs1/fs2" "$dstds/$FS/fs1/fs2" \ + "$POOL/pclone" "$dstds/pclone" \ + "$POOL/$FS/fs1/fclone" "$dstds/$FS/fs1/fclone" + +if is_global_zone ; then + typeset -i n=${#pair[@]} + pair[((n))]="$POOL/vol"; pair[((n+1))]="$dstds/vol" + pair[((n+2))]="$POOL/$FS/vol" pair[((n+3))]="$dstds/$FS/vol" +fi + +# +# Verify all the sub-datasets can be properly received. +# +log_must cmp_ds_subs $POOL $dstds +typeset -i i=0 +while ((i < ${#pair[@]})); do + log_must cmp_ds_cont ${pair[$i]} ${pair[((i+1))]} + log_must cmp_ds_prop ${pair[$i]} ${pair[((i+1))]} + + ((i += 2)) +done + +# Verify the original filesystem can be promoted +log_must zfs promote $dstds +if is_global_zone ; then + log_must zfs promote $dstds/$FS/vol +fi +log_must zfs promote $dstds/$FS/fs1/fs2 + +log_pass "Changes made by 'zfs promote' can be properly received." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh new file mode 100755 index 000000000000..7d858d92d511 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh @@ -0,0 +1,92 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# zfs receive can handle out of space correctly. +# +# STRATEGY: +# 1. Create two pools, one is big and another is small. +# 2. Fill the big pool with data. +# 3. Take snapshot and backup the whole pool. +# 4. Receive this stream in small pool. +# 5. Verify zfs receive can handle the out of space error correctly. +# + +verify_runnable "global" + +function cleanup +{ + if datasetexists bpool ; then + log_must_busy zpool destroy -f bpool + fi + if datasetexists spool ; then + log_must_busy zpool destroy -f spool + fi +} + +log_assert "Verify zfs receive can handle out of space correctly." +log_onexit cleanup + +log_must mkfile $MINVDEVSIZE $TESTDIR/bfile +log_must mkfile $SPA_MINDEVSIZE $TESTDIR/sfile +log_must zpool create bpool $TESTDIR/bfile +log_must zpool create spool $TESTDIR/sfile + +# +# Test out of space on sub-filesystem +# +log_must zfs create bpool/fs +log_must mkfile 30M /bpool/fs/file + +log_must zfs snapshot bpool/fs@snap +log_must eval "zfs send -R bpool/fs@snap > $BACKDIR/fs-R" +log_mustnot eval "zfs receive -d -F spool < $BACKDIR/fs-R" + +log_must datasetnonexists spool/fs +log_must ismounted spool + +# +# Test out of space on top filesystem +# +log_must mv /bpool/fs/file /bpool +log_must_busy zfs destroy -rf bpool/fs + +log_must zfs snapshot bpool@snap +log_must eval "zfs send -R bpool@snap > $BACKDIR/bpool-R" +log_mustnot eval "zfs receive -d -F spool < $BACKDIR/bpool-R" + +log_must datasetnonexists spool/fs +log_must ismounted spool + +log_pass "zfs receive can handle out of space correctly." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_010_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_010_pos.ksh new file mode 100755 index 000000000000..e76f5c5c8620 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_010_pos.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# ZFS can handle stream with multiple identical (same GUID) snapshots +# +# STRATEGY: +# 1. Recursively backup snapshot +# 2. Restore it to the given filesystem +# 3. Resend the snapshot again +# 4. Verify this stream can be restore to this filesystem again +# + +verify_runnable "both" + +log_assert "ZFS can handle stream with multiple identical (same GUID) snapshots" +log_onexit cleanup_pool $POOL2 + +log_must zfs create $POOL2/$FS +log_must zfs snapshot $POOL2/$FS@snap + +# +# First round restore the stream +# +log_must eval "zfs send -R $POOL2/$FS@snap > $BACKDIR/fs-R" +log_must eval "zfs receive -d -F $POOL2/$FS < $BACKDIR/fs-R" + +# +# In order to avoid 'zfs send -R' failed, create snapshot for +# all the sub-systems +# +list=$(zfs list -r -H -o name -t filesystem $POOL2/$FS) +for item in $list ; do + if datasetnonexists $item@snap ; then + log_must zfs snapshot $item@snap + fi +done + +# +# Second round restore the stream +# +log_must eval "zfs send -R $POOL2/$FS@snap > $BACKDIR/fs-R" +dstds=$(get_dst_ds $POOL2/$FS $POOL2/$FS) +log_must eval "zfs receive -d -F $dstds < $BACKDIR/fs-R" + +log_pass "ZFS can handle stream with multiple identical (same GUID) snapshots" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh new file mode 100755 index 000000000000..68f0e13927dc --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh @@ -0,0 +1,120 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Changes made by 'zfs inherit' can be properly received. +# +# STRATEGY: +# 1. Inherit property for filesystem and volume +# 2. Send and restore them in the target pool +# 3. Verify all the datasets can be properly backup and receive +# + +verify_runnable "both" + +function cleanup +{ + log_must cleanup_pool $POOL + log_must cleanup_pool $POOL2 + + log_must setup_test_model $POOL +} + +log_assert "Verify changes made by 'zfs inherit' can be properly received." +log_onexit cleanup + +# +# Setting all the $FS properties as local value, +# +for prop in $(fs_inherit_prop); do + value=$(get_prop $prop $POOL/$FS) + log_must zfs set $prop=$value $POOL/$FS +done + +# +# Inherit properties in sub-datasets +# +for ds in "$POOL/$FS/fs1" "$POOL/$FS/fs1/fs2" "$POOL/$FS/fs1/fclone" ; do + for prop in $(fs_inherit_prop) ; do + zfs inherit $prop $ds + if (($? !=0 )); then + log_fail "zfs inherit $prop $ds" + fi + done +done +if is_global_zone ; then + for prop in $(vol_inherit_prop) ; do + zfs inherit $prop $POOL/$FS/vol + if (($? !=0 )); then + log_fail "zfs inherit $prop $POOL/$FS/vol" + fi + done +fi + +# +# Verify datasets can be backup and restore correctly +# Unmount $POOL/$FS to avoid two fs mount in the same mountpoint +# +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-R" +log_must zfs unmount -f $POOL/$FS +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-R" + +dstds=$(get_dst_ds $POOL $POOL2) +# +# Define all the POOL/POOL2 datasets pair +# +set -A pair "$POOL" "$dstds" \ + "$POOL/$FS" "$dstds/$FS" \ + "$POOL/$FS/fs1" "$dstds/$FS/fs1" \ + "$POOL/$FS/fs1/fs2" "$dstds/$FS/fs1/fs2" \ + "$POOL/pclone" "$dstds/pclone" \ + "$POOL/$FS/fs1/fclone" "$dstds/$FS/fs1/fclone" + +if is_global_zone ; then + typeset -i n=${#pair[@]} + pair[((n))]="$POOL/vol"; pair[((n+1))]="$dstds/vol" + pair[((n+2))]="$POOL/$FS/vol" pair[((n+3))]="$dstds/$FS/vol" +fi + +# +# Verify all the sub-datasets can be properly received. +# +log_must cmp_ds_subs $POOL $dstds +typeset -i i=0 +while ((i < ${#pair[@]})); do + log_must cmp_ds_prop ${pair[$i]} ${pair[((i+1))]} + ((i += 2)) +done + +log_pass "Changes made by 'zfs inherit' can be properly received." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_012_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_012_pos.ksh new file mode 100755 index 000000000000..0441f7ff32be --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_012_pos.ksh @@ -0,0 +1,186 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved. +# Copyright (c) 2013, 2016, Delphix. All rights reserved. +# Use is subject to license terms. +# + +. $STF_SUITE/include/properties.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# zfs send -R will backup all the filesystem properties correctly. +# +# STRATEGY: +# 1. Setting properties for all the filesystem and volumes randomly +# 2. Backup all the data from POOL by send -R +# 3. Restore all the data in POOL2 +# 4. Verify all the properties in the two pools are the same +# + +verify_runnable "global" + +function edited_prop +{ + typeset behaviour=$1 + typeset ds=$2 + typeset backfile=$TESTDIR/edited_prop_$ds + + case $behaviour in + "get") + typeset props=$(zfs inherit 2>&1 | \ + awk '$2=="YES" {print $1}' | \ + egrep -v "^vol|\.\.\.$") + for item in $props ; do + if [[ $item == "mlslabel" ]] && \ + ! is_te_enabled ; then + continue + fi + zfs get -H -o property,value $item $ds >> \ + $backfile + if (($? != 0)); then + log_fail "zfs get -H -o property,value"\ + "$item $ds > $backfile" + fi + done + ;; + "set") + if [[ ! -f $backfile ]] ; then + log_fail "$ds need backup properties firstly." + fi + + typeset prop value + while read prop value ; do + eval zfs set $prop='$value' $ds + if (($? != 0)); then + log_fail "zfs set $prop=$value $ds" + fi + done < $backfile + ;; + *) + log_fail "Unrecognized behaviour: $behaviour" + esac +} + +function cleanup +{ + log_must cleanup_pool $POOL + log_must cleanup_pool $POOL2 + + log_must edited_prop "set" $POOL + log_must edited_prop "set" $POOL2 + + typeset prop + for prop in $(fs_inherit_prop) ; do + log_must zfs inherit $prop $POOL + log_must zfs inherit $prop $POOL2 + done + + log_must setup_test_model $POOL + + if [[ -d $TESTDIR ]]; then + log_must rm -rf $TESTDIR/* + fi +} + +log_assert "Verify zfs send -R will backup all the filesystem properties " \ + "correctly." +log_onexit cleanup + +log_must edited_prop "get" $POOL +log_must edited_prop "get" $POOL2 + +for fs in "$POOL" "$POOL/pclone" "$POOL/$FS" "$POOL/$FS/fs1" \ + "$POOL/$FS/fs1/fs2" "$POOL/$FS/fs1/fclone" ; do + rand_set_prop $fs aclinherit "discard" "noallow" "secure" "passthrough" + rand_set_prop $fs checksum "on" "off" "fletcher2" "fletcher4" "sha256" + rand_set_prop $fs acltype "off" "noacl" "posixacl" + rand_set_prop $fs atime "on" "off" + rand_set_prop $fs checksum "on" "off" "fletcher2" "fletcher4" "sha256" + rand_set_prop $fs compression "${compress_prop_vals[@]}" + rand_set_prop $fs copies "1" "2" "3" + rand_set_prop $fs devices "on" "off" + rand_set_prop $fs exec "on" "off" + rand_set_prop $fs quota "512M" "1024M" + rand_set_prop $fs recordsize "512" "2K" "8K" "32K" "128K" + rand_set_prop $fs dnodesize "legacy" "auto" "1k" "2k" "4k" "8k" "16k" + rand_set_prop $fs setuid "on" "off" + rand_set_prop $fs snapdir "hidden" "visible" + if ! is_freebsd; then + rand_set_prop $fs xattr "on" "off" + fi + rand_set_prop $fs user:prop "aaa" "bbb" "23421" "()-+?" +done + +for vol in "$POOL/vol" "$POOL/$FS/vol" ; do + rand_set_prop $vol checksum "on" "off" "fletcher2" "fletcher4" "sha256" + rand_set_prop $vol compression "${compress_prop_vals[@]}" + rand_set_prop $vol readonly "on" "off" + rand_set_prop $vol copies "1" "2" "3" + rand_set_prop $vol user:prop "aaa" "bbb" "23421" "()-+?" +done + + +# Verify inherited property can be received +rand_set_prop $POOL redundant_metadata "all" "most" +rand_set_prop $POOL sync "standard" "always" "disabled" + +# +# Duplicate POOL2 for testing +# +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R" + +# +# Define all the POOL/POOL2 datasets pair +# +set -A pair "$POOL" "$POOL2" \ + "$POOL/$FS" "$POOL2/$FS" \ + "$POOL/$FS/fs1" "$POOL2/$FS/fs1" \ + "$POOL/$FS/fs1/fs2" "$POOL2/$FS/fs1/fs2" \ + "$POOL/pclone" "$POOL2/pclone" \ + "$POOL/$FS/fs1/fclone" "$POOL2/$FS/fs1/fclone" \ + "$POOL/vol" "$POOL2/vol" \ + "$POOL/$FS/vol" "$POOL2/$FS/vol" + +typeset -i i=0 +while ((i < ${#pair[@]})); do + log_must cmp_ds_prop ${pair[$i]} ${pair[((i+1))]} + + ((i += 2)) +done + + +zpool upgrade -v | grep "Snapshot properties" > /dev/null 2>&1 +if (( $? == 0 )) ; then + i=0 + while ((i < ${#pair[@]})); do + log_must cmp_ds_prop ${pair[$i]}@final ${pair[((i+1))]}@final + ((i += 2)) + done +fi + +log_pass "Verify zfs send -R will backup all the filesystem properties " \ + "correctly." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh new file mode 100755 index 000000000000..110e56144674 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh @@ -0,0 +1,86 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# zfs receive -dF with incremental stream will destroy all the +# dataset that not exist on the sender side. +# +# STRATEGY: +# 1. Setup test model +# 2. Send -R @final on pool +# 3. Destroy some dataset within the @final, and create @destroy +# 4. Send -R -I @final @destroy on pool +# 5. Verify receive -dF will destroy all the dataset that not exist +# on the sender side. +# + +verify_runnable "both" + +function cleanup +{ + cleanup_pool $POOL2 + cleanup_pool $POOL + log_must setup_test_model $POOL +} + +log_assert "zfs receive -dF will destroy all the dataset that not exist" \ + "on the sender side" +log_onexit cleanup + +cleanup + +# +# Duplicate POOL2 for testing +# +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R" +log_must eval "zfs receive -dF $POOL2 < $BACKDIR/pool-final-R" + +log_must_busy zfs destroy -Rf $POOL/$FS +log_must_busy zfs destroy -Rf $POOL/pclone + +if is_global_zone ; then + log_must_busy zfs destroy -Rf $POOL/vol +fi +log_must zfs snapshot -r $POOL@destroy + +log_must eval "zfs send -R -I @final $POOL@destroy > " \ + "$BACKDIR/pool-final-destroy-IR" +log_must eval "zfs receive -dF $POOL2 < $BACKDIR/pool-final-destroy-IR" + +dstds=$(get_dst_ds $POOL $POOL2) +log_must cmp_ds_subs $POOL $dstds +log_must cmp_ds_cont $POOL $dstds + +log_pass "zfs receive -dF will destroy all the dataset that not exist" \ + "on the sender side" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_014_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_014_pos.ksh new file mode 100755 index 000000000000..47fc2f1bead4 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_014_pos.ksh @@ -0,0 +1,56 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2014, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that a pool imported readonly can be sent and received. +# +# Strategy: +# 1. Make the source pool readonly, and receive it into pool2. +# 2. Reset pool2, and repeat the send from a non-root fs of the source pool. +# 3. Make the source pool read-write again. +# + +verify_runnable "both" + +log_assert "zfs send will work on filesystems and volumes in a read-only pool." +log_onexit cleanup_pool $POOL2 + +log_must zpool export $POOL +log_must zpool import -o readonly=on $POOL +log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R" + +dstds=$(get_dst_ds $POOL $POOL2) +log_must cmp_ds_subs $POOL $dstds +log_must cmp_ds_cont $POOL $dstds + +log_must cleanup_pool $POOL2 + +log_must eval "zfs send -R $POOL/$FS@final > $BACKDIR/fs-final-R" +log_must eval "zfs receive -d $POOL2 < $BACKDIR/fs-final-R" +log_must_busy zpool export $POOL +log_must zpool import $POOL + +dstds=$(get_dst_ds $POOL/$FS $POOL2) +log_must cmp_ds_subs $POOL/$FS $dstds +log_must cmp_ds_cont $POOL/$FS $dstds + +log_pass "zfs send will work on filesystems and volumes in a read-only pool." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh new file mode 100755 index 000000000000..6dfb3423fdf9 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh @@ -0,0 +1,33 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2014, 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that error conditions don't cause panics in zfs send +# +# Strategy: +# 1. Perform a zfs incremental send from a bookmark that doesn't exist +# + +verify_runnable "both" + +log_neg eval "zfs send -i \#bla $POOl/$FS@final > /dev/null" + +log_pass "Ensure that error conditions cause appropriate failures." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_019_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_019_pos.ksh new file mode 100755 index 000000000000..673d1d072be4 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_019_pos.ksh @@ -0,0 +1,54 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2014, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify resumability of a full and incremental ZFS send/receive in the +# presence of a corrupted stream. +# +# Strategy: +# 1. Start a full ZFS send, redirect output to a file +# 2. Mess up the contents of the stream state file on disk +# 3. Try ZFS receive, which should fail with a checksum mismatch error +# 4. ZFS send to the stream state file again using the receive_resume_token +# 5. ZFS receive and verify the receive completes successfully +# 6. Repeat steps on an incremental ZFS send +# 7. Repeat the entire procedure for a dataset at the pool root +# + +verify_runnable "both" + +log_assert "Verify resumability of a full and incremental ZFS send/receive " \ + "in the presence of a corrupted stream" +log_onexit resume_cleanup $sendfs $streamfs + +sendfs=$POOL/sendfs +recvfs=$POOL3/recvfs +streamfs=$POOL2/stream + +for sendfs in $POOL2/sendfs $POOL3; do + test_fs_setup $sendfs $recvfs $streamfs + resume_test "zfs send -v $sendfs@a" $streamfs $recvfs + resume_test "zfs send -v -i @a $sendfs@b" $streamfs $recvfs + file_check $sendfs $recvfs +done + +log_pass "Verify resumability of a full and incremental ZFS send/receive " \ + "in the presence of a corrupted stream" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh new file mode 100755 index 000000000000..b7ecf90916ba --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh @@ -0,0 +1,50 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2014, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify resumability of a full ZFS send/receive with the -D (dedup) flag in +# the presence of a corrupted stream. +# +# Strategy: +# 1. Start a full ZFS send with the -D flag (dedup), redirect output to a file +# 2. Mess up the contents of the stream state file on disk +# 3. Try ZFS receive, which should fail with a checksum mismatch error +# 4. ZFS send to the stream state file again using the receive_resume_token +# 5. ZFS receive and verify the receive completes successfully +# + +verify_runnable "both" + +log_assert "Verify resumability of full ZFS send/receive with the -D " \ + "(dedup) flag" + +sendfs=$POOL/sendfs +recvfs=$POOL2/recvfs +streamfs=$POOL/stream + +log_onexit resume_cleanup $sendfs $streamfs + +test_fs_setup $sendfs $recvfs $streamfs +resume_test "zfs send -D -v $sendfs@a" $streamfs $recvfs +file_check $sendfs $recvfs + +log_pass "Verify resumability of full ZFS send/receive with the -D " \ + "(dedup) flag" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh new file mode 100755 index 000000000000..50f2b8890a5c --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh @@ -0,0 +1,53 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2014, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify resumability of a full and incremental ZFS send/receive with the +# -e (embedded) flag in the presence of a corrupted stream. +# +# Strategy: +# 1. Start a full ZFS send with the -e flag (embedded), redirect output to +# a file +# 2. Mess up the contents of the stream state file on disk +# 3. Try ZFS receive, which should fail with a checksum mismatch error +# 4. ZFS send to the stream state file again using the receive_resume_token +# 5. ZFS receive and verify the receive completes successfully +# 6. Repeat steps on an incremental ZFS send +# + +verify_runnable "both" + +log_assert "Verify resumability of a full and incremental ZFS send/receive " \ + "with the -e (embedded) flag" + +sendfs=$POOL/sendfs +recvfs=$POOL2/recvfs +streamfs=$POOL/stream + +log_onexit resume_cleanup $sendfs $streamfs + +test_fs_setup $sendfs $recvfs $streamfs +resume_test "zfs send -v -e $sendfs@a" $streamfs $recvfs +resume_test "zfs send -v -e -i @a $sendfs@b" $streamfs $recvfs +file_check $sendfs $recvfs + +log_pass "Verify resumability of a full and incremental ZFS send/receive " \ + "with the -e (embedded) flag" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh new file mode 100755 index 000000000000..cb68b1c3b27d --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh @@ -0,0 +1,61 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2014, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify resumability of an incremental ZFS send/receive with ZFS bookmarks in +# the presence of a corrupted stream. +# +# Strategy: +# 1. Bookmark a ZFS snapshot +# 2. Destroy the ZFS snapshot +# 3. Destroy the filesystem for the receive +# 4. Verify receive of the full send stream +# 5. Start an incremental ZFS send of the ZFS bookmark, redirect output to a +# file +# 6. Mess up the contents of the stream state file on disk +# 7. Try ZFS receive, which should fail with a checksum mismatch error +# 8. ZFS send to the stream state file again using the receive_resume_token +# 9. ZFS receive and verify the receive completes successfully +# + +verify_runnable "both" + +log_assert "Verify resumability of an incremental ZFS send/receive with ZFS " \ + "bookmarks" + +sendfs=$POOL/sendfs +recvfs=$POOL2/recvfs +streamfs=$POOL/stream + +log_onexit resume_cleanup $sendfs $streamfs + +test_fs_setup $sendfs $recvfs $streamfs +log_must zfs bookmark $sendfs@a $sendfs#bm_a +log_must_busy zfs destroy $sendfs@a +log_must zfs receive -v $recvfs </$POOL/initial.zsend +resume_test "zfs send -i \#bm_a $sendfs@b" $streamfs $recvfs +log_must_busy zfs destroy -r -f $sendfs +log_must zfs receive -v $sendfs </$POOL/initial.zsend +log_must zfs receive -v $sendfs </$POOL/incremental.zsend +file_check $sendfs $recvfs + +log_pass "Verify resumability of an incremental ZFS send/receive with ZFS " \ + "bookmarks" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh new file mode 100755 index 000000000000..2d9fb01af10f --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh @@ -0,0 +1,53 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2014, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify resumability of a full ZFS send/receive with the source filesystem +# unmounted. +# +# Strategy: +# 1. Destroy the filesystem for the receive +# 2. Unmount the source filesystem +# 3. Start a full ZFS send, redirect output to a file +# 4. Mess up the contents of the stream state file on disk +# 5. Try ZFS receive, which should fail with a checksum mismatch error +# 6. ZFS send to the stream state file again using the receive_resume_token +# 7. ZFS receive and verify the receive completes successfully +# + +verify_runnable "both" + +log_assert "Verify resumability of a full ZFS send/receive with the source " \ + "filesystem unmounted" + +sendfs=$POOL/sendfs +recvfs=$POOL2/recvfs +streamfs=$POOL/stream + +log_onexit resume_cleanup $sendfs $streamfs + +test_fs_setup $sendfs $recvfs $streamfs +log_must zfs unmount -f $sendfs +resume_test "zfs send $sendfs" $streamfs $recvfs +file_check $sendfs $recvfs + +log_pass "Verify resumability of a full ZFS send/receive with the source " \ + "filesystem unmounted" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-L_toggle.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-L_toggle.ksh new file mode 100755 index 000000000000..483efcc60548 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-L_toggle.ksh @@ -0,0 +1,65 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that send -L can be changed to on in an incremental. +# Verify that send -L can not be turned off in an incremental. +# + +function cleanup +{ + log_must_busy zfs destroy -r $TESTPOOL/fs + log_must_busy zfs destroy -r $TESTPOOL/recv +} + +verify_runnable "both" + +log_assert "Verify toggling send -L works as expected" +log_onexit cleanup + +log_must zfs create -o compression=on -o recordsize=1m $TESTPOOL/fs + +log_must dd if=/dev/urandom of=/$TESTPOOL/fs/file bs=1024 count=1500 + +log_must zfs snapshot $TESTPOOL/fs@snap + +log_must dd if=/dev/urandom of=/$TESTPOOL/fs/file bs=1024 count=1500 conv=notrunc seek=2048 + +log_must zfs snapshot $TESTPOOL/fs@snap2 + +log_must zfs create $TESTPOOL/recv + +log_must zfs send -c $TESTPOOL/fs@snap | zfs recv $TESTPOOL/recv/noL-noL +log_must zfs send -c -i @snap $TESTPOOL/fs@snap2| zfs recv $TESTPOOL/recv/noL-noL +log_must diff /$TESTPOOL/fs/file /$TESTPOOL/recv/noL-noL/file + +log_must zfs send -c -L $TESTPOOL/fs@snap | zfs recv $TESTPOOL/recv/L-L +log_must zfs send -c -L -i @snap $TESTPOOL/fs@snap2 | zfs recv $TESTPOOL/recv/L-L +log_must diff /$TESTPOOL/fs/file /$TESTPOOL/recv/L-L/file + +log_must zfs send -c $TESTPOOL/fs@snap | zfs recv $TESTPOOL/recv/noL-L +log_must zfs send -c -L -i @snap $TESTPOOL/fs@snap2 | zfs recv $TESTPOOL/recv/noL-L +log_must diff /$TESTPOOL/fs/file /$TESTPOOL/recv/noL-L/file + +log_must zfs send -c -L $TESTPOOL/fs@snap | zfs recv $TESTPOOL/recv/L-noL +log_mustnot zfs send -c -i @snap $TESTPOOL/fs@snap2 | zfs recv $TESTPOOL/recv/L-noL +log_must diff /$TESTPOOL/fs/.zfs/snapshot/snap/file /$TESTPOOL/recv/L-noL/file + +log_pass "Verify toggling send -L works as expected" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_embedded_blocks.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_embedded_blocks.ksh new file mode 100755 index 000000000000..70f79b3173b7 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_embedded_blocks.ksh @@ -0,0 +1,109 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/properties.shlib + +# +# Description: +# Verify that compressed streams can contain embedded blocks. +# +# Strategy: +# 1. Create a filesystem with compressible data and embedded blocks. +# 2. Verify the created streams can be received correctly. +# 3. Verify the presence / absence of embedded blocks in the compressed stream, +# as well as the receiving file system. +# + +verify_runnable "both" + +log_assert "Verify that compressed streams can contain embedded blocks." +log_onexit cleanup_pool $POOL2 + +typeset objs obj recsize +typeset sendfs=$POOL2/sendfs +typeset recvfs=$POOL2/recvfs +typeset stream=$BACKDIR/stream +typeset dump=$BACKDIR/dump +typeset recvfs2=$POOL2/recvfs2 +typeset stream2=$BACKDIR/stream2 +typeset dump2=$BACKDIR/dump2 +log_must zfs create -o compress=lz4 $sendfs +log_must zfs create -o compress=lz4 $recvfs +log_must zfs create -o compress=lz4 $recvfs2 +typeset dir=$(get_prop mountpoint $sendfs) + +# Populate the send dataset with compressible data and embedded block files. +write_compressible $dir 16m +for recsize in "${recsize_prop_vals[@]}"; do + # For lz4, this method works for blocks up to 16k, but not larger + [[ $recsize -eq $((32 * 1024)) ]] && break + + if is_linux || is_freebsd; then + log_must truncate -s $recsize $dir/$recsize + log_must dd if=/dev/urandom of=$dir/$recsize \ + seek=$((recsize - 8)) bs=1 count=8 conv=notrunc + else + log_must mkholes -h 0:$((recsize - 8)) -d $((recsize - 8)):8 \ + $dir/$recsize + fi +done + +# Generate the streams and zstreamdump output. +log_must zfs snapshot $sendfs@now +log_must eval "zfs send -c $sendfs@now >$stream" +log_must eval "zstreamdump -v <$stream >$dump" +log_must eval "zfs recv -d $recvfs <$stream" +cmp_ds_cont $sendfs $recvfs +verify_stream_size $stream $sendfs +log_mustnot stream_has_features $stream embed_data + +log_must eval "zfs send -c -e $sendfs@now >$stream2" +log_must eval "zstreamdump -v <$stream2 >$dump2" +log_must eval "zfs recv -d $recvfs2 <$stream2" +cmp_ds_cont $sendfs $recvfs2 +verify_stream_size $stream2 $sendfs +log_must stream_has_features $stream2 embed_data + +# Verify embedded blocks are present only when expected. +for recsize in "${recsize_prop_vals[@]}"; do + [[ $recsize -eq $((32 * 1024)) ]] && break + + typeset send_obj=$(get_objnum $(get_prop mountpoint $sendfs)/$recsize) + typeset recv_obj=$(get_objnum \ + $(get_prop mountpoint $recvfs/sendfs)/$recsize) + typeset recv2_obj=$(get_objnum \ + $(get_prop mountpoint $recvfs2/sendfs)/$recsize) + + log_must eval "zdb -ddddd $sendfs $send_obj >$BACKDIR/sendfs.zdb" + log_must eval "zdb -ddddd $recvfs/sendfs $recv_obj >$BACKDIR/recvfs.zdb" + log_must eval "zdb -ddddd $recvfs2/sendfs $recv2_obj >$BACKDIR/recvfs2.zdb" + + grep -q "EMBEDDED" $BACKDIR/sendfs.zdb || \ + log_fail "Obj $send_obj not embedded in $sendfs" + grep -q "EMBEDDED" $BACKDIR/recvfs.zdb || \ + log_fail "Obj $recv_obj not embedded in $recvfs" + grep -q "EMBEDDED" $BACKDIR/recvfs2.zdb || \ + log_fail "Obj $recv2_obj not embedded in $recvfs2" + + grep -q "WRITE_EMBEDDED object = $send_obj offset = 0" $dump && \ + log_fail "Obj $obj embedded in zstreamdump output" + grep -q "WRITE_EMBEDDED object = $send_obj offset = 0" $dump2 || \ + log_fail "Obj $obj not embedded in zstreamdump output" +done + +log_pass "Compressed streams can contain embedded blocks." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_incremental.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_incremental.ksh new file mode 100755 index 000000000000..d673282c9355 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_incremental.ksh @@ -0,0 +1,100 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that compressed send works correctly with incremental sends. +# +# Strategy: +# 1. Randomly choose either a -i or -I incremental. +# 2. Generate compressed incremental replication streams for a pool, a +# descendant dataset, and a volume. +# 3. Receive these streams verifying both the contents, and intermediate +# snapshots are present or absent as appropriate to the -i or -I option. +# + +verify_runnable "both" + +log_assert "Verify compressed send works with incremental send streams." +log_onexit cleanup_pool $POOL2 + +typeset opt=$(random_get "-i" "-I") +typeset final dstlist list vol + +log_must eval "zfs send -R $POOL@final > $BACKDIR/final" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/final" + +function do_checks +{ + log_must cmp_ds_cont $POOL $POOL2 + [[ $opt = "-I" ]] && log_must cmp_ds_subs $POOL $POOL2 + [[ $opt = "-i" ]] && log_mustnot cmp_ds_subs $POOL $POOL2 + + [[ $1 != "clean" ]] && return + + cleanup_pool $POOL2 + log_must eval "zfs send -R $POOL@final > $BACKDIR/final" + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/final" +} + +if is_global_zone; then + # Send from the pool root + final=$(getds_with_suffix $POOL2 @final) + list="$final $(getds_with_suffix $POOL2 @snapA)" + list="$list $(getds_with_suffix $POOL2 @snapB)" + list="$list $(getds_with_suffix $POOL2 @snapC)" + + log_must eval "zfs send -c -R $opt @init $POOL2@final >$BACKDIR/pool" + log_must destroy_tree $list + log_must eval "zfs recv -d -F $POOL2 <$BACKDIR/pool" + + dstlist=$(getds_with_suffix $POOL2 @final) + [[ $final != $dstlist ]] && log_fail "$final != $dstlist" + + do_checks clean + + # Send of a volume + vol=$POOL2/$FS/vol + final=$(getds_with_suffix $vol @final) + log_must eval "zfs send -c -R $opt @init $vol@final >$BACKDIR/vol" + log_must destroy_tree $vol@snapB $vol@snapC $vol@final + log_must eval "zfs recv -d -F $POOL2 <$BACKDIR/vol" + + dstlist=$(getds_with_suffix $POOL2/$FS/vol @final) + [[ $final != $dstlist ]] && log_fail "$final != $dstlist" + + do_checks clean +fi + +# Send of a descendant fs +final=$(getds_with_suffix $POOL2/$FS @final) +list="$final $(getds_with_suffix $POOL2/$FS @snapA)" +list="$list $(getds_with_suffix $POOL2/$FS @snapB)" +list="$list $(getds_with_suffix $POOL2/$FS @snapC)" + +log_must eval "zfs send -c -R $opt @init $POOL2/$FS@final >$BACKDIR/fs" +log_must destroy_tree $list +log_must eval "zfs recv -d -F $POOL2 <$BACKDIR/fs" + +dstlist=$(getds_with_suffix $POOL2/$FS @final) +[[ $final != $dstlist ]] && log_fail "$final != $dstlist" + +do_checks + +log_pass "Compressed send works with incremental send streams." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh new file mode 100755 index 000000000000..666e11f702f8 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh @@ -0,0 +1,73 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify a pool without the lz4 feature enabled can create compressed send +# streams, and that they can be received into pools with or without the +# lz4 feature. +# +# Strategy: +# 1. For each of an uncompressed, and gzip dataset created from a pool with +# the lz4 feature disabled, receive the stream into a pool with and without +# the feature enabled. +# + +verify_runnable "both" + +log_assert "Verify compressed streams are rejected if incompatible." + +typeset send_ds=$POOL2/testds +typeset recv_ds=$POOL3/testds + +function cleanup +{ + poolexists $POOL2 && destroy_pool $POOL2 + poolexists $POOL3 && destroy_pool $POOL3 + log_must zpool create $POOL2 $DISK2 +} +log_onexit cleanup + +datasetexists $POOL2 && log_must zpool destroy $POOL2 +log_must zpool create -d $POOL2 $DISK2 + +for compress in off gzip; do + for pool_opt in '' -d; do + poolexists $POOL3 && destroy_pool $POOL3 + log_must zpool create $pool_opt $POOL3 $DISK3 + + datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds + datasetexists $recv_ds && log_must_busy zfs destroy -r $recv_ds + + log_must zfs create -o compress=$compress $send_ds + typeset dir=$(get_prop mountpoint $send_ds) + write_compressible $dir 16m + log_must zfs snapshot $send_ds@full + + log_must eval "zfs send -c $send_ds@full >$BACKDIR/full-c" + log_must eval "zfs recv $recv_ds <$BACKDIR/full-c" + + log_must_busy zfs destroy -r $recv_ds + + log_must eval "zfs send $send_ds@full >$BACKDIR/full" + log_must eval "zfs recv $recv_ds <$BACKDIR/full" + done +done + +log_pass "Compressed streams are rejected if incompatible." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_mixed_compression.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_mixed_compression.ksh new file mode 100755 index 000000000000..22e91c011b31 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_mixed_compression.ksh @@ -0,0 +1,54 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/properties.shlib + +# +# Description: +# Verify datasets using mixed compression algorithms can be received. +# +# Strategy: +# 1. Write data with each of the available compression algorithms +# 2. Receive a full compressed send, and verify the data and compression ratios +# + +verify_runnable "both" + +log_assert "Verify datasets using mixed compression algorithms can be received." +log_onexit cleanup_pool $POOL2 + +send_ds=$POOL2/sendfs +recv_ds=$POOL2/recvfs + +log_must zfs create $send_ds + +for prop in "${compress_prop_vals[@]}"; do + log_must zfs set compress=$prop $send_ds + write_compressible $(get_prop mountpoint $send_ds) 16m +done + +log_must zfs set compress=off $send_ds +log_must zfs snapshot $send_ds@full +log_must eval "zfs send -c $send_ds@full >$BACKDIR/full" +log_must eval "zfs recv $recv_ds <$BACKDIR/full" + +verify_stream_size $BACKDIR/full $send_ds +verify_stream_size $BACKDIR/full $recv_ds +log_must cmp_ds_cont $send_ds $recv_ds + +log_pass "Datasets using mixed compression algorithms can be received." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_props.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_props.ksh new file mode 100755 index 000000000000..6e95c2c30b67 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_props.ksh @@ -0,0 +1,67 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/properties.shlib + +# +# Description: +# Verify compressed send streams can still preserve properties +# +# Strategy: +# 1. Randomly modify the properties in the src pool +# 2. Send a full compressed stream with -p to preserve properties +# 3. Verify all the received properties match the source datasets +# 4. Repeat the process with -R instead of -p +# + +verify_runnable "global" + +function cleanup +{ + destroy_pool $POOL + destroy_pool $POOL2 + log_must zpool create $POOL $DISK1 + log_must zpool create $POOL2 $DISK2 + log_must setup_test_model $POOL +} + +log_assert "Compressed send doesn't interfere with preservation of properties" +log_onexit cleanup + +typeset -a datasets=("" "/pclone" "/$FS" "/$FS/fs1" "/$FS/fs1/fs2" + "/$FS/fs1/fclone" "/vol" "/$FS/vol") + +typeset ds +for opt in "-p" "-R"; do + for ds in ${datasets[@]}; do + randomize_ds_props $POOL$ds + done + + log_must eval "zfs send -c $opt $POOL@final > $BACKDIR/pool-final$opt" + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final$opt" + + for ds in ${datasets[@]}; do + log_must cmp_ds_prop $POOL$ds $POOL2$ds + log_must cmp_ds_prop $POOL$ds@final $POOL2$ds@final + done + + # Don't cleanup the second time, since we do that on exit anyway. + [[ $opt = "-p" ]] && cleanup +done + +log_pass "Compressed send doesn't interfere with preservation of properties" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_recv_dedup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_recv_dedup.ksh new file mode 100755 index 000000000000..cef81617b13b --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_recv_dedup.ksh @@ -0,0 +1,55 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that we can receive a compressed stream into a deduped filesystem. +# +# Strategy: +# 1. Write heavily duplicated data to a filesystem and create a compressed +# full stream. +# 2. Verify that the stream can be received correctly into a dedup=verify +# filesystem. +# + +verify_runnable "both" + +log_pass "Verify a compressed stream can be received into a deduped filesystem" +log_onexit cleanup_pool $POOL2 + +typeset sendfs=$POOL2/sendfs +typeset recvfs=$POOL2/recvfs +typeset stream0=$BACKDIR/stream.0 +typeset stream1=$BACKDIR/stream.1 +typeset inc=$BACKDIR/stream.inc + +log_must zfs create -o compress=lz4 $sendfs +log_must zfs create -o compress=lz4 -o dedup=verify $recvfs +typeset dir=$(get_prop mountpoint $sendfs) +for i in {0..10}; do + log_must file_write -o overwrite -f $dir/file.$i -d R -b 4096 -c 1000 +done +log_must zfs snapshot $sendfs@snap0 +log_must eval "zfs send -c $sendfs@snap0 >$stream0" + +# Finally, make sure the receive works correctly. +log_must eval "zfs recv -d $recvfs <$stream0" +cmp_ds_cont $sendfs $recvfs + +log_pass "The compressed stream could be received into a deduped filesystem" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh new file mode 100755 index 000000000000..20a8c5481224 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh @@ -0,0 +1,68 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015, Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/properties.shlib + +# +# Description: +# Verify a pool without the lz4 feature gracefully rejects a compressed stream +# because on any sending pool that supports it, metadata will be compressed +# with lz4 even if user data is not compressed. +# +# Strategy: +# 1. For each of an uncompressed, gzip and lz4 dataset, do the following +# receives into a pool without the lz4 feature: +# 2. Attempt to receive the compressed stream (should fail) +# 3. Attempt to receive the uncompressed stream (should succeed) +# + +verify_runnable "both" + +log_assert "Verify compressed streams are rejected if incompatible." + +typeset send_ds=$POOL2/testds +typeset recv_ds=$POOL3/testds + +function cleanup +{ + poolexists $POOL2 && destroy_pool $POOL2 + poolexists $POOL3 && destroy_pool $POOL3 + log_must zpool create $POOL2 $DISK2 +} +log_onexit cleanup + +datasetexists $POOL3 && log_must zpool destroy $POOL3 +log_must zpool create -d $POOL3 $DISK3 + +for compress in "${compress_prop_vals[@]}"; do + datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds + datasetexists $recv_ds && log_must_busy zfs destroy -r $recv_ds + + log_must zfs create -o compress=$compress $send_ds + typeset dir=$(get_prop mountpoint $send_ds) + write_compressible $dir 16m + log_must zfs snapshot $send_ds@full + + log_must eval "zfs send -c $send_ds@full >$BACKDIR/full-c" + log_mustnot eval "zfs recv $recv_ds <$BACKDIR/full-c" + + log_must eval "zfs send $send_ds@full >$BACKDIR/full" + log_must eval "zfs recv $recv_ds <$BACKDIR/full" +done + +log_pass "Compressed streams are rejected if incompatible." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_resume.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_resume.ksh new file mode 100755 index 000000000000..05ba5ed244d9 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_resume.ksh @@ -0,0 +1,49 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify resumability of full and incremental ZFS send/receive with the -c +# (compress) flag in the presence of a corrupted stream. +# +# Strategy: +# 1. Start a full ZFS send with the -c flag (compress), redirect output to +# a file +# 2. Mess up the contents of the stream state file on disk +# 3. Try ZFS receive, which should fail with a checksum mismatch error +# 4. ZFS send to the stream state file again using the receive_resume_token +# 5. ZFS receive and verify the receive completes successfully +# 6. Repeat steps on an incremental ZFS send +# + +verify_runnable "both" + +sendfs=$POOL/sendfs +recvfs=$POOL2/recvfs +streamfs=$POOL/stream + +log_assert "Verify compressed send streams can be resumed if interrupted" +log_onexit resume_cleanup $sendfs $streamfs + +test_fs_setup $sendfs $recvfs $streamfs +resume_test "zfs send -c -v $sendfs@a" $streamfs $recvfs +resume_test "zfs send -c -v -i @a $sendfs@b" $streamfs $recvfs +file_check $sendfs $recvfs + +log_pass "Compressed send streams can be resumed if interrupted" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh new file mode 100755 index 000000000000..840b5f085967 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh @@ -0,0 +1,97 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015, Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/properties.shlib + +# +# Description: +# Verify the stream size estimate given by -P accounts for compressed send. +# Verify the stream size given by -P accounts for compressed send." +# +# Strategy: +# 1. For datasets of varied compression types do the following: +# 2. Write data, verify stream size estimates with and without -c +# + +verify_runnable "both" +typeset send_ds="$POOL2/testfs" +typeset send_vol="$POOL2/vol" +typeset send_voldev="$ZVOL_DEVDIR/$POOL2/vol" +typeset file="$BACKDIR/file.0" +typeset megs="16" +typeset compress + +function get_estimated_size +{ + typeset cmd=$1 + typeset ds=${cmd##* } + if is_freebsd; then + mkdir -p $BACKDIR + typeset tmpfile=$(TMPDIR=$BACKDIR mktemp) + else + typeset tmpfile=$(mktemp -p $BACKDIR) + fi + + eval "$cmd >$tmpfile" + [[ $? -eq 0 ]] || log_fail "get_estimated_size: $cmd" + typeset size=$(eval "awk '\$2 == \"$ds\" {print \$3}' $tmpfile") + rm -f $tmpfile + + echo $size +} + +log_assert "Verify the stream size given by -P accounts for compressed send." +log_onexit cleanup_pool $POOL2 + +write_compressible $BACKDIR ${megs}m + +for compress in "${compress_prop_vals[@]}"; do + datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds + datasetexists $send_vol && log_must_busy zfs destroy -r $send_vol + log_must zfs create -o compress=$compress $send_ds + log_must zfs create -V 1g -o compress=$compress $send_vol + block_device_wait + + typeset dir=$(get_prop mountpoint $send_ds) + log_must cp $file $dir + log_must zfs snapshot $send_ds@snap + log_must dd if=$file of=$send_voldev + log_must zfs snapshot $send_vol@snap + + typeset ds_size=$(get_estimated_size "zfs send -nP $send_ds@snap") + typeset ds_lrefer=$(get_prop lrefer $send_ds) + within_percent $ds_size $ds_lrefer 90 || log_fail \ + "$ds_size and $ds_lrefer differed by too much" + + typeset vol_size=$(get_estimated_size "zfs send -nP $send_vol@snap") + typeset vol_lrefer=$(get_prop lrefer $send_vol) + within_percent $vol_size $vol_lrefer 90 || log_fail \ + "$vol_size and $vol_lrefer differed by too much" + + typeset ds_csize=$(get_estimated_size "zfs send -nP -c $send_ds@snap") + typeset ds_refer=$(get_prop refer $send_ds) + within_percent $ds_csize $ds_refer 90 || log_fail \ + "$ds_csize and $ds_refer differed by too much" + + typeset vol_csize=$(get_estimated_size "zfs send -nP -c $send_vol@snap") + typeset vol_refer=$(get_prop refer $send_vol) + within_percent $vol_csize $vol_refer 90 || log_fail \ + "$vol_csize and $vol_refer differed by too much" +done + +log_pass "The stream size given by -P accounts for compressed send." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_verify_contents.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_verify_contents.ksh new file mode 100755 index 000000000000..0a5d56bebe32 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_verify_contents.ksh @@ -0,0 +1,55 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify compressed send streams replicate data and datasets +# +# Strategy: +# 1. Back up all the data from POOL/FS +# 2. Verify all the datasets and data can be recovered in POOL2 +# 3. Back up all the data from root filesystem POOL2 +# 4. Verify all the data can be recovered, too +# + +verify_runnable "both" + +log_assert "zfs send -c -R send replication stream up to the named snap." +log_onexit cleanup_pool $POOL2 + +# Verify the entire pool and descendants can be backed up and restored. +log_must eval "zfs send -c -R $POOL@final > $BACKDIR/pool-final-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R" + +dstds=$(get_dst_ds $POOL $POOL2) +log_must cmp_ds_subs $POOL $dstds +log_must cmp_ds_cont $POOL $dstds + +# Cleanup POOL2 +log_must cleanup_pool $POOL2 + +# Verify all the filesystems and descendants can be backed up and restored. +log_must eval "zfs send -c -R $POOL/$FS@final > $BACKDIR/fs-final-R" +log_must eval "zfs receive -d $POOL2 < $BACKDIR/fs-final-R" + +dstds=$(get_dst_ds $POOL/$FS $POOL2) +log_must cmp_ds_subs $POOL/$FS $dstds +log_must cmp_ds_cont $POOL/$FS $dstds + +log_pass "zfs send -c -R send replication stream up to the named snap." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh new file mode 100755 index 000000000000..b7d978624f2b --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh @@ -0,0 +1,67 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015, Delphix. All rights reserved. +# Copyright (c) 2019, Kjeld Schouten-Lebbing. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/properties.shlib + +# +# Description: +# Verify that the amount of data in a send -c stream matches compressratio. +# +# Strategy: +# 1. For random compression types, and compressible / incompressible data: +# 2. Create a snap with data +# 3. Compare the size of the stream with the data on the dataset, adjusted +# by compressratio for normal send, and compared to used for send -c. +# + +verify_runnable "both" + +log_assert "Verify send -c streams are compressed" +log_onexit cleanup_pool $POOL2 + +typeset sendfs=$POOL2/$FS +typeset megs=128 + +for prop in "${compress_prop_vals[@]}"; do + for compressible in 'yes' 'no'; do + log_must zfs create -o compress=$prop $sendfs + + if [[ $compressible = 'yes' ]]; then + write_compressible $(get_prop mountpoint $sendfs) \ + ${megs}m + else + typeset file="$(get_prop mountpoint $sendfs)/ddfile" + log_must dd if=/dev/urandom of=$file bs=1024k count=$megs + fi + + log_must zfs snapshot $sendfs@snap + + # Calculate the sizes and verify the compression ratio. + log_must eval "zfs send $sendfs@snap >$BACKDIR/uncompressed" + verify_stream_size $BACKDIR/uncompressed $sendfs + + log_must eval "zfs send -c $sendfs@snap >$BACKDIR/compressed" + verify_stream_size $BACKDIR/compressed $sendfs + + log_must rm $BACKDIR/uncompressed $BACKDIR/compressed + log_must_busy zfs destroy -r $sendfs + done +done + +log_pass "Verify send -c streams are compressed" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh new file mode 100755 index 000000000000..988ed91b9918 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh @@ -0,0 +1,78 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that compressed send correctly handles volumes +# +# Strategy: +# 1. Write compressible data into a volume, take a snap +# 2. Verify the compressed stream is the correct size, and has the correct data +# 3. Repeat step 2 for an incremental compressed stream +# + +function cleanup +{ + log_must_busy zfs destroy -r $vol + cleanup_pool $POOL2 +} + +verify_runnable "both" + +log_assert "Verify compressed send works with volumes" +log_onexit cleanup + +typeset vol="$POOL/newvol" +typeset vol2="$POOL2/newvol" +typeset voldev="$ZVOL_DEVDIR/$POOL/newvol" +typeset voldev2="$ZVOL_DEVDIR/$POOL2/newvol" +typeset data1=$BACKDIR/file.0 +typeset data2=$BACKDIR/file.1 +typeset megs=8 + +log_must zfs create -V 256m -o compress=lz4 $vol + +write_compressible $BACKDIR ${megs}m 2 +md5_1=$(md5digest $data1) +md5_2=$(md5digest $data2) + +log_must dd if=$data1 of=$voldev bs=1024k +log_must zfs snapshot $vol@snap + +log_must eval "zfs send -c $vol@snap >$BACKDIR/full" +log_must eval "zfs recv -d $POOL2 <$BACKDIR/full" + +verify_stream_size $BACKDIR/full $vol +verify_stream_size $BACKDIR/full $vol2 +md5=$(dd if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5digest) +[[ $md5 = $md5_1 ]] || log_fail "md5 mismatch: $md5 != $md5_1" + +# Repeat, for an incremental send +log_must dd seek=$megs if=$data2 of=$voldev bs=1024k +log_must zfs snapshot $vol@snap2 + +log_must eval "zfs send -c -i snap $vol@snap2 >$BACKDIR/inc" +log_must eval "zfs recv -d $POOL2 <$BACKDIR/inc" + +verify_stream_size $BACKDIR/inc $vol 90 $vol@snap +verify_stream_size $BACKDIR/inc $vol2 90 $vol2@snap +md5=$(dd skip=$megs if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5digest) +[[ $md5 = $md5_2 ]] || log_fail "md5 mismatch: $md5 != $md5_2" + +log_pass "Verify compressed send works with volumes" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh new file mode 100755 index 000000000000..b4dc00cec4e7 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh @@ -0,0 +1,75 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# Copyright (c) 2020 by Datto, Inc. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/math.shlib + +# +# Description: +# Verify compression features show up in zstreamdump +# +# Strategy: +# 1. Create a full compressed send stream +# 2. Verify zstreamdump shows this stream has the relevant features +# 3. Verify zstreamdump's accounting of logical and compressed size is correct +# 4. Verify the toname from a resume token +# 5. Verify it fails with corrupted resume token +# 6. Verify it fails with missing resume token +# + +verify_runnable "both" + +log_assert "Verify zstreamdump correctly interprets compressed send streams." +log_onexit cleanup_pool $POOL2 + +typeset sendfs=$POOL2/fs +typeset streamfs=$POOL2/fs2 +typeset recvfs=$POOL2/fs3 + +log_must zfs create -o compress=lz4 $sendfs +log_must zfs create -o compress=lz4 $streamfs +typeset dir=$(get_prop mountpoint $sendfs) +write_compressible $dir 16m +log_must zfs snapshot $sendfs@full + +log_must eval "zfs send -c $sendfs@full >$BACKDIR/full" +log_must stream_has_features $BACKDIR/full lz4 compressed +cat $BACKDIR/full | zstreamdump -v > $BACKDIR/dump.out + +lsize=$(awk '/^WRITE [^0]/ {lsize += $24} END {printf("%d", lsize)}' \ + $BACKDIR/dump.out) +lsize_prop=$(get_prop logicalused $sendfs) +within_percent $lsize $lsize_prop 90 || log_fail \ + "$lsize and $lsize_prop differed by too much" + +csize=$(awk '/^WRITE [^0]/ {csize += $27} END {printf("%d", csize)}' \ + $BACKDIR/dump.out) +csize_prop=$(get_prop used $sendfs) +within_percent $csize $csize_prop 90 || log_fail \ + "$csize and $csize_prop differed by too much" + +x=$(get_resume_token "zfs send -c $sendfs@full" $streamfs $recvfs) +resume_token=$(cat /$streamfs/resume_token) +to_name_fs=$sendfs +log_must eval "zstream token $resume_token | grep $to_name_fs" + +bad_resume_token="1-1162e8285b-100789c6360" +log_mustnot eval "zstream token $bad_resume_token 2>&1" +log_mustnot eval "zstream token 2>&1" + +log_pass "zstreamdump correctly interprets compressed send streams." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh new file mode 100755 index 000000000000..e1ac00c79c96 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh @@ -0,0 +1,203 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify compressed send works correctly with datasets of varying recsize. +# +# Strategy: +# 1. Check the recv behavior (into pools with features enabled and disabled) +# of all combinations of -c -p and -L. Verify the stream is compressed, +# and that the recsize property and that of a received file is correct +# according to this matrix: +# +# +---------+--------+------------+------------+-----------+-----------+ +# | send | send | received | received | received | received | +# | stream | stream | file bs | prop | file bs | props | +# | recsize | flags | (disabled) | (disabled) | (enabled) | (enabled) | +# +---------+--------+------------+------------+-----------+-----------+ +# | 128k | | 128k | 128k | 128k | 128k | +# | 128k | -c | Fails | Fails | 128k | 128k | +# | 128k | -p | 128k | 128k | 128k | 128k | +# | 128k | -L | 128k | 128k | 128k | 128k | +# | 128k | -cp | Fails | Fails | 128k | 128k | +# | 128k | -cL | Fails | Fails | 128k | 128k | +# | 128k | -pL | 128k | 128k | 128k | 128k | +# | 128k | -cpL | Fails | Fails | 128k | 128k | +# | 1m | | Fails | Fails | 128k | 128k | +# | 1m | -c | Fails | Fails | 128k | 128k | +# | 1m | -p | 128k | 128k | 128k | 1m | +# | 1m | -L | Fails | Fails | 1m | 128k | +# | 1m | -cp | Fails | Fails | 128k | 1m | +# | 1m | -cL | Fails | Fails | 1m | 128k | +# | 1m | -pL | Fails | Fails | 1m | 1m | +# | 1m | -cpL | Fails | Fails | 1m | 1m | +# +---------+--------+------------+------------+-----------+-----------+ +# + +verify_runnable "both" + +function cleanup +{ + datasetexists $TESTPOOL/128k && log_must_busy zfs destroy $TESTPOOL/128k + datasetexists $TESTPOOL/1m && log_must_busy zfs destroy $TESTPOOL/1m + cleanup_pool $POOL2 + destroy_pool $POOL3 +} + +# For a received stream, verify the recsize (prop and file) match expectations. +function check_recsize +{ + typeset recv_ds=$1 + typeset expected_file_bs=$2 + typeset expected_recsize=$3 + typeset file="$(get_prop mountpoint $recv_ds)/testfile" + + [[ -f $file ]] || log_fail "file '$file' doesn't exist" + + typeset read_recsize=$(get_prop recsize $recv_ds) + if is_freebsd; then + typeset read_file_bs=$(stat -f "%k" $file) + else + typeset read_file_bs=$(stat $file | sed -n \ + 's/.*IO Block: \([0-9]*\).*/\1/p') + fi + + [[ $read_recsize = $expected_recsize ]] || log_fail \ + "read_recsize: $read_recsize expected_recsize: $expected_recsize" + [[ $read_file_bs = $expected_file_bs ]] || log_fail \ + "read_file_bs: $read_file_bs expected_file_bs: $expected_file_bs" +} + +# +# This function does a zfs send and receive according to the parameters +# below, and verifies the data shown in the strategy section. +# +# -[cpL] flags to pass through to 'zfs send' +# -d Receive into a pool with all features disabled +# +# $1 The recordsize of the send dataset +# $2 Whether or not the recv should work. +# $3 The blocksize expected in a received file (default 128k) +# $4 The recordsize property expected in a received dataset (default 128k) +# +function check +{ + typeset recv_pool=$POOL2 + typeset flags='-' + + while getopts "cdpL" opt; do + case $opt in + c) + flags+='c' + ;; + d) + recv_pool=$POOL3 + ;; + p) + flags+='p' + ;; + L) + flags+='L' + ;; + esac + done + shift $(($OPTIND - 1)) + [[ ${#flags} -eq 1 ]] && flags='' + + typeset recsize=$1 + typeset verify=$2 + typeset expected_file_bs=${3-131072} + typeset expected_recsize=${4-131072} + typeset send_ds=$TESTPOOL/$recsize + typeset send_snap=$send_ds@snap + typeset recv_ds=$recv_pool/$recsize + typeset stream=$BACKDIR/stream.out + + datasetexists $send_ds || log_fail "send ds: $send_ds doesn't exist" + [[ -f $stream ]] && log_must rm $stream + log_must eval "zfs send $flags $send_snap >$stream" + $verify eval "zfs recv $recv_ds <$stream" + typeset stream_size=$(cat $stream | zstreamdump | sed -n \ + 's/ Total write size = \(.*\) (0x.*)/\1/p') + + # + # Special case: For a send dataset with large blocks, don't try to + # verify the stream size is correct if the compress flag is present + # but the large blocks flag isn't. In these cases, the user data + # isn't compressed in the stream (though metadata is) so the + # verification would fail. + # + typeset do_size_test=true + [[ $recsize = $large && $flags =~ 'c' && ! $flags =~ 'L' ]] && \ + do_size_test=false + + $do_size_test && verify_stream_size $stream $send_ds + + if [[ $verify = "log_mustnot" ]]; then + datasetnonexists $recv_ds || log_fail "$recv_ds shouldn't exist" + return + fi + + check_recsize $recv_ds $expected_file_bs $expected_recsize + $do_size_test && verify_stream_size $stream $recv_ds + log_must_busy zfs destroy -r $recv_ds +} + +log_assert "Verify compressed send works with datasets of varying recsize." +log_onexit cleanup +typeset recsize opts dir +typeset small=$((128 * 1024)) +typeset large=$((1024 * 1024)) + +# Create POOL3 with features disabled and datasets to create test send streams +datasetexists $POOL3 && log_must zpool destroy $POOL3 +log_must zpool create -d $POOL3 $DISK3 +write_compressible $BACKDIR 32m +for recsize in $small $large; do + log_must zfs create -o compress=gzip -o recsize=$recsize \ + $TESTPOOL/$recsize + dir=$(get_prop mountpoint $TESTPOOL/$recsize) + log_must cp $BACKDIR/file.0 $dir/testfile + log_must zfs snapshot $TESTPOOL/$recsize@snap +done + +# Run tests for send streams without large blocks +for opts in '' -d -c -p -dp -L -dL -cp -cL -pL -dpL -cpL; do + check $opts $small log_must +done +for opts in -dc -dcp -dcL -dcpL; do + check $opts $small log_mustnot +done + +# Run tests for send streams with large blocks +for opts in '' -d -dp -c; do + check $opts $large log_must +done +for opts in -dc -dL -dcp -dcL -dpL -dcpL; do + check $opts $large log_mustnot +done +check -p $large log_must $small $large +check -L $large log_must $large $small +check -cp $large log_must $small $large +check -cL $large log_must $large $small +check -pL $large log_must $large $large +check -cpL $large log_must $large $large + +log_pass "Compressed send works with datasets of varying recsize." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-wR_encrypted_zvol.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-wR_encrypted_zvol.ksh new file mode 100755 index 000000000000..b95fc3da30c3 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-wR_encrypted_zvol.ksh @@ -0,0 +1,108 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Datto Inc. All rights reserved. +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Verify that zvols with encryption=on can be sent and received with a raw +# send stream. +# +# STRATEGY: +# 1. Create a zvol with encryption on and put a filesystem on it +# 2. Copy a file into the zvol a few times and take a snapshot +# 3. Repeat step 2 a few times to create more snapshots +# 4. Send all snapshots in a recursive, raw send stream +# 5. Mount the received zvol and verify that all of the data there is correct +# + +verify_runnable "both" + +function cleanup +{ + ismounted $recvmnt $fstype && log_must umount $recvmnt + ismounted $mntpnt $fstype && log_must umount $mntpnt + [[ -d $recvmnt ]] && log_must rm -rf $keyfile + [[ -d $mntpnt ]] && log_must rm -rf $keyfile + destroy_dataset $TESTPOOL/recv "-r" + destroy_dataset $TESTPOOL/$TESTVOL "-r" + [[ -f $keyfile ]] && log_must rm $keyfile + [[ -f $sendfile ]] && log_must rm $sendfile +} +log_onexit cleanup + +log_assert "Verify zfs can receive raw, recursive send streams" + +typeset keyfile=/$TESTPOOL/pkey +typeset snap_count=5 +typeset zdev=$ZVOL_DEVDIR/$TESTPOOL/$TESTVOL +typeset mntpnt=$TESTDIR/$TESTVOL +typeset recvdev=$ZVOL_DEVDIR/$TESTPOOL/recv +typeset recvmnt=$TESTDIR/recvmnt +typeset sendfile=$TESTDIR/sendfile +typeset fstype=none + +log_must eval "echo 'password' > $keyfile" + +log_must zfs create -o dedup=on -o encryption=on -o keyformat=passphrase \ + -o keylocation=file://$keyfile -V 128M $TESTPOOL/$TESTVOL +block_device_wait + +if is_linux; then + # ext4 only supported on Linux + log_must new_fs -t ext4 $zdev + fstype=ext4 + typeset remount_ro="-o remount,ro" + typeset remount_rw="-o remount,rw" +else + log_must new_fs $zdev + fstype=$NEWFS_DEFAULT_FS + typeset remount_ro="-ur" + typeset remount_rw="-uw" +fi +log_must mkdir -p $mntpnt +log_must mkdir -p $recvmnt +log_must mount $zdev $mntpnt + +for ((i = 1; i <= $snap_count; i++)); do + log_must dd if=/dev/urandom of=$mntpnt/file bs=1M count=1 + for ((j = 0; j < 10; j++)); do + log_must cp $mntpnt/file $mntpnt/file$j + done + + log_must sync + log_must mount $remount_ro $zdev $mntpnt + log_must zfs snap $TESTPOOL/$TESTVOL@snap$i + log_must mount $remount_rw $zdev $mntpnt +done + +log_must eval "zfs send -wR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile" +log_must eval "zfs recv $TESTPOOL/recv < $sendfile" +log_must zfs load-key $TESTPOOL/recv +block_device_wait + +log_must mount $recvdev $recvmnt + +md5_1=$(cat $mntpnt/* | md5digest) +md5_2=$(cat $recvmnt/* | md5digest) +[[ "$md5_1" == "$md5_2" ]] || log_fail "md5 mismatch: $md5_1 != $md5_2" + +log_pass "zfs can receive raw, recursive send streams" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh new file mode 100755 index 000000000000..f89cb3b31bee --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh @@ -0,0 +1,120 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Datto Inc. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Verify that a raw zfs send and receive can deal with several different +# types of file layouts. +# +# STRATEGY: +# 1. Create a new encrypted filesystem +# 2. Add an empty file to the filesystem +# 3. Add a small 512 byte file to the filesystem +# 4. Add a larger 32M file to the filesystem +# 5. Add a large sparse file to the filesystem +# 6. Add 1000 empty files to the filesystem +# 7. Add a file with a large xattr value +# 8. Use xattrtest to create files with random xattrs (with and without xattrs=on) +# 9. Take a snapshot of the filesystem +# 10. Remove the 1000 empty files to the filesystem +# 11. Take another snapshot of the filesystem +# 12. Send and receive both snapshots +# 13. Mount the filesystem and check the contents +# + +verify_runnable "both" + +function cleanup +{ + datasetexists $TESTPOOL/$TESTFS2 && \ + log_must zfs destroy -r $TESTPOOL/$TESTFS2 + datasetexists $TESTPOOL/recv && \ + log_must zfs destroy -r $TESTPOOL/recv + [[ -f $keyfile ]] && log_must rm $keyfile + [[ -f $sendfile ]] && log_must rm $sendfile +} +log_onexit cleanup + +log_assert "Verify 'zfs send -w' works with many different file layouts" + +typeset keyfile=/$TESTPOOL/pkey +typeset sendfile=/$TESTPOOL/sendfile +typeset sendfile2=/$TESTPOOL/sendfile2 + +# Create an encrypted dataset +log_must eval "echo 'password' > $keyfile" +log_must zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=file://$keyfile $TESTPOOL/$TESTFS2 + +# Create files with varied layouts on disk +log_must touch /$TESTPOOL/$TESTFS2/empty +log_must mkfile 512 /$TESTPOOL/$TESTFS2/small +log_must mkfile 32M /$TESTPOOL/$TESTFS2/full +log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/sparse \ + bs=512 count=1 seek=1048576 >/dev/null 2>&1 + +log_must mkdir -p /$TESTPOOL/$TESTFS2/dir +for i in {1..1000}; do + log_must mkfile 512 /$TESTPOOL/$TESTFS2/dir/file-$i +done + +log_must mkdir -p /$TESTPOOL/$TESTFS2/xattrondir +log_must zfs set xattr=on $TESTPOOL/$TESTFS2 +log_must xattrtest -f 10 -x 3 -s 32768 -r -k -p /$TESTPOOL/$TESTFS2/xattrondir +log_must mkdir -p /$TESTPOOL/$TESTFS2/xattrsadir +log_must zfs set xattr=sa $TESTPOOL/$TESTFS2 +log_must xattrtest -f 10 -x 3 -s 32768 -r -k -p /$TESTPOOL/$TESTFS2/xattrsadir + +# ZoL issue #7432 +log_must zfs set compression=on xattr=sa $TESTPOOL/$TESTFS2 +log_must touch /$TESTPOOL/$TESTFS2/attrs +log_must eval "python -c 'print \"a\" * 4096' | \ + set_xattr_stdin bigval /$TESTPOOL/$TESTFS2/attrs" +log_must zfs set compression=off xattr=on $TESTPOOL/$TESTFS2 + +log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1 + +# Remove the empty files created in the first snapshot +for i in {1..1000}; do + log_must rm /$TESTPOOL/$TESTFS2/dir/file-$i +done +sync + +log_must zfs snapshot $TESTPOOL/$TESTFS2@snap2 +expected_cksum=$(recursive_cksum /$TESTPOOL/$TESTFS2) + +log_must eval "zfs send -wp $TESTPOOL/$TESTFS2@snap1 > $sendfile" +log_must eval "zfs send -wp -i @snap1 $TESTPOOL/$TESTFS2@snap2 > $sendfile2" + +log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile" +log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile2" +log_must zfs load-key $TESTPOOL/recv + +log_must zfs mount -a +actual_cksum=$(recursive_cksum /$TESTPOOL/recv) +[[ "$expected_cksum" != "$actual_cksum" ]] && \ + log_fail "Recursive checksums differ ($expected_cksum != $actual_cksum)" + +log_must xattrtest -f 10 -o3 -y -p /$TESTPOOL/recv/xattrondir +log_must xattrtest -f 10 -o3 -y -p /$TESTPOOL/recv/xattrsadir + +log_pass "Verified 'zfs send -w' works with many different file layouts" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_hierarchy.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_hierarchy.ksh new file mode 100755 index 000000000000..8417afc88d33 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_hierarchy.ksh @@ -0,0 +1,96 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2017 by Datto Inc. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib + +# +# DESCRIPTION: +# Raw recursive sends preserve filesystem structure. +# +# STRATEGY: +# 1. Create an encrypted filesystem with a clone and a child +# 2. Snapshot and send the filesystem tree +# 3. Verify that the filesystem structure was correctly received +# 4. Change the child to an encryption root and promote the clone +# 5. Snapshot and send the filesystem tree again +# 6. Verify that the new structure is received correctly +# + +verify_runnable "both" + +function cleanup +{ + log_must cleanup_pool $POOL + log_must cleanup_pool $POOL2 + log_must setup_test_model $POOL +} + +log_assert "Raw recursive sends preserve filesystem structure." +log_onexit cleanup + +# Create the filesystem hierarchy +log_must cleanup_pool $POOL +log_must eval "echo $PASSPHRASE | zfs create -o encryption=on" \ + "-o keyformat=passphrase $POOL/$FS" +log_must zfs snapshot $POOL/$FS@snap +log_must zfs clone $POOL/$FS@snap $POOL/clone +log_must zfs create $POOL/$FS/child + +# Back up the tree and verify the structure +log_must zfs snapshot -r $POOL@before +log_must eval "zfs send -wR $POOL@before > $BACKDIR/fs-before-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/fs-before-R" +dstds=$(get_dst_ds $POOL/$FS $POOL2) +log_must cmp_ds_subs $POOL/$FS $dstds + +log_must verify_encryption_root $POOL/$FS $POOL/$FS +log_must verify_keylocation $POOL/$FS "prompt" +log_must verify_origin $POOL/$FS "-" + +log_must verify_encryption_root $POOL/clone $POOL/$FS +log_must verify_keylocation $POOL/clone "none" +log_must verify_origin $POOL/clone "$POOL/$FS@snap" + +log_must verify_encryption_root $POOL/$FS/child $POOL/$FS +log_must verify_keylocation $POOL/$FS/child "none" + +# Alter the hierarchy and re-send +log_must eval "echo $PASSPHRASE1 | zfs change-key -o keyformat=passphrase" \ + "$POOL/$FS/child" +log_must zfs promote $POOL/clone +log_must zfs snapshot -r $POOL@after +log_must eval "zfs send -wR -i $POOL@before $POOL@after >" \ + "$BACKDIR/fs-after-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/fs-after-R" +log_must cmp_ds_subs $POOL/$FS $dstds + +log_must verify_encryption_root $POOL/$FS $POOL/clone +log_must verify_keylocation $POOL/$FS "none" +log_must verify_origin $POOL/$FS "$POOL/clone@snap" + +log_must verify_encryption_root $POOL/clone $POOL/clone +log_must verify_keylocation $POOL/clone "prompt" +log_must verify_origin $POOL/clone "-" + +log_must verify_encryption_root $POOL/$FS/child $POOL/$FS/child +log_must verify_keylocation $POOL/$FS/child "prompt" + +log_pass "Raw recursive sends preserve filesystem structure." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_props.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_props.ksh new file mode 100755 index 000000000000..793904db91ca --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_props.ksh @@ -0,0 +1,215 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Datto Inc. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Verify that zfs properly handles encryption properties when receiving +# send streams. +# +# STRATEGY: +# 1. Create a few unencrypted and encrypted test datasets with some data +# 2. Take snapshots of these datasets in preparation for sending +# 3. Verify that 'zfs recv -o keylocation=prompt' fails +# 4. Verify that 'zfs recv -x <encryption prop>' fails on a raw send stream +# 5. Verify that encryption properties cannot be changed on incrementals +# 6. Verify that a simple send can be received as an encryption root +# 7. Verify that an unencrypted props send can be received as an +# encryption root +# 8. Verify that an unencrypted recursive send can be received as an +# encryption root +# 9. Verify that an unencrypted props send can be received as an +# encryption child +# 10. Verify that an unencrypted recursive send can be received as an +# encryption child +# + +verify_runnable "both" + +function cleanup +{ + destroy_dataset $TESTPOOL/recv "-r" + destroy_dataset $TESTPOOL/crypt "-r" + destroy_dataset $TESTPOOL/ds "-r" + [[ -f $sendfile ]] && log_must rm $sendfile + [[ -f $keyfile ]] && log_must rm $keyfile +} +log_onexit cleanup + +log_assert "'zfs recv' must properly handle encryption properties" + +typeset keyfile=/$TESTPOOL/pkey +typeset sendfile=/$TESTPOOL/sendfile +typeset snap=$TESTPOOL/ds@snap1 +typeset snap2=$TESTPOOL/ds@snap2 +typeset esnap=$TESTPOOL/crypt@snap1 +typeset esnap2=$TESTPOOL/crypt@snap2 + +log_must eval "echo 'password' > $keyfile" + +log_must zfs create $TESTPOOL/ds +log_must zfs create $TESTPOOL/ds/ds1 + +log_must zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=file://$keyfile $TESTPOOL/crypt +log_must zfs create $TESTPOOL/crypt/ds1 +log_must zfs create -o keyformat=passphrase -o keylocation=file://$keyfile \ + $TESTPOOL/crypt/ds2 + +log_must mkfile 1M /$TESTPOOL/ds/$TESTFILE0 +log_must cp /$TESTPOOL/ds/$TESTFILE0 /$TESTPOOL/crypt/$TESTFILE0 +typeset cksum=$(md5digest /$TESTPOOL/ds/$TESTFILE0) + +log_must zfs snap -r $snap +log_must zfs snap -r $snap2 +log_must zfs snap -r $esnap +log_must zfs snap -r $esnap2 + +# Embedded data is incompatible with encrypted datasets, so we cannot +# allow embedded streams to be received. +log_note "Must not be able to receive an embedded stream as encrypted" +log_mustnot eval "zfs send -e $TESTPOOL/crypt/ds1 | zfs recv $TESTPOOL/recv" + +# We currently don't have an elegant and secure way to pass the passphrase +# in via prompt because the send stream itself is using stdin. +log_note "Must not be able to use 'keylocation=prompt' on receive" +log_must eval "zfs send $snap > $sendfile" +log_mustnot eval "zfs recv -o encryption=on -o keyformat=passphrase" \ + "$TESTPOOL/recv < $sendfile" +log_mustnot eval "zfs recv -o encryption=on -o keyformat=passphrase" \ + "-o keylocation=prompt $TESTPOOL/recv < $sendfile" + +# Raw sends do not have access to the decrypted data so we cannot override +# the encryption settings without losing the data. +log_note "Must not be able to disable encryption properties on raw send" +log_must eval "zfs send -w $esnap > $sendfile" +log_mustnot eval "zfs recv -x encryption $TESTPOOL/recv < $sendfile" +log_mustnot eval "zfs recv -x keyformat $TESTPOOL/recv < $sendfile" +log_mustnot eval "zfs recv -x pbkdf2iters $TESTPOOL/recv < $sendfile" + +# Encryption properties are set upon creating the dataset. Changing them +# afterwards requires using 'zfs change-key' or an update from a raw send. +log_note "Must not be able to change encryption properties on incrementals" +log_must eval "zfs send $esnap | zfs recv -o encryption=on" \ + "-o keyformat=passphrase -o keylocation=file://$keyfile $TESTPOOL/recv" +log_mustnot eval "zfs send -i $esnap $esnap2 |" \ + "zfs recv -o encryption=aes-128-ccm $TESTPOOL/recv" +log_mustnot eval "zfs send -i $esnap $esnap2 |" \ + "zfs recv -o keyformat=hex $TESTPOOL/recv" +log_mustnot eval "zfs send -i $esnap $esnap2 |" \ + "zfs recv -o pbkdf2iters=100k $TESTPOOL/recv" +log_must zfs destroy -r $TESTPOOL/recv + +# Test that we can receive a simple stream as an encryption root. +log_note "Must be able to receive stream as encryption root" +ds=$TESTPOOL/recv +log_must eval "zfs send $snap > $sendfile" +log_must eval "zfs recv -o encryption=on -o keyformat=passphrase" \ + "-o keylocation=file://$keyfile $ds < $sendfile" +log_must test "$(get_prop 'encryption' $ds)" == "aes-256-gcm" +log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds" +log_must test "$(get_prop 'keyformat' $ds)" == "passphrase" +log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile" +log_must test "$(get_prop 'mounted' $ds)" == "yes" +recv_cksum=$(md5digest /$ds/$TESTFILE0) +log_must test "$recv_cksum" == "$cksum" +log_must zfs destroy -r $ds + +# Test that we can override encryption properties on a properties stream +# of an unencrypted dataset, turning it into an encryption root. +log_note "Must be able to receive stream with props as encryption root" +ds=$TESTPOOL/recv +log_must eval "zfs send -p $snap > $sendfile" +log_must eval "zfs recv -o encryption=on -o keyformat=passphrase" \ + "-o keylocation=file://$keyfile $ds < $sendfile" +log_must test "$(get_prop 'encryption' $ds)" == "aes-256-gcm" +log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds" +log_must test "$(get_prop 'keyformat' $ds)" == "passphrase" +log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile" +log_must test "$(get_prop 'mounted' $ds)" == "yes" +recv_cksum=$(md5digest /$ds/$TESTFILE0) +log_must test "$recv_cksum" == "$cksum" +log_must zfs destroy -r $ds + +# Test that we can override encryption properties on a recursive stream +# of an unencrypted dataset, turning it into an encryption root. The root +# dataset of the stream should become an encryption root with all children +# inheriting from it. +log_note "Must be able to receive recursive stream as encryption root" +ds=$TESTPOOL/recv +log_must eval "zfs send -R $snap > $sendfile" +log_must eval "zfs recv -o encryption=on -o keyformat=passphrase" \ + "-o keylocation=file://$keyfile $ds < $sendfile" +log_must test "$(get_prop 'encryption' $ds)" == "aes-256-gcm" +log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds" +log_must test "$(get_prop 'keyformat' $ds)" == "passphrase" +log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile" +log_must test "$(get_prop 'mounted' $ds)" == "yes" +recv_cksum=$(md5digest /$ds/$TESTFILE0) +log_must test "$recv_cksum" == "$cksum" +log_must zfs destroy -r $ds + +# Test that we can override an unencrypted properties stream's encryption +# settings, receiving it as an encrypted child. +log_note "Must be able to receive stream with props to encrypted child" +ds=$TESTPOOL/crypt/recv +log_must eval "zfs send -p $snap > $sendfile" +log_must eval "zfs recv -x encryption $ds < $sendfile" +log_must test "$(get_prop 'encryptionroot' $ds)" == "$TESTPOOL/crypt" +log_must test "$(get_prop 'encryption' $ds)" == "aes-256-gcm" +log_must test "$(get_prop 'keyformat' $ds)" == "passphrase" +log_must test "$(get_prop 'mounted' $ds)" == "yes" +recv_cksum=$(md5digest /$ds/$TESTFILE0) +log_must test "$recv_cksum" == "$cksum" +log_must zfs destroy -r $ds + +# Test that we can override an unencrypted recursive stream's encryption +# settings, receiving all datasets as encrypted children. +log_note "Must be able to receive recursive stream to encrypted child" +ds=$TESTPOOL/crypt/recv +log_must eval "zfs send -R $snap > $sendfile" +log_must eval "zfs recv -x encryption $ds < $sendfile" +log_must test "$(get_prop 'encryptionroot' $ds)" == "$TESTPOOL/crypt" +log_must test "$(get_prop 'encryption' $ds)" == "aes-256-gcm" +log_must test "$(get_prop 'keyformat' $ds)" == "passphrase" +log_must test "$(get_prop 'mounted' $ds)" == "yes" +recv_cksum=$(md5digest /$ds/$TESTFILE0) +log_must test "$recv_cksum" == "$cksum" +log_must zfs destroy -r $ds + +# Test that we can override an unencrypted, incremental, recursive stream's +# encryption settings, receiving all datasets as encrypted children. +log_note "Must be able to receive recursive stream to encrypted child" +ds=$TESTPOOL/crypt/recv +log_must eval "zfs send -R $snap2 > $sendfile" +log_must eval "zfs recv -x encryption $ds < $sendfile" +log_must test "$(get_prop 'encryptionroot' $ds)" == "$TESTPOOL/crypt" +log_must test "$(get_prop 'encryption' $ds)" == "aes-256-gcm" +log_must test "$(get_prop 'keyformat' $ds)" == "passphrase" +log_must test "$(get_prop 'mounted' $ds)" == "yes" +recv_cksum=$(md5digest /$ds/$TESTFILE0) +log_must test "$recv_cksum" == "$cksum" +log_must zfs destroy -r $ds + +# Check that we haven't printed the key to the zpool history log +log_mustnot eval "zpool history -i | grep -q 'wkeydata'" + +log_pass "'zfs recv' properly handles encryption properties" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh new file mode 100755 index 000000000000..c710b3d911e7 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh @@ -0,0 +1,126 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Datto Inc. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# +# +# STRATEGY: +# 1. Create a new encrypted filesystem +# 2. Add a 4 files that are to be truncated later +# 3. Take a snapshot of the filesystem +# 4. Truncate one of the files from 32M to 128k +# 5. Truncate one of the files from 512k to 384k +# 6. Truncate one of the files from 512k to 0 to 384k via reallocation +# 7. Truncate one of the files from 1k to 0 to 512b via reallocation +# 8. Take another snapshot of the filesystem +# 9. Send and receive both snapshots +# 10. Mount the filesystem and check the contents +# + +verify_runnable "both" + +function cleanup +{ + datasetexists $TESTPOOL/$TESTFS2 && \ + log_must zfs destroy -r $TESTPOOL/$TESTFS2 + datasetexists $TESTPOOL/recv && \ + log_must zfs destroy -r $TESTPOOL/recv + [[ -f $keyfile ]] && log_must rm $keyfile + [[ -f $sendfile ]] && log_must rm $sendfile +} +log_onexit cleanup + +function recursive_cksum +{ + case "$(uname)" in + FreeBSD) + find $1 -type f -exec sha256 -q {} \; | \ + sort | sha256digest + ;; + *) + find $1 -type f -exec sha256sum {} \; | \ + sort -k 2 | awk '{ print $1 }' | sha256digest + ;; + esac +} + +log_assert "Verify 'zfs send -w' works with many different file layouts" + +typeset keyfile=/$TESTPOOL/pkey +typeset sendfile=/$TESTPOOL/sendfile +typeset sendfile2=/$TESTPOOL/sendfile2 + +# Create an encrypted dataset +log_must eval "echo 'password' > $keyfile" +log_must zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=file://$keyfile $TESTPOOL/$TESTFS2 + +# Explicitly set the recordsize since the truncation sizes below depend on +# this value being 128k. This is currently same as the default recordsize. +log_must zfs set recordsize=128k $TESTPOOL/$TESTFS2 + +# Create files with varied layouts on disk +log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated +log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2 +log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3 +log_must mkfile 1024 /$TESTPOOL/$TESTFS2/truncated4 + +log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1 + +# +# Truncate files created in the first snapshot. The first tests +# truncating a large file to a single block. The second tests +# truncating one block off the end of a file without changing +# the required nlevels to hold it. The third tests handling +# of a maxblkid that is dropped and then raised again. The +# fourth tests an object that is truncated from a single block +# to a smaller single block. +# +log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated +log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2 +log_must rm -f /$TESTPOOL/$TESTFS2/truncated3 +log_must rm -f /$TESTPOOL/$TESTFS2/truncated4 +log_must zpool sync $TESTPOOL +log_must zfs umount $TESTPOOL/$TESTFS2 +log_must zfs mount $TESTPOOL/$TESTFS2 +log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \ + bs=128k count=3 iflag=fullblock +log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated4 \ + bs=512 count=1 iflag=fullblock + +log_must zfs snapshot $TESTPOOL/$TESTFS2@snap2 +expected_cksum=$(recursive_cksum /$TESTPOOL/$TESTFS2) + +log_must eval "zfs send -wp $TESTPOOL/$TESTFS2@snap1 > $sendfile" +log_must eval "zfs send -wp -i @snap1 $TESTPOOL/$TESTFS2@snap2 > $sendfile2" + +log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile" +log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile2" +log_must zfs load-key $TESTPOOL/recv + +log_must zfs mount -a +actual_cksum=$(recursive_cksum /$TESTPOOL/recv) +[[ "$expected_cksum" != "$actual_cksum" ]] && \ + log_fail "Recursive checksums differ ($expected_cksum != $actual_cksum)" + +log_pass "Verified 'zfs send -w' works with many different file layouts" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_freeobjects.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_freeobjects.ksh new file mode 100755 index 000000000000..6533352a9a14 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_freeobjects.ksh @@ -0,0 +1,81 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify FREEOBJECTS record frees sequential objects (See +# https://github.com/zfsonlinux/zfs/issues/6694) +# +# Strategy: +# 1. Create three files with sequential object numbers, f1 f2 and f3 +# 2. Delete f2 +# 3. Take snapshot A +# 4. Delete f3 +# 5. Take snapshot B +# 6. Receive a full send of A +# 7. Receive an incremental send of B +# 8. Fail test if f3 exists on received snapshot B +# + +verify_runnable "both" + +log_assert "Verify FREEOBJECTS record frees sequential objects" + +sendds=sendfo +recvds=recvfo +f1=/$POOL/$sendds/f1 +f2=/$POOL/$sendds/f2 +f3=/$POOL/$sendds/f3 + +# +# We need to set xattr=sa and dnodesize=legacy to guarantee sequential +# object numbers for this test. Otherwise, if we used directory-based +# xattrs, SELinux extended attributes might consume intervening object +# numbers. +# +log_must zfs create -o xattr=sa -o dnodesize=legacy $POOL/$sendds + +tries=100 +for ((i=0; i<$tries; i++)); do + touch $f1 $f2 $f3 + o1=$(ls -li $f1 | awk '{print $1}') + o2=$(ls -li $f2 | awk '{print $1}') + o3=$(ls -li $f3 | awk '{print $1}') + + if [[ $o2 -ne $(( $o1 + 1 )) ]] || [[ $o3 -ne $(( $o2 + 1 )) ]]; then + rm -f $f1 $f2 $f3 + else + break + fi +done + +if [[ $i -eq $tries ]]; then + log_fail "Failed to create three sequential objects" +fi + +log_must rm $f2 +log_must zfs snap $POOL/$sendds@A +log_must rm $f3 +log_must zfs snap $POOL/$sendds@B +log_must eval "zfs send $POOL/$sendds@A | zfs recv $POOL/$recvds" +log_must eval "zfs send -i $POOL/$sendds@A $POOL/$sendds@B |" \ + "zfs recv $POOL/$recvds" +log_mustnot zdb $POOL/$recvds@B $o3 +log_pass "Verify FREEOBJECTS record frees sequential objects" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_holds.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_holds.ksh new file mode 100755 index 000000000000..5dcf0e2a0a7a --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_holds.ksh @@ -0,0 +1,177 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2012, 2016 by Delphix. All rights reserved. +# Copyright (c) 2018 Datto, Inc. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/tests/functional/cli_root/cli_common.kshlib + +# +# DESCRIPTION: +# Verify 'zfs send' can send dataset holds. +# Verify 'zfs receive' can receive dataset holds. +# +# STRATEGY: +# 1. Create a snapshot. +# 2. Create a full send stream with the fs, including holds. +# 3. Receive the send stream and verify the data integrity. +# 4. Fill in fs with some new data. +# 5. Create an incremental send stream with the fs, including holds. +# 6. Receive the incremental send stream and verify the data integrity. +# 7. Verify the holds have been received as expected. +# 8. Create an incremental snap with no holds, and send that with -h. +# 9. Confirm the snapshot was received as expected. +# 10. Create an incremental snapshot and place a hold on it. +# 11. Receive the incremental stream with -h and verify holds skipped. +# + +verify_runnable "both" + +function cleanup +{ + eval "zfs holds $init_snap |grep -q hold1-1" && + log_must zfs release hold1-1 $init_snap + eval "zfs holds $init_snap |grep -q hold1-2" && + log_must zfs release hold1-2 $init_snap + eval "zfs holds $recv_snap |grep -q hold1-1" && + log_must zfs release hold1-1 $recv_snap + eval "zfs holds $recv_snap |grep -q hold1-2" && + log_must zfs release hold1-2 $recv_snap + eval "zfs holds $inc_snap |grep -q hold2-1" && + log_must zfs release hold2-1 $inc_snap + eval "zfs holds $recv_inc_snap |grep -q hold2-1" && + log_must zfs release hold2-1 $recv_inc_snap + eval "zfs holds $inc_snap3 |grep -q hold3-1" && + log_must zfs release hold3-1 $inc_snap3 + + # destroy datasets + datasetexists $recv_root/$TESTFS1 && + log_must destroy_dataset "$recv_root/$TESTFS1" "-Rf" + datasetexists $recv_root && log_must destroy_dataset "$recv_root" "-Rf" + datasetexists $TESTPOOL/$TESTFS1 && log_must destroy_dataset "$TESTPOOL/$TESTFS1" "-Rf" + + [[ -e $full_bkup ]] && log_must rm -f $full_bkup + [[ -e $inc_bkup ]] && log_must rm -f $inc_bkup + [[ -e $inc_data2 ]] && log_must rm -f $inc_data2 + [[ -d $TESTDIR1 ]] && log_must rm -rf $TESTDIR1 + +} + +# +# Check if hold exists on the specified dataset. +# +function check_hold #<snapshot> <hold> +{ + typeset dataset=$1 + typeset hold=$2 + + log_note "checking $dataset for $hold" + eval "zfs holds $dataset |grep -q $hold" +} + +log_assert "Verify 'zfs send/recv' can send and receive dataset holds." +log_onexit cleanup + +init_snap=$TESTPOOL/$TESTFS1@init_snap +inc_snap=$TESTPOOL/$TESTFS1@inc_snap +inc_snap2=$TESTPOOL/$TESTFS1@inc_snap2 +inc_snap3=$TESTPOOL/$TESTFS1@inc_snap3 +full_bkup=$TEST_BASE_DIR/fullbkup.$$ +inc_bkup=$TEST_BASE_DIR/incbkup.$$ +init_data=$TESTDIR/$TESTFILE1 +inc_data=$TESTDIR/$TESTFILE2 +inc_data2=$TESTDIR/testfile3 +recv_root=$TESTPOOL/rst_ctr +recv_snap=$recv_root/$TESTFS1@init_snap +recv_inc_snap=$recv_root/$TESTFS1@inc_snap +recv_inc_snap2=$recv_root/$TESTFS1@inc_snap2 +recv_inc_snap3=$recv_root/$TESTFS1@inc_snap3 +recv_data=$TESTDIR1/$TESTFS1/$TESTFILE1 +recv_inc_data=$TESTDIR1/$TESTFS1/$TESTFILE2 +recv_inc_data2=$TESTDIR1/$TESTFS1/testfile3 + +log_note "Verify 'zfs send' can create full send stream." + +# Preparation +if ! datasetexists $TESTPOOL/$TESTFS1; then + log_must zfs create $TESTPOOL/$TESTFS1 +fi +[[ -e $init_data ]] && log_must rm -f $init_data +log_must zfs create $recv_root +[[ ! -d $TESTDIR1 ]] && log_must mkdir -p $TESTDIR1 +[[ ! -d $TESTDIR ]] && log_must mkdir -p $TESTDIR +log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS1 +log_must zfs set mountpoint=$TESTDIR1 $recv_root + +file_write -o create -f $init_data -b $BLOCK_SIZE -c $WRITE_COUNT + +log_must zfs snapshot $init_snap +log_must zfs hold hold1-1 $init_snap +log_must zfs hold hold1-2 $init_snap +log_must eval "zfs send -h $init_snap > $full_bkup" + +log_note "Verify the send stream is valid to receive." + +log_must zfs recv -F $recv_snap <$full_bkup +log_must datasetexists $recv_snap +receive_check $recv_snap ${recv_snap%%@*} + +log_note "Verify the holds were received." +log_must check_hold $recv_snap hold1-1 +log_must check_hold $recv_snap hold1-2 +compare_cksum $init_data $recv_data + +log_note "Verify 'zfs send -i' can create incremental send stream." + +file_write -o create -f $inc_data -b $BLOCK_SIZE -c $WRITE_COUNT -d 0 + +log_must zfs snapshot $inc_snap +log_must zfs hold hold2-1 $inc_snap +log_must eval "zfs send -h -i $init_snap $inc_snap > $inc_bkup" + +log_note "Verify the incremental send stream is valid to receive." + +log_must zfs recv -F $recv_inc_snap <$inc_bkup +log_must datasetexists $recv_inc_snap +log_note "Verify the hold was received from the incremental send." + +log_must check_hold $recv_inc_snap hold2-1 + +compare_cksum $inc_data $recv_inc_data + +log_note "Verify send -h works when there are no holds." +file_write -o create -f $inc_data2 -b $BLOCK_SIZE -c $WRITE_COUNT -d 0 +log_must zfs snapshot $inc_snap2 +log_must eval "zfs send -h -i $inc_snap $inc_snap2 > $inc_bkup" +log_must zfs recv -F $recv_inc_snap2 <$inc_bkup +log_must datasetexists $recv_inc_snap2 +compare_cksum $inc_data2 $recv_inc_data2 + +log_note "Verify send -h fails properly when receive dataset already exists" +log_must zfs recv -F $recv_inc_snap2 <$inc_bkup + +log_note "Verify recv -h skips the receive of holds" +log_must zfs snapshot $inc_snap3 +log_must zfs hold hold3-1 $inc_snap3 +log_must eval "zfs send -h -i $inc_snap2 $inc_snap3 > $inc_bkup" +log_must zfs recv -F -h $recv_inc_snap3 <$inc_bkup +log_must datasetexists $recv_inc_snap3 +log_mustnot check_hold $recv_inc_snap3 hold3-1 + +log_pass "'zfs send/recv' can send and receive dataset holds." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_hole_birth.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_hole_birth.ksh new file mode 100755 index 000000000000..1dfa97e77358 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_hole_birth.ksh @@ -0,0 +1,123 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# Verify send streams which contain holes. +# +# STRATEGY: +# 1. Create an initial file for the full send and snapshot. +# 2. Permute the file with holes and snapshot. +# 3. Send the full and incremental snapshots to a new pool. +# 4. Verify the contents of the files match. +# + +sendpool=$POOL +sendfs=$sendpool/sendfs + +recvpool=$POOL2 +recvfs=$recvpool/recvfs + +verify_runnable "both" + +log_assert "Test hole_birth" +log_onexit cleanup + +function cleanup +{ + cleanup_pool $sendpool + cleanup_pool $recvpool + set_tunable64 SEND_HOLES_WITHOUT_BIRTH_TIME 1 +} + +function send_and_verify +{ + log_must eval "zfs send $sendfs@snap1 > $BACKDIR/pool-snap1" + log_must eval "zfs receive -F $recvfs < $BACKDIR/pool-snap1" + + log_must eval "zfs send -i $sendfs@snap1 $sendfs@snap2 " \ + ">$BACKDIR/pool-snap1-snap2" + log_must eval "zfs receive $recvfs < $BACKDIR/pool-snap1-snap2" + + log_must cmp_md5s /$sendfs/file1 /$recvfs/file1 +} + +# By default sending hole_birth times is disabled. This functionality needs +# to be re-enabled for this test case to verify correctness. Once we're +# comfortable that all hole_birth bugs has been resolved this behavior may +# be re-enabled by default. +log_must set_tunable64 SEND_HOLES_WITHOUT_BIRTH_TIME 0 + +# Incremental send truncating the file and adding new data. +log_must zfs create -o recordsize=4k $sendfs + +log_must truncate -s 1G /$sendfs/file1 +log_must dd if=/dev/urandom of=/$sendfs/file1 bs=4k count=11264 seek=1152 +log_must zfs snapshot $sendfs@snap1 + +log_must truncate -s 4M /$sendfs/file1 +log_must dd if=/dev/urandom of=/$sendfs/file1 bs=4k count=152 seek=384 \ + conv=notrunc +log_must dd if=/dev/urandom of=/$sendfs/file1 bs=4k count=10 seek=1408 \ + conv=notrunc +log_must zfs snapshot $sendfs@snap2 + +send_and_verify +log_must cleanup_pool $sendpool +log_must cleanup_pool $recvpool + +# Incremental send appending a hole and data. +log_must zfs create -o recordsize=512 $sendfs + +log_must dd if=/dev/urandom of=/$sendfs/file1 bs=128k count=1 seek=1 +log_must zfs snapshot $sendfs@snap1 + +log_must dd if=/dev/urandom of=/$sendfs/file1 bs=128k count=1 +log_must dd if=/dev/urandom of=/$sendfs/file1 bs=128k count=1 seek=3 +log_must zfs snapshot $sendfs@snap2 + +send_and_verify +log_must cleanup_pool $sendpool +log_must cleanup_pool $recvpool + +# Incremental send truncating the file and adding new data. +log_must zfs create -o recordsize=512 $sendfs + +log_must truncate -s 300M /$sendfs/file1 +log_must dd if=/dev/urandom of=/$sendfs/file1 bs=512 count=128k conv=notrunc +log_must zfs snapshot $sendfs@snap1 + +log_must truncate -s 10M /$sendfs/file1 +log_must dd if=/dev/urandom of=/$sendfs/file1 bs=512 count=1 seek=96k \ + conv=notrunc +log_must zfs snapshot $sendfs@snap2 + +send_and_verify + +log_pass "Test hole_birth" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_mixed_raw.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_mixed_raw.ksh new file mode 100755 index 000000000000..eea535af1100 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_mixed_raw.ksh @@ -0,0 +1,118 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# +# +# Copyright (c) 2019 Datto, Inc. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# Verify that 'zfs receive' produces an error when mixing +# raw and non-raw sends in a way that would break IV set +# consistency. +# +# STRATEGY: +# 1. Create an initial dataset with 3 snapshots. +# 2. Perform a raw send of the first snapshot to 2 other datasets. +# 3. Perform a non-raw send of the second snapshot to one of +# the other datasets. Perform a raw send from this dataset to +# the last one. +# 4. Attempt to raw send the final snapshot of the first dataset +# to the other 2 datasets, which should fail. +# 5. Repeat steps 1-4, but using bookmarks for incremental sends. +# +# +# A B C notes +# ------------------------------------------------------------------------------ +# snap1 ---raw---> snap1 --raw--> snap1 # all snaps initialized via raw send +# snap2 -non-raw-> snap2 --raw--> snap2 # A sends non-raw to B, B sends raw to C +# snap3 ------------raw---------> snap3 # attempt send to C (should fail) +# + + +verify_runnable "both" + +function cleanup +{ + datasetexists $TESTPOOL/$TESTFS3 && \ + log_must zfs destroy -r $TESTPOOL/$TESTFS3 + datasetexists $TESTPOOL/$TESTFS2 && \ + log_must zfs destroy -r $TESTPOOL/$TESTFS2 + datasetexists $TESTPOOL/$TESTFS1 && \ + log_must zfs destroy -r $TESTPOOL/$TESTFS1 +} +log_onexit cleanup + +log_assert "Mixing raw and non-raw receives should fail" + +typeset passphrase="password" + +log_must eval "echo $passphrase | zfs create -o encryption=on" \ + "-o keyformat=passphrase $TESTPOOL/$TESTFS1" + +log_must zfs snapshot $TESTPOOL/$TESTFS1@1 +log_must touch /$TESTPOOL/$TESTFS1/a +log_must zfs snapshot $TESTPOOL/$TESTFS1@2 +log_must touch /$TESTPOOL/$TESTFS1/b +log_must zfs snapshot $TESTPOOL/$TESTFS1@3 + +# Testing with snapshots +log_must eval "zfs send -w $TESTPOOL/$TESTFS1@1 |" \ + "zfs receive $TESTPOOL/$TESTFS2" +log_must eval "echo $passphrase | zfs load-key $TESTPOOL/$TESTFS2" +log_must eval "zfs send -w $TESTPOOL/$TESTFS2@1 |" \ + "zfs receive $TESTPOOL/$TESTFS3" +log_must eval "echo $passphrase | zfs load-key $TESTPOOL/$TESTFS3" + +log_must eval "zfs send -i $TESTPOOL/$TESTFS1@1 $TESTPOOL/$TESTFS1@2 |" \ + "zfs receive $TESTPOOL/$TESTFS2" +log_must eval "zfs send -w -i $TESTPOOL/$TESTFS2@1 $TESTPOOL/$TESTFS2@2 |" \ + "zfs receive $TESTPOOL/$TESTFS3" + +log_mustnot eval "zfs send -w -i $TESTPOOL/$TESTFS1@2 $TESTPOOL/$TESTFS1@3 |" \ + "zfs receive $TESTPOOL/$TESTFS2" +log_mustnot eval "zfs send -w -i $TESTPOOL/$TESTFS2@2 $TESTPOOL/$TESTFS2@3 |" \ + "zfs receive $TESTPOOL/$TESTFS3" + +log_must zfs destroy -r $TESTPOOL/$TESTFS3 +log_must zfs destroy -r $TESTPOOL/$TESTFS2 + +# Testing with bookmarks +log_must zfs bookmark $TESTPOOL/$TESTFS1@1 $TESTPOOL/$TESTFS1#b1 +log_must zfs bookmark $TESTPOOL/$TESTFS1@2 $TESTPOOL/$TESTFS1#b2 + +log_must eval "zfs send -w $TESTPOOL/$TESTFS1@1 |" \ + "zfs receive $TESTPOOL/$TESTFS2" +log_must eval "echo $passphrase | zfs load-key $TESTPOOL/$TESTFS2" + +log_must zfs bookmark $TESTPOOL/$TESTFS2@1 $TESTPOOL/$TESTFS2#b1 + +log_must eval "zfs send -w $TESTPOOL/$TESTFS2@1 |" \ + "zfs receive $TESTPOOL/$TESTFS3" +log_must eval "echo $passphrase | zfs load-key $TESTPOOL/$TESTFS3" + +log_must eval "zfs send -i $TESTPOOL/$TESTFS1#b1 $TESTPOOL/$TESTFS1@2 |" \ + "zfs receive $TESTPOOL/$TESTFS2" +log_must eval "zfs send -w -i $TESTPOOL/$TESTFS2#b1 $TESTPOOL/$TESTFS2@2 |" \ + "zfs receive $TESTPOOL/$TESTFS3" + +log_mustnot eval "zfs send -w -i $TESTPOOL/$TESTFS1#b2" \ + "$TESTPOOL/$TESTFS1@3 | zfs receive $TESTPOOL/$TESTFS2" +log_mustnot eval "zfs send -w -i $TESTPOOL/$TESTFS2#b2" \ + "$TESTPOOL/$TESTFS2@3 | zfs receive $TESTPOOL/$TESTFS3" + +log_pass "Mixing raw and non-raw receives fail as expected" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh new file mode 100755 index 000000000000..d5eb9a0edc11 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh @@ -0,0 +1,110 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version a.0. +# You may only use this file in accordance with the terms of version +# a.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019 Datto Inc. +# Copyright (c) 2020 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that a partially received dataset can be sent with +# 'zfs send --saved'. +# +# Strategy: +# 1. Setup a pool with partially received filesystem +# 2. Perform saved send without incremental +# 3. Perform saved send with incremental +# 4. Perform saved send with incremental, resuming from a token +# 5. Perform negative tests for invalid command inputs +# + +verify_runnable "both" + +log_assert "Verify that a partially received dataset can be sent with " \ + "'zfs send --saved'." + +function cleanup +{ + destroy_dataset $POOL/testfs2 "-r" + destroy_dataset $POOL/stream "-r" + destroy_dataset $POOL/recvfs "-r" + destroy_dataset $POOL/partialfs "-r" +} +log_onexit cleanup + +log_must zfs create $POOL/testfs2 +log_must zfs create $POOL/stream +mntpnt=$(get_prop mountpoint $POOL/testfs2) + +# Setup a pool with partially received filesystems +log_must mkfile 1m $mntpnt/filea +log_must zfs snap $POOL/testfs2@a +log_must mkfile 1m $mntpnt/fileb +log_must zfs snap $POOL/testfs2@b +log_must eval "zfs send $POOL/testfs2@a | zfs recv $POOL/recvfs" +log_must eval "zfs send -i $POOL/testfs2@a $POOL/testfs2@b > " \ + "/$POOL/stream/inc.send" +log_must eval "zfs send $POOL/testfs2@b > /$POOL/stream/full.send" +mess_send_file /$POOL/stream/full.send +mess_send_file /$POOL/stream/inc.send +log_mustnot zfs recv -s $POOL/recvfullfs < /$POOL/stream/full.send +log_mustnot zfs recv -s $POOL/recvfs < /$POOL/stream/inc.send + +# Perform saved send without incremental +log_mustnot eval "zfs send --saved $POOL/recvfullfs | zfs recv -s " \ + "$POOL/partialfs" +token=$(zfs get -Hp -o value receive_resume_token $POOL/partialfs) +log_must eval "zfs send -t $token | zfs recv -s $POOL/partialfs" +file_check $POOL/recvfullfs $POOL/partialfs +log_must zfs destroy -r $POOL/partialfs + +# Perform saved send with incremental +log_must eval "zfs send $POOL/recvfs@a | zfs recv $POOL/partialfs" +log_mustnot eval "zfs send --saved $POOL/recvfs | " \ + "zfs recv -s $POOL/partialfs" +token=$(zfs get -Hp -o value receive_resume_token $POOL/partialfs) +log_must eval "zfs send -t $token | zfs recv -s $POOL/partialfs" +file_check $POOL/recvfs $POOL/partialfs +log_must zfs destroy -r $POOL/partialfs + +# Perform saved send with incremental, resuming from token +log_must eval "zfs send $POOL/recvfs@a | zfs recv $POOL/partialfs" +log_must eval "zfs send --saved $POOL/recvfs > " \ + "/$POOL/stream/partial.send" +mess_send_file /$POOL/stream/partial.send +log_mustnot zfs recv -s $POOL/partialfs < /$POOL/stream/partial.send +token=$(zfs get -Hp -o value receive_resume_token $POOL/partialfs) +log_must eval "zfs send -t $token | zfs recv -s $POOL/partialfs" +file_check $POOL/recvfs $POOL/partialfs + +# Perform negative tests for invalid command inputs +set -A badargs \ + "" \ + "$POOL/recvfs@a" \ + "-i $POOL/recvfs@a $POOL/recvfs@b" \ + "-R $POOL/recvfs" \ + "-p $POOL/recvfs" \ + "-I $POOL/recvfs" \ + "-h $POOL/recvfs" + +while (( i < ${#badargs[*]} )) +do + log_mustnot eval "zfs send --saved ${badargs[i]} >/dev/null" + (( i = i + 1 )) +done + +log_pass "A partially received dataset can be sent with 'zfs send --saved'." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_realloc_dnode_size.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_realloc_dnode_size.ksh new file mode 100755 index 000000000000..394fe95bb9b7 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_realloc_dnode_size.ksh @@ -0,0 +1,112 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. +# Copyright (c) 2018 Datto Inc. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify incremental receive properly handles objects with changed +# dnode slot count. +# +# Strategy: +# 1. Populate a dataset with 1k byte dnodes and snapshot +# 2. Remove objects, set dnodesize=legacy, and remount dataset so new objects +# get recycled numbers and formerly "interior" dnode slots get assigned +# to new objects +# 3. Remove objects, set dnodesize=2k, and remount dataset so new objects +# overlap with recently recycled and formerly "normal" dnode slots get +# assigned to new objects +# 4. Create an empty file and add xattrs to it to exercise reclaiming a +# dnode that requires more than 1 slot for its bonus buffer (Zol #7433) +# 5. Generate initial and incremental streams +# 6. Verify initial and incremental streams can be received +# + +verify_runnable "both" + +log_assert "Verify incremental receive handles objects with changed dnode size" + +function cleanup +{ + rm -f $BACKDIR/fs-dn-legacy + rm -f $BACKDIR/fs-dn-1k + rm -f $BACKDIR/fs-dn-2k + rm -f $BACKDIR/fs-attr + + if datasetexists $POOL/fs ; then + log_must zfs destroy -rR $POOL/fs + fi + + if datasetexists $POOL/newfs ; then + log_must zfs destroy -rR $POOL/newfs + fi +} + +log_onexit cleanup + +# 1. Populate a dataset with 1k byte dnodes and snapshot +log_must zfs create -o dnodesize=1k $POOL/fs +log_must mk_files 200 262144 0 $POOL/fs +log_must zfs snapshot $POOL/fs@a + +# 2. Remove objects, set dnodesize=legacy, and remount dataset so new objects +# get recycled numbers and formerly "interior" dnode slots get assigned +# to new objects +rm /$POOL/fs/* + +log_must zfs unmount $POOL/fs +log_must zfs set dnodesize=legacy $POOL/fs +log_must zfs mount $POOL/fs + +log_must mk_files 200 262144 0 $POOL/fs +log_must zfs snapshot $POOL/fs@b + +# 3. Remove objects, set dnodesize=2k, and remount dataset so new objects +# overlap with recently recycled and formerly "normal" dnode slots get +# assigned to new objects +rm /$POOL/fs/* + +log_must zfs unmount $POOL/fs +log_must zfs set dnodesize=2k $POOL/fs +log_must zfs mount $POOL/fs + +log_must touch /$POOL/fs/attrs +mk_files 200 262144 0 $POOL/fs +log_must zfs snapshot $POOL/fs@c + +# 4. Create an empty file and add xattrs to it to exercise reclaiming a +# dnode that requires more than 1 slot for its bonus buffer (Zol #7433) +log_must zfs set compression=on xattr=sa $POOL/fs +log_must eval "python -c 'print \"a\" * 512' | + set_xattr_stdin bigval /$POOL/fs/attrs" +log_must zfs snapshot $POOL/fs@d + +# 5. Generate initial and incremental streams +log_must eval "zfs send $POOL/fs@a > $BACKDIR/fs-dn-1k" +log_must eval "zfs send -i $POOL/fs@a $POOL/fs@b > $BACKDIR/fs-dn-legacy" +log_must eval "zfs send -i $POOL/fs@b $POOL/fs@c > $BACKDIR/fs-dn-2k" +log_must eval "zfs send -i $POOL/fs@c $POOL/fs@d > $BACKDIR/fs-attr" + +# 6. Verify initial and incremental streams can be received +log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs-dn-1k" +log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs-dn-legacy" +log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs-dn-2k" +log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs-attr" + +log_pass "Verify incremental receive handles objects with changed dnode size" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh new file mode 100755 index 000000000000..a653f8b3f15e --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh @@ -0,0 +1,124 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019, Lawrence Livermore National Security LLC. +# Use is subject to license terms. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/include/properties.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify encrypted raw incremental receives handle dnode reallocation. + +# Strategy: +# 1. Create a pool containing an encrypted filesystem. +# 2. Use 'zfs send -wp' to perform a raw send of the initial filesystem. +# 3. Repeat the following steps N times to verify raw incremental receives. +# a) Randomly change several key dataset properties. +# b) Modify the contents of the filesystem such that dnode reallocation +# is likely during the 'zfs receive', and receive_object() exercises +# as much of its functionality as possible. +# c) Create a new snapshot and generate an raw incremental stream. +# d) Receive the raw incremental stream and verify the received contents. +# e) Destroy the incremental stream and old snapshot. +# + +verify_runnable "both" + +log_assert "Verify encrypted raw incremental receive handles reallocation" + +function cleanup +{ + rm -f $BACKDIR/fs@* + rm -f $keyfile + destroy_dataset $POOL/fs "-rR" + destroy_dataset $POOL/newfs "-rR" +} + +log_onexit cleanup + +typeset keyfile=/$TESTPOOL/pkey + +# Create an encrypted dataset +log_must eval "echo 'password' > $keyfile" +log_must zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=file://$keyfile $POOL/fs + +last_snap=1 +log_must zfs snapshot $POOL/fs@snap${last_snap} +log_must eval "zfs send -wp $POOL/fs@snap${last_snap} \ + >$BACKDIR/fs@snap${last_snap}" +log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs@snap${last_snap}" + +# Set atime=off to prevent the recursive_cksum from modifying newfs. +log_must zfs set atime=off $POOL/newfs + +if is_kmemleak; then + # Use fewer files and passes on debug kernels + # to avoid timeout due to reduced performance. + nr_files=100 + passes=2 +elif is_freebsd; then + # Use fewer files and passes on FreeBSD to avoid timeout. + nr_files=500 + passes=2 +else + nr_files=1000 + passes=3 +fi + +for i in {1..$passes}; do + # Randomly modify several dataset properties in order to generate + # more interesting incremental send streams. + rand_set_prop $POOL/fs checksum "off" "fletcher4" "sha256" + rand_set_prop $POOL/fs compression "${compress_prop_vals[@]}" + rand_set_prop $POOL/fs recordsize "32K" "128K" + rand_set_prop $POOL/fs dnodesize "legacy" "auto" "4k" + rand_set_prop $POOL/fs xattr "on" "sa" + + # Churn the filesystem in such a way that we're likely to be both + # allocating and reallocating objects in the incremental stream. + log_must churn_files $nr_files 524288 $POOL/fs + expected_cksum=$(recursive_cksum /$POOL/fs) + + # Create a snapshot and use it to send an incremental stream. + this_snap=$((last_snap + 1)) + log_must zfs snapshot $POOL/fs@snap${this_snap} + log_must eval "zfs send -wp -i $POOL/fs@snap${last_snap} \ + $POOL/fs@snap${this_snap} > $BACKDIR/fs@snap${this_snap}" + + # Receive the incremental stream and verify the received contents. + log_must eval "zfs recv -Fu $POOL/newfs < $BACKDIR/fs@snap${this_snap}" + + log_must zfs load-key $POOL/newfs + log_must zfs mount $POOL/newfs + actual_cksum=$(recursive_cksum /$POOL/newfs) + log_must zfs umount $POOL/newfs + log_must zfs unload-key $POOL/newfs + + if [[ "$expected_cksum" != "$actual_cksum" ]]; then + log_fail "Checksums differ ($expected_cksum != $actual_cksum)" + fi + + # Destroy the incremental stream and old snapshot. + rm -f $BACKDIR/fs@snap${last_snap} + log_must zfs destroy $POOL/fs@snap${last_snap} + log_must zfs destroy $POOL/newfs@snap${last_snap} + last_snap=$this_snap +done + +log_pass "Verify encrypted raw incremental receive handles reallocation" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_realloc_files.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_realloc_files.ksh new file mode 100755 index 000000000000..083a2bec9daa --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_realloc_files.ksh @@ -0,0 +1,111 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/properties.shlib +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify incremental receive properly handles reallocation. +# +# Strategy: +# 1. Create a pool containing an encrypted filesystem. +# 2. Use 'zfs send -wp' to perform a raw send of the initial filesystem. +# 3. Repeat the following steps N times to verify raw incremental receives. +# a) Randomly change several key dataset properties. +# b) Modify the contents of the filesystem such that dnode reallocation +# is likely during the 'zfs receive', and receive_object() exercises +# as much of its functionality as possible. +# c) Create a new snapshot and generate an raw incremental stream. +# d) Receive the raw incremental stream and verify the received contents. +# e) Destroy the incremental stream and old snapshot. +# + +verify_runnable "both" + +log_assert "Verify incremental receive handles reallocation" + +function cleanup +{ + rm -f $BACKDIR/fs@* + destroy_dataset $POOL/fs "-rR" + destroy_dataset $POOL/newfs "-rR" +} + +log_onexit cleanup + +log_must zfs create $POOL/fs + +last_snap=1 +log_must zfs snapshot $POOL/fs@snap${last_snap} +log_must eval "zfs send $POOL/fs@snap${last_snap} >$BACKDIR/fs@snap${last_snap}" +log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs@snap${last_snap}" + +# Set atime=off to prevent the recursive_cksum from modifying newfs. +log_must zfs set atime=off $POOL/newfs + +if is_kmemleak; then + # Use fewer files and passes on debug kernels + # to avoid timeout due to reduced performance. + nr_files=100 + passes=2 +elif is_freebsd; then + # Use fewer passes and files on FreeBSD to avoid timeout. + nr_files=500 + passes=2 +else + nr_files=1000 + passes=3 +fi + +for i in {1..$passes}; do + # Randomly modify several dataset properties in order to generate + # more interesting incremental send streams. + rand_set_prop $POOL/fs checksum "off" "fletcher4" "sha256" + rand_set_prop $POOL/fs compression "${compress_prop_vals[@]}" + rand_set_prop $POOL/fs recordsize "32K" "128K" + rand_set_prop $POOL/fs dnodesize "legacy" "auto" "4k" + rand_set_prop $POOL/fs xattr "on" "sa" + + # Churn the filesystem in such a way that we're likely to be both + # allocating and reallocating objects in the incremental stream. + log_must churn_files $nr_files 524288 $POOL/fs + expected_cksum=$(recursive_cksum /$POOL/fs) + + # Create a snapshot and use it to send an incremental stream. + this_snap=$((last_snap + 1)) + log_must zfs snapshot $POOL/fs@snap${this_snap} + log_must eval "zfs send -i $POOL/fs@snap${last_snap} \ + $POOL/fs@snap${this_snap} > $BACKDIR/fs@snap${this_snap}" + + # Receive the incremental stream and verify the received contents. + log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs@snap${this_snap}" + actual_cksum=$(recursive_cksum /$POOL/newfs) + + if [[ "$expected_cksum" != "$actual_cksum" ]]; then + log_fail "Checksums differ ($expected_cksum != $actual_cksum)" + fi + + # Destroy the incremental stream and old snapshot. + rm -f $BACKDIR/fs@snap${last_snap} + log_must zfs destroy $POOL/fs@snap${last_snap} + log_must zfs destroy $POOL/newfs@snap${last_snap} + last_snap=$this_snap +done + +log_pass "Verify incremental receive handles dnode reallocation" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_spill_block.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_spill_block.ksh new file mode 100755 index 000000000000..73f164852f91 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_spill_block.ksh @@ -0,0 +1,155 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify spill blocks are correctly preserved. +# +# Strategy: +# 1) Create a set of files each containing some file data. +# 2) Add enough xattrs to the file to require a spill block. +# 3) Snapshot and send these files to a new dataset. +# 4) Modify the files and spill blocks in a variety of ways. +# 5) Send the changes using an incremental send stream. +# 6) Verify that all the xattrs (and thus the spill block) were +# preserved when receiving the incremental stream. +# + +verify_runnable "both" + +log_assert "Verify spill blocks are correctly preserved" + +function cleanup +{ + rm -f $BACKDIR/fs@* + destroy_dataset $POOL/fs "-rR" + destroy_dataset $POOL/newfs "-rR" +} + +attrvalue="abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + +log_onexit cleanup + +log_must zfs create $POOL/fs +log_must zfs set xattr=sa $POOL/fs +log_must zfs set dnodesize=legacy $POOL/fs +log_must zfs set recordsize=128k $POOL/fs + +# Create 40 files each with a spill block containing xattrs. Each file +# will be modified in a different way to validate the incremental receive. +for i in {1..40}; do + file="/$POOL/fs/file$i" + + log_must mkfile 16384 $file + for j in {1..20}; do + log_must set_xattr "testattr$j" "$attrvalue" $file + done +done + +# Snapshot the pool and send it to the new dataset. +log_must zfs snapshot $POOL/fs@snap1 +log_must eval "zfs send -e $POOL/fs@snap1 >$BACKDIR/fs@snap1" +log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs@snap1" + +# +# Modify file[1-6]'s contents but not the spill blocks. +# +# file1 - Increase record size; single block +# file2 - Increase record size; multiple blocks +# file3 - Truncate file to zero size; single block +# file4 - Truncate file to smaller size; single block +# file5 - Truncate file to much larger size; add holes +# file6 - Truncate file to embedded size; embedded data +# +log_must mkfile 32768 /$POOL/fs/file1 +log_must mkfile 1048576 /$POOL/fs/file2 +log_must truncate -s 0 /$POOL/fs/file3 +log_must truncate -s 8192 /$POOL/fs/file4 +log_must truncate -s 1073741824 /$POOL/fs/file5 +log_must truncate -s 50 /$POOL/fs/file6 + +# +# Modify file[11-16]'s contents and their spill blocks. +# +# file11 - Increase record size; single block +# file12 - Increase record size; multiple blocks +# file13 - Truncate file to zero size; single block +# file14 - Truncate file to smaller size; single block +# file15 - Truncate file to much larger size; add holes +# file16 - Truncate file to embedded size; embedded data +# +log_must mkfile 32768 /$POOL/fs/file11 +log_must mkfile 1048576 /$POOL/fs/file12 +log_must truncate -s 0 /$POOL/fs/file13 +log_must truncate -s 8192 /$POOL/fs/file14 +log_must truncate -s 1073741824 /$POOL/fs/file15 +log_must truncate -s 50 /$POOL/fs/file16 + +for i in {11..20}; do + log_must rm_xattr testattr1 /$POOL/fs/file$i +done + +# +# Modify file[21-26]'s contents and remove their spill blocks. +# +# file21 - Increase record size; single block +# file22 - Increase record size; multiple blocks +# file23 - Truncate file to zero size; single block +# file24 - Truncate file to smaller size; single block +# file25 - Truncate file to much larger size; add holes +# file26 - Truncate file to embedded size; embedded data +# +log_must mkfile 32768 /$POOL/fs/file21 +log_must mkfile 1048576 /$POOL/fs/file22 +log_must truncate -s 0 /$POOL/fs/file23 +log_must truncate -s 8192 /$POOL/fs/file24 +log_must truncate -s 1073741824 /$POOL/fs/file25 +log_must truncate -s 50 /$POOL/fs/file26 + +for i in {21..30}; do + for j in {1..20}; do + log_must rm_xattr testattr$j /$POOL/fs/file$i + done +done + +# +# Modify file[31-40]'s spill blocks but not the file contents. +# +for i in {31..40}; do + file="/$POOL/fs/file$i" + log_must rm_xattr testattr$(((RANDOM % 20) + 1)) $file + log_must set_xattr testattr$(((RANDOM % 20) + 1)) "$attrvalue" $file +done + +# Calculate the expected recursive checksum for the source. +expected_cksum=$(recursive_cksum /$POOL/fs) + +# Snapshot the pool and send the incremental snapshot. +log_must zfs snapshot $POOL/fs@snap2 +log_must eval "zfs send -e -i $POOL/fs@snap1 $POOL/fs@snap2 >$BACKDIR/fs@snap2" +log_must eval "zfs recv -F $POOL/newfs < $BACKDIR/fs@snap2" + +# Validate the received copy using the received recursive checksum. +actual_cksum=$(recursive_cksum /$POOL/newfs) +if [[ "$expected_cksum" != "$actual_cksum" ]]; then + log_fail "Checksums differ ($expected_cksum != $actual_cksum)" +fi + +log_pass "Verify spill blocks are correctly preserved" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/setup.ksh new file mode 100755 index 000000000000..dbf4eecede88 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/setup.ksh @@ -0,0 +1,46 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +verify_runnable "both" +verify_disk_count "$DISKS" 2 + +if is_global_zone ; then + log_must zpool create $POOL $DISK1 + log_must zpool create $POOL2 $DISK2 + log_must zpool create $POOL3 $DISK3 +fi +log_must mkdir $BACKDIR $TESTDIR + +log_must setup_test_model $POOL + +log_pass |