diff options
Diffstat (limited to 'sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send')
22 files changed, 2058 insertions, 0 deletions
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/Makefile.am b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/Makefile.am new file mode 100644 index 000000000000..dd6b4eb679a3 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/Makefile.am @@ -0,0 +1,25 @@ +pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/redacted_send +dist_pkgdata_SCRIPTS = \ + setup.ksh \ + cleanup.ksh \ + redacted_compressed.ksh \ + redacted_contents.ksh \ + redacted_deleted.ksh \ + redacted_disabled_feature.ksh \ + redacted_embedded.ksh \ + redacted_holes.ksh \ + redacted_incrementals.ksh \ + redacted_largeblocks.ksh \ + redacted_many_clones.ksh \ + redacted_mixed_recsize.ksh \ + redacted_mounts.ksh \ + redacted_negative.ksh \ + redacted_origin.ksh \ + redacted_props.ksh \ + redacted_resume.ksh \ + redacted_size.ksh \ + redacted_volume.ksh + +dist_pkgdata_DATA = \ + redacted.cfg \ + redacted.kshlib diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh new file mode 100755 index 000000000000..1a7c142b8551 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh @@ -0,0 +1,33 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +destroy_pool $POOL +destroy_pool $POOL2 +log_must set_tunable32 ALLOW_REDACTED_DATASET_MOUNT 0 + +log_pass diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted.cfg b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted.cfg new file mode 100644 index 000000000000..f964b37bad3b --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted.cfg @@ -0,0 +1,86 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +export DISK1=$(echo $DISKS | awk '{print $1}') +export DISK2=$(echo $DISKS | awk '{print $2}') + +export POOL=$TESTPOOL +export POOL2=$TESTPOOL2 +export FS=$TESTFS +export FS2=$TESTFS2 + +# +# These are the byte ranges that differ between files and their redacted +# counterparts. See compare_files() for more detail. +# +typeset RANGE0="0,2097152" +typeset RANGE1="0,131072" +typeset RANGE2="1048576,2097152" +typeset RANGE3="0,131072 +1966080,131072 +3932160,131072" +typeset RANGE4="0,131072 +262144,131072 +524288,131072 +786432,131072" +typeset RANGE5="0,1048576 +7340032,1048576" +typeset RANGE6="393216,131072 +655360,131072 +917504,131072 +1179648,131072 +1441792,393216 +1966080,393216 +2621440,262144 +3145728,262144 +3670016,262144 +4194304,262144 +4718592,262144 +5242880,262144" +typeset RANGE7="1048576,6291456" +typeset RANGE8="4063232,131072" +typeset RANGE9="0,131072 +262144,131072 +524288,131072 +786432,131072 +1048576,131072 +1310720,131072 +1572864,131072 +1835008,131072 +2097152,131072 +2359296,131072 +2621440,131072 +2883584,131072 +3145728,131072 +3407872,131072 +3670016,131072 +3932160,131072" +typeset RANGE10="0,393216" +typeset RANGE11="0,1048576" +typeset RANGE12="0,2097152" +typeset RANGE13="0,16384" +typeset RANGE14="" +typeset RANGE15="0,4194304" +typeset RANGE16="0,6291456"
\ No newline at end of file diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib new file mode 100644 index 000000000000..30101939db64 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib @@ -0,0 +1,266 @@ +#!/bin/ksh + +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2016, 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/tests/functional/redacted_send/redacted.cfg + +function setup_dataset +{ + typeset ds_name=$1 + typeset opts=$2 + typeset file_create_func=$3 + typeset sendfs="$POOL/$ds_name" + [[ -n $file_create_func ]] || file_create_func=setup_common + + log_must zfs create $opts $sendfs + + $file_create_func $sendfs + + log_must zfs snapshot $sendfs@snap + log_must zfs clone $opts $sendfs@snap $POOL/${ds_name}_clone + log_must zfs snapshot $POOL/${ds_name}_clone@snap +} + +function setup_common +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + typeset bs=$(get_prop recsize $sendfs) + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=$bs count=16 + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=$bs count=32 +} + +function setup_embedded +{ + typeset sendfs=$1 + + typeset recsize + typeset mntpnt=$(get_prop mountpoint $sendfs) + for recsize in 512 1024 2048 4096 8192 16384; do + if is_illumos; then + log_must mkholes -d $((recsize - 8)):8 $mntpnt/$recsize + else + log_must dd if=/dev/urandom of=$mntpnt/$recsize bs=8 \ + count=1 seek=$(((recsize / 8) - 1)) + fi + done +} + +function setup_holes +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + typeset M=$((1024 * 1024)) + + if is_illumos; then + log_must mkholes -d 0:$((8 * M)) $mntpnt/f1 + log_must mkholes -d 0:$M -d $((7 * M)):$M $mntpnt/f2 + log_must mkholes -d $M:$((6 * M)) -h $((7 * M)):$M $mntpnt/f3 + log_must mkholes -h 0:$((8 * M)) $mntpnt/f4 + else + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=8M count=1 + + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1M count=1 + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1M count=1 seek=7 \ + conv=notrunc + + log_must dd if=/dev/urandom of=$mntpnt/f3 bs=1M count=6 seek=1 + log_must truncate -s $((8 * M)) $mntpnt/f3 + + log_must truncate -s $((8 * M)) $mntpnt/f4 + fi + + log_must zfs create $sendfs/manyrm + for i in {1..256}; do + log_must stride_dd -i /dev/urandom -o $mntpnt/manyrm/f$i -b 512 \ + -c $(random_int_between 1 100) -s $(random_int_between 1 4) + done + + log_must zfs snapshot $sendfs/manyrm@snap + log_must zfs clone $sendfs/manyrm@snap $sendfs/manyrm_clone + log_must zfs snapshot $sendfs/manyrm_clone@snap +} + +function setup_incrementals +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + typeset bs=$(get_prop recsize $sendfs) + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=$bs count=16 + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=$bs count=32 + log_must mkdir $mntpnt/d1 + log_must eval "cat $mntpnt/f1 $mntpnt/f2 >$mntpnt/d1/f1" + log_must zfs snapshot $sendfs@snap0 + + log_must zfs clone $sendfs@snap0 $POOL/hole + mntpnt=$(get_prop mountpoint $POOL/hole) + log_must dd if=/dev/zero of=$mntpnt/f2 bs=$bs count=16 conv=notrunc + log_must zfs snapshot $POOL/hole@snap + + log_must zfs clone $sendfs@snap0 $POOL/stride3 + mntpnt=$(get_prop mountpoint $POOL/stride3) + log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $bs -c 11 -s 3 + log_must zfs snapshot $POOL/stride3@snap + + log_must zfs clone $sendfs@snap0 $POOL/stride5 + mntpnt=$(get_prop mountpoint $POOL/stride5) + log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $bs -c 7 -s 5 + log_must zfs snapshot $POOL/stride5@snap + + log_must zfs clone $sendfs@snap0 $POOL/int + log_must zfs snapshot $POOL/int@snap + + log_must zfs clone $POOL/int@snap $POOL/rm + mntpnt=$(get_prop mountpoint $POOL/rm) + log_must rm -rf $mntpnt/[df][12] + log_must zfs snapshot $POOL/rm@snap + + log_must zfs clone $POOL/int@snap $POOL/write + mntpnt=$(get_prop mountpoint $POOL/write) + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=16 conv=notrunc + log_must dd if=/dev/urandom of=$mntpnt/d1/f1 bs=512 count=16 seek=16 \ + conv=notrunc + log_must zfs snapshot $POOL/write@snap +} + +function setup_mounts +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + log_must touch $mntpnt/empty + log_must dd if=/dev/urandom of=$mntpnt/contents1 bs=512 count=2 + log_must dd if=/dev/urandom of=$mntpnt/contents2 bs=512 count=2 + log_must mkdir $mntpnt/dir1 + log_must touch $mntpnt/dir1/empty + log_must dd if=/dev/urandom of=$mntpnt/dir1/contents1 bs=512 count=2 + log_must dd if=/dev/urandom of=$mntpnt/dir1/contents2 bs=512 count=2 + log_must mkdir $mntpnt/dir1/dir2 + log_must touch $mntpnt/dir1/dir2/empty + log_must dd if=/dev/urandom of=$mntpnt/dir1/dir2/file bs=512 count=2 + + log_must zfs create -s -V 16p $sendfs/vol + log_must zfs snapshot $sendfs/vol@snap + log_must zfs clone $sendfs/vol@snap $sendfs/vol_clone + log_must zfs snapshot $sendfs/vol_clone@snap +} + +function mount_redacted +{ + typeset flag='' + while getopts "f" opt; do + case $opt in + f) + flag='-f' + ;; + esac + done + shift $(($OPTIND - 1)) + + typeset ds=$1 + log_must set_tunable32 ALLOW_REDACTED_DATASET_MOUNT 1 + zfs mount $flag -oro $ds || return 1 + log_must set_tunable32 ALLOW_REDACTED_DATASET_MOUNT 0 + return 0 +} + +function unmount_redacted +{ + typeset ds=$1 + + zfs unmount $ds +} + +# +# This function calls a utility that prints out the ranges where a file +# and its redacted counterpart differ, each range on a new line like this: +# +# 0,131072 +# 1966080,131072 +# 3932160,131072 +# +# The output is then checked against a variable containing the expected +# output to verify the redacted ranges are the ones expected. +# +function compare_files +{ + typeset sendfs=$1 + typeset recvfs=$2 + typeset file=$3 + typeset expected="$4" + typeset tmpfile="$tmpdir/get_file.out" + + log_must mount_redacted -f $recvfs + + typeset file1="$(get_prop mountpoint $sendfs)/$file" + typeset file2="$(get_prop mountpoint $recvfs)/$file" + log_note "Comparing $file1 and $file2" + [[ -f $file1 ]] || log_fail "File $file1 does not exist." + [[ -f $file2 ]] || log_fail "File $file2 does not exist." + + log_must eval "get_diff $file1 $file2 >$tmpfile" + typeset range="$(cat $tmpfile)" + log_must unmount_redacted $recvfs + [[ "$expected" = "$range" ]] || log_fail "Unexpected range: $range" +} + +function redacted_cleanup +{ + typeset ds_list=$@ + typeset ds + + for ds in $ds_list; do + zfs destroy -R $ds + done + + set_tunable32 ALLOW_REDACTED_DATASET_MOUNT 0 + rm -f $(get_prop mountpoint $POOL)/tmp/* +} + +# Retrieve the redaction list of a bookmark or snapshot, using +# the property or zdb output, as requested. +function get_guid_list +{ + typeset filename=$1 + typeset dataset=$2 + typeset use_zdb=${3:-false} + + if $use_zdb; then + guid_list=$(zdb -vvvv $dataset | sed -e 's/,//g' \ + -ne 's/^.*Snapshots: \[\(.*\)\]/\1/p') + else + guid_list=$(get_prop redact_snaps $dataset) + fi + + for guid in $(echo $guid_list | tr ',' ' '); do + echo $guid + done | sort >$filename +} diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh new file mode 100755 index 000000000000..0a8bf3903c28 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh @@ -0,0 +1,71 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that compressed send streams are redacted correctly. +# +# Strategy: +# 1. Receive a redacted compressed send stream, verifying compression and +# redaction. +# 2. Receive an incremental on the full receive, verifying compression and +# redaction. +# + +typeset ds_name="compressed" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name "-o compress=lz4" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset clone_mnt="$(get_prop mountpoint $clone)" + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must stride_dd -i /dev/urandom -o $clone_mnt/f1 -b $((128 * 1024)) -c 4 -s 2 +log_must zfs snapshot $clone@snap1 +log_must rm $clone_mnt/f2 +log_must zfs snapshot $clone@snap2 + +log_must zfs redact $sendfs@snap book1 $clone@snap1 $clone@snap2 +log_must eval "zfs send -c --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must stream_has_features $stream compressed lz4 redacted +compare_files $sendfs $recvfs "f1" "$RANGE4" +verify_stream_size $stream $sendfs +log_must mount_redacted -f $recvfs +verify_stream_size $stream $recvfs +log_must unmount_redacted $recvfs + +log_must eval "zfs send -c -i $sendfs@snap $clone@snap1 >$stream" +log_must eval "zfs recv $POOL2/inc1 <$stream" +log_must stream_has_features $stream compressed lz4 +typeset mntpnt=$(get_prop mountpoint $POOL2) +log_must diff $clone_mnt/f1 $mntpnt/inc1/f1 +log_must diff $send_mnt/f2 $mntpnt/inc1/f2 + +log_must eval "zfs send -c -i $sendfs@snap $clone@snap2 >$stream" +log_must eval "zfs recv $POOL2/inc2 <$stream" +log_must stream_has_features $stream compressed lz4 +log_must diff $clone_mnt/f1 $mntpnt/inc1/f1 +[[ -f $mntpnt/inc2/f2 ]] && log_fail "File f2 should not exist." + +log_pass "Compressed send streams are redacted correctly." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh new file mode 100755 index 000000000000..fb12862c9531 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh @@ -0,0 +1,162 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redaction works as expected for various scenarios. +# +# Strategy: +# 1. An unmodified file does not get redacted at all. +# 2. Empty redaction list redacts everything. +# 3. A file removed in the clone redacts the whole file. +# 4. A file moved in the clone does not redact the file. +# 5. A copied, then removed file in the clone redacts the whole file. +# 6. Overwriting a file with identical contents redacts the file. +# 7. A partially modified block redacts the entire block. +# 8. Only overlapping areas of modified ranges are redacted. +# 9. Send from the root dataset of a pool work correctly. +# + +typeset ds_name="contents" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +# An unmodified file does not get redacted at all. +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must diff $send_mnt/f2 $recv_mnt/f2 +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Removing a file in the clone redacts the entire file. +log_must rm "$clone_mnt/f1" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book3 $clone@snap1 +log_must eval "zfs send --redact book3 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE0" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Moving a file in the clone does not redact the file. +log_must mv "$clone_mnt/f1" "$clone_mnt/f1.moved" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book4 $clone@snap1 +log_must eval "zfs send --redact book4 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +[[ -f $recv_mnt/f1.moved ]] && log_fail "Found moved file in redacted receive." +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Copying, then removing a file in the clone does redact the file. +log_must cp "$clone_mnt/f1" "$clone_mnt/f1.copied" +log_must rm "$clone_mnt/f1" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book5 $clone@snap1 +log_must eval "zfs send --redact book5 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE0" +log_must mount_redacted -f $recvfs +[[ -f $recv_mnt/f1.copied ]] && log_fail "Found moved file in redacted receive." +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Overwriting the contents of a block with identical contents redacts the file. +log_must cp "$clone_mnt/f1" "$clone_mnt/f1.copied" +log_must cp "$clone_mnt/f1.copied" "$clone_mnt/f1" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book6 $clone@snap1 +log_must eval "zfs send --redact book6 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE0" +log_must mount_redacted -f $recvfs +[[ -f $recv_mnt/f1.copied ]] && log_fail "Found moved file in redacted receive." +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Modifying some of a block redacts the whole block. +log_must dd if=/dev/urandom of=$clone_mnt/f1 conv=notrunc seek=2 count=1 bs=32k +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book7 $clone@snap1 +log_must eval "zfs send --redact book7 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE1" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Only overlapping areas of modified ranges are redacted. +log_must dd if=/dev/urandom of=$clone_mnt/f2 bs=1024k count=3 conv=notrunc +log_must zfs snapshot $clone@snap1 +log_must zfs clone $sendfs@snap $clone/new +typeset mntpnt="$(get_prop mountpoint $clone/new)" +log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1024k seek=1 count=3 \ + conv=notrunc +log_must zfs snapshot $clone/new@snap +log_must zfs redact $sendfs@snap book8 $clone@snap1 $clone/new@snap +log_must eval "zfs send --redact book8 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE2" +log_must zfs destroy -R $clone/new +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# FizzBuzz version +log_must zfs clone $sendfs@snap $POOL/stride3 +mntpnt="$(get_prop mountpoint $POOL/stride3)" +log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $((128 * 1024)) -c 11 -s 3 +log_must zfs snapshot $POOL/stride3@snap +log_must zfs clone $sendfs@snap $POOL/stride5 +mntpnt="$(get_prop mountpoint $POOL/stride5)" +log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $((128 * 1024)) -c 7 -s 5 +log_must zfs snapshot $POOL/stride5@snap +log_must zfs redact $sendfs@snap book8a $POOL/stride3@snap $POOL/stride5@snap +log_must eval "zfs send --redact book8a $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE3" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Send from the root dataset of a pool work correctly. +log_must dd if=/dev/urandom of=/$POOL/f1 bs=128k count=4 +log_must zfs snapshot $POOL@snap +log_must zfs clone $POOL@snap $POOL/clone +log_must dd if=/dev/urandom of=/$POOL/clone/f1 bs=128k count=1 conv=notrunc +log_must zfs snapshot $POOL/clone@snap +log_must zfs redact $POOL@snap book9 $POOL/clone@snap +log_must eval "zfs send --redact book9 $POOL@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $POOL $recvfs "f1" "$RANGE1" +log_must zfs destroy -R $POOL@snap + +log_pass "Redaction works as expected for various scenarios." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh new file mode 100755 index 000000000000..3e2aeb733546 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh @@ -0,0 +1,103 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017, 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redaction works as expected with respect to deleted files +# +# Strategy: +# 1. A file on the delete queue counts as deleted when using it to calculate +# redaction. +# 2. A file that is removed in the tosnap of an incremental, where the fromsnap +# is a redaction bookmark that contains references to that file, does not +# result in records for that file. +# + +typeset ds_name="deleted" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset clone2="$POOL/${ds_name}_clone2" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +# +# A file on the delete queue counts as deleted when using it to calculate +# redaction. +# + +# +# Open file descriptor 5 for appending to $clone_mnt/f1 so that it will go on +# the delete queue when we rm it. +# +exec 5>>$clone_mnt/f1 +log_must dd if=/dev/urandom of=$clone_mnt/f1 bs=512 count=1 conv=notrunc +log_must rm $clone_mnt/f1 +log_must zfs snapshot $clone@snap1 +# Close file descriptor 5 +exec 5>&- +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +# +# We have temporarily disabled redaction blkptrs, so this will not +# fail as was originally intended. We should uncomment this line +# when we re-enable redaction blkptrs. +# +#log_mustnot dd if=$recv_mnt/f1 of=/dev/null bs=512 count=1 +log_must diff $send_mnt/f2 $recv_mnt/f2 +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# +# A file that is removed in the tosnap of an incremental, where the fromsnap +# is a redaction bookmark that contains references to that file, does not +# result in records for that file. +# +log_must zfs clone $sendfs@snap $clone2 +typeset clone2_mnt="$(get_prop mountpoint $clone2)" +log_must rm -rf $clone2_mnt/* +log_must zfs snapshot $clone2@snap +log_must zfs redact $sendfs@snap book2 $clone2@snap +log_must zfs destroy -R $clone2 +log_must eval "zfs send --redact book2 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must rm $send_mnt/f1 +log_must zfs snapshot $sendfs@snap2 +log_must zfs clone $sendfs@snap2 $clone2 +typeset clone2_mnt="$(get_prop mountpoint $clone2)" +log_must rm $clone2_mnt/* +log_must zfs snapshot $clone2@snap +log_must zfs redact $sendfs@snap2 book3 $clone2@snap +log_must zfs destroy -R $clone2 +log_must eval "zfs send -i $sendfs#book2 --redact book3 $sendfs@snap2 >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +log_must diff <(ls $send_mnt) <(ls $recv_mnt) +log_must zfs destroy -R $recvfs +log_must zfs rollback -R $sendfs@snap + +log_pass "Verify Redaction works as expected with respect to deleted files." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh new file mode 100755 index 000000000000..3cf73f00167e --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh @@ -0,0 +1,71 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify the functionality of the redaction_bookmarks and redacted_datasets +# features. +# +# Strategy: +# 1. Create a pool with all features disabled. +# 2. Verify redacted send fails. +# 3. Enable redaction_bookmarks and verify redacted sends works. +# 4. Verify receipt of a redacted stream fails. +# 5. Enable recacted_datasets and verify zfs receive works. +# + +typeset ds_name="disabled" +typeset sendfs="$POOL/$ds_name" +typeset sendfs1="$POOL2/${ds_name}1" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset clone1="$POOL2/${ds_name}_clone1" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' + +function cleanup +{ + destroy_pool $POOL2 + create_pool $POOL2 $DISK2 + log_must zfs snapshot $POOL2@init + redacted_cleanup $sendfs $recvfs +} + +log_onexit cleanup + +destroy_pool $POOL2 +log_must zpool create -d $POOL2 $DISK2 + +log_must zfs create $sendfs1 +log_must zfs snapshot $sendfs1@snap +log_must zfs clone $sendfs1@snap $clone1 +log_must zfs snapshot $clone1@snap + +log_mustnot zfs redact $sendfs1@snap book1 $clone1@snap +log_must zpool set feature@redaction_bookmarks=enabled $POOL2 +log_must zfs redact $sendfs1@snap book1 $clone1@snap + +log_must zfs redact $sendfs@snap book1 $clone@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_mustnot eval "zfs recv $recvfs <$stream" +log_must zpool set feature@redacted_datasets=enabled $POOL2 +log_must eval "zfs recv $recvfs <$stream" + +log_pass "The redacted send/recv features work correctly." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh new file mode 100755 index 000000000000..94937a2f79ab --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh @@ -0,0 +1,103 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify embedded blocks and redacted send work correctly together. +# +# Strategy: +# 1. Create recsize sized files with embedded blocks from size 512b to 16k. +# 2. Receive a redacted send stream with nothing redacted. +# 3. Verify the received files match the source, contain embedded blocks, and +# that the stream has the redacted and embedded data features. +# 4. Receive a redacted send stream with files 512, 2048 and 8192 redacted. +# 5. Verify that the redacted files no longer match, but the others still +# contain embedded blocks and the stream has the redacted and embedded +# data features. +# + +typeset ds_name="embedded" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '-o compress=lz4' setup_embedded +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset recsize send_obj recv_obj + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must zfs redact $sendfs@snap book1 $clone@snap +log_must eval "zfs send -e --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must stream_has_features $stream redacted embed_data + +log_must mount_redacted -f $recvfs +for recsize in 512 1024 2048 4096 8192 16384; do + send_obj=$(get_objnum $send_mnt/$recsize) + recv_obj=$(get_objnum $recv_mnt/$recsize) + + log_must diff $send_mnt/$recsize $recv_mnt/$recsize + log_must eval "zdb -ddddd $sendfs $send_obj >$tmpdir/send.zdb" + log_must eval "zdb -ddddd $recvfs $recv_obj >$tmpdir/recv.zdb" + + grep -q "EMBEDDED" $tmpdir/send.zdb || \ + log_fail "Obj $send_obj not embedded in $sendfs" + grep -q "EMBEDDED" $tmpdir/recv.zdb || \ + log_fail "Obj $recv_obj not embedded in $recvfs" + + cat $stream | zstreamdump -v | log_must grep -q \ + "WRITE_EMBEDDED object = $send_obj offset = 0" +done + +log_must zfs destroy -R $recvfs +for recsize in 512 2048 8192; do + log_must dd if=/dev/urandom of=$clone_mnt/$recsize bs=$recsize count=1 +done +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book2 $clone@snap1 +log_must eval "zfs send -e --redact book2 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must stream_has_features $stream redacted embed_data + +log_must mount_redacted -f $recvfs +for recsize in 512 2048 8192; do + log_mustnot diff $send_mnt/$recsize $recv_mnt/$recsize +done +for recsize in 1024 4096 16384; do + send_obj=$(get_objnum $send_mnt/$recsize) + recv_obj=$(get_objnum $recv_mnt/$recsize) + + log_must diff $send_mnt/$recsize $recv_mnt/$recsize + log_must eval "zdb -ddddd $sendfs $send_obj >$tmpdir/send.zdb" + log_must eval "zdb -ddddd $recvfs $recv_obj >$tmpdir/recv.zdb" + + grep -q "EMBEDDED" $tmpdir/send.zdb || \ + log_fail "Obj $send_obj not embedded in $sendfs" + grep -q "EMBEDDED" $tmpdir/recv.zdb || \ + log_fail "Obj $recv_obj not embedded in $recvfs" + + cat $stream | zstreamdump -v | log_must grep -q \ + "WRITE_EMBEDDED object = $send_obj offset = 0" +done + +log_pass "Embedded blocks and redacted send work correctly together." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh new file mode 100755 index 000000000000..d111aa0ef608 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh @@ -0,0 +1,120 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redacted send streams reliably handle holes. +# +# Strategy: +# 1. Holes written at the beginning and end of a non-sparse file in the +# redacted list are correctly redacted. +# 2. Holes written throughout a non-sparse file in the redacted list are +# correctly redacted. +# 3. Data written into a hole in a sparse file in the redacted list are +# correctly redacted. +# 4. Holes in metadata blocks. +# + +typeset ds_name="holes" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_holes +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset M=$((1024 * 1024)) + +log_onexit redacted_cleanup $sendfs $recvfs + +# Write holes at the start and end of a non-sparse file. +if is_illumos; then + log_must mkholes -h 0:$M -h $((7 * M)):$M $clone_mnt/f1 +else + log_must dd if=/dev/zero of=$clone_mnt/f1 bs=1M count=1 conv=notrunc + log_must dd if=/dev/zero of=$clone_mnt/f1 bs=1M count=1 conv=notrunc seek=7 +fi +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE5" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Write two overlapping sets of holes into the same non-sparse file. +log_must stride_dd -i /dev/zero -o $clone_mnt/f1 -b $((128 * 1024)) -c 8 -s 2 -k 3 +log_must stride_dd -i /dev/zero -o $clone_mnt/f1 -b $((256 * 1024)) -c 8 -s 2 -k 6 +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book2 $clone@snap1 +log_must eval "zfs send --redact book2 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE6" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Write data into the middle of a hole. +if is_illumos; then + log_must mkholes -d $((3 * M)):$((2 * M)) $clone_mnt/f2 +else + log_must dd if=/dev/urandom of=$clone_mnt/f2 bs=1M count=2 seek=3 \ + conv=notrunc +fi +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book3 $clone@snap1 +log_must eval "zfs send --redact book3 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE14" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Remove a file with holes. +log_must rm $clone_mnt/f3 +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book4 $clone@snap1 +log_must eval "zfs send --redact book4 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f3" "$RANGE7" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Create a hole in a L0 metadata block by removing files. +log_must rm $send_mnt/manyrm_clone/f{32..96} +log_must zfs snapshot $sendfs/manyrm_clone@snap1 + +log_must zfs redact $sendfs/manyrm@snap book6 $sendfs/manyrm_clone@snap1 +log_must eval "zfs send --redact book6 $sendfs/manyrm@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +for i in {1..31} {97..256}; do + diff $send_mnt/manyrm/f$i $recv_mnt/f$i || log_fail \ + "File f$i did not match in the send and recv datasets." +done +for i in {32..96}; do + file_size=$(stat_size $send_mnt/manyrm/f$i) + redacted_size=$(stat_size $recv_mnt/f$i) + [[ $file_size -eq $redacted_size ]] || log_fail \ + "File f$i has size $file_size and redacted size $redacted_size" +done +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +log_pass "Redacted send streams reliably handle holes." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh new file mode 100755 index 000000000000..1d2ed3a687be --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh @@ -0,0 +1,152 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that incrementals (redacted and normal) work with redacted datasets. +# +# Strategy: +# 1. Test normal incrementals from the original snap to a subset of the +# redaction list. +# 2. Test receipt of intermediate clones, and their children. +# 3. Test receipt with origin snap specified by '-o origin='. +# 4. Test incrementals from redaction bookmarks. +# + +typeset ds_name="incrementals" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_incrementals +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs $POOL2/rfs + +# Setup a redacted send using a redaction list at varying depth. +log_must zfs redact $sendfs@snap0 book1 $POOL/rm@snap $POOL/stride3@snap \ + $POOL/stride5@snap +log_must eval "zfs send --redact book1 $sendfs@snap0 >$stream" +log_must eval "zfs receive $POOL2/rfs <$stream" + +# Verify receipt of normal incrementals to redaction list members. +log_must eval "zfs send -i $sendfs@snap0 $POOL/stride3@snap >$stream" +log_must eval "zfs recv $POOL2/rstride3 <$stream" +log_must diff -r /$POOL/stride3 /$POOL2/rstride3 +log_must eval "zfs send -i $sendfs@snap0 $POOL/stride5@snap >$stream" +log_must eval "zfs recv $POOL2/rstride5 <$stream" +log_must diff -r /$POOL/stride5 /$POOL2/rstride5 + +# But not a normal child that we weren't redacted with respect to. +log_must eval "zfs send -i $sendfs@snap0 $POOL/hole@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rhole@snap <$stream" + +# Verify we can receive an intermediate clone redacted with respect to a +# subset of the original redaction list. +log_must zfs redact $POOL/int@snap book2 $POOL/rm@snap +log_must eval "zfs send -i $sendfs@snap0 --redact book2 $POOL/int@snap >$stream" +log_must eval "zfs recv $POOL2/rint <$stream" +compare_files $POOL/int $POOL2/rint "f1" "$RANGE0" +compare_files $POOL/int $POOL2/rint "f2" "$RANGE15" +compare_files $POOL/int $POOL2/rint "d1/f1" "$RANGE16" +log_must mount_redacted -f $POOL2/rint + +# Verify we can receive grandchildren on the child. +log_must eval "zfs send -i $POOL/int@snap $POOL/rm@snap >$stream" +log_must eval "zfs receive $POOL2/rrm <$stream" +log_must diff -r /$POOL/rm /$POOL2/rrm + +# But not a grandchild that the received child wasn't redacted with respect to. +log_must eval "zfs send -i $POOL/int@snap $POOL/write@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rwrite<$stream" + +# Verify we cannot receive an intermediate clone that isn't redacted with +# respect to a subset of the original redaction list. +log_must zfs redact $POOL/int@snap book4 $POOL/rm@snap $POOL/write@snap +log_must eval "zfs send -i $sendfs@snap0 --redact book4 $POOL/int@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rint <$stream" +log_must zfs redact $POOL/int@snap book5 $POOL/write@snap +log_must eval "zfs send -i $sendfs@snap0 --redact book5 $POOL/int@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rint <$stream" +log_mustnot zfs redact $POOL/int@snap book6 $POOL/hole@snap + +# Verify we can receive a full clone of the grandchild on the child. +log_must eval "zfs send $POOL/write@snap >$stream" +log_must eval "zfs recv -o origin=$POOL2/rint@snap $POOL2/rwrite <$stream" +log_must diff -r /$POOL/write /$POOL2/rwrite + +# Along with other origins. +log_must eval "zfs recv -o origin=$POOL2/rfs@snap0 $POOL2/rwrite1 <$stream" +log_must diff -r /$POOL/write /$POOL2/rwrite1 +log_must eval "zfs recv -o origin=$POOL2@init $POOL2/rwrite2 <$stream" +log_must diff -r /$POOL/write /$POOL2/rwrite2 +log_must zfs destroy -R $POOL2/rwrite2 + +log_must zfs destroy -R $POOL2/rfs + +# Write some data for tests of incremental sends from bookmarks +log_must zfs snapshot $sendfs@snap1 +log_must zfs clone $sendfs@snap1 $POOL/hole1 +typeset mntpnt=$(get_prop mountpoint $POOL/hole1) +log_must dd if=/dev/zero of=$mntpnt/f2 bs=128k count=16 conv=notrunc +log_must zfs snapshot $POOL/hole1@snap +log_must zfs clone $sendfs@snap1 $POOL/write1 +mntpnt=$(get_prop mountpoint $POOL/write1) +log_must dd if=/dev/urandom of=$mntpnt/f2 bs=128k count=16 conv=notrunc +log_must zfs snapshot $POOL/write1@snap +log_must zfs clone $POOL/int@snap $POOL/write2 +mntpnt=$(get_prop mountpoint $POOL/write2) +log_must dd if=/dev/urandom of=$mntpnt/f2 bs=128k count=16 conv=notrunc +log_must zfs snapshot $POOL/write2@snap + +# Setup a redacted send using a redaction list at varying depth. +log_must zfs redact $sendfs@snap0 book7 $POOL/rm@snap $POOL/stride3@snap \ + $POOL/stride5@snap +log_must eval "zfs send --redact book7 $sendfs@snap0 >$stream" +log_must eval "zfs receive $POOL2/rfs <$stream" + +# Verify we can receive a redacted incremental sending from the bookmark. +log_must zfs redact $sendfs@snap1 book8 $POOL/write1@snap +log_must eval "zfs send -i $sendfs#book7 --redact book8 $sendfs@snap1 >$stream" +log_must eval "zfs receive $POOL2/rfs <$stream" +# The stride3 and stride5 snaps redact 3 128k blocks at block offsets 0 15 and +# 30 of f2. The write1 snap only covers the first two of those three blocks. +compare_files $sendfs $POOL2/rfs "f2" "$RANGE12" +log_must mount_redacted -f $POOL2/rfs +log_must diff $send_mnt/f1 /$POOL2/rfs/f1 +log_must diff $send_mnt/d1/f1 /$POOL2/rfs/d1/f1 +unmount_redacted $POOL2/rfs + +# Verify we can receive a normal child we weren't redacted with respect to by +# sending from the bookmark. +log_must eval "zfs send -i $sendfs#book7 $POOL/hole1@snap >$stream" +log_must eval "zfs recv $POOL2/rhole1 <$stream" +log_must diff -r /$POOL/hole1 /$POOL2/rhole1 + +# Verify we can receive an intermediate clone redacted with respect to a +# non-subset if we send from the bookmark. +log_must zfs redact $POOL/int@snap book9 $POOL/write2@snap +log_must eval "zfs send -i $sendfs#book7 --redact book9 $POOL/int@snap >$stream" +log_must eval "zfs receive $POOL2/rint <$stream" +compare_files $sendfs $POOL2/rint "f2" "$RANGE12" + +log_pass "Incrementals (redacted and normal) work with redacted datasets." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh new file mode 100755 index 000000000000..caccdd360061 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh @@ -0,0 +1,63 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify large blocks and redacted send work correctly together. +# +# Strategy: +# 1. Create a dataset and clone with a 1m recordsize, modifying a few k +# within the first 1m of a 16m file. +# 2. Verify that the whole first 1m of the file is redacted. +# 3. Receive an incremental stream from the original snap to the snap it +# was redacted with respect to. +# 4. Verify that the received dataset matches the clone +# + +typeset ds_name="largeblocks" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '-o recsize=1m' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must dd if=/dev/urandom of=$clone_mnt/f1 bs=32k count=3 seek=8 conv=notrunc +log_must zfs snapshot $clone@snap1 + +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send -L --redact book1 $sendfs@snap >$stream" +log_must stream_has_features $stream redacted large_blocks +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE11" +log_must mount_redacted -f $recvfs +log_must diff $send_mnt/f2 $recv_mnt/f2 +unmount_redacted $recvfs + +log_must eval "zfs send -L -i $sendfs@snap $clone@snap1 >$stream" +log_must stream_has_features $stream large_blocks +log_must eval "zfs recv $recvfs/new <$stream" +log_must diff -r $clone_mnt $recv_mnt/new + +log_pass "Large blocks and redacted send work correctly together." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh new file mode 100755 index 000000000000..3386643b295e --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh @@ -0,0 +1,68 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redacted send can deal with a large redaction list. +# +# Strategy: +# 1. Create 64 clones of sendfs each of which modifies two blocks in a file. +# The first modification is at an offset unique to each clone, and the +# second (the last block in the file) is common to them all. +# 2. Verify a redacted stream with a reasonable redaction list length can +# be correctly processed. +# 3. Verify that if the list is too long, the send fails gracefully. +# + +typeset ds_name="many_clones" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset redaction_list='' +typeset mntpnt + +log_onexit redacted_cleanup $sendfs $recvfs + +# Fill in both the last block, and a different block in every clone. +for i in {1..64}; do + log_must zfs clone $sendfs@snap ${clone}$i + mntpnt=$(get_prop mountpoint ${clone}$i) + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=$i \ + conv=notrunc + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=63 \ + conv=notrunc + log_must zfs snapshot ${clone}$i@snap +done + +# The limit isn't necessarily 32 snapshots. The maximum number of snapshots in +# the redacted list is determined in dsl_bookmark_create_redacted_check(). +log_must zfs redact $sendfs@snap book1 $clone{1..32}@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE8" + +log_mustnot zfs redact $sendfs@snap book2 $clone{1..64}@snap + +log_pass "Redacted send can deal with a large redaction list." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh new file mode 100755 index 000000000000..e1cd09e17d59 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redacted send works with datasets of different sizes. +# +# Strategy: +# 1. Create two dataset one with recsize 512, and one 1m and create a 2m file. +# 2. For each dataset, create clones of both 512 and 1m recsize and modify +# the first 16k of the file. +# 3. Send each original dataset, redacted with respect to each of the clones +# into both a dataset inheriting a 512 recsize and a 1m one. +# 4. Verify that the smallest unit of redaction is that of the origin fs. +# + +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +typeset mntpnt + +log_onexit redacted_cleanup $POOL/512 $POOL/1m $POOL2/512 $POOL2/1m + +# Set up the datasets we'll send and redact from. +log_must zfs create -o recsize=512 $POOL/512 +mntpnt=$(get_prop mountpoint $POOL/512) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=1024k count=2 +log_must zfs snapshot $POOL/512@snap +log_must zfs clone -o recsize=1m $POOL/512@snap $POOL/1mclone +mntpnt=$(get_prop mountpoint $POOL/1mclone) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=32 conv=notrunc +log_must zfs snapshot $POOL/1mclone@snap + +log_must zfs create -o recsize=1m $POOL/1m +mntpnt=$(get_prop mountpoint $POOL/1m) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=1024k count=2 +log_must zfs snapshot $POOL/1m@snap +log_must zfs clone -o recsize=512 $POOL/1m@snap $POOL/512clone +mntpnt=$(get_prop mountpoint $POOL/512clone) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=32 conv=notrunc +log_must zfs snapshot $POOL/512clone@snap + +# Create datasets that allow received datasets to inherit recordsize. +log_must zfs create -o recsize=512 $POOL2/512 +log_must zfs create -o recsize=1m $POOL2/1m + +# Do the sends and verify the contents. +log_must zfs redact $POOL/512@snap book1 $POOL/1mclone@snap +log_must eval "zfs send --redact book1 $POOL/512@snap>$stream" +log_must eval "zfs recv $POOL2/512/recva <$stream" +compare_files $POOL/512 $POOL2/512/recva "f1" "$RANGE13" +log_must eval "zfs recv $POOL2/1m/recvb <$stream" +compare_files $POOL/512 $POOL2/1m/recvb "f1" "$RANGE13" + +log_must zfs redact $POOL/1m@snap book2 $POOL/512clone@snap +log_must eval "zfs send --redact book2 $POOL/1m@snap >$stream" +log_must eval "zfs recv $POOL2/512/recvc <$stream" +compare_files $POOL/1m $POOL2/512/recvc "f1" "$RANGE11" +log_must eval "zfs recv $POOL2/1m/recvd <$stream" +compare_files $POOL/1m $POOL2/1m/recvd "f1" "$RANGE11" + +log_pass "Redaction works correctly with different recordsizes." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh new file mode 100755 index 000000000000..0bc4bf461747 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh @@ -0,0 +1,109 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that received redacted datasets are not mounted by default, but +# can still be mounted after setting ALLOW_REDACTED_DATASET_MOUNT. +# +# Strategy: +# 1. Verify a received redacted stream isn't mounted by default. +# 2. Set ALLOW_REDACTED_DATASET_MOUNT and verify it can't be mounted +# without the -f flag, but can with -f. +# 3. Receive a redacted volume. +# 4. Verify the device file isn't present until the kernel variable is set. +# 5. Verify the files in the send fs are also present in the recv fs. +# + +typeset ds_name="mounts" +typeset sendfs="$POOL/$ds_name" +typeset sendvol="$sendfs/vol" +typeset recvfs="$POOL2/$ds_name" +typeset recvvol="$POOL2/vol" +typeset clone="$POOL/${ds_name}_clone" +typeset clonevol="${sendvol}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_mounts +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset recv_vol_file="/dev/zvol/$recvvol" + +log_onexit redacted_cleanup $sendfs $recvfs $recvvol + +log_must rm $clone_mnt/empty $clone_mnt/contents1 +log_must dd if=/dev/urandom of=$clone_mnt/contents2 bs=512 count=1 conv=notrunc +log_must rm $clone_mnt/dir1/contents1 +log_must rm -rf $clone_mnt/dir1/dir2 +log_must dd if=/dev/urandom of=$clone_mnt/dir1/contents2 bs=512 count=1 \ + conv=notrunc +log_must dd if=/dev/urandom of=$clone_mnt/dir1/empty bs=512 count=1 +log_must zfs snapshot $clone@snap1 + +log_must zfs redact $sendfs@snap book1 $clone@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs receive $recvfs <$stream" +log_mustnot ismounted $recvfs +log_mustnot mount_redacted $recvfs +log_mustnot ismounted $recvfs +log_must mount_redacted -f $recvfs +log_must ismounted $recvfs + +# Verify that the send and recv fs both have the same files under their +# mountpoints by comparing find output with the name of the mountpoint +# deleted. +contents=$(log_must find $recv_mnt) +contents_orig=$(log_must find $send_mnt) +log_must diff <(echo ${contents//$recv_mnt/}) \ + <(echo ${contents_orig//$send_mnt/}) +log_must zfs redact $sendvol@snap book2 $clonevol@snap +log_must eval "zfs send --redact book2 $sendvol@snap >$stream" +log_must eval "zfs receive $recvvol <$stream" +is_disk_device $recv_vol_file && log_fail "Volume device file should not exist." +log_must set_tunable32 ALLOW_REDACTED_DATASET_MOUNT 1 +log_must zpool export $POOL2 +log_must zpool import $POOL2 +udevadm settle + +# The device file isn't guaranteed to show up right away. +if ! is_disk_device $recv_vol_file; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + is_disk_device $recv_vol_file && break + done +fi +is_disk_device $recv_vol_file || log_fail "Volume device file should exist." + +log_must dd if=/dev/urandom of=$send_mnt/dir1/contents1 bs=512 count=2 +log_must rm $send_mnt/dir1/dir2/empty +log_must zfs snapshot $sendfs@snap2 +log_must eval "zfs send -i $sendfs#book1 $sendfs@snap2 >$stream" +log_must eval "zfs receive $recvfs <$stream" +log_must mount_redacted -f $recvfs +log_must ismounted $recvfs +contents=$(log_must find $recv_mnt) +contents_orig=$(log_must find $send_mnt) +log_must diff <(echo ${contents//$recv_mnt/}) \ + <(echo ${contents_orig//$send_mnt/}) + +log_pass "Received redacted streams can be mounted." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh new file mode 100755 index 000000000000..56b990be1bee --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh @@ -0,0 +1,92 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Test that redacted send correctly detects invalid arguments. +# + +typeset sendfs="$POOL2/sendfs" +typeset recvfs="$POOL2/recvfs" +typeset clone1="$POOL2/clone1" +typeset clone2="$POOL2/clone2" +typeset clone3="$POOL2/clone3" +typeset clone3="$POOL2/clone4" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) + +log_onexit redacted_cleanup $sendfs $recvfs $clone3 + +log_must zfs create $sendfs +log_must zfs snapshot $sendfs@snap1 +log_must zfs snapshot $sendfs@snap2 +log_must zfs snapshot $sendfs@snap3 +log_must zfs clone $sendfs@snap2 $clone1 +log_must zfs snapshot $clone1@snap +log_must zfs bookmark $clone1@snap $clone1#book +log_must zfs clone $sendfs@snap2 $clone2 +log_must zfs snapshot $clone2@snap + +# Incompatible flags +log_must zfs redact $sendfs@snap2 book $clone1@snap +log_mustnot eval "zfs send -R --redact book $sendfs@snap2 >/dev/null" + +typeset arg +for arg in "$sendfs" "$clone1#book"; do + log_mustnot eval "zfs send --redact book $arg >/dev/null" +done + +# Bad redaction list arguments +log_mustnot zfs redact $sendfs@snap1 +log_mustnot zfs redact $sendfs@snap1 book +log_mustnot zfs redact $sendfs#book1 book4 $clone1 +log_mustnot eval "zfs send --redact $sendfs#book $sendfs@snap >/dev/null" + +# Redaction snapshots not a descendant of tosnap +log_mustnot zfs redact $sendfs@snap2 book $sendfs@snap2 +log_must zfs redact $sendfs@snap2 book2 $clone1@snap $clone2@snap +log_must eval "zfs send --redact book2 $sendfs@snap2 >$stream" +log_must zfs redact $sendfs@snap2 book3 $clone1@snap $clone2@snap +log_must eval "zfs send -i $sendfs@snap1 --redact book3 $sendfs@snap2 \ + >/dev/null" +log_mustnot zfs redact $sendfs@snap3 $sendfs@snap3 $clone1@snap + +# Full redacted sends of redacted datasets are not allowed. +log_must eval "zfs recv $recvfs <$stream" +log_must zfs snapshot $recvfs@snap +log_must zfs clone $recvfs@snap $clone3 +log_must zfs snapshot $clone3@snap +log_mustnot zfs redact $recvfs@snap book5 $clone3@snap + +# Nor may a redacted dataset appear in the redaction list. +log_mustnot zfs redact testpool2/recvfs@snap2 book7 testpool2/recvfs@snap + +# Non-redaction bookmark cannot be sent and produces invalid argument error +log_must zfs bookmark "$sendfs@snap1" "$sendfs#book8" +log_must eval "zfs send --redact book8 -i $sendfs@snap1 $sendfs@snap2 2>&1 | head -n 100 | grep 'not a redaction bookmark'" + +# Error messages for common usage errors +log_mustnot_expect "not contain '#'" zfs redact $sendfs@snap1 \#book $sendfs@snap2 +log_mustnot_expect "not contain '#'" zfs redact $sendfs@snap1 $sendfs#book $sendfs@snap2 +log_mustnot_expect "full dataset names" zfs redact $sendfs@snap1 book @snap2 +log_mustnot_expect "full dataset names" zfs redact $sendfs@snap1 book @snap2 +log_mustnot_expect "full dataset names" zfs redact $sendfs@snap1 \#book @snap2 +log_mustnot_expect "descendent of snapshot" zfs redact $sendfs@snap2 book $sendfs@snap1 + +log_pass "Verify that redacted send correctly detects invalid arguments." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh new file mode 100755 index 000000000000..74e5914f2d88 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh @@ -0,0 +1,87 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Test that receiving sends from redaction bookmarks and redacted datasets +# works correctly in certain edge cases. +# 1. Send A(B,C,D) to pool2. +# 2. Verify send from A(B, C, D) can be received onto it. +# 3. Verify send from A(B, C) can be received onto it. +# 4. Verify send from A() can be received onto it. +# 5. Verify send from A(E) cannot be received onto it. +# 6. Verify send from redaction bookmark for A(B, C) can be received onto it. +# 7. Verify send from redaction bookmark for A() can be received onto it. +# 8. Verify send from redaction bookmark for A(E) cannot be received onto it. +# + +typeset ds_name="origin" +typeset sendfs="$POOL/$ds_name" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_incrementals +typeset dsA=$sendfs@snap0 +typeset dsB=$POOL/hole@snap +typeset dsC=$POOL/rm@snap +typeset dsD=$POOL/write@snap +typeset dsE=$POOL/stride3@snap +typeset dsF=$POOL/stride5@snap +typeset targ=$POOL2/targfs@snap + +log_onexit redacted_cleanup $sendfs $POOL2/rBCD $POOL2/targfs \ + $POOL2/rBC $POOL2/rE + +# Set up all the filesystems and clones. +log_must zfs redact $dsA BCD $dsB $dsC $dsD +log_must eval "zfs send --redact BCD $dsA >$stream" +log_must eval "zfs receive $POOL2/rBCD <$stream" +log_must eval "zfs receive $targ <$stream" + +log_must zfs redact $dsA BC $dsB $dsC +log_must eval "zfs send --redact BC $dsA >$stream" +log_must eval "zfs receive $POOL2/rBC <$stream" + +log_must zfs redact $dsA E $dsE +log_must eval "zfs send --redact E $dsA >$stream" +log_must eval "zfs receive $POOL2/rE <$stream" + +log_must eval "zfs send $dsF >$stream" +log_must eval "zfs receive -o origin=$POOL2/rBCD@snap0 $POOL2/BCDrF <$stream" +log_must eval "zfs receive -o origin=$POOL2/rBC@snap0 $POOL2/BCrF <$stream" +log_must eval "zfs receive -o origin=$POOL2/rE@snap0 $POOL2/ErF <$stream" + +# Run tests from redacted datasets. +log_must eval "zfs send -i $POOL2/rBCD@snap0 $POOL2/BCDrF@snap >$stream" +log_must eval "zfs receive -o origin=$targ $POOL2/tdBCD <$stream" + +log_must eval "zfs send -i $POOL2/rBC@snap0 $POOL2/BCrF@snap >$stream" +log_must eval "zfs receive -o origin=$targ $POOL2/tdBC <$stream" + +log_must eval "zfs send -i $POOL2/rE@snap0 $POOL2/ErF@snap >$stream" +log_mustnot eval "zfs receive -o origin=$targ $POOL2/tdE <$stream" + +# Run tests from redaction bookmarks. +log_must eval "zfs send -i $sendfs#BC $dsF >$stream" +log_must eval "zfs receive -o origin=$targ $POOL2/tbBC <$stream" + +log_must eval "zfs send -i $sendfs#E $dsF >$stream" +log_mustnot eval "zfs receive -o origin=$targ $POOL2/tbE <$stream" + +log_pass "Verify sends from redacted datasets and bookmarks work correctly." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh new file mode 100755 index 000000000000..e4163c4ef8da --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify the list of redacted snapshot guids as properties. +# +# Strategy: +# 1. Create a redacted dataset and receive it into another pool. +# 2. Verify that the redaction list in the book mark (according to zdb) +# matches the list shown in the redact_snaps property. +# 3. Verify that the received snapshot has a matching redaction list. +# + +typeset ds_name="props" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset mntpnt + +log_onexit redacted_cleanup $sendfs $recvfs + +# Verify a plain dataset, snapshot or bookmark has an empty list. +log_must zfs snapshot $sendfs@empty_snapshot +log_must zfs bookmark $sendfs@empty_snapshot $sendfs#empty_bookmark +found_list=$(get_prop redact_snaps $sendfs) +[[ $found_list = "-" ]] || log_fail "Unexpected dataset list: $found_list" +found_list=$(get_prop redact_snaps $sendfs@empty_snapshot) +[[ $found_list = "-" ]] || log_fail "Unexpected snapshot list: $found_list" +found_list=$(get_prop redact_snaps $sendfs#empty_bookmark) +[[ $found_list = "-" ]] || log_fail "Unexpected bookmark list: $found_list" + +# Fill in a different block in every clone. +for i in {1..16}; do + log_must zfs clone $sendfs@snap ${clone}$i + mntpnt=$(get_prop mountpoint ${clone}$i) + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=$i \ + conv=notrunc + log_must zfs snapshot ${clone}$i@snap +done + +log_must zfs redact $sendfs@snap book1 $clone{1..16}@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" + +get_guid_list $tmpdir/prop_list $sendfs#book1 +get_guid_list $tmpdir/zdb_list $sendfs#book1 true +get_guid_list $tmpdir/recvd_prop_list $recvfs@snap + +count=$(wc -l $tmpdir/prop_list | awk '{print $1}') +[[ $count -eq 16 ]] || log_fail "Found incorrect number of redaction snapshots." + +diff $tmpdir/prop_list $tmpdir/zdb_list || \ + log_fail "Property list differed from zdb output" +diff $tmpdir/prop_list $tmpdir/recvd_prop_list || \ + log_fail "Received property list differed from sent" + +log_pass "The redaction list is consistent between sent and received datasets." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh new file mode 100755 index 000000000000..8118ea59ec8b --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh @@ -0,0 +1,88 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that resumable send works correctly with redacted streams. +# +# Strategy: +# 1. Do a full redacted resumable send. +# 2. Verify the received contents are correct. +# 3. Do an incremental redacted resumable send. +# 4. Verify the received contents are correct. +# 5. Verify that recv -A removes a partially received dataset. +# + +typeset ds_name="resume" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset clone1="$POOL/${ds_name}_clone1" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must stride_dd -i /dev/urandom -o $clone_mnt/f2 -b 512 -c 64 -s 512 +log_must zfs snapshot $clone@snap1 + +# Do the full resumable send +log_must zfs redact $sendfs@snap book1 $clone@snap1 +resume_test "zfs send --redact book1 $sendfs@snap" $tmpdir $recvfs +log_must mount_redacted -f $recvfs +log_must set_tunable32 ALLOW_REDACTED_DATASET_MOUNT 1 +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must eval "get_diff $send_mnt/f2 $recv_mnt/f2 >$tmpdir/get_diff.out" +typeset range=$(cat $tmpdir/get_diff.out) +[[ "$RANGE9" = "$range" ]] || log_fail "Unexpected range: $range" + +log_must dd if=/dev/urandom of=$send_mnt/f3 bs=1024k count=3 +log_must zfs snapshot $sendfs@snap2 +log_must zfs clone $sendfs@snap2 $clone1 +typeset clone1_mnt="$(get_prop mountpoint $clone1)" +log_must dd if=/dev/urandom of=$clone1_mnt/f3 bs=128k count=3 conv=notrunc +log_must zfs snapshot $clone1@snap + +# Do the incremental resumable send +log_must zfs redact $sendfs@snap2 book2 $clone1@snap +resume_test "zfs send --redact book2 -i $sendfs#book1 $sendfs@snap2" \ + $tmpdir $recvfs +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must diff $send_mnt/f2 $recv_mnt/f2 +log_must eval "get_diff $send_mnt/f3 $recv_mnt/f3 >$tmpdir/get_diff.out" +range=$(cat $tmpdir/get_diff.out) +[[ "$RANGE10" = "$range" ]] || log_fail "Unexpected range: $range" + +# Test recv -A works properly and verify saved sends are not allowed +log_mustnot zfs recv -A $recvfs +log_must zfs destroy -R $recvfs +log_mustnot zfs recv -A $recvfs +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +dd if=$stream bs=64k count=1 | log_mustnot zfs receive -s $recvfs +[[ "-" = $(get_prop receive_resume_token $recvfs) ]] && \ + log_fail "Receive token not found." +log_mustnot eval "zfs send --saved --redact book1 $recvfs > /dev/null" +log_must zfs recv -A $recvfs +log_must datasetnonexists $recvfs + +log_pass "Resumable send works correctly with redacted streams." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh new file mode 100755 index 000000000000..7456084b04bc --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh @@ -0,0 +1,64 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that send size estimates of redacted sends work correctly +# +# Strategy: +# 1. Perform a redacted send with -nv and without, and verify the +# size estimate is the same as the size of the actual send. +# 2. Receive an incremental send from the redaction bookmark with +# -nv and without, and verify the size estimate is the same as +# the size of the actual send. +# + +ds_name="sizes" +typeset sendfs="$POOL/$ds_name" +typeset clone="$POOL/${ds_name}_clone2" +setup_dataset $ds_name "-o compress=lz4" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset size=$(mktemp $tmpdir/size.XXXX) +typeset size2=$(mktemp $tmpdir/size.XXXX) + +log_onexit redacted_cleanup $sendfs $clone +log_must zfs clone $sendfs@snap $clone +typeset clone_mnt="$(get_prop mountpoint $clone)" +log_must rm -rf $clone_mnt/* +log_must zfs snapshot $clone@snap +log_must zfs redact $sendfs@snap book $clone@snap +log_must eval "zfs send -nvP --redact book $sendfs@snap | \ + grep '^size' | awk '{print \$2}' >$size" +log_must eval "zfs send --redact book $sendfs@snap | wc -c \ + >$size2" +bytes1=$(cat $size | tr -d '[[:space:]]') +bytes2=$(cat $size2 | tr -d '[[:space:]]') +[[ "$bytes1" -eq "$bytes2" ]] || \ + log_fail "Full sizes differ: estimate $bytes1 and actual $bytes2" + +log_must zfs snapshot $sendfs@snap2 +log_must eval "zfs send -nvP -i $sendfs#book $sendfs@snap2 | \ + grep '^size' | awk '{print \$2}' >$size" +log_must eval "zfs send -i $sendfs#book $sendfs@snap2 | wc -c >$size2" +bytes1=$(cat $size | tr -d '[[:space:]]') +bytes2=$(cat $size2 | tr -d '[[:space:]]') +[[ "$bytes1" -eq "$bytes2" ]] || \ + log_fail "Incremental sizes differ: estimate $bytes1 and actual $bytes2" + +log_pass "Size estimates of redacted sends estimate accurately." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh new file mode 100755 index 000000000000..2ea10638ce31 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh @@ -0,0 +1,105 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that redacted send works on volumes. +# +# Strategy: +# 1. Write to a volume, then make a clone of that volume. +# 2. Receive a redacted stream that sends all blocks. +# 3. Receive a redacted stream that redacts the first half of the written area. +# + +typeset ds_name="volume" +typeset sendvol="$POOL/$ds_name" +typeset recvvol="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +typeset send_file="/dev/zvol/$sendvol" +typeset recv_file="/dev/zvol/$recvvol" +typeset clone_file="/dev/zvol/$clone" + +log_onexit redacted_cleanup $sendvol $recvvol + +log_must zfs create -b 8k -V 1g $sendvol +sleep 10 +log_must zpool export $POOL +log_must zpool import $POOL +udevadm settle +if ! is_disk_device $send_file; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + is_disk_device $send_file && break + done +fi +log_must dd if=/dev/urandom of=$send_file bs=8k count=64 +log_must zfs snapshot $sendvol@snap +log_must zfs clone $sendvol@snap $clone +log_must zfs snapshot $clone@snap + +log_must set_tunable32 ALLOW_REDACTED_DATASET_MOUNT 1 +log_must zfs redact $sendvol@snap book1 $clone@snap +log_must eval "zfs send --redact book1 $sendvol@snap >$stream" +log_must eval "zfs recv $recvvol <$stream" +sleep 10 +log_must zpool export $POOL2 +log_must zpool import $POOL2 +udevadm settle +if ! is_disk_device $recv_file; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + is_disk_device $recv_file && break + done +fi +log_must dd if=$send_file of=$tmpdir/send.dd bs=8k count=64 +log_must dd if=$recv_file of=$tmpdir/recv.dd bs=8k count=64 +log_must diff $tmpdir/send.dd $tmpdir/recv.dd +log_must zfs destroy -R $recvvol + +log_must dd if=/dev/urandom of=$clone_file bs=8k count=32 +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendvol@snap book2 $clone@snap1 +log_must eval "zfs send --redact book2 $sendvol@snap >$stream" +log_must eval "zfs recv $recvvol <$stream" +sleep 10 +log_must zpool export $POOL2 +log_must zpool import $POOL2 +udevadm settle +if ! is_disk_device $recv_file; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + is_disk_device $recv_file && break + done +fi +log_must dd if=$send_file of=$tmpdir/send.dd bs=8k count=32 skip=32 +log_must dd if=$recv_file of=$tmpdir/recv.dd bs=8k count=32 skip=32 +log_must diff $tmpdir/send.dd $tmpdir/recv.dd + +log_pass "Redacted send works correctly with volumes." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/setup.ksh new file mode 100755 index 000000000000..3f537f813db0 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/redacted_send/setup.ksh @@ -0,0 +1,36 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +verify_disk_count "$DISKS" 2 + +create_pool $POOL $DISK1 +log_must zfs snapshot $POOL@init +create_pool $POOL2 $DISK2 +log_must zfs snapshot $POOL2@init +log_must zfs create $POOL/tmp +log_pass |