diff options
| author | Martin Matuska <mm@FreeBSD.org> | 2026-06-27 06:15:42 +0000 |
|---|---|---|
| committer | Martin Matuska <mm@FreeBSD.org> | 2026-06-27 08:51:33 +0000 |
| commit | d0b3ecdc274930e190ea233b6b69ff03782eaf8d (patch) | |
| tree | 8f7bc49621e339a1a9baa14a3221e2d90c7a0c5d /sys/contrib | |
| parent | 8e61d8707f8a10acc143210089fea2502a3f2922 (diff) | |
| parent | 37af899488652c55c456d3f160fb8b295db2ec70 (diff) | |
zfs: merge openzfs/zfs@37af89948
Notable upstream pull request merges:
#18509 f16b3744d zstream: refactor common functions
#18573 -multiple Persist z_seq across znode eviction
s18611 eb0c674c2 zfs_ioctl: fix EBUSY race between quota queries and mount
#18637 77e64d86e Fix self-deadlock when setting the "allocating"/"path"
vdev property
#18645 e3082b923 freebsd: set mnt_time on the rootfs at mountroot time
#18652 50d012b2a zbookmark_compare: handle "marker" bookmarks with negative
levels
#18664 520eeeaa6 Improve performance of "zpool offline" for log devices
#18668 6b8f79877 Avoid more abd_t allocations in RAIDZ/dRAID
#18669 99ab859c3 Optimize metaslab_set_selected_txg()
#18673 97b9ba7a9 delegate: add 'send:encrypted' permission
#18687 2ea519c2a Avoid lookup overhead for nonexistent xattr directories
#18688 87593ea2b Fix handling of _PC_HAS_HIDDENSYSTEM for FreeBSD
#18693 0483a8e0c Clean up embedded slog metaslab across txgs
#18695 41311c665 RAIDZ: Optimize single data column writes
#18706 37af89948 ddt_log: Fix refcount tagging for begin/commit
Obtained from: OpenZFS
OpenZFS commit: 37af899488652c55c456d3f160fb8b295db2ec70
Diffstat (limited to 'sys/contrib')
158 files changed, 6960 insertions, 2479 deletions
diff --git a/sys/contrib/openzfs/.github/workflows/README.md b/sys/contrib/openzfs/.github/workflows/README.md index 78774aac52f9..434fc92b573c 100644 --- a/sys/contrib/openzfs/.github/workflows/README.md +++ b/sys/contrib/openzfs/.github/workflows/README.md @@ -75,7 +75,7 @@ Auto-selected: - Linux: almalinux 8/9/10, centos-stream 9/10, debian 11/12/13, fedora 43/44, ubuntu 22/24/26 -- FreeBSD: 14.4-RELEASE/STABLE, 15.0-RELEASE, 15.1-STABLE, 16.0-CURRENT +- FreeBSD: 14.4-RELEASE/STABLE, 15.1-RELEASE/STABLE, 16.0-CURRENT Available via `specific_os` or `ZTS_OS_OVERRIDE`: diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh index 7e72030adb99..6d62160dca4c 100755 --- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh +++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh @@ -107,8 +107,8 @@ case "$OS" in URLxz="$FREEBSD_REL/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI.raw.xz" KSRC="$FREEBSD_REL/../amd64/$FreeBSD/src.txz" ;; - freebsd15-0r) - FreeBSD="15.0-RELEASE" + freebsd15-1r) + FreeBSD="15.1-RELEASE" OSNAME="FreeBSD $FreeBSD" OSv="freebsd14.0" URLxz="$FREEBSD_REL/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI-ufs.raw.xz" diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh index dfe70c4f1ef4..879e98efd5a4 100755 --- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh +++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh @@ -400,13 +400,12 @@ case "$OS" in ;; alma*|centos*) rpm_build_and_install "--with-spec=redhat $extra" + if [ -n "$TARBALL" ] ; then + build_tarball + fi ;; fedora*) rpm_build_and_install "$extra" - - # Historically, we've always built the release tarballs on Fedora, since - # there was one instance long ago where we built them on CentOS 7, and they - # didn't work correctly for everyone. if [ -n "$TARBALL" ] ; then build_tarball fi diff --git a/sys/contrib/openzfs/.github/workflows/unit-tests.yml b/sys/contrib/openzfs/.github/workflows/unit-tests.yml new file mode 100644 index 000000000000..0ee27029c166 --- /dev/null +++ b/sys/contrib/openzfs/.github/workflows/unit-tests.yml @@ -0,0 +1,29 @@ +name: unit-tests + +on: + push: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + unit-tests: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Install dependencies + run: | + ONLY_DEPS=1 .github/workflows/scripts/qemu-3-deps-vm.sh ubuntu24 + - name: Autogen + run: | + ./autogen.sh + - name: Configure + run: | + ./configure --with-config=user --enable-debug --enable-debuginfo + - name: Unit tests + run: | + make -j$(nproc) unit diff --git a/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml b/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml index 64ffee484a57..46bb70691ff5 100644 --- a/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml +++ b/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml @@ -23,7 +23,7 @@ concurrency: jobs: test-config: name: Setup - if: github.event_name == 'pull_request' || github.repository != 'openzfs/zfs' + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || github.repository != 'openzfs/zfs' runs-on: ubuntu-24.04 outputs: test_os: ${{ steps.os.outputs.os }} @@ -56,11 +56,11 @@ jobs: os_selection='["almalinux8", "almalinux9", "almalinux10", "centos-stream9", "centos-stream10", "debian11", "debian12", "debian13", "fedora43", "fedora44", "ubuntu22", "ubuntu24", "ubuntu26"]' ;; freebsd) - os_selection='["freebsd14-4r", "freebsd14-4s", "freebsd15-0r", "freebsd15-1s", "freebsd16-0c"]' + os_selection='["freebsd14-4r", "freebsd14-4s", "freebsd15-1r", "freebsd15-1s", "freebsd16-0c"]' ;; *) # default list - os_selection='["almalinux8", "almalinux9", "almalinux10", "centos-stream9", "centos-stream10", "debian12", "debian13", "fedora43", "fedora44", "freebsd14-4r", "freebsd15-0r", "freebsd15-1s", "freebsd16-0c", "ubuntu22", "ubuntu24", "ubuntu26"]' + os_selection='["almalinux8", "almalinux9", "almalinux10", "centos-stream9", "centos-stream10", "debian12", "debian13", "fedora43", "fedora44", "freebsd14-4r", "freebsd15-1r", "freebsd15-1s", "freebsd16-0c", "ubuntu22", "ubuntu24", "ubuntu26"]' ;; esac @@ -106,7 +106,7 @@ jobs: # debian: debian12, debian13, ubuntu22, ubuntu24, ubuntu26 # misc: archlinux, tumbleweed # FreeBSD variants of november 2025: - # FreeBSD Release: freebsd14-4r, freebsd15-0r + # FreeBSD Release: freebsd14-4r, freebsd15-1r # FreeBSD Stable: freebsd14-4s, freebsd15-1s # FreeBSD Current: freebsd16-0c os: ${{ fromJson(needs.test-config.outputs.test_os) }} diff --git a/sys/contrib/openzfs/META b/sys/contrib/openzfs/META index ab1b8955d245..3c688e54a11f 100644 --- a/sys/contrib/openzfs/META +++ b/sys/contrib/openzfs/META @@ -6,5 +6,5 @@ Release: 1 Release-Tags: relext License: CDDL Author: OpenZFS -Linux-Maximum: 7.0 +Linux-Maximum: 7.1 Linux-Minimum: 4.18 diff --git a/sys/contrib/openzfs/README.md b/sys/contrib/openzfs/README.md index 6a77cedb6355..087d7402d6c5 100644 --- a/sys/contrib/openzfs/README.md +++ b/sys/contrib/openzfs/README.md @@ -68,4 +68,4 @@ Generally, if a distribution is following an LTS kernel, it should work well wit All FreeBSD releases receiving [security support](https://www.freebsd.org/security/#sup) are supported by OpenZFS. -**Supported FreeBSD releases**: **15.0**, **14.4**. +**Supported FreeBSD releases**: **15.1**, **14.4**. diff --git a/sys/contrib/openzfs/cmd/zfs/zfs_main.c b/sys/contrib/openzfs/cmd/zfs/zfs_main.c index d448a3df60ef..99f0f1818aec 100644 --- a/sys/contrib/openzfs/cmd/zfs/zfs_main.c +++ b/sys/contrib/openzfs/cmd/zfs/zfs_main.c @@ -33,6 +33,7 @@ * Copyright 2019 Joyent, Inc. * Copyright (c) 2019, 2020 by Christian Schwarz. All rights reserved. * Copyright 2026 Oxide Computer Company + * Copyright (c) 2026, TrueNAS. */ #include <assert.h> @@ -5361,6 +5362,7 @@ zfs_do_receive(int argc, char **argv) #define ZFS_DELEG_PERM_SHARE "share" #define ZFS_DELEG_PERM_SEND "send" #define ZFS_DELEG_PERM_SEND_RAW "send:raw" +#define ZFS_DELEG_PERM_SEND_ENCRYPTED "send:encrypted" #define ZFS_DELEG_PERM_RECEIVE "receive" #define ZFS_DELEG_PERM_RECEIVE_APPEND "receive:append" #define ZFS_DELEG_PERM_ALLOW "allow" @@ -5404,6 +5406,7 @@ static zfs_deleg_perm_tab_t zfs_deleg_perm_tbl[] = { { ZFS_DELEG_PERM_ROLLBACK, ZFS_DELEG_NOTE_ROLLBACK }, { ZFS_DELEG_PERM_SEND, ZFS_DELEG_NOTE_SEND }, { ZFS_DELEG_PERM_SEND_RAW, ZFS_DELEG_NOTE_SEND_RAW }, + { ZFS_DELEG_PERM_SEND_ENCRYPTED, ZFS_DELEG_NOTE_SEND_ENCRYPTED }, { ZFS_DELEG_PERM_SHARE, ZFS_DELEG_NOTE_SHARE }, { ZFS_DELEG_PERM_SNAPSHOT, ZFS_DELEG_NOTE_SNAPSHOT }, { ZFS_DELEG_PERM_BOOKMARK, ZFS_DELEG_NOTE_BOOKMARK }, @@ -5902,11 +5905,15 @@ deleg_perm_comment(zfs_deleg_note_t note) str = gettext(""); break; case ZFS_DELEG_NOTE_SEND: - str = gettext(""); + str = gettext("Allow sending datasets"); break; case ZFS_DELEG_NOTE_SEND_RAW: - str = gettext("Allow sending ONLY encrypted (raw) replication" - "\n\t\t\t\tstreams"); + str = gettext("Allow sending datasets, but only in 'raw'" + "\n\t\t\t\treplication mode"); + break; + case ZFS_DELEG_NOTE_SEND_ENCRYPTED: + str = gettext("Allow sending only encrypted datasets, and" + "\n\t\t\t\tonly in 'raw' replication mode"); break; case ZFS_DELEG_NOTE_SHARE: str = gettext("Allows sharing file systems over NFS or SMB" diff --git a/sys/contrib/openzfs/cmd/zstream/Makefile.am b/sys/contrib/openzfs/cmd/zstream/Makefile.am index 6c629ff5aa59..0be7298c8655 100644 --- a/sys/contrib/openzfs/cmd/zstream/Makefile.am +++ b/sys/contrib/openzfs/cmd/zstream/Makefile.am @@ -7,14 +7,26 @@ CPPCHECKTARGETS += zstream zstream_SOURCES = \ %D%/zstream.c \ %D%/zstream.h \ + %D%/zstream_byteswap.c \ + %D%/zstream_byteswap.h \ + %D%/zstream_chain.c \ + %D%/zstream_chain.h \ %D%/zstream_decompress.c \ %D%/zstream_drop_record.c \ %D%/zstream_dump.c \ + %D%/zstream_fletcher4.c \ + %D%/zstream_fletcher4.h \ + %D%/zstream_io.c \ + %D%/zstream_io.h \ + %D%/zstream_modules.h \ %D%/zstream_recompress.c \ + %D%/zstream_recompress.h \ %D%/zstream_redup.c \ %D%/zstream_token.c \ %D%/zstream_util.c \ - %D%/zstream_util.h + %D%/zstream_util.h \ + %D%/zstream_validate.c \ + %D%/zstream_validate.h zstream_LDADD = \ libzfs.la \ diff --git a/sys/contrib/openzfs/cmd/zstream/zstream.c b/sys/contrib/openzfs/cmd/zstream/zstream.c index da74ab6e1e59..546c3d99ca60 100644 --- a/sys/contrib/openzfs/cmd/zstream/zstream.c +++ b/sys/contrib/openzfs/cmd/zstream/zstream.c @@ -18,19 +18,11 @@ * Copyright (c) 2020 by Delphix. All rights reserved. * Copyright (c) 2020 by Datto Inc. All rights reserved. */ -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <ctype.h> + #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <unistd.h> -#include <libintl.h> -#include <stddef.h> -#include <libzfs.h> -#include <signal.h> -#include <sys/backtrace.h> + #include "zstream.h" void @@ -55,43 +47,9 @@ zstream_usage(void) exit(1); } -static void sig_handler(int signo) -{ - struct sigaction action; - libspl_backtrace(STDERR_FILENO); - - /* - * Restore default action and re-raise signal so SIGSEGV and - * SIGABRT can trigger a core dump. - */ - action.sa_handler = SIG_DFL; - sigemptyset(&action.sa_mask); - action.sa_flags = 0; - (void) sigaction(signo, &action, NULL); - raise(signo); -} - - int main(int argc, char *argv[]) { - /* - * Set up signal handlers, so if we crash due to bad data in the stream - * we can get more info. Unlike ztest, we don't bail out if we can't - * set up signal handlers, because zstream is very useful without them. - */ - struct sigaction action = { .sa_handler = sig_handler }; - sigemptyset(&action.sa_mask); - action.sa_flags = 0; - if (sigaction(SIGSEGV, &action, NULL) < 0) { - (void) fprintf(stderr, "zstream: cannot catch SIGSEGV: %s\n", - strerror(errno)); - } - if (sigaction(SIGABRT, &action, NULL) < 0) { - (void) fprintf(stderr, "zstream: cannot catch SIGABRT: %s\n", - strerror(errno)); - } - char *basename = strrchr(argv[0], '/'); basename = basename ? (basename + 1) : argv[0]; if (argc >= 1 && strcmp(basename, "zstreamdump") == 0) diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_byteswap.c b/sys/contrib/openzfs/cmd/zstream/zstream_byteswap.c new file mode 100644 index 000000000000..bbd76162370e --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_byteswap.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#include <err.h> +#include <stdint.h> +#include <stdlib.h> +#include <sys/byteorder.h> +#include <sys/spa_checksum.h> +#include <sys/stdtypes.h> +#include <sys/zfs_ioctl.h> + +#include "zstream_modules.h" + +/* + * Mostly from dmu_recv.c + */ + +#define DO64(X) (drr->drr_u.X = BSWAP_64(drr->drr_u.X)) +#define DO32(X) (drr->drr_u.X = BSWAP_32(drr->drr_u.X)) + +typedef byteswap_stage_t byteswap_context_t; + +static byteswap_context_t byteswap_contexts[MAX_BYTESWAP]; +static int next_context = 0; + +static disposition_t +chain_byteswap(drr_packet_t *item, byteswap_context_t *context) +{ + if (item == NULL) { + return (D_OK); + } + + struct dmu_replay_record *drr = &item->dp_drr; + boolean_t input_swapped = *context == BS_INCOMING && + ATTR_IS_SET(CA_BYTESWAPPED); + boolean_t swap = input_swapped || (*context == BS_OUTGOING && + OPTION_ENABLED(CA_BYTESWAP_ON_OUTPUT)); + uint32_t drr_type = + input_swapped ? BSWAP_32(drr->drr_type) : drr->drr_type; + + if (swap) { + byteswap_record(drr, drr_type); + } + return (D_OK); +} + +/* + * Unconditionally byteswap a DMU replay record. drr_type is passed in + * separately because we don't know whether we're doing input or output + * swapping. + */ +void +byteswap_record(dmu_replay_record_t *drr, uint32_t drr_type) +{ + drr->drr_type = BSWAP_32(drr->drr_type); + drr->drr_payloadlen = BSWAP_32(drr->drr_payloadlen); + + switch (drr_type) { + + case DRR_BEGIN: + DO64(drr_begin.drr_magic); + DO64(drr_begin.drr_versioninfo); + DO64(drr_begin.drr_creation_time); + DO32(drr_begin.drr_type); + DO32(drr_begin.drr_flags); + DO64(drr_begin.drr_toguid); + DO64(drr_begin.drr_fromguid); + break; + + case DRR_END: + DO64(drr_end.drr_toguid); + ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_end.drr_checksum); + break; + + case DRR_OBJECT: + DO64(drr_object.drr_object); + DO32(drr_object.drr_type); + DO32(drr_object.drr_bonustype); + DO32(drr_object.drr_blksz); + DO32(drr_object.drr_bonuslen); + DO32(drr_object.drr_raw_bonuslen); + DO64(drr_object.drr_toguid); + DO64(drr_object.drr_maxblkid); + break; + + case DRR_FREEOBJECTS: + DO64(drr_freeobjects.drr_firstobj); + DO64(drr_freeobjects.drr_numobjs); + DO64(drr_freeobjects.drr_toguid); + break; + + case DRR_WRITE: + DO64(drr_write.drr_object); + DO32(drr_write.drr_type); + DO64(drr_write.drr_offset); + DO64(drr_write.drr_logical_size); + DO64(drr_write.drr_toguid); + ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write.drr_key.ddk_cksum); + DO64(drr_write.drr_key.ddk_prop); + DO64(drr_write.drr_compressed_size); + break; + + case DRR_WRITE_BYREF: + DO64(drr_write_byref.drr_object); + DO64(drr_write_byref.drr_offset); + DO64(drr_write_byref.drr_length); + DO64(drr_write_byref.drr_toguid); + DO64(drr_write_byref.drr_refguid); + DO64(drr_write_byref.drr_refobject); + DO64(drr_write_byref.drr_refoffset); + ZIO_CHECKSUM_BSWAP( + &drr->drr_u.drr_write_byref.drr_key.ddk_cksum); + DO64(drr_write_byref.drr_key.ddk_prop); + break; + + case DRR_FREE: + DO64(drr_free.drr_object); + DO64(drr_free.drr_offset); + DO64(drr_free.drr_length); + /* Note: toguid not byte-swapped in original zstream_dump.c */ + DO64(drr_free.drr_toguid); + break; + + case DRR_SPILL: + DO64(drr_spill.drr_object); + DO64(drr_spill.drr_length); + /* Note: toguid not byte-swapped in original zstream_dump.c */ + DO64(drr_spill.drr_toguid); + DO64(drr_spill.drr_compressed_size); + DO32(drr_spill.drr_type); + break; + + case DRR_WRITE_EMBEDDED: + DO64(drr_write_embedded.drr_object); + DO64(drr_write_embedded.drr_offset); + DO64(drr_write_embedded.drr_length); + DO64(drr_write_embedded.drr_toguid); + DO32(drr_write_embedded.drr_lsize); + DO32(drr_write_embedded.drr_psize); + break; + + case DRR_OBJECT_RANGE: + DO64(drr_object_range.drr_firstobj); + DO64(drr_object_range.drr_numslots); + DO64(drr_object_range.drr_toguid); + break; + + case DRR_REDACT: + DO64(drr_redact.drr_object); + DO64(drr_redact.drr_offset); + DO64(drr_redact.drr_length); + DO64(drr_redact.drr_toguid); + break; + + default: + errx(1, "unknown record type %llu, aborting...", + (u_longlong_t)drr_type); + } + + if (drr_type != DRR_BEGIN) { + ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_checksum.drr_checksum); + } +} + +chain_step_t +serial_byteswap(byteswap_stage_t stage) +{ + int context_ix = next_context++ % MAX_BYTESWAP; + byteswap_context_t *bsc = &byteswap_contexts[context_ix]; + + *bsc = stage; + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = sizeof (drr_packet_t), + .cs_context = bsc, + .cs_serial = { + .process = (zc_serial_process_f *)chain_byteswap, + } + }; + return (step); +} diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_byteswap.h b/sys/contrib/openzfs/cmd/zstream/zstream_byteswap.h new file mode 100644 index 000000000000..2ed806ddf2e1 --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_byteswap.h @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#ifndef _ZSTREAM_BYTESWAP_H +#define _ZSTREAM_BYTESWAP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "zstream_io.h" + +#define MAX_BYTESWAP 4 /* Most swapping ops in a chain */ + +/* + * Byteswapping is generally done both on input and on output. By default, + * the stream's endianness is preserved. That is, opposite-endian streams + * are byteswapped for processing by other modules, then ultimately + * de-byteswapped for output. + */ +typedef enum { BS_INCOMING, BS_OUTGOING } byteswap_stage_t; + +chain_step_t +serial_byteswap(byteswap_stage_t stage); + +/* + * Unconditionally swap a record. drr_type is passed in separately because + * we don't know whether we're doing input or output swapping. We need + * that value in native byte order to know how to swap the rest of the + * record. + */ +extern void +byteswap_record(dmu_replay_record_t *drr, uint32_t drr_type); + +#ifdef __cplusplus +} +#endif + +#endif /* _ZSTREAM_BYTESWAP_H */ diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_chain.c b/sys/contrib/openzfs/cmd/zstream/zstream_chain.c new file mode 100644 index 000000000000..22317dbf1b23 --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_chain.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#include <assert.h> +#include <err.h> +#include <libspl.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/abd.h> +#include <sys/param.h> +#include <sys/stdtypes.h> +#include <sys/zio.h> +#include <sys/zstd/zstd.h> +#include <zfs_fletcher.h> + +#include "zstream_chain.h" + +#define MAX_CHAIN_LENGTH 32 + +chain_attrs_t *chain_attrs; + +static disposition_t +chain_null_step(void *item, void *context) +{ + (void) item; + (void) context; + return (D_OK); +} + +chain_step_t +serial_null_step(void) +{ + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_serial = { + .process = (zc_serial_process_f *)chain_null_step + } + }; + return (step); +} + +chain_step_t +chain_terminator(void) +{ + chain_step_t step = { .cs_type = CS_TERMINATE }; + return (step); +} + +static void +libraries_init(void) +{ + zfs_refcount_init(); + abd_init(); + zio_init(); + zstd_init(); + libspl_init(); + fletcher_4_init(); +} + +static void +libraries_fini(void) +{ + fletcher_4_fini(); + libspl_fini(); + zio_fini(); + zstd_fini(); + abd_fini(); + zfs_refcount_fini(); +} + +/* + * Execute a chain of serial processing steps. + * + * For simplicity, we normalize the chain item size to that of the largest + * output of any step. Packets with data beyond the base drr_record_t should + * add their additional data to the end of the packet, and this area may be + * reused for different purposes as items travel down the chain. + * + * Each item traverses the entire chain before the next item is read. + */ +void +zstream_chain_exec(zstream_chain_t chain, chain_attrs_t *attrs) +{ + int num_steps = 0; + size_t packet_size = 0; + chain_attrs_t backup_attrs = {0}; + + chain_attrs = attrs ? attrs : &backup_attrs; + + while (chain[num_steps].cs_type != CS_TERMINATE) { + packet_size = MAX(packet_size, chain[num_steps].cs_out_size); + num_steps++; + if (num_steps >= MAX_CHAIN_LENGTH) { + errx(1, "unterminated zstream_chain"); + } + } + VERIFY3U(num_steps, >, 0); + + /* + * Check for consistency of input and output packet sizes in + * adjacent steps. A declared packet size of zero waives this check. + */ + for (int i = 0; i < num_steps; i++) { + boolean_t mismatch = i > 0 && + chain[i].cs_in_size != 0 && + chain[i-1].cs_out_size != 0 && + chain[i].cs_in_size != chain[i-1].cs_out_size; + if (mismatch) { + warnx("note - chain steps %d and %d have " + "mismatched packet sizes", i - 1, i); + } + } + + libraries_init(); + + uint8_t buffer[packet_size]; + boolean_t done = B_FALSE; + + while (!done) { + for (int i = 0; i < num_steps; i++) { + if (done) { + (void) chain[i].cs_serial.process(NULL, + chain[i].cs_context); + } else { + disposition_t dispo = + chain[i].cs_serial.process(buffer, + chain[i].cs_context); + if (dispo == D_EOF) { + VERIFY0(i); + done = B_TRUE; + } else if (dispo == D_DROP) { + break; + } + } + } + } + + libraries_fini(); +} diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_chain.h b/sys/contrib/openzfs/cmd/zstream/zstream_chain.h new file mode 100644 index 000000000000..2e3c90cbecc5 --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_chain.h @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#ifndef _ZSTREAM_CHAIN_H +#define _ZSTREAM_CHAIN_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stddef.h> +#include <stdint.h> +#include <sys/zfs_ioctl.h> + +/* + * A chain is a linear series of steps that process packets of data. It's + * designed to modularize common functionality, reduce code duplication, and + * separate processing structure from implementation. + * + * Some terms: + * + * **STEP** - A chain_step_t struct that represents a packet-processing + * module and any arguments or context that it needs. Chain modules + * generally define a function named serial_* that produces a chain_step_t + * that can be incorporated directly into a chain. + * + * **CHAIN** - An array of chain_step_t's. It's just data, so you can create + * the array however you like. But normally you'd just declare the whole + * thing: + * + * zstream_chain_t dump_chain = { + * serial_read_stream(infile), + * serial_validate_fletcher4(), + * serial_byteswap(BS_INCOMING), + * serial_validate_records(), + * serial_dump_records(&dump_args), + * serial_null_output(), + * chain_terminator() + * } + * + * Or more succinctly: + * + * zstream_chain_t dump_chain = { + * STANDARD_INPUT_STACK(infile), + * serial_dump_records(&dump_args), + * NULL_OUTPUT_STACK() + * }; + * + * Chains must be terminated by a step of type CS_TERMINATE. + * + * **ITEMS** - The data packets that flow through a chain. Each step accepts + * items of one size and emits items of another size, which may be smaller, + * larger, or the same size. Items will generally be structs that start with + * a drr_packet_t (defined in zstream_io.h) and may include additional + * module-specific fields. + * + * **PROCESSING FUNCTION** - Each step names a processing function that does + * the actual work of transforming an input buffer into an output buffer. + * The transformation happens in place, in a single buffer provided by the + * chain. + * + * The processing function should return a disposition_t, normally D_OK. A + * function can return D_DROP to remove an item from the stream entirely. It + * can also return D_EOF to indicate that no more data will be forthcoming. + * However, only the first step in the chain should ever return D_EOF. + * + * Functions are called with a NULL packet pointer when the end of the + * stream passes by them. + * + * **CONTEXT** - An arbitrary void * that the chain passes along to the + * processing function as an argument. + * + * **CHAIN ATTRIBUTES** - A global set of flags available to all steps. + */ + +#define CA_BYTESWAPPED (1ULL << 0) /* ca_attrs */ +#define CA_BIG_ENDIAN_INPUT (1ULL << 1) +#define CA_LITTLE_ENDIAN_INPUT (1ULL << 2) + +#define CA_VERBOSE (1ULL << 0) /* ca_command_opts */ +#define CA_VERY_VERBOSE (1ULL << 1) +#define CA_DUMP_DATA (1ULL << 2) +#define CA_IGNORE_CKSUMS (1ULL << 3) +#define CA_DO_NOT_VALIDATE (1ULL << 4) +#define CA_FORBID_DEDUP (1ULL << 5) +#define CA_REQUIRE_DEDUP (1ULL << 6) +#define CA_REQUIRE_NATIVE_ENDIAN (1ULL << 7) +#define CA_BYTESWAP_ON_OUTPUT (1ULL << 8) +#define CA_BIG_ENDIAN_OUT (1ULL << 9) +#define CA_LITTLE_ENDIAN_OUT (1ULL << 10) +#define CA_OPPOSITE_ENDIAN_OUT (1ULL << 11) + +#define OPTION_ENABLED(option) (!!(chain_attrs->ca_command_opts & (option))) +#define STREAM_HAS_FEATURE(feat) (!!(chain_attrs->ca_feature_flags & (feat))) +#define ATTR_IS_SET(attr) (!!(chain_attrs->ca_attrs & (attr))) + +#define ENABLE_OPTION(attrs, opt) ((attrs)->ca_command_opts |= (opt)) +#define SET_ATTR(attr) (chain_attrs->ca_attrs |= (attr)) + +typedef struct { + uint64_t rs_num_records; + uint64_t rs_total_header_bytes; + uint64_t rs_total_payload_bytes; +} record_stats_t; + +/* + * Chain attribute flags that describe the stream. Statistics are maintained + * by the zstream_io modules. + */ +typedef struct { + uint64_t ca_feature_flags; /* From drr_versioninfo */ + uint64_t ca_attrs; /* Discovered attributes */ + uint64_t ca_command_opts; /* Command line options */ + record_stats_t ca_totals_in; + record_stats_t ca_totals_out; + record_stats_t ca_stats_in[DRR_NUMTYPES]; + record_stats_t ca_stats_out[DRR_NUMTYPES]; +} chain_attrs_t; + +typedef enum { CS_SERIAL, CS_TERMINATE } step_type_t; +typedef enum { D_OK, D_EOF, D_DROP } disposition_t; + +typedef disposition_t +zc_serial_process_f(void *item, void *context); + +typedef struct chain_step +{ + step_type_t cs_type; + size_t cs_in_size; + size_t cs_out_size; + void *cs_context; + struct { + zc_serial_process_f *process; + } cs_serial; +} chain_step_t; + +typedef chain_step_t zstream_chain_t[]; + +/* + * Chain attributes accessible to any step on the chain. In theory this + * could cause a race condition between reading and setting, but all + * attributes are typically set by the time the first record has been read. + * Ergo, nobody else will be executing while that first chain_read() runs. + */ +extern chain_attrs_t *chain_attrs; + +/* + * Execute a chain. Returns once execution is complete. You can pass NULL + * for the attrs if you're not interested in preserving them after the chain + * has run. + */ +void +zstream_chain_exec(zstream_chain_t chain, chain_attrs_t *attrs); + +chain_step_t +serial_null_step(void); + +chain_step_t +chain_terminator(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _ZSTREAM_CHAIN_H */ diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_decompress.c b/sys/contrib/openzfs/cmd/zstream/zstream_decompress.c index a9dbe30798d8..d9855b8f4c79 100644 --- a/sys/contrib/openzfs/cmd/zstream/zstream_decompress.c +++ b/sys/contrib/openzfs/cmd/zstream/zstream_decompress.c @@ -25,40 +25,135 @@ * Use is subject to license terms. * * Copyright (c) 2024, Klara, Inc. + * Copyright (c) 2026 by Garth Snyder */ #include <err.h> +#include <errno.h> #include <search.h> +#include <stdint.h> #include <stdio.h> #include <stdlib.h> -#include <unistd.h> +#include <string.h> +#include <sys/stdtypes.h> #include <sys/zfs_ioctl.h> -#include <sys/zio_checksum.h> -#include <sys/zstd/zstd.h> -#include "zfs_fletcher.h" +#include <sys/zio_compress.h> +#include <unistd.h> + #include "zstream.h" +#include "zstream_modules.h" #include "zstream_util.h" +#define KEYSIZE 64 + +static disposition_t +chain_decompress_named_writes(drr_packet_t *item, void *context) +{ + (void) context; + if (item == NULL) { + return (D_OK); + } + + dmu_replay_record_t *drr = &item->dp_drr; + struct drr_write *drrw = &drr->drr_u.drr_write; + char key[KEYSIZE]; + uint8_t *dcbuff; + + if (drr->drr_type != DRR_WRITE) { + return (D_OK); + } + + snprintf(key, KEYSIZE, "%llu,%llu", + (u_longlong_t)drrw->drr_object, (u_longlong_t)drrw->drr_offset); + ENTRY e = { .key = key }; + ENTRY *p = hsearch(e, FIND); + if (p == NULL) { + return (D_OK); + } + + enum zio_compress ctype = (enum zio_compress)(intptr_t)p->data; + if (ctype == ZIO_COMPRESS_INHERIT) { + /* Unspecified */ + ctype = drrw->drr_compressiontype; + } + if (ctype_is_uncompressed(ctype)) { + drrw->drr_compressiontype = 0; + drrw->drr_logical_size = drrw->drr_compressed_size; + drrw->drr_compressed_size = 0; + if (OPTION_ENABLED(CA_VERBOSE)) { + fprintf(stderr, + "Resetting compression type to " + "off for ino %llu offset %llu\n", + (u_longlong_t)drrw->drr_object, + (u_longlong_t)drrw->drr_offset); + } + return (D_OK); + } + + if (write_is_encrypted(drrw)) { + warnx("the write for ino %llu offset %llu is marked " + "as encrypted. Attempting decompression anyway...", + (u_longlong_t)drrw->drr_object, + (u_longlong_t)drrw->drr_offset); + } + + dcbuff = decompress_buffer(item->dp_payload, item->dp_payload_size, + drrw->drr_logical_size, ctype); + + if (dcbuff == NULL) { + /* + * The block must not be compressed, at least not with this + * compression type, possibly because it gets written + * multiple times in this stream. + */ + warnx("decompression failed for ino %llu offset %llu", + (u_longlong_t)drrw->drr_object, + (u_longlong_t)drrw->drr_offset); + } else { + free(item->dp_payload); + item->dp_payload = dcbuff; + item->dp_payload_size = drrw->drr_logical_size; + drrw->drr_compressiontype = 0; + drrw->drr_compressed_size = 0; + if (OPTION_ENABLED(CA_VERBOSE)) { + fprintf(stderr, + "Successfully decompressed ino %llu offset %llu\n", + (u_longlong_t)drrw->drr_object, + (u_longlong_t)drrw->drr_offset); + } + } + return (D_OK); +} + +static chain_step_t +serial_decompress_named_writes(void) +{ + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = sizeof (drr_packet_t), + .cs_context = NULL, + .cs_serial = { + .process = + (zc_serial_process_f *)chain_decompress_named_writes + } + }; + return (step); +} + int zstream_do_decompress(int argc, char *argv[]) { - const int KEYSIZE = 64; - int bufsz = SPA_MAXBLOCKSIZE; - char *buf = safe_malloc(bufsz); - dmu_replay_record_t thedrr; - dmu_replay_record_t *drr = &thedrr; - zio_cksum_t stream_cksum; + chain_attrs_t attrs = {0}; int c; - boolean_t verbose = B_FALSE; while ((c = getopt(argc, argv, "v")) != -1) { switch (c) { case 'v': - verbose = B_TRUE; + ENABLE_OPTION(&attrs, CA_VERBOSE); break; case '?': - (void) fprintf(stderr, "invalid option '%c'\n", - optopt); + fprintf(stderr, "invalid option '%c'\n", optopt); zstream_usage(); break; } @@ -69,16 +164,16 @@ zstream_do_decompress(int argc, char *argv[]) if (argc < 0) zstream_usage(); - if (hcreate(argc) == 0) - errx(1, "hcreate"); + errx(1, "hcreate failed"); + for (int i = 0; i < argc; i++) { uint64_t object, offset; char *obj_str; char *offset_str; char *key; char *end; - enum zio_compress type = ZIO_COMPRESS_LZ4; + enum zio_compress type = ZIO_COMPRESS_INHERIT; obj_str = strsep(&argv[i], ","); if (argv[i] == NULL) { @@ -107,266 +202,33 @@ zstream_do_decompress(int argc, char *argv[]) else if (0 == strcmp("zstd", argv[i])) type = ZIO_COMPRESS_ZSTD; else { - fprintf(stderr, "Invalid compression type %s.\n" + errx(2, "invalid compression type %s. " "Supported types are off, lz4, lzjb, gzip, " - "zle, and zstd\n", - argv[i]); - exit(2); + "zle, and zstd", argv[i]); } } - if (asprintf(&key, "%llu,%llu", (u_longlong_t)object, - (u_longlong_t)offset) < 0) { + int n_chars = asprintf(&key, "%llu,%llu", (u_longlong_t)object, + (u_longlong_t)offset); + if (n_chars < 0) err(1, "asprintf"); - } - ENTRY e = {.key = key}; + ENTRY e = { .key = key }; ENTRY *p; - p = hsearch(e, ENTER); if (p == NULL) - errx(1, "hsearch"); - p->data = (void*)(intptr_t)type; - } - - if (isatty(STDIN_FILENO)) { - (void) fprintf(stderr, - "Error: The send stream is a binary format " - "and can not be read from a\n" - "terminal. Standard input must be redirected.\n"); - exit(1); + errx(1, "hsearch failed"); + p->data = (void *)(intptr_t)type; } - fletcher_4_init(); - int begin = 0; - boolean_t seen = B_FALSE; - while (sfread(drr, sizeof (*drr), stdin) != 0) { - struct drr_write *drrw; - uint64_t payload_size = 0; - - /* - * We need to regenerate the checksum. - */ - if (drr->drr_type != DRR_BEGIN) { - memset(&drr->drr_u.drr_checksum.drr_checksum, 0, - sizeof (drr->drr_u.drr_checksum.drr_checksum)); - } - - switch (drr->drr_type) { - case DRR_BEGIN: - { - ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); - VERIFY0(begin++); - seen = B_TRUE; - - uint32_t sz = drr->drr_payloadlen; - - VERIFY3U(sz, <=, 1U << 28); - - if (sz != 0) { - if (sz > bufsz) { - buf = realloc(buf, sz); - if (buf == NULL) - err(1, "realloc"); - bufsz = sz; - } - (void) sfread(buf, sz, stdin); - } - payload_size = sz; - break; - } - case DRR_END: - { - struct drr_end *drre = &drr->drr_u.drr_end; - /* - * We would prefer to just check --begin == 0, but - * replication streams have an end of stream END - * record, so we must avoid tripping it. - */ - VERIFY3B(seen, ==, B_TRUE); - begin--; - /* - * Use the recalculated checksum, unless this is - * the END record of a stream package, which has - * no checksum. - */ - if (!ZIO_CHECKSUM_IS_ZERO(&drre->drr_checksum)) - drre->drr_checksum = stream_cksum; - break; - } - - case DRR_OBJECT: - { - struct drr_object *drro = &drr->drr_u.drr_object; - VERIFY3S(begin, ==, 1); - - if (drro->drr_bonuslen > 0) { - payload_size = DRR_OBJECT_PAYLOAD_SIZE(drro); - (void) sfread(buf, payload_size, stdin); - } - break; - } - - case DRR_SPILL: - { - struct drr_spill *drrs = &drr->drr_u.drr_spill; - VERIFY3S(begin, ==, 1); - payload_size = DRR_SPILL_PAYLOAD_SIZE(drrs); - (void) sfread(buf, payload_size, stdin); - break; - } - - case DRR_WRITE_BYREF: - VERIFY3S(begin, ==, 1); - fprintf(stderr, - "Deduplicated streams are not supported\n"); - exit(1); - break; - - case DRR_WRITE: - { - VERIFY3S(begin, ==, 1); - drrw = &thedrr.drr_u.drr_write; - payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); - ENTRY *p; - char key[KEYSIZE]; - - snprintf(key, KEYSIZE, "%llu,%llu", - (u_longlong_t)drrw->drr_object, - (u_longlong_t)drrw->drr_offset); - ENTRY e = {.key = key}; - - p = hsearch(e, FIND); - if (p == NULL) { - /* - * Read the contents of the block unaltered - */ - (void) sfread(buf, payload_size, stdin); - break; - } - - /* - * Read and decompress the block - */ - enum zio_compress c = - (enum zio_compress)(intptr_t)p->data; - - if (c == ZIO_COMPRESS_OFF) { - (void) sfread(buf, payload_size, stdin); - drrw->drr_compressiontype = 0; - drrw->drr_compressed_size = 0; - if (verbose) - fprintf(stderr, - "Resetting compression type to " - "off for ino %llu offset %llu\n", - (u_longlong_t)drrw->drr_object, - (u_longlong_t)drrw->drr_offset); - break; - } + ENABLE_OPTION(&attrs, CA_FORBID_DEDUP); - uint64_t lsize = drrw->drr_logical_size; - ASSERT3U(payload_size, <=, lsize); + zstream_chain_t decompress_chain = { + STANDARD_INPUT_STACK(NULL), + serial_decompress_named_writes(), + STANDARD_OUTPUT_STACK(NULL) + }; + zstream_chain_exec(decompress_chain, &attrs); - char *lzbuf = safe_calloc(payload_size); - (void) sfread(lzbuf, payload_size, stdin); - - abd_t sabd, dabd; - abd_get_from_buf_struct(&sabd, lzbuf, payload_size); - abd_get_from_buf_struct(&dabd, buf, lsize); - int err = zio_decompress_data(c, &sabd, &dabd, - payload_size, lsize, NULL); - abd_free(&dabd); - abd_free(&sabd); - - if (err == 0) { - drrw->drr_compressiontype = 0; - drrw->drr_compressed_size = 0; - payload_size = lsize; - if (verbose) { - fprintf(stderr, - "successfully decompressed " - "ino %llu offset %llu\n", - (u_longlong_t)drrw->drr_object, - (u_longlong_t)drrw->drr_offset); - } - } else { - /* - * The block must not be compressed, at least - * not with this compression type, possibly - * because it gets written multiple times in - * this stream. - */ - warnx("decompression failed for " - "ino %llu offset %llu", - (u_longlong_t)drrw->drr_object, - (u_longlong_t)drrw->drr_offset); - memcpy(buf, lzbuf, payload_size); - } - - free(lzbuf); - break; - } - - case DRR_WRITE_EMBEDDED: - { - VERIFY3S(begin, ==, 1); - struct drr_write_embedded *drrwe = - &drr->drr_u.drr_write_embedded; - payload_size = - P2ROUNDUP((uint64_t)drrwe->drr_psize, 8); - (void) sfread(buf, payload_size, stdin); - break; - } - - case DRR_FREEOBJECTS: - case DRR_FREE: - case DRR_OBJECT_RANGE: - VERIFY3S(begin, ==, 1); - break; - - default: - (void) fprintf(stderr, "INVALID record type 0x%x\n", - drr->drr_type); - /* should never happen, so assert */ - assert(B_FALSE); - } - - if (feof(stdout)) { - fprintf(stderr, "Error: unexpected end-of-file\n"); - exit(1); - } - if (ferror(stdout)) { - fprintf(stderr, "Error while reading file: %s\n", - strerror(errno)); - exit(1); - } - - /* - * We need to recalculate the checksum, and it needs to be - * initially zero to do that. BEGIN records don't have - * a checksum. - */ - if (drr->drr_type != DRR_BEGIN) { - memset(&drr->drr_u.drr_checksum.drr_checksum, 0, - sizeof (drr->drr_u.drr_checksum.drr_checksum)); - } - if (dump_record(drr, buf, payload_size, - &stream_cksum, STDOUT_FILENO) != 0) - break; - if (drr->drr_type == DRR_END) { - /* - * Typically the END record is either the last - * thing in the stream, or it is followed - * by a BEGIN record (which also zeros the checksum). - * However, a stream package ends with two END - * records. The last END record's checksum starts - * from zero. - */ - ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); - } - } - free(buf); - fletcher_4_fini(); hdestroy(); - return (0); } diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_drop_record.c b/sys/contrib/openzfs/cmd/zstream/zstream_drop_record.c index b6895bd52cc0..53ca3694e260 100644 --- a/sys/contrib/openzfs/cmd/zstream/zstream_drop_record.c +++ b/sys/contrib/openzfs/cmd/zstream/zstream_drop_record.c @@ -26,37 +26,104 @@ */ #include <err.h> +#include <errno.h> #include <search.h> +#include <stdint.h> #include <stdio.h> #include <stdlib.h> -#include <unistd.h> +#include <string.h> +#include <sys/stdtypes.h> #include <sys/zfs_ioctl.h> -#include <sys/zio_checksum.h> -#include <sys/zstd/zstd.h> -#include "zfs_fletcher.h" +#include <unistd.h> + #include "zstream.h" -#include "zstream_util.h" +#include "zstream_modules.h" + +#define KEYSIZE 64 + +static disposition_t +chain_drop_records(drr_packet_t *item, void *context) +{ + (void) context; + if (item == NULL) + return (D_OK); + + dmu_replay_record_t *drr = &item->dp_drr; + struct drr_write *drrw = &drr->drr_u.drr_write; + struct drr_write_embedded *drrwe = &drr->drr_u.drr_write_embedded; + char key[KEYSIZE]; + u_longlong_t object, offset; + const char *record_type; + ENTRY e = {.key = key}; + + if (drr->drr_type == DRR_WRITE) { + object = drrw->drr_object; + offset = drrw->drr_offset; + record_type = "WRITE"; + } else if (drr->drr_type == DRR_WRITE_EMBEDDED) { + object = drrwe->drr_object; + offset = drrwe->drr_offset; + record_type = "WRITE_EMBEDDED"; + } else { + return (D_OK); + } + + snprintf(key, KEYSIZE, "%llu,%llu", object, offset); + if (hsearch(e, FIND) != NULL) { + if (OPTION_ENABLED(CA_VERBOSE)) { + warnx("dropping %s record for object %llu " + "offset %llu", record_type, object, offset); + } + /* + * It really feels like the chain executor ought to be + * responsible for freeing this payload. However, it + * operates at a more abstract level and knows nothing about + * DMU records and their payloads, so this'll have to be + * done here when the drop decision is made. + * + * Fine for now, but if another case like this comes up in + * the future, the issue probably needs to be handled + * through a more clearly defined path. + */ + if (item->dp_payload_size && item->dp_payload != NULL) { + free(item->dp_payload); + item->dp_payload = NULL; + item->dp_payload_size = 0; + } + return (D_DROP); + } + + return (D_OK); +} + +static chain_step_t +serial_drop_records(void) +{ + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = sizeof (drr_packet_t), + .cs_context = NULL, + .cs_serial = { + .process = (zc_serial_process_f *)chain_drop_records + } + }; + return (step); +} int zstream_do_drop_record(int argc, char *argv[]) { - const int KEYSIZE = 64; - int bufsz = SPA_MAXBLOCKSIZE; - char *buf = safe_malloc(bufsz); - dmu_replay_record_t thedrr; - dmu_replay_record_t *drr = &thedrr; - zio_cksum_t stream_cksum; int c; - boolean_t verbose = B_FALSE; + chain_attrs_t attrs = {0}; while ((c = getopt(argc, argv, "v")) != -1) { switch (c) { case 'v': - verbose = B_TRUE; + ENABLE_OPTION(&attrs, CA_VERBOSE); break; case '?': - (void) fprintf(stderr, "invalid option '%c'\n", - optopt); + warnx("invalid option '%c'\n", optopt); zstream_usage(); break; } @@ -67,10 +134,11 @@ zstream_do_drop_record(int argc, char *argv[]) if (argc < 0) zstream_usage(); - if (hcreate(argc) == 0) - errx(1, "hcreate"); + errx(1, "hcreate failed"); + for (int i = 0; i < argc; i++) { + uint64_t object, offset; char *obj_str; char *offset_str; @@ -97,228 +165,21 @@ zstream_do_drop_record(int argc, char *argv[]) } ENTRY e = {.key = key}; ENTRY *p; - p = hsearch(e, ENTER); if (p == NULL) errx(1, "hsearch"); - p->data = (void*)(intptr_t)B_TRUE; - } - - if (isatty(STDIN_FILENO)) { - (void) fprintf(stderr, - "Error: The send stream is a binary format " - "and can not be read from a\n" - "terminal. Standard input must be redirected.\n"); - exit(1); + p->data = (void *)(intptr_t)B_TRUE; } - fletcher_4_init(); - int begin = 0; - boolean_t seen = B_FALSE; - while (sfread(drr, sizeof (*drr), stdin) != 0) { - struct drr_write *drrw; - uint64_t payload_size = 0; - - /* - * We need to regenerate the checksum. - */ - if (drr->drr_type != DRR_BEGIN) { - memset(&drr->drr_u.drr_checksum.drr_checksum, 0, - sizeof (drr->drr_u.drr_checksum.drr_checksum)); - } - - switch (drr->drr_type) { - case DRR_BEGIN: - { - ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); - VERIFY0(begin++); - seen = B_TRUE; - - uint32_t sz = drr->drr_payloadlen; - - VERIFY3U(sz, <=, 1U << 28); - - if (sz != 0) { - if (sz > bufsz) { - buf = realloc(buf, sz); - if (buf == NULL) - err(1, "realloc"); - bufsz = sz; - } - (void) sfread(buf, sz, stdin); - } - payload_size = sz; - break; - } - case DRR_END: - { - struct drr_end *drre = &drr->drr_u.drr_end; - /* - * We would prefer to just check --begin == 0, but - * replication streams have an end of stream END - * record, so we must avoid tripping it. - */ - VERIFY3B(seen, ==, B_TRUE); - begin--; - /* - * Use the recalculated checksum, unless this is - * the END record of a stream package, which has - * no checksum. - */ - if (!ZIO_CHECKSUM_IS_ZERO(&drre->drr_checksum)) - drre->drr_checksum = stream_cksum; - break; - } - - case DRR_OBJECT: - { - struct drr_object *drro = &drr->drr_u.drr_object; - VERIFY3S(begin, ==, 1); - - if (drro->drr_bonuslen > 0) { - payload_size = DRR_OBJECT_PAYLOAD_SIZE(drro); - (void) sfread(buf, payload_size, stdin); - } - break; - } - - case DRR_SPILL: - { - struct drr_spill *drrs = &drr->drr_u.drr_spill; - VERIFY3S(begin, ==, 1); - payload_size = DRR_SPILL_PAYLOAD_SIZE(drrs); - (void) sfread(buf, payload_size, stdin); - break; - } - - case DRR_WRITE_BYREF: - VERIFY3S(begin, ==, 1); - fprintf(stderr, - "Deduplicated streams are not supported\n"); - exit(1); - break; - - case DRR_WRITE: - { - VERIFY3S(begin, ==, 1); - drrw = &thedrr.drr_u.drr_write; - payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); - ENTRY *p; - char key[KEYSIZE]; - - snprintf(key, KEYSIZE, "%llu,%llu", - (u_longlong_t)drrw->drr_object, - (u_longlong_t)drrw->drr_offset); - ENTRY e = {.key = key}; - - (void) sfread(buf, payload_size, stdin); - p = hsearch(e, FIND); - if (p == NULL) { - /* - * Dump the contents of the block unaltered - */ - } else { - /* - * Read and discard the block - */ - if (verbose) - fprintf(stderr, - "Dropping WRITE record for object " - "%llu offset %llu\n", - (u_longlong_t)drrw->drr_object, - (u_longlong_t)drrw->drr_offset); - continue; - } - break; - } - - case DRR_WRITE_EMBEDDED: - { - ENTRY *p; - char key[KEYSIZE]; + ENABLE_OPTION(&attrs, CA_FORBID_DEDUP); - VERIFY3S(begin, ==, 1); - struct drr_write_embedded *drrwe = - &drr->drr_u.drr_write_embedded; - payload_size = - P2ROUNDUP((uint64_t)drrwe->drr_psize, 8); + zstream_chain_t drop_chain = { + STANDARD_INPUT_STACK(NULL), + serial_drop_records(), + STANDARD_OUTPUT_STACK(NULL) + }; + zstream_chain_exec(drop_chain, &attrs); - snprintf(key, KEYSIZE, "%llu,%llu", - (u_longlong_t)drrwe->drr_object, - (u_longlong_t)drrwe->drr_offset); - ENTRY e = {.key = key}; - - (void) sfread(buf, payload_size, stdin); - p = hsearch(e, FIND); - if (p == NULL) { - /* - * Dump the contents of the block unaltered - */ - } else { - /* - * Read and discard the block - */ - if (verbose) - fprintf(stderr, - "Dropping WRITE_EMBEDDED record for" - " object %llu offset %llu\n", - (u_longlong_t)drrwe->drr_object, - (u_longlong_t)drrwe->drr_offset); - continue; - } - break; - } - - case DRR_FREEOBJECTS: - case DRR_FREE: - case DRR_OBJECT_RANGE: - VERIFY3S(begin, ==, 1); - break; - - default: - (void) fprintf(stderr, "INVALID record type 0x%x\n", - drr->drr_type); - /* should never happen, so assert */ - assert(B_FALSE); - } - - if (feof(stdout)) { - fprintf(stderr, "Error: unexpected end-of-file\n"); - exit(1); - } - if (ferror(stdout)) { - fprintf(stderr, "Error while reading file: %s\n", - strerror(errno)); - exit(1); - } - - /* - * We need to recalculate the checksum, and it needs to be - * initially zero to do that. BEGIN records don't have - * a checksum. - */ - if (drr->drr_type != DRR_BEGIN) { - memset(&drr->drr_u.drr_checksum.drr_checksum, 0, - sizeof (drr->drr_u.drr_checksum.drr_checksum)); - } - if (dump_record(drr, buf, payload_size, - &stream_cksum, STDOUT_FILENO) != 0) - break; - if (drr->drr_type == DRR_END) { - /* - * Typically the END record is either the last - * thing in the stream, or it is followed - * by a BEGIN record (which also zeros the checksum). - * However, a stream package ends with two END - * records. The last END record's checksum starts - * from zero. - */ - ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); - } - } - free(buf); - fletcher_4_fini(); hdestroy(); - return (0); } diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_dump.c b/sys/contrib/openzfs/cmd/zstream/zstream_dump.c index 7757ee3b1754..3fa02c6e2178 100644 --- a/sys/contrib/openzfs/cmd/zstream/zstream_dump.c +++ b/sys/contrib/openzfs/cmd/zstream/zstream_dump.c @@ -25,26 +25,26 @@ * Use is subject to license terms. * * Portions Copyright 2012 Martin Matuska <martin@matuska.org> - */ - -/* * Copyright (c) 2013, 2015 by Delphix. All rights reserved. + * Portions copyright 2026 by Garth Snyder <garth@garthsnyder.com> */ #include <ctype.h> +#include <err.h> #include <libnvpair.h> +#include <stdint.h> #include <stdio.h> -#include <stdlib.h> #include <string.h> -#include <unistd.h> -#include <stddef.h> - -#include <sys/dmu.h> +#include <sys/nvpair.h> +#include <sys/param.h> +#include <sys/spa_checksum.h> +#include <sys/stdtypes.h> #include <sys/zfs_ioctl.h> #include <sys/zio.h> -#include <zfs_fletcher.h> +#include <unistd.h> + #include "zstream.h" -#include "zstream_util.h" +#include "zstream_modules.h" /* * If dump mode is enabled, the number of bytes to print per line @@ -56,88 +56,46 @@ */ #define DUMP_GROUPING 4 -static uint64_t total_stream_len = 0; -static FILE *send_stream = 0; -static boolean_t do_byteswap = B_FALSE; -static boolean_t do_cksum = B_TRUE; - -/* - * ssread - send stream read. - * - * Read while computing incremental checksum - */ -static size_t -ssread(void *buf, size_t len, zio_cksum_t *cksum) -{ - size_t outlen; +typedef struct { + uint8_t drr_salt[ZIO_DATA_SALT_LEN]; + uint8_t drr_iv[ZIO_DATA_IV_LEN]; + uint8_t drr_mac[ZIO_DATA_MAC_LEN]; +} crypto_fields_t; - if ((outlen = fread(buf, len, 1, send_stream)) == 0) - return (0); +typedef void dumper_f(drr_packet_t *item); - if (do_cksum) { - if (do_byteswap) - fletcher_4_incremental_byteswap(buf, len, cksum); - else - fletcher_4_incremental_native(buf, len, cksum); - } - total_stream_len += len; - return (outlen); -} +typedef struct { + const char *rt_typename; + dumper_f *rt_dumper; +} record_type_t; -static size_t -read_hdr(dmu_replay_record_t *drr, zio_cksum_t *cksum) -{ - ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), - ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t)); - size_t r = ssread(drr, sizeof (*drr) - sizeof (zio_cksum_t), cksum); - if (r == 0) - return (0); - zio_cksum_t saved_cksum = *cksum; - r = ssread(&drr->drr_u.drr_checksum.drr_checksum, - sizeof (zio_cksum_t), cksum); - if (r == 0) - return (0); - if (do_cksum && - !ZIO_CHECKSUM_IS_ZERO(&drr->drr_u.drr_checksum.drr_checksum) && - !ZIO_CHECKSUM_EQUAL(saved_cksum, - drr->drr_u.drr_checksum.drr_checksum)) { - fprintf(stderr, "invalid checksum\n"); - (void) printf("Incorrect checksum in record header.\n"); - (void) printf("Expected checksum = %llx/%llx/%llx/%llx\n", - (longlong_t)saved_cksum.zc_word[0], - (longlong_t)saved_cksum.zc_word[1], - (longlong_t)saved_cksum.zc_word[2], - (longlong_t)saved_cksum.zc_word[3]); - return (0); - } - return (sizeof (*drr)); -} +static int stream_error; /* * Print part of a block in ASCII characters */ static void -print_ascii_block(char *subbuf, int length) +print_ascii_block(uint8_t *subbuf, int length) { int i; for (i = 0; i < length; i++) { char char_print = isprint(subbuf[i]) ? subbuf[i] : '.'; if (i != 0 && i % DUMP_GROUPING == 0) { - (void) printf(" "); + printf(" "); } - (void) printf("%c", char_print); + printf("%c", char_print); } - (void) printf("\n"); + printf("\n"); } /* * print_block - Dump the contents of a modified block to STDOUT * - * Assume that buf has capacity evenly divisible by BYTES_PER_LINE + * Assumes that buf has capacity evenly divisible by BYTES_PER_LINE */ static void -print_block(char *buf, int length) +print_block(uint8_t *buf, uint32_t length) { int i; /* @@ -194,623 +152,425 @@ sprintf_bytes(char *str, uint8_t *buf, uint_t buf_len) str[0] = '\0'; } -int -zstream_do_dump(int argc, char *argv[]) +static void +maybe_dump_payload(drr_packet_t *item) { - char *buf = safe_malloc(SPA_MAXBLOCKSIZE); - uint64_t drr_record_count[DRR_NUMTYPES] = { 0 }; - uint64_t total_payload_size = 0; - uint64_t total_overhead_size = 0; - uint64_t drr_byte_count[DRR_NUMTYPES] = { 0 }; - char salt[ZIO_DATA_SALT_LEN * 2 + 1]; - char iv[ZIO_DATA_IV_LEN * 2 + 1]; - char mac[ZIO_DATA_MAC_LEN * 2 + 1]; - uint64_t total_records = 0; - uint64_t payload_size; - dmu_replay_record_t thedrr; - dmu_replay_record_t *drr = &thedrr; - struct drr_begin *drrb = &thedrr.drr_u.drr_begin; - struct drr_end *drre = &thedrr.drr_u.drr_end; - struct drr_object *drro = &thedrr.drr_u.drr_object; - struct drr_freeobjects *drrfo = &thedrr.drr_u.drr_freeobjects; - struct drr_write *drrw = &thedrr.drr_u.drr_write; - struct drr_write_byref *drrwbr = &thedrr.drr_u.drr_write_byref; - struct drr_free *drrf = &thedrr.drr_u.drr_free; - struct drr_spill *drrs = &thedrr.drr_u.drr_spill; - struct drr_write_embedded *drrwe = &thedrr.drr_u.drr_write_embedded; - struct drr_object_range *drror = &thedrr.drr_u.drr_object_range; - struct drr_redact *drrr = &thedrr.drr_u.drr_redact; - struct drr_checksum *drrc = &thedrr.drr_u.drr_checksum; - int c; - boolean_t verbose = B_FALSE; - boolean_t very_verbose = B_FALSE; - boolean_t first = B_TRUE; - /* - * dump flag controls whether the contents of any modified data blocks - * are printed to the console during processing of the stream. Warning: - * for large streams, this can obviously lead to massive prints. - */ - boolean_t dump = B_FALSE; - int err; - zio_cksum_t zc = { { 0 } }; - zio_cksum_t pcksum = { { 0 } }; - - while ((c = getopt(argc, argv, ":vCd")) != -1) { - switch (c) { - case 'C': - do_cksum = B_FALSE; - break; - case 'v': - if (verbose) - very_verbose = B_TRUE; - verbose = B_TRUE; - break; - case 'd': - dump = B_TRUE; - verbose = B_TRUE; - very_verbose = B_TRUE; - break; - case ':': - (void) fprintf(stderr, - "missing argument for '%c' option\n", optopt); - zstream_usage(); - break; - case '?': - (void) fprintf(stderr, "invalid option '%c'\n", - optopt); - zstream_usage(); - break; - } + if (OPTION_ENABLED(CA_DUMP_DATA)) { + print_block(item->dp_payload, item->dp_payload_size); } +} - if (argc > optind) { - const char *filename = argv[optind]; - send_stream = fopen(filename, "r"); - if (send_stream == NULL) { - (void) fprintf(stderr, - "Error while opening file '%s': %s\n", - filename, strerror(errno)); - exit(1); - } - } else { - if (isatty(STDIN_FILENO)) { - (void) fprintf(stderr, - "Error: The send stream is a binary format " - "and can not be read from a\n" - "terminal. Standard input must be redirected, " - "or a file must be\n" - "specified as a command-line argument.\n"); - exit(1); - } - send_stream = stdin; - } +static char * +stringify_encryption_fields(void *crypto_in) +{ + crypto_fields_t *crypto = crypto_in; + char salt[sizeof (crypto->drr_salt) * 2 + 1]; + char iv[sizeof (crypto->drr_iv) * 2 + 1]; + char mac[sizeof (crypto->drr_mac) * 2 + 1]; + static char buff[sizeof (salt) + sizeof (iv) + sizeof (mac) + 32]; + + sprintf_bytes(salt, crypto->drr_salt, sizeof (crypto->drr_salt)); + sprintf_bytes(iv, crypto->drr_iv, sizeof (crypto->drr_iv)); + sprintf_bytes(mac, crypto->drr_mac, sizeof (crypto->drr_mac)); + snprintf(buff, sizeof (buff), "salt = %s iv = %s mac = %s", + salt, iv, mac); + return (buff); +} + +static void +dump_begin_record(drr_packet_t *item) +{ + dmu_replay_record_t *drr = &item->dp_drr; + struct drr_begin *drrb = &item->dp_drr.drr_u.drr_begin; - fletcher_4_init(); - while (read_hdr(drr, &zc)) { - uint64_t featureflags = 0; + printf("BEGIN record\n"); + printf("\thdrtype = %llu\n", + DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo)); + printf("\tfeatures = %llx\n", + DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo)); + printf("\tmagic = %llx\n", (u_longlong_t)drrb->drr_magic); + printf("\tcreation_time = %llx\n", + (u_longlong_t)drrb->drr_creation_time); + printf("\ttype = %u\n", drrb->drr_type); + printf("\tflags = 0x%x\n", drrb->drr_flags); + printf("\ttoguid = %llx\n", (u_longlong_t)drrb->drr_toguid); + printf("\tfromguid = %llx\n", (u_longlong_t)drrb->drr_fromguid); + printf("\ttoname = %s\n", drrb->drr_toname); + printf("\tpayloadlen = %u\n", drr->drr_payloadlen); + if (OPTION_ENABLED(CA_VERBOSE)) + printf("\n"); + + if (drr->drr_payloadlen >= 2) { + nvlist_t *nv; /* - * If this is the first DMU record being processed, check for - * the magic bytes and figure out the endian-ness based on them. + * It looks like zfs send or the ioctls it's using are + * generating packed nvlists with NV_ENCODE_NATIVE encoding + * in some circumstances. I don't think these can be decoded + * on an opposite-endian system, even by the core ZFS code. */ - if (first) { - if (drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) { - do_byteswap = B_TRUE; - if (do_cksum) { - ZIO_SET_CHECKSUM(&zc, 0, 0, 0, 0); - /* - * recalculate header checksum now - * that we know it needs to be - * byteswapped. - */ - fletcher_4_incremental_byteswap(drr, - sizeof (dmu_replay_record_t), &zc); - } - } else if (drrb->drr_magic != DMU_BACKUP_MAGIC) { - (void) fprintf(stderr, "Invalid stream " - "(bad magic number)\n"); - exit(1); - } - first = B_FALSE; + uint8_t *nvlist_header = item->dp_payload; + uint8_t nvlist_encoding = nvlist_header[0]; + boolean_t big_endian = nvlist_header[1] == 0; + if (nvlist_encoding == NV_ENCODE_XDR) { + printf("nvlist encoding = NV_ENCODE_XDR\n"); + } else { + printf("nvlist encoding = NV_ENCODE_NATIVE (%s)\n", + big_endian ? "big-endian" : "little-endian"); } - if (do_byteswap) { - drr->drr_type = BSWAP_32(drr->drr_type); - drr->drr_payloadlen = - BSWAP_32(drr->drr_payloadlen); + int err = nvlist_unpack((char *)item->dp_payload, + drr->drr_payloadlen, &nv, 0); + if (err) { + printf("failed to unpack DRR_BEGIN nvlist: %s\n", + strerror(err)); + if (!stream_error) + stream_error = err; + } else { + nvlist_print(stdout, nv); + nvlist_free(nv); } + } else if (drr->drr_payloadlen != 0) { + printf("unexpected packed nvlist length %d\n", + drr->drr_payloadlen); + } +} - /* - * At this point, the leading fields of the replay record - * (drr_type and drr_payloadlen) have been byte-swapped if - * necessary, but the rest of the data structure (the - * union of type-specific structures) is still in its - * original state. - */ - if (drr->drr_type >= DRR_NUMTYPES) { - (void) printf("INVALID record found: type 0x%x\n", - drr->drr_type); - (void) printf("Aborting.\n"); - exit(1); - } +static void +dump_end_record(drr_packet_t *item) +{ + struct drr_end *drre = &item->dp_drr.drr_u.drr_end; - drr_record_count[drr->drr_type]++; - total_overhead_size += sizeof (*drr); - total_records++; - payload_size = 0; + printf("END checksum = %llx/%llx/%llx/%llx\n", + (u_longlong_t)drre->drr_checksum.zc_word[0], + (u_longlong_t)drre->drr_checksum.zc_word[1], + (u_longlong_t)drre->drr_checksum.zc_word[2], + (u_longlong_t)drre->drr_checksum.zc_word[3]); +} - switch (drr->drr_type) { - case DRR_BEGIN: - if (do_byteswap) { - drrb->drr_magic = BSWAP_64(drrb->drr_magic); - drrb->drr_versioninfo = - BSWAP_64(drrb->drr_versioninfo); - drrb->drr_creation_time = - BSWAP_64(drrb->drr_creation_time); - drrb->drr_type = BSWAP_32(drrb->drr_type); - drrb->drr_flags = BSWAP_32(drrb->drr_flags); - drrb->drr_toguid = BSWAP_64(drrb->drr_toguid); - drrb->drr_fromguid = - BSWAP_64(drrb->drr_fromguid); - } +static void +dump_object_record(drr_packet_t *item) +{ + struct drr_object *drro = &item->dp_drr.drr_u.drr_object; - (void) printf("BEGIN record\n"); - (void) printf("\thdrtype = %lld\n", - DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo)); - (void) printf("\tfeatures = %llx\n", - DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo)); - (void) printf("\tmagic = %llx\n", - (u_longlong_t)drrb->drr_magic); - (void) printf("\tcreation_time = %llx\n", - (u_longlong_t)drrb->drr_creation_time); - (void) printf("\ttype = %u\n", drrb->drr_type); - (void) printf("\tflags = 0x%x\n", drrb->drr_flags); - (void) printf("\ttoguid = %llx\n", - (u_longlong_t)drrb->drr_toguid); - (void) printf("\tfromguid = %llx\n", - (u_longlong_t)drrb->drr_fromguid); - (void) printf("\ttoname = %s\n", drrb->drr_toname); - (void) printf("\tpayloadlen = %u\n", - drr->drr_payloadlen); - if (verbose) - (void) printf("\n"); + if (OPTION_ENABLED(CA_VERBOSE)) { + printf("OBJECT object = %llu type = %u " + "bonustype = %u blksz = %u bonuslen = %u " + "dn_slots = %u raw_bonuslen = %u " + "flags = %u maxblkid = %llu " + "indblkshift = %u nlevels = %u " + "nblkptr = %u\n", + (u_longlong_t)drro->drr_object, + drro->drr_type, + drro->drr_bonustype, + drro->drr_blksz, + drro->drr_bonuslen, + drro->drr_dn_slots, + drro->drr_raw_bonuslen, + drro->drr_flags, + (u_longlong_t)drro->drr_maxblkid, + drro->drr_indblkshift, + drro->drr_nlevels, + drro->drr_nblkptr); + } + if (drro->drr_bonuslen > 0) { + maybe_dump_payload(item); + } +} - if (drr->drr_payloadlen != 0) { - nvlist_t *nv; - int sz = drr->drr_payloadlen; +static void +dump_freeobjects_record(drr_packet_t *item) +{ + struct drr_freeobjects *drrfo = &item->dp_drr.drr_u.drr_freeobjects; - if (sz > SPA_MAXBLOCKSIZE) { - free(buf); - buf = safe_malloc(sz); - } - (void) ssread(buf, sz, &zc); - if (ferror(send_stream)) - perror("fread"); + if (OPTION_ENABLED(CA_VERBOSE)) { + printf("FREEOBJECTS firstobj = %llu numobjs = %llu\n", + (u_longlong_t)drrfo->drr_firstobj, + (u_longlong_t)drrfo->drr_numobjs); + } +} - uint8_t *nv_header = (uint8_t *)buf; - boolean_t xdr = nv_header[0] == NV_ENCODE_XDR; - boolean_t big_endian = nv_header[1] == 0; - const char *nc; - if (xdr) { - nc = "NV_ENCODE_XDR"; - } else if (big_endian) { - nc = "NV_ENCODE_NATIVE (big-endian)"; - } else { - nc = "NV_ENCODE_NATIVE (little-endian)"; - } - printf("nvlist encoding = %s\n", nc); +static void +dump_write_record(drr_packet_t *item) +{ + struct drr_write *drrw = &item->dp_drr.drr_u.drr_write; - err = nvlist_unpack(buf, sz, &nv, 0); - if (err) { - perror(strerror(err)); - } else { - nvlist_print(stdout, nv); - nvlist_free(nv); - } - payload_size = sz; - } - break; + if (OPTION_ENABLED(CA_VERBOSE)) { + printf("WRITE object = %llu type = %u " + "checksum type = %u compression type = %u " + "flags = %u offset = %llu " + "logical_size = %llu " + "compressed_size = %llu " + "payload_size = %u props = %llx " + "%s\n", + (u_longlong_t)drrw->drr_object, + drrw->drr_type, + drrw->drr_checksumtype, + drrw->drr_compressiontype, + drrw->drr_flags, + (u_longlong_t)drrw->drr_offset, + (u_longlong_t)drrw->drr_logical_size, + (u_longlong_t)drrw->drr_compressed_size, + item->dp_payload_size, + (u_longlong_t)drrw->drr_key.ddk_prop, + stringify_encryption_fields(&drrw->drr_salt)); + } + maybe_dump_payload(item); +} - case DRR_END: - if (do_byteswap) { - drre->drr_checksum.zc_word[0] = - BSWAP_64(drre->drr_checksum.zc_word[0]); - drre->drr_checksum.zc_word[1] = - BSWAP_64(drre->drr_checksum.zc_word[1]); - drre->drr_checksum.zc_word[2] = - BSWAP_64(drre->drr_checksum.zc_word[2]); - drre->drr_checksum.zc_word[3] = - BSWAP_64(drre->drr_checksum.zc_word[3]); - } - /* - * We compare against the *previous* checksum - * value, because the stored checksum is of - * everything before the DRR_END record. - */ - if (do_cksum && !ZIO_CHECKSUM_EQUAL(drre->drr_checksum, - pcksum)) { - (void) printf("Expected checksum differs from " - "checksum in stream.\n"); - (void) printf("Expected checksum = " - "%llx/%llx/%llx/%llx\n", - (long long unsigned int)pcksum.zc_word[0], - (long long unsigned int)pcksum.zc_word[1], - (long long unsigned int)pcksum.zc_word[2], - (long long unsigned int)pcksum.zc_word[3]); - } - (void) printf("END checksum = %llx/%llx/%llx/%llx\n", - (long long unsigned int) - drre->drr_checksum.zc_word[0], - (long long unsigned int) - drre->drr_checksum.zc_word[1], - (long long unsigned int) - drre->drr_checksum.zc_word[2], - (long long unsigned int) - drre->drr_checksum.zc_word[3]); +static void +dump_write_byref_record(drr_packet_t *item) +{ + struct drr_write_byref *drrwbr = &item->dp_drr.drr_u.drr_write_byref; - ZIO_SET_CHECKSUM(&zc, 0, 0, 0, 0); - break; + if (OPTION_ENABLED(CA_VERBOSE)) { + printf("WRITE_BYREF object = %llu " + "checksum type = %u props = %llx " + "offset = %llu length = %llu " + "toguid = %llx refguid = %llx " + "refobject = %llu refoffset = %llu\n", + (u_longlong_t)drrwbr->drr_object, + drrwbr->drr_checksumtype, + (u_longlong_t)drrwbr->drr_key.ddk_prop, + (u_longlong_t)drrwbr->drr_offset, + (u_longlong_t)drrwbr->drr_length, + (u_longlong_t)drrwbr->drr_toguid, + (u_longlong_t)drrwbr->drr_refguid, + (u_longlong_t)drrwbr->drr_refobject, + (u_longlong_t)drrwbr->drr_refoffset); + } +} - case DRR_OBJECT: - if (do_byteswap) { - drro->drr_object = BSWAP_64(drro->drr_object); - drro->drr_type = BSWAP_32(drro->drr_type); - drro->drr_bonustype = - BSWAP_32(drro->drr_bonustype); - drro->drr_blksz = BSWAP_32(drro->drr_blksz); - drro->drr_bonuslen = - BSWAP_32(drro->drr_bonuslen); - drro->drr_raw_bonuslen = - BSWAP_32(drro->drr_raw_bonuslen); - drro->drr_toguid = BSWAP_64(drro->drr_toguid); - drro->drr_maxblkid = - BSWAP_64(drro->drr_maxblkid); - } +static void +dump_free_record(drr_packet_t *item) +{ + struct drr_free *drrf = &item->dp_drr.drr_u.drr_free; - featureflags = - DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo); + if (OPTION_ENABLED(CA_VERBOSE)) { + printf("FREE object = %llu " + "offset = %llu length = %lld\n", + (u_longlong_t)drrf->drr_object, + (u_longlong_t)drrf->drr_offset, + (longlong_t)drrf->drr_length); + } +} - if (featureflags & DMU_BACKUP_FEATURE_RAW && - drro->drr_bonuslen > drro->drr_raw_bonuslen) { - (void) fprintf(stderr, - "Warning: Object %llu has bonuslen = " - "%u > raw_bonuslen = %u\n\n", - (u_longlong_t)drro->drr_object, - drro->drr_bonuslen, drro->drr_raw_bonuslen); - } +static void +dump_spill_record(drr_packet_t *item) +{ + struct drr_spill *drrs = &item->dp_drr.drr_u.drr_spill; - payload_size = DRR_OBJECT_PAYLOAD_SIZE(drro); + if (OPTION_ENABLED(CA_VERBOSE)) { + printf("SPILL block for object = %llu " + "length = %llu flags = %u " + "compression type = %u " + "compressed_size = %llu " + "payload_size = %u " + "%s\n", + (u_longlong_t)drrs->drr_object, + (u_longlong_t)drrs->drr_length, + drrs->drr_flags, + drrs->drr_compressiontype, + (u_longlong_t)drrs->drr_compressed_size, + item->dp_payload_size, + stringify_encryption_fields(&drrs->drr_salt)); + } + maybe_dump_payload(item); +} - if (verbose) { - (void) printf("OBJECT object = %llu type = %u " - "bonustype = %u blksz = %u bonuslen = %u " - "dn_slots = %u raw_bonuslen = %u " - "flags = %u maxblkid = %llu " - "indblkshift = %u nlevels = %u " - "nblkptr = %u\n", - (u_longlong_t)drro->drr_object, - drro->drr_type, - drro->drr_bonustype, - drro->drr_blksz, - drro->drr_bonuslen, - drro->drr_dn_slots, - drro->drr_raw_bonuslen, - drro->drr_flags, - (u_longlong_t)drro->drr_maxblkid, - drro->drr_indblkshift, - drro->drr_nlevels, - drro->drr_nblkptr); - } - if (drro->drr_bonuslen > 0) { - (void) ssread(buf, payload_size, &zc); - if (dump) - print_block(buf, payload_size); - } - break; +static void +dump_write_embedded_record(drr_packet_t *item) +{ + struct drr_write_embedded *drrwe = + &item->dp_drr.drr_u.drr_write_embedded; - case DRR_FREEOBJECTS: - if (do_byteswap) { - drrfo->drr_firstobj = - BSWAP_64(drrfo->drr_firstobj); - drrfo->drr_numobjs = - BSWAP_64(drrfo->drr_numobjs); - drrfo->drr_toguid = BSWAP_64(drrfo->drr_toguid); - } - if (verbose) { - (void) printf("FREEOBJECTS firstobj = %llu " - "numobjs = %llu\n", - (u_longlong_t)drrfo->drr_firstobj, - (u_longlong_t)drrfo->drr_numobjs); - } - break; + if (OPTION_ENABLED(CA_VERBOSE)) { + printf("WRITE_EMBEDDED object = %llu " + "offset = %llu length = %llu " + "toguid = %llx comp = %u etype = %u " + "lsize = %u psize = %u\n", + (u_longlong_t)drrwe->drr_object, + (u_longlong_t)drrwe->drr_offset, + (u_longlong_t)drrwe->drr_length, + (u_longlong_t)drrwe->drr_toguid, + drrwe->drr_compression, + drrwe->drr_etype, + drrwe->drr_lsize, + drrwe->drr_psize); + } + maybe_dump_payload(item); +} - case DRR_WRITE: - if (do_byteswap) { - drrw->drr_object = BSWAP_64(drrw->drr_object); - drrw->drr_type = BSWAP_32(drrw->drr_type); - drrw->drr_offset = BSWAP_64(drrw->drr_offset); - drrw->drr_logical_size = - BSWAP_64(drrw->drr_logical_size); - drrw->drr_toguid = BSWAP_64(drrw->drr_toguid); - drrw->drr_key.ddk_prop = - BSWAP_64(drrw->drr_key.ddk_prop); - drrw->drr_compressed_size = - BSWAP_64(drrw->drr_compressed_size); - } +static void +dump_object_range_record(drr_packet_t *item) +{ + struct drr_object_range *drror = &item->dp_drr.drr_u.drr_object_range; - payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); + if (OPTION_ENABLED(CA_VERBOSE)) { + printf("OBJECT_RANGE firstobj = %llu " + "numslots = %llu flags = %u " + "%s\n", + (u_longlong_t)drror->drr_firstobj, + (u_longlong_t)drror->drr_numslots, + drror->drr_flags, + stringify_encryption_fields(&drror->drr_salt)); + } +} - /* - * If this is verbose and/or dump output, - * print info on the modified block - */ - if (verbose) { - sprintf_bytes(salt, drrw->drr_salt, - ZIO_DATA_SALT_LEN); - sprintf_bytes(iv, drrw->drr_iv, - ZIO_DATA_IV_LEN); - sprintf_bytes(mac, drrw->drr_mac, - ZIO_DATA_MAC_LEN); +static void +dump_redact_record(drr_packet_t *item) +{ + struct drr_redact *drrr = &item->dp_drr.drr_u.drr_redact; - (void) printf("WRITE object = %llu type = %u " - "checksum type = %u compression type = %u " - "flags = %u offset = %llu " - "logical_size = %llu " - "compressed_size = %llu " - "payload_size = %llu props = %llx " - "salt = %s iv = %s mac = %s\n", - (u_longlong_t)drrw->drr_object, - drrw->drr_type, - drrw->drr_checksumtype, - drrw->drr_compressiontype, - drrw->drr_flags, - (u_longlong_t)drrw->drr_offset, - (u_longlong_t)drrw->drr_logical_size, - (u_longlong_t)drrw->drr_compressed_size, - (u_longlong_t)payload_size, - (u_longlong_t)drrw->drr_key.ddk_prop, - salt, - iv, - mac); - } + if (OPTION_ENABLED(CA_VERBOSE)) { + printf("REDACT object = %llu offset = " + "%llu length = %llu\n", + (u_longlong_t)drrr->drr_object, + (u_longlong_t)drrr->drr_offset, + (u_longlong_t)drrr->drr_length); + } +} - /* - * Read the contents of the block in from STDIN to buf - */ - (void) ssread(buf, payload_size, &zc); - /* - * If in dump mode - */ - if (dump) { - print_block(buf, payload_size); - } - break; +static disposition_t +chain_dump_record(drr_packet_t *item, record_type_t *context) +{ + if (item == NULL) { + return (D_OK); + } - case DRR_WRITE_BYREF: - if (do_byteswap) { - drrwbr->drr_object = - BSWAP_64(drrwbr->drr_object); - drrwbr->drr_offset = - BSWAP_64(drrwbr->drr_offset); - drrwbr->drr_length = - BSWAP_64(drrwbr->drr_length); - drrwbr->drr_toguid = - BSWAP_64(drrwbr->drr_toguid); - drrwbr->drr_refguid = - BSWAP_64(drrwbr->drr_refguid); - drrwbr->drr_refobject = - BSWAP_64(drrwbr->drr_refobject); - drrwbr->drr_refoffset = - BSWAP_64(drrwbr->drr_refoffset); - drrwbr->drr_key.ddk_prop = - BSWAP_64(drrwbr->drr_key.ddk_prop); - } - if (verbose) { - (void) printf("WRITE_BYREF object = %llu " - "checksum type = %u props = %llx " - "offset = %llu length = %llu " - "toguid = %llx refguid = %llx " - "refobject = %llu refoffset = %llu\n", - (u_longlong_t)drrwbr->drr_object, - drrwbr->drr_checksumtype, - (u_longlong_t)drrwbr->drr_key.ddk_prop, - (u_longlong_t)drrwbr->drr_offset, - (u_longlong_t)drrwbr->drr_length, - (u_longlong_t)drrwbr->drr_toguid, - (u_longlong_t)drrwbr->drr_refguid, - (u_longlong_t)drrwbr->drr_refobject, - (u_longlong_t)drrwbr->drr_refoffset); - } - break; + dmu_replay_record_t *drr = &item->dp_drr; + zio_cksum_t *cksum = &drr->drr_u.drr_checksum.drr_checksum; + int type = (int)drr->drr_type; - case DRR_FREE: - if (do_byteswap) { - drrf->drr_object = BSWAP_64(drrf->drr_object); - drrf->drr_offset = BSWAP_64(drrf->drr_offset); - drrf->drr_length = BSWAP_64(drrf->drr_length); - } - if (verbose) { - (void) printf("FREE object = %llu " - "offset = %llu length = %lld\n", - (u_longlong_t)drrf->drr_object, - (u_longlong_t)drrf->drr_offset, - (longlong_t)drrf->drr_length); - } - break; - case DRR_SPILL: - if (do_byteswap) { - drrs->drr_object = BSWAP_64(drrs->drr_object); - drrs->drr_length = BSWAP_64(drrs->drr_length); - drrs->drr_compressed_size = - BSWAP_64(drrs->drr_compressed_size); - drrs->drr_type = BSWAP_32(drrs->drr_type); - } + context[type].rt_dumper(item); - payload_size = DRR_SPILL_PAYLOAD_SIZE(drrs); + if (type != DRR_BEGIN && OPTION_ENABLED(CA_VERY_VERBOSE)) { + printf(" checksum = %llx/%llx/%llx/%llx\n", + (u_longlong_t)cksum->zc_word[0], + (u_longlong_t)cksum->zc_word[1], + (u_longlong_t)cksum->zc_word[2], + (u_longlong_t)cksum->zc_word[3]); + } - if (verbose) { - sprintf_bytes(salt, drrs->drr_salt, - ZIO_DATA_SALT_LEN); - sprintf_bytes(iv, drrs->drr_iv, - ZIO_DATA_IV_LEN); - sprintf_bytes(mac, drrs->drr_mac, - ZIO_DATA_MAC_LEN); + return (D_OK); +} - (void) printf("SPILL block for object = %llu " - "length = %llu flags = %u " - "compression type = %u " - "compressed_size = %llu " - "payload_size = %llu " - "salt = %s iv = %s mac = %s\n", - (u_longlong_t)drrs->drr_object, - (u_longlong_t)drrs->drr_length, - drrs->drr_flags, - drrs->drr_compressiontype, - (u_longlong_t)drrs->drr_compressed_size, - (u_longlong_t)payload_size, - salt, - iv, - mac); - } - (void) ssread(buf, payload_size, &zc); - if (dump) { - print_block(buf, payload_size); - } +static chain_step_t +serial_dump_records(record_type_t *context) +{ + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = sizeof (drr_packet_t), + .cs_context = context, + .cs_serial = { + .process = (zc_serial_process_f *)chain_dump_record + } + }; + return (step); +} + +int +zstream_do_dump(int argc, char *argv[]) +{ + chain_attrs_t attrs = {0}; + const char *input_file = NULL; + int c; + + record_type_t record_types[] = { + { "DRR_BEGIN", dump_begin_record }, + { "DRR_OBJECT", dump_object_record }, + { "DRR_FREEOBJECTS", dump_freeobjects_record }, + { "DRR_WRITE", dump_write_record }, + { "DRR_FREE", dump_free_record }, + { "DRR_END", dump_end_record }, + { "DRR_WRITE_BYREF", dump_write_byref_record }, + { "DRR_SPILL", dump_spill_record }, + { "DRR_WRITE_EMBEDDED", dump_write_embedded_record }, + { "DRR_OBJECT_RANGE", dump_object_range_record }, + { "DRR_REDACT", dump_redact_record } + }; + + while ((c = getopt(argc, argv, ":vCd")) != -1) { + switch (c) { + case 'C': + ENABLE_OPTION(&attrs, CA_IGNORE_CKSUMS); break; - case DRR_WRITE_EMBEDDED: - if (do_byteswap) { - drrwe->drr_object = - BSWAP_64(drrwe->drr_object); - drrwe->drr_offset = - BSWAP_64(drrwe->drr_offset); - drrwe->drr_length = - BSWAP_64(drrwe->drr_length); - drrwe->drr_toguid = - BSWAP_64(drrwe->drr_toguid); - drrwe->drr_lsize = - BSWAP_32(drrwe->drr_lsize); - drrwe->drr_psize = - BSWAP_32(drrwe->drr_psize); - } - if (verbose) { - (void) printf("WRITE_EMBEDDED object = %llu " - "offset = %llu length = %llu " - "toguid = %llx comp = %u etype = %u " - "lsize = %u psize = %u\n", - (u_longlong_t)drrwe->drr_object, - (u_longlong_t)drrwe->drr_offset, - (u_longlong_t)drrwe->drr_length, - (u_longlong_t)drrwe->drr_toguid, - drrwe->drr_compression, - drrwe->drr_etype, - drrwe->drr_lsize, - drrwe->drr_psize); - } - (void) ssread(buf, - P2ROUNDUP(drrwe->drr_psize, 8), &zc); - if (dump) { - print_block(buf, - P2ROUNDUP(drrwe->drr_psize, 8)); + case 'v': + if (attrs.ca_command_opts & CA_VERBOSE) { + ENABLE_OPTION(&attrs, CA_VERY_VERBOSE); + } else { + ENABLE_OPTION(&attrs, CA_VERBOSE); } - payload_size = P2ROUNDUP(drrwe->drr_psize, 8); break; - case DRR_OBJECT_RANGE: - if (do_byteswap) { - drror->drr_firstobj = - BSWAP_64(drror->drr_firstobj); - drror->drr_numslots = - BSWAP_64(drror->drr_numslots); - drror->drr_toguid = BSWAP_64(drror->drr_toguid); - } - if (verbose) { - sprintf_bytes(salt, drror->drr_salt, - ZIO_DATA_SALT_LEN); - sprintf_bytes(iv, drror->drr_iv, - ZIO_DATA_IV_LEN); - sprintf_bytes(mac, drror->drr_mac, - ZIO_DATA_MAC_LEN); - - (void) printf("OBJECT_RANGE firstobj = %llu " - "numslots = %llu flags = %u " - "salt = %s iv = %s mac = %s\n", - (u_longlong_t)drror->drr_firstobj, - (u_longlong_t)drror->drr_numslots, - drror->drr_flags, - salt, - iv, - mac); - } + case 'd': + ENABLE_OPTION(&attrs, CA_VERBOSE); + ENABLE_OPTION(&attrs, CA_VERY_VERBOSE); + ENABLE_OPTION(&attrs, CA_DUMP_DATA); break; - case DRR_REDACT: - if (do_byteswap) { - drrr->drr_object = BSWAP_64(drrr->drr_object); - drrr->drr_offset = BSWAP_64(drrr->drr_offset); - drrr->drr_length = BSWAP_64(drrr->drr_length); - drrr->drr_toguid = BSWAP_64(drrr->drr_toguid); - } - if (verbose) { - (void) printf("REDACT object = %llu offset = " - "%llu length = %llu\n", - (u_longlong_t)drrr->drr_object, - (u_longlong_t)drrr->drr_offset, - (u_longlong_t)drrr->drr_length); - } + case ':': + warnx("missing argument for '%c' option\n", optopt); + zstream_usage(); + break; + case '?': + warnx("invalid option '%c'\n", optopt); + zstream_usage(); break; - case DRR_NUMTYPES: - /* should never be reached */ - exit(1); - } - if (drr->drr_type != DRR_BEGIN && very_verbose) { - (void) printf(" checksum = %llx/%llx/%llx/%llx\n", - (longlong_t)drrc->drr_checksum.zc_word[0], - (longlong_t)drrc->drr_checksum.zc_word[1], - (longlong_t)drrc->drr_checksum.zc_word[2], - (longlong_t)drrc->drr_checksum.zc_word[3]); } - pcksum = zc; - drr_byte_count[drr->drr_type] += payload_size; - total_payload_size += payload_size; } - free(buf); - fletcher_4_fini(); - /* Print final summary */ + if (argc > optind) { + input_file = argv[optind]; + } + + zstream_chain_t dump_chain = { + STANDARD_INPUT_STACK(input_file), + serial_dump_records(record_types), + NULL_OUTPUT_STACK() + }; + + stream_error = 0; + zstream_chain_exec(dump_chain, &attrs); + + /* + * Match previous zstream dump summary order + */ + int print_order[] = { + DRR_BEGIN, DRR_END, DRR_OBJECT, DRR_FREEOBJECTS, + DRR_WRITE, DRR_WRITE_BYREF, DRR_WRITE_EMBEDDED, + DRR_FREE, DRR_SPILL, DRR_OBJECT_RANGE, DRR_REDACT + }; - (void) printf("SUMMARY:\n"); - (void) printf("\tTotal DRR_BEGIN records = %lld (%llu bytes)\n", - (u_longlong_t)drr_record_count[DRR_BEGIN], - (u_longlong_t)drr_byte_count[DRR_BEGIN]); - (void) printf("\tTotal DRR_END records = %lld (%llu bytes)\n", - (u_longlong_t)drr_record_count[DRR_END], - (u_longlong_t)drr_byte_count[DRR_END]); - (void) printf("\tTotal DRR_OBJECT records = %lld (%llu bytes)\n", - (u_longlong_t)drr_record_count[DRR_OBJECT], - (u_longlong_t)drr_byte_count[DRR_OBJECT]); - (void) printf("\tTotal DRR_FREEOBJECTS records = %lld (%llu bytes)\n", - (u_longlong_t)drr_record_count[DRR_FREEOBJECTS], - (u_longlong_t)drr_byte_count[DRR_FREEOBJECTS]); - (void) printf("\tTotal DRR_WRITE records = %lld (%llu bytes)\n", - (u_longlong_t)drr_record_count[DRR_WRITE], - (u_longlong_t)drr_byte_count[DRR_WRITE]); - (void) printf("\tTotal DRR_WRITE_BYREF records = %lld (%llu bytes)\n", - (u_longlong_t)drr_record_count[DRR_WRITE_BYREF], - (u_longlong_t)drr_byte_count[DRR_WRITE_BYREF]); - (void) printf("\tTotal DRR_WRITE_EMBEDDED records = %lld (%llu " - "bytes)\n", (u_longlong_t)drr_record_count[DRR_WRITE_EMBEDDED], - (u_longlong_t)drr_byte_count[DRR_WRITE_EMBEDDED]); - (void) printf("\tTotal DRR_FREE records = %lld (%llu bytes)\n", - (u_longlong_t)drr_record_count[DRR_FREE], - (u_longlong_t)drr_byte_count[DRR_FREE]); - (void) printf("\tTotal DRR_SPILL records = %lld (%llu bytes)\n", - (u_longlong_t)drr_record_count[DRR_SPILL], - (u_longlong_t)drr_byte_count[DRR_SPILL]); - (void) printf("\tTotal records = %lld\n", - (u_longlong_t)total_records); - (void) printf("\tTotal payload size = %lld (0x%llx)\n", - (u_longlong_t)total_payload_size, (u_longlong_t)total_payload_size); - (void) printf("\tTotal header overhead = %lld (0x%llx)\n", - (u_longlong_t)total_overhead_size, - (u_longlong_t)total_overhead_size); - (void) printf("\tTotal stream length = %lld (0x%llx)\n", - (u_longlong_t)total_stream_len, (u_longlong_t)total_stream_len); + printf("SUMMARY:\n"); + for (int i = 0; i < DRR_NUMTYPES; i++) { + int type = print_order[i]; + record_type_t *rec = &record_types[type]; + record_stats_t *stats = &attrs.ca_stats_in[type]; + printf("\tTotal %s records = %llu (%llu bytes)\n", + rec->rt_typename, + (u_longlong_t)stats->rs_num_records, + (u_longlong_t)stats->rs_total_payload_bytes); + } + + uint64_t total_payload = + attrs.ca_totals_in.rs_total_payload_bytes; + uint64_t total_header = + attrs.ca_totals_in.rs_total_header_bytes; + + printf("\tTotal records = %llu\n", + (u_longlong_t)attrs.ca_totals_in.rs_num_records); + printf("\tTotal payload size = %llu (0x%llx)\n", + (u_longlong_t)total_payload, (u_longlong_t)total_payload); + printf("\tTotal header overhead = %llu (0x%llx)\n", + (u_longlong_t)total_header, (u_longlong_t)total_header); + printf("\tTotal stream length = %llu (0x%llx)\n", + (u_longlong_t)(total_header + total_payload), + (u_longlong_t)(total_header + total_payload)); + + if (stream_error) { + fflush(stdout); + fprintf(stderr, "\nzstream dump completed with errors (first " + "error code %d)\n", stream_error); + exit(stream_error); + } return (0); } diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_fletcher4.c b/sys/contrib/openzfs/cmd/zstream/zstream_fletcher4.c new file mode 100644 index 000000000000..d7083e507a57 --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_fletcher4.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#include <assert.h> +#include <stddef.h> +#include <stdint.h> +#include <sys/byteorder.h> +#include <sys/spa_checksum.h> +#include <sys/stdtypes.h> +#include <sys/types.h> +#include <sys/zfs_ioctl.h> +#include <zfs_fletcher.h> + +#include "zstream_modules.h" +#include "zstream_util.h" + +#define CK_OFFSET offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum) +#define END_CK_OFFSET offsetof(dmu_replay_record_t, drr_u.drr_end.drr_checksum) + +typedef enum { F4_SET, F4_VALIDATE } fletcher4_op_t; + +typedef zio_cksum_t fletcher4_context_t; + +static fletcher4_context_t fletcher4_contexts[MAX_FLETCHER_4]; +static int next_context = 0; + +static inline int +fletcher_4_incremental(boolean_t swap, void *buff, size_t size, void *cksum) +{ + if (swap) { + return (fletcher_4_incremental_byteswap(buff, size, cksum)); + } else { + return (fletcher_4_incremental_native(buff, size, cksum)); + } +} + +/* + * Emit or validate (below) a replay record with proper checksums and with + * proper maintenance of the stream checksum. That is: + * + * 1) Update stream checksum with the record header up to drr_checksum. + * 2) Update drr_checksum field in the record header from stream checksum. + * 3) Update stream checksum with the checksum field in the record header. + * 4) Update stream checksum with the contents of the payload. + * + * DRR_BEGIN records do not have record checksums. They can't, because the + * drr_begin struct overlaps with space that would otherwise be used for the + * end-record checksum. + */ +static disposition_t +chain_add_fletcher4(drr_packet_t *item, zio_cksum_t *stream_cksum) +{ + if (item == NULL) + return (D_OK); + + dmu_replay_record_t *drr = &item->dp_drr; + struct drr_end *drre = &item->dp_drr.drr_u.drr_end; + zio_cksum_t *record_cksum = &drr->drr_u.drr_checksum.drr_checksum; + zio_cksum_t *end_cksum = &drre->drr_checksum; + + boolean_t swap = OPTION_ENABLED(CA_BYTESWAP_ON_OUTPUT); + uint32_t drr_type = swap ? BSWAP_32(drr->drr_type) : drr->drr_type; + + if (drr_type == DRR_BEGIN) { + ZIO_SET_CHECKSUM(stream_cksum, 0, 0, 0, 0); + } else if (drr_type == DRR_END) { + *end_cksum = *stream_cksum; + if (swap) + ZIO_CHECKSUM_BSWAP(end_cksum); + } + fletcher_4_incremental(swap, drr, CK_OFFSET, stream_cksum); + if (drr_type != DRR_BEGIN && !IS_CONCLUSION(drr, drr_type)) { + *record_cksum = *stream_cksum; + if (swap) + ZIO_CHECKSUM_BSWAP(record_cksum); + } + if (drr_type == DRR_END) { + ZIO_SET_CHECKSUM(stream_cksum, 0, 0, 0, 0); + } else { + fletcher_4_incremental(swap, record_cksum, + sizeof (drr->drr_u.drr_checksum.drr_checksum), + stream_cksum); + if (item->dp_payload_size > 0) { + fletcher_4_incremental(swap, item->dp_payload, + item->dp_payload_size, stream_cksum); + } + } + return (D_OK); +} + +static disposition_t +chain_validate_fletcher4(drr_packet_t *item, zio_cksum_t *stream_cksum) +{ + if (item == NULL || OPTION_ENABLED(CA_IGNORE_CKSUMS)) { + return (D_OK); + } + + dmu_replay_record_t *drr = &item->dp_drr; + struct drr_end *drre = &item->dp_drr.drr_u.drr_end; + zio_cksum_t *record_cksum = &drr->drr_u.drr_checksum.drr_checksum; + zio_cksum_t *end_cksum = &drre->drr_checksum; + + boolean_t swap = ATTR_IS_SET(CA_BYTESWAPPED); + uint32_t drr_type = swap ? BSWAP_32(drr->drr_type) : drr->drr_type; + + if (drr_type == DRR_BEGIN) { + ZIO_SET_CHECKSUM(stream_cksum, 0, 0, 0, 0); + } else if (drr_type == DRR_END) { + off_t stream_offset = item->dp_stream_offset + END_CK_OFFSET; + validate_or_exit(stream_cksum, end_cksum, swap, + "in DRR_END record", stream_offset); + } + fletcher_4_incremental(swap, drr, CK_OFFSET, stream_cksum); + if (drr_type != DRR_BEGIN && !IS_CONCLUSION(drr, drr_type)) { + off_t stream_offset = item->dp_stream_offset + CK_OFFSET; + validate_or_exit(stream_cksum, record_cksum, + swap, "at DRR record end", stream_offset); + } + if (drr_type == DRR_END) { + ZIO_SET_CHECKSUM(stream_cksum, 0, 0, 0, 0); + } else { + fletcher_4_incremental(swap, record_cksum, + sizeof (drr->drr_u.drr_checksum.drr_checksum), + stream_cksum); + if (item->dp_payload_size > 0) { + fletcher_4_incremental(swap, item->dp_payload, + item->dp_payload_size, stream_cksum); + } + } + return (D_OK); +} + +static chain_step_t +fletcher4_serial_step(fletcher4_op_t operation) +{ + int context_ix = next_context++ % MAX_FLETCHER_4; + fletcher4_context_t *context = &fletcher4_contexts[context_ix]; + + ZIO_SET_CHECKSUM(context, 0, 0, 0, 0); + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = sizeof (drr_packet_t), + .cs_context = context, + .cs_serial = { + .process = (zc_serial_process_f *) + ((operation == F4_VALIDATE) ? + chain_validate_fletcher4 : chain_add_fletcher4) + } + }; + return (step); +} + +chain_step_t +serial_add_fletcher4(void) +{ + return (fletcher4_serial_step(F4_SET)); +} + +chain_step_t +serial_validate_fletcher4(void) +{ + return (fletcher4_serial_step(F4_VALIDATE)); +} diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_fletcher4.h b/sys/contrib/openzfs/cmd/zstream/zstream_fletcher4.h new file mode 100644 index 000000000000..70c797ccb06b --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_fletcher4.h @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#ifndef _ZSTREAM_FLETCHER4_H +#define _ZSTREAM_FLETCHER4_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "zstream_io.h" + +/* + * zstream_chain module for calculating, validating, and inscribing + * Fletcher4 checksums. + * + * serial_validate_fletcher4() validates record checksums against the + * running stream checksum and fails loudly on any mismatch. + * + * serial_add_fletcher4() inscribes record checksums from the running + * stream checksum, in theory replacing whatever was there before. But + * see note in zstream_fletcher4.c regarding zero checksums generated + * by send_conclusion_record(), which are preserved. + */ + +/* + * DRR_END records normally do have end-record checksums. However, records + * emitted by send_conclusion_record() in libzfs_sendrecv.c have the + * checksum set to zero. zfs receive ignores those checksums. DRR_END + * records also have an internal checksum that applies to the stream-to-date + * since the most recent DRR_BEGIN. + * + * Ideally, null zstream transformations should be idempotent. E.g., a + * zstream redup that does not redup anything should yield a stream that is + * identical to the original stream. So, it's helpful to emulate zfs send's + * checksumming practices just to minimize spurious differences between + * input and output streams. + * + * The IS_CONCLUSION macro recognizes the records generated by + * send_conclusion_record() so that they can be treated specially. + */ +#define IS_CONCLUSION(drr, type) \ + ((type) == DRR_END && \ + (drr)->drr_u.drr_end.drr_toguid == 0 && \ + ZIO_CHECKSUM_IS_ZERO(&(drr)->drr_u.drr_checksum.drr_checksum)) + +/* + * Maximum number of checksum operations in one chain + */ +#define MAX_FLETCHER_4 8 + +chain_step_t +serial_validate_fletcher4(void); + +chain_step_t +serial_add_fletcher4(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _ZSTREAM_FLETCHER4_H */ diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_io.c b/sys/contrib/openzfs/cmd/zstream/zstream_io.c new file mode 100644 index 000000000000..47a0a1fe7773 --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_io.c @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#include <arpa/inet.h> +#include <err.h> +#include <errno.h> +#include <libzutil.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/byteorder.h> +#include <sys/stdtypes.h> +#include <sys/sysmacros.h> +#include <sys/types.h> +#include <sys/zfs_ioctl.h> +#include <time.h> +#include <unistd.h> + +#include "zstream_chain.h" +#include "zstream_modules.h" +#include "zstream_util.h" + +/* + * Init only the filename; chain functions will prepare the FILE * + */ +typedef struct { + const char *ic_filename; + FILE *ic_fp; + boolean_t ic_for_reading; + off_t ic_offset; +} io_context_t; + +typedef struct { + const char *cc_name; + double cc_last_sec; + double cc_period_sec; + uint64_t cc_last_bytes; +} checkpoint_context_t; + +static io_context_t io_contexts[MAX_IO_STREAMS]; +static int next_io_context = 0; + +static checkpoint_context_t checkpoint_contexts[MAX_IO_STREAMS]; +static int next_checkpoint_context = 0; + +/* + * Run from within chain execution to initialize I/O. A NULL filename + * indicates stdin or stdout. + */ +static void +open_file(io_context_t *context) +{ + if (context->ic_filename) { + context->ic_fp = fopen(context->ic_filename, + context->ic_for_reading ? "rb" : "wb+"); + if (!context->ic_fp) { + perror(context->ic_filename); + exit(1); + } + } else if (context->ic_for_reading && isatty(STDIN_FILENO)) { + errx(1, "stream cannot be read from a terminal. " + "Name a file or take input from a pipe."); + } else if (context->ic_for_reading) { + context->ic_fp = stdin; + } else if (isatty(STDOUT_FILENO)) { + errx(1, "stream cannot be written to a terminal. " + "Capture output to a file or pipe to another command."); + } else { + context->ic_fp = stdout; + } +} + +/* + * Extract the payload size from a replay record that is potentially + * byteswapped. We want to leave the bulk of byteswapping to another module, + * so just take a quick, nondestructive peek. + * + * Record-specific macros such as DRR_WRITE_PAYLOAD_SIZE do not seem to be + * byteswap-aware. However, with the exception of DRR_OBJECT_PAYLOAD_SIZE, + * they happen to work with post-swapping since they are switching on either + * a uint8_t value or 0. + * + * DRR_WRITE and DRR_SPILL use 64-bit sizes. The other two record types have + * 32-bit sizes. The drr_payloadlen field shared by all record types (but + * used only by BEGIN records is also 32 bits. + */ +static size_t +calc_payload_size(dmu_replay_record_t *drr) +{ + struct drr_object *drro = &drr->drr_u.drr_object; + struct drr_write *drrw = &drr->drr_u.drr_write; + struct drr_spill *drrs = &drr->drr_u.drr_spill; + struct drr_write_embedded *drrwe = &drr->drr_u.drr_write_embedded; + + boolean_t swap = ATTR_IS_SET(CA_BYTESWAPPED); + uint32_t drr_type = swap ? BSWAP_32(drr->drr_type) : drr->drr_type; + uint64_t size, size64 = 0; + uint32_t size32 = 0; + boolean_t round = B_FALSE; + + if (drr_type == DRR_OBJECT) { + round = drro->drr_raw_bonuslen == 0; + size32 = round ? drro->drr_bonuslen : drro->drr_raw_bonuslen; + } else if (drr_type == DRR_WRITE) { + size64 = DRR_WRITE_PAYLOAD_SIZE(drrw); + } else if (drr_type == DRR_SPILL) { + size64 = DRR_SPILL_PAYLOAD_SIZE(drrs); + } else if (drr_type == DRR_WRITE_EMBEDDED) { + size32 = drrwe->drr_psize; + round = B_TRUE; + } else if (drr_type == DRR_BEGIN) { + size32 = drr->drr_payloadlen; + } else { + return (0); + } + if (size32 != 0) { + size = swap ? BSWAP_32(size32) : size32; + } else { + size = swap ? BSWAP_64(size64) : size64; + } + return (round ? P2ROUNDUP(size, 8) : size); +} + +/* + * Must be called only with the first record in a stream. Must be a + * DRR_BEGIN record or we'll terminate with "invalid stream". + */ +static void +set_stream_attributes(drr_packet_t *item) +{ + dmu_replay_record_t *drr = &item->dp_drr; + struct drr_begin *drrb = &drr->drr_u.drr_begin; + uint64_t magic = drrb->drr_magic; + uint64_t versioninfo = drrb->drr_versioninfo; + boolean_t i_am_big_endian = htonl(0xFF00) == 0xFF00; + + boolean_t swap_on_output, is_deduped; + + if (magic == BSWAP_64(DMU_BACKUP_MAGIC)) { + SET_ATTR(CA_BYTESWAPPED); + versioninfo = BSWAP_64(versioninfo); + } else if (magic != DMU_BACKUP_MAGIC) { + errx(1, "invalid ZFS stream, bad magic number %llx", + (u_longlong_t)magic); + } + if (i_am_big_endian == ATTR_IS_SET(CA_BYTESWAPPED)) { + SET_ATTR(CA_LITTLE_ENDIAN_INPUT); + } else { + SET_ATTR(CA_BIG_ENDIAN_INPUT); + } + chain_attrs->ca_feature_flags = DMU_GET_FEATUREFLAGS(versioninfo); + + is_deduped = + STREAM_HAS_FEATURE(DMU_BACKUP_FEATURE_DEDUP) || + STREAM_HAS_FEATURE(DMU_BACKUP_FEATURE_DEDUPPROPS); + + if (OPTION_ENABLED(CA_FORBID_DEDUP) && is_deduped) { + errx(1, "input stream is deduplicated, but this subcommand " + "does not support deduplicated streams. Use 'zstream " + "redup' to reduplicate."); + } + boolean_t req_dedup = OPTION_ENABLED(CA_REQUIRE_DEDUP); + boolean_t is_dedup = STREAM_HAS_FEATURE(DMU_BACKUP_FEATURE_DEDUP); + if (req_dedup && !is_dedup) { + errx(1, "this subcommand requires a deduplicated input " + "stream, but the stream is not deduplicated"); + } + boolean_t req_native = OPTION_ENABLED(CA_REQUIRE_NATIVE_ENDIAN); + boolean_t is_byteswapped = ATTR_IS_SET(CA_BYTESWAPPED); + if (req_native && is_byteswapped) { + errx(1, "this subcommand requires a native-endian " + "input stream"); + } + + /* + * Figure out output endianness. In the absence of explicit byte + * order instructions, we default to preserving the input byte + * order. Record headers are always converted to native byte order + * for processing, but they can be swapped back on output. + * + * zfs receive inspects the endianness of each DRR record + * and assumes, at least in some cases, that payload data has the + * same order as the DMU wrappers. + */ + if (OPTION_ENABLED(CA_BIG_ENDIAN_OUT)) + swap_on_output = !i_am_big_endian; + else if (OPTION_ENABLED(CA_LITTLE_ENDIAN_OUT)) + swap_on_output = i_am_big_endian; + else if (OPTION_ENABLED(CA_OPPOSITE_ENDIAN_OUT)) + swap_on_output = !ATTR_IS_SET(CA_BYTESWAPPED); + else + swap_on_output = ATTR_IS_SET(CA_BYTESWAPPED); + + if (swap_on_output) { + ENABLE_OPTION(chain_attrs, CA_BYTESWAP_ON_OUTPUT); + } +} + +static disposition_t +chain_read(drr_packet_t *item, io_context_t *context) +{ + if (item == NULL) + return (D_OK); + + dmu_replay_record_t *drr = &item->dp_drr; + + if (!context->ic_fp) + open_file(context); + + if (fread(drr, sizeof (dmu_replay_record_t), 1, context->ic_fp) != 1) { + if (ferror(context->ic_fp)) { + err(1, "error reading record header at offset %llu", + (u_longlong_t)context->ic_offset); + } + fclose(context->ic_fp); + return (D_EOF); + } + + if (context->ic_offset == 0) + set_stream_attributes(item); + + size_t payload_size = calc_payload_size(&item->dp_drr); + if (payload_size > UINT32_MAX) { + errx(1, "stated packet size is greater than uint32_t" + "at offset %llu", (u_longlong_t)context->ic_offset); + } + item->dp_payload_size = payload_size; + if (item->dp_payload_size > 0) { + item->dp_payload = safe_malloc(item->dp_payload_size); + size_t n_read = fread(item->dp_payload, item->dp_payload_size, + 1, context->ic_fp); + if (n_read != 1) { + if (ferror(context->ic_fp)) { + err(1, "error reading record payload at " + " offset %llu", + (u_longlong_t)context->ic_offset); + } else { + /* + * We can't exit here because the ZFS test + * suite depends on being able to process + * streams truncated at random places. + */ + warnx("input ends mid-record at offset %llu " + "- stream is likely corrupt", + (u_longlong_t)context->ic_offset); + fclose(context->ic_fp); + free(item->dp_payload); + return (D_EOF); + } + } + } else { + item->dp_payload = NULL; + } + item->dp_stream_offset = context->ic_offset; + + uint32_t drr_type = ATTR_IS_SET(CA_BYTESWAPPED) ? + BSWAP_32(drr->drr_type) : drr->drr_type; + + if (drr_type >= DRR_NUMTYPES) { + err(1, "invalid record type %llu found at offset %llu", + (u_longlong_t)drr_type, (u_longlong_t)context->ic_offset); + } + + context->ic_offset += sizeof (*drr) + item->dp_payload_size; + + record_stats_t *stats = &chain_attrs->ca_stats_in[drr_type]; + stats->rs_num_records++; + stats->rs_total_header_bytes += sizeof (dmu_replay_record_t); + stats->rs_total_payload_bytes += item->dp_payload_size; + + stats = &chain_attrs->ca_totals_in; + stats->rs_num_records++; + stats->rs_total_header_bytes += sizeof (dmu_replay_record_t); + stats->rs_total_payload_bytes += item->dp_payload_size; + + return (D_OK); +} + +static disposition_t +chain_write(drr_packet_t *item, io_context_t *context) +{ + if (item == NULL) { + if (context->ic_fp) { + if (fclose(context->ic_fp) != 0) + err(1, "error closing output stream"); + context->ic_fp = NULL; + } + return (D_OK); + } + + if (!context->ic_fp) { + open_file(context); + } + + dmu_replay_record_t *drr = &item->dp_drr; + + if (fwrite(drr, sizeof (dmu_replay_record_t), 1, context->ic_fp) != 1) { + err(1, "error writing record header"); + } else if (item->dp_payload_size > 0) { + size_t n_written = fwrite(item->dp_payload, + item->dp_payload_size, 1, context->ic_fp); + if (n_written != 1) { + err(1, "error writing payload"); + } else { + free(item->dp_payload); + item->dp_payload = NULL; + } + } + + uint32_t drr_type = OPTION_ENABLED(CA_BYTESWAP_ON_OUTPUT) ? + BSWAP_32(drr->drr_type) : drr->drr_type; + + record_stats_t *stats = &chain_attrs->ca_stats_out[drr_type]; + stats->rs_num_records++; + stats->rs_total_header_bytes += sizeof (dmu_replay_record_t); + stats->rs_total_payload_bytes += item->dp_payload_size; + + stats = &chain_attrs->ca_totals_out; + stats->rs_num_records++; + stats->rs_total_header_bytes += sizeof (dmu_replay_record_t); + stats->rs_total_payload_bytes += item->dp_payload_size; + + return (D_OK); +} + +/* + * Even if the chain doesn't write out a stream, payloads still need freed. + */ +static disposition_t +chain_null_output(drr_packet_t *item, void *context) +{ + (void) context; + if (item && item->dp_payload != NULL && item->dp_payload_size > 0) { + free(item->dp_payload); + item->dp_payload = NULL; + item->dp_payload_size = 0; + } + return (D_OK); +} + +/* + * Storage for the filename must remain valid during chain execution + */ +static chain_step_t +setup_io(const char *filename, boolean_t for_reading) +{ + int context_num = next_io_context++ % MAX_IO_STREAMS; + + io_context_t context = { + .ic_filename = filename, + .ic_for_reading = for_reading + }; + io_contexts[context_num] = context; + + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = 0, + .cs_out_size = sizeof (drr_packet_t), + .cs_context = &io_contexts[context_num], + .cs_serial = { + .process = (zc_serial_process_f *) + (for_reading ? chain_read : chain_write), + } + }; + return (step); +} + +chain_step_t +serial_read_stream(const char *filename) +{ + return (setup_io(filename, B_TRUE)); +} + +chain_step_t +serial_write_stream(const char *filename) +{ + return (setup_io(filename, B_FALSE)); +} + +chain_step_t +serial_null_output(void) +{ + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = 0, + .cs_context = NULL, + .cs_serial = { + .process = (zc_serial_process_f *)chain_null_output + } + }; + return (step); +} + +static disposition_t +chain_checkpoint(drr_packet_t *item, checkpoint_context_t *ctxt) +{ + struct timespec now; + char buff[32]; + uint64_t delta_b, dbdt; + double now_sec, delta_t; + + if (item == NULL) + return (D_OK); + + clock_gettime(CLOCK_MONOTONIC, &now); + now_sec = now.tv_sec + (double)now.tv_nsec / 1E9; + if (ctxt->cc_last_sec > 1E-9) { + delta_t = now_sec - ctxt->cc_last_sec; + if (delta_t < ctxt->cc_period_sec) + return (D_OK); + delta_b = item->dp_stream_offset - ctxt->cc_last_bytes; + dbdt = delta_b / delta_t; + zfs_nicenum(dbdt, buff, sizeof (buff)); + fprintf(stderr, "Checkpoint %s: %s/s\n", ctxt->cc_name, buff); + } + ctxt->cc_last_sec = now_sec; + ctxt->cc_last_bytes = item->dp_stream_offset; + return (D_OK); +} + +/* + * Storage for name must remain valid throughout chain execution + */ +chain_step_t +serial_checkpoint(const char *name) +{ + int context_no = next_checkpoint_context++ % MAX_IO_STREAMS; + + checkpoint_context_t context = { + .cc_name = name, + .cc_period_sec = 1.0 + }; + checkpoint_contexts[context_no] = context; + + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = sizeof (drr_packet_t), + .cs_context = &checkpoint_contexts[context_no], + .cs_serial = { + .process = (zc_serial_process_f *)chain_checkpoint + }, + }; + return (step); +} diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_io.h b/sys/contrib/openzfs/cmd/zstream/zstream_io.h new file mode 100644 index 000000000000..e5239f92cd72 --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_io.h @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#ifndef _ZSTREAM_IO_H +#define _ZSTREAM_IO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include <sys/types.h> +#include <sys/zfs_ioctl.h> + +#include "zstream_chain.h" + +#define MAX_IO_STREAMS 4 + +/* + * The stream offset is the offset within the original source stream. + * Changes to the stream (e.g., recompression) will necessarily change + * offsets within the final stream. The original stream offset is raw data; + * it should never be updated. + */ +typedef struct { + dmu_replay_record_t dp_drr; + uint8_t *dp_payload; + uint32_t dp_payload_size; + off_t dp_stream_offset; +} drr_packet_t; + +/* + * In the following, the filename or checkpoint names must remain valid + * as long as the chain is executing. + */ + +chain_step_t +serial_read_stream(const char *filename); + +chain_step_t +serial_write_stream(const char *filename); + +/* Report throughput periodically */ +chain_step_t +serial_checkpoint(const char *name); + +/* + * Usually the output step is responsible for freeing payloads. Subcommands + * that don't have stream outputs still need to free this memory. A + * serial_null_output step does this and nothing more. + */ +chain_step_t +serial_null_output(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _ZSTREAM_IO_H */ diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_modules.h b/sys/contrib/openzfs/cmd/zstream/zstream_modules.h new file mode 100644 index 000000000000..143beef5c793 --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_modules.h @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#ifndef _ZSTREAM_MODULES_H +#define _ZSTREAM_MODULES_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This file aggregates all zstream_chain utility modules into a single + * header and defines macros for standard input and output operations. + */ + +#include "zstream_byteswap.h" +#include "zstream_chain.h" +#include "zstream_fletcher4.h" +#include "zstream_io.h" +#include "zstream_recompress.h" +#include "zstream_util.h" +#include "zstream_validate.h" + +#define READ_STEP 0 + +#define STANDARD_INPUT_STACK(infile) \ + serial_read_stream(infile), \ + serial_validate_fletcher4(), \ + serial_byteswap(BS_INCOMING), \ + serial_validate_records() + +#define STANDARD_OUTPUT_STACK(outfile) \ + serial_byteswap(BS_OUTGOING), \ + serial_add_fletcher4(), \ + serial_write_stream(outfile), \ + chain_terminator() + +#define NULL_OUTPUT_STACK() \ + serial_null_output(), \ + chain_terminator() + +#ifdef __cplusplus +} +#endif + +#endif /* _ZSTREAM_MODULES_H */ diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_recompress.c b/sys/contrib/openzfs/cmd/zstream/zstream_recompress.c index f5abfa98b18f..160deea05ffb 100644 --- a/sys/contrib/openzfs/cmd/zstream/zstream_recompress.c +++ b/sys/contrib/openzfs/cmd/zstream/zstream_recompress.c @@ -26,335 +26,269 @@ * * Copyright (c) 2022 by Delphix. All rights reserved. * Copyright (c) 2024, Klara, Inc. + * Copyright (c) 2026 by Garth Snyder */ +#include <assert.h> #include <err.h> +#include <stdint.h> #include <stdio.h> #include <stdlib.h> -#include <unistd.h> +#include <string.h> #include <sys/zfs_ioctl.h> -#include <sys/zio_checksum.h> +#include <sys/zio_compress.h> #include <sys/zstd/zstd.h> -#include "zfs_fletcher.h" +#include <unistd.h> +#include <sys/stdtypes.h> + #include "zstream.h" +#include "zstream_chain.h" +#include "zstream_modules.h" #include "zstream_util.h" -int -zstream_do_recompress(int argc, char *argv[]) +#define MAX_COMPRESSION_STEPS 4 + +static compression_spec_t specs[MAX_COMPRESSION_STEPS]; +static int next_spec = 0; + +/* + * Item is known to be a DRR_WRITE packet. Determine whether current + * compression is compatible with desired compression and whether the + * current record is modifiable at all. + */ +static boolean_t +needs_modification(drr_packet_t *item, compression_spec_t *target) { - int bufsz = SPA_MAXBLOCKSIZE; - char *buf = safe_malloc(bufsz); - dmu_replay_record_t thedrr; - dmu_replay_record_t *drr = &thedrr; - zio_cksum_t stream_cksum; - int c; - int level = 0; + dmu_replay_record_t *drr = &item->dp_drr; + struct drr_write *drrw = &drr->drr_u.drr_write; + enum zio_compress ctype = drrw->drr_compressiontype; + uint8_t cur_level; - while ((c = getopt(argc, argv, "l:")) != -1) { - switch (c) { - case 'l': - if (sscanf(optarg, "%d", &level) != 1) { - fprintf(stderr, - "failed to parse level '%s'\n", - optarg); - zstream_usage(); - } - break; - case '?': - (void) fprintf(stderr, "invalid option '%c'\n", - optopt); - zstream_usage(); - break; + /* + * Do not modify metadata records. It's a general stream invariant + * that metadata is never compressed. See comments at + * dmu_receive.c:flush_write_batch_impl(). + */ + if (DMU_OT_IS_METADATA(drrw->drr_type)) { + return (B_FALSE); + } + boolean_t ctype_uncompressed = ctype_is_uncompressed(ctype); + if (target == NULL) { + return (!ctype_uncompressed && !write_is_encrypted(drrw)); + } + boolean_t target_uncompressed = ctype_is_uncompressed(target->cs_type); + if (target_uncompressed && ctype_uncompressed) { + return (B_FALSE); + } + /* + * In order to recompress an encrypted block, you have to decrypt, + * decompress, recompress, and re-encrypt. That can be a future + * enhancement (along with decryption or re-encryption), but for now + * we skip encrypted blocks. + */ + if (write_is_encrypted(drrw)) { + return (B_FALSE); + } + if (ctype != target->cs_type) { + return (B_TRUE); + } + if (target->cs_type == ZIO_COMPRESS_ZSTD) { + cur_level = zfs_get_hdrlevel((void *)item->dp_payload); + if (target->cs_level == ZIO_COMPLEVEL_DEFAULT) { + return (cur_level != ZIO_ZSTD_LEVEL_DEFAULT); } + return (target->cs_level != cur_level); } + return (B_FALSE); +} - argc -= optind; - argv += optind; +static boolean_t +needs_compression(drr_packet_t *item, compression_spec_t *context) +{ + return (needs_modification(item, context)); +} - if (argc != 1) - zstream_usage(); +/* + * Don't decompress packets that aren't compressed. And don't decompress + * them if their ultimate fate is to be recompressed using the compression + * profile that's already in use. + */ +static boolean_t +needs_decompression(drr_packet_t *item, compression_spec_t *context) +{ + dmu_replay_record_t *drr = &item->dp_drr; + struct drr_write *drrw = &drr->drr_u.drr_write; + enum zio_compress ctype = drrw->drr_compressiontype; - enum zio_compress ctype; - if (strcmp(argv[0], "off") == 0) { - ctype = ZIO_COMPRESS_OFF; - } else { - for (ctype = 0; ctype < ZIO_COMPRESS_FUNCTIONS; ctype++) { - if (strcmp(argv[0], - zio_compress_table[ctype].ci_name) == 0) - break; - } - if (ctype == ZIO_COMPRESS_FUNCTIONS || - zio_compress_table[ctype].ci_compress == NULL) { - fprintf(stderr, "Invalid compression type %s.\n", - argv[0]); - exit(2); - } - } + if (ctype_is_uncompressed(ctype)) + return (B_FALSE); + return (needs_modification(item, context)); +} - if (isatty(STDIN_FILENO)) { - (void) fprintf(stderr, - "Error: The send stream is a binary format " - "and can not be read from a\n" - "terminal. Standard input must be redirected.\n"); - exit(1); - } +static disposition_t +chain_decompress_writes(drr_packet_t *item, compression_spec_t *context) +{ + if (item == NULL) + return (D_OK); - zfs_refcount_init(); - abd_init(); - fletcher_4_init(); - zio_init(); - zstd_init(); - int begin = 0; - boolean_t seen = B_FALSE; - while (sfread(drr, sizeof (*drr), stdin) != 0) { - struct drr_write *drrw; - uint64_t payload_size = 0; + dmu_replay_record_t *drr = &item->dp_drr; + struct drr_write *drrw = &drr->drr_u.drr_write; + uint8_t *debuff; - /* - * We need to regenerate the checksum. - */ - if (drr->drr_type != DRR_BEGIN) { - memset(&drr->drr_u.drr_checksum.drr_checksum, 0, - sizeof (drr->drr_u.drr_checksum.drr_checksum)); - } + if (drr->drr_type != DRR_WRITE || !needs_decompression(item, context)) { + return (D_OK); + } + debuff = decompress_buffer(item->dp_payload, item->dp_payload_size, + drrw->drr_logical_size, drrw->drr_compressiontype); + if (debuff == NULL) { + errx(4, "decompression type %d failed for ino %llu offset %llu", + drrw->drr_compressiontype, + (u_longlong_t)drrw->drr_object, + (u_longlong_t)drrw->drr_offset); + } + free(item->dp_payload); + item->dp_payload = debuff; + item->dp_payload_size = drrw->drr_logical_size; + drrw->drr_compressed_size = 0; + drrw->drr_compressiontype = 0; + return (D_OK); +} - switch (drr->drr_type) { - case DRR_BEGIN: - { - ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); - VERIFY0(begin++); - seen = B_TRUE; +static disposition_t +chain_compress_writes(drr_packet_t *item, compression_spec_t *context) +{ + if (item == NULL) + return (D_OK); - uint32_t sz = drr->drr_payloadlen; + dmu_replay_record_t *drr = &item->dp_drr; - VERIFY3U(sz, <=, 1U << 28); + if (drr->drr_type != DRR_WRITE || !needs_compression(item, context)) { + return (D_OK); + } - if (sz != 0) { - if (sz > bufsz) { - buf = realloc(buf, sz); - if (buf == NULL) - err(1, "realloc"); - bufsz = sz; - } - (void) sfread(buf, sz, stdin); - } - payload_size = sz; - break; - } - case DRR_END: - { - struct drr_end *drre = &drr->drr_u.drr_end; - /* - * We would prefer to just check --begin == 0, but - * replication streams have an end of stream END - * record, so we must avoid tripping it. - */ - VERIFY3B(seen, ==, B_TRUE); - begin--; - /* - * Use the recalculated checksum, unless this is - * the END record of a stream package, which has - * no checksum. - */ - if (!ZIO_CHECKSUM_IS_ZERO(&drre->drr_checksum)) - drre->drr_checksum = stream_cksum; - break; - } + struct drr_write *drrw = &drr->drr_u.drr_write; + enum zio_compress ctype = drrw->drr_compressiontype; + uint8_t *cbuff; + size_t csize; - case DRR_OBJECT: - { - struct drr_object *drro = &drr->drr_u.drr_object; - VERIFY3S(begin, ==, 1); + VERIFY3B(ctype_is_uncompressed(ctype), ==, B_TRUE); + cbuff = compress_buffer(item->dp_payload, item->dp_payload_size, + *context, &csize); + if (cbuff == NULL) { + drrw->drr_compressiontype = 0; + drrw->drr_compressed_size = 0; + } else { + free(item->dp_payload); + item->dp_payload = cbuff; + item->dp_payload_size = csize; + drrw->drr_compressed_size = csize; + drrw->drr_compressiontype = context->cs_type; + } + return (D_OK); +} - if (drro->drr_bonuslen > 0) { - payload_size = DRR_OBJECT_PAYLOAD_SIZE(drro); - (void) sfread(buf, payload_size, stdin); - } - break; - } +/* + * Decompress writes, but only if they don't match a target compression + * type. Pass NULL to uncompress unconditionally (if not already + * uncompressed). + */ +chain_step_t +serial_decompress_writes(compression_spec_t *target) +{ + int this_spec = next_spec++ % MAX_COMPRESSION_STEPS; + compression_spec_t *context = &specs[this_spec]; - case DRR_SPILL: - { - struct drr_spill *drrs = &drr->drr_u.drr_spill; - VERIFY3S(begin, ==, 1); - payload_size = DRR_SPILL_PAYLOAD_SIZE(drrs); - (void) sfread(buf, payload_size, stdin); - break; + if (target == NULL) { + context = NULL; + } else { + *context = *target; + } + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = sizeof (drr_packet_t), + .cs_context = context, + .cs_serial = { + .process = (zc_serial_process_f *)chain_decompress_writes } + }; + return (step); +} - case DRR_WRITE_BYREF: - VERIFY3S(begin, ==, 1); - fprintf(stderr, - "Deduplicated streams are not supported\n"); - exit(1); - break; - - case DRR_WRITE: - { - VERIFY3S(begin, ==, 1); - drrw = &thedrr.drr_u.drr_write; - payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); - /* - * In order to recompress an encrypted block, you have - * to decrypt, decompress, recompress, and - * re-encrypt. That can be a future enhancement (along - * with decryption or re-encryption), but for now we - * skip encrypted blocks. - */ - boolean_t encrypted = B_FALSE; - for (int i = 0; i < ZIO_DATA_SALT_LEN; i++) { - if (drrw->drr_salt[i] != 0) { - encrypted = B_TRUE; - break; - } - } - if (encrypted) { - (void) sfread(buf, payload_size, stdin); - break; - } - enum zio_compress dtype = drrw->drr_compressiontype; - if (dtype >= ZIO_COMPRESS_FUNCTIONS) { - fprintf(stderr, "Invalid compression type in " - "stream: %d\n", dtype); - exit(3); - } - if (zio_compress_table[dtype].ci_decompress == NULL) - dtype = ZIO_COMPRESS_OFF; +chain_step_t +serial_compress_writes(compression_spec_t *target) +{ + int this_spec = next_spec++ % MAX_COMPRESSION_STEPS; + compression_spec_t *context = &specs[this_spec]; - /* Set up buffers to minimize memcpys */ - char *cbuf, *dbuf; - if (ctype == ZIO_COMPRESS_OFF) - dbuf = buf; - else - dbuf = safe_calloc(bufsz); + VERIFY3P(target, !=, NULL); + *context = *target; + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = sizeof (drr_packet_t), + .cs_context = context, + .cs_serial = { + .process = + (zc_serial_process_f *)chain_compress_writes + } + }; + return (step); +} - if (dtype == ZIO_COMPRESS_OFF) - cbuf = dbuf; - else - cbuf = safe_calloc(payload_size); +int +zstream_do_recompress(int argc, char *argv[]) +{ + int c; + int level = ZIO_COMPLEVEL_DEFAULT; - /* Read and decompress the payload */ - (void) sfread(cbuf, payload_size, stdin); - if (dtype != ZIO_COMPRESS_OFF) { - abd_t cabd, dabd; - abd_get_from_buf_struct(&cabd, - cbuf, payload_size); - abd_get_from_buf_struct(&dabd, dbuf, - MIN(bufsz, drrw->drr_logical_size)); - if (zio_decompress_data(dtype, &cabd, &dabd, - payload_size, abd_get_size(&dabd), - NULL) != 0) { - warnx("decompression type %d failed " - "for ino %llu offset %llu", - dtype, - (u_longlong_t)drrw->drr_object, - (u_longlong_t)drrw->drr_offset); - exit(4); - } - payload_size = drrw->drr_logical_size; - abd_free(&dabd); - abd_free(&cabd); - free(cbuf); - } + chain_attrs_t attrs = { .ca_command_opts = CA_FORBID_DEDUP }; - /* Recompress the payload */ - if (ctype != ZIO_COMPRESS_OFF) { - abd_t dabd, abd; - abd_get_from_buf_struct(&dabd, - dbuf, drrw->drr_logical_size); - abd_t *pabd = - abd_get_from_buf_struct(&abd, buf, bufsz); - size_t csize = zio_compress_data(ctype, &dabd, - &pabd, drrw->drr_logical_size, - drrw->drr_logical_size, level); - size_t rounded = - P2ROUNDUP(csize, SPA_MINBLOCKSIZE); - if (rounded >= drrw->drr_logical_size) { - memcpy(buf, dbuf, payload_size); - drrw->drr_compressiontype = 0; - drrw->drr_compressed_size = 0; - } else { - abd_zero_off(pabd, csize, - rounded - csize); - drrw->drr_compressiontype = ctype; - drrw->drr_compressed_size = - payload_size = rounded; - } - abd_free(&abd); - abd_free(&dabd); - free(dbuf); - } else { - drrw->drr_compressiontype = 0; - drrw->drr_compressed_size = 0; + while ((c = getopt(argc, argv, "l:")) != -1) { + switch (c) { + case 'l': + if (sscanf(optarg, "%d", &level) != 1) { + warnx("failed to parse level '%s'", optarg); + zstream_usage(); } break; - } - - case DRR_WRITE_EMBEDDED: - { - struct drr_write_embedded *drrwe = - &drr->drr_u.drr_write_embedded; - VERIFY3S(begin, ==, 1); - payload_size = - P2ROUNDUP((uint64_t)drrwe->drr_psize, 8); - (void) sfread(buf, payload_size, stdin); + case '?': + warnx("invalid option '%c'", optopt); + zstream_usage(); break; } + } - case DRR_FREEOBJECTS: - case DRR_FREE: - case DRR_OBJECT_RANGE: - VERIFY3S(begin, ==, 1); - break; - - default: - (void) fprintf(stderr, "INVALID record type 0x%x\n", - drr->drr_type); - /* should never happen, so assert */ - assert(B_FALSE); - } + argc -= optind; + argv += optind; - if (feof(stdout)) { - fprintf(stderr, "Error: unexpected end-of-file\n"); - exit(1); - } - if (ferror(stdout)) { - fprintf(stderr, "Error while reading file: %s\n", - strerror(errno)); - exit(1); - } + if (argc != 1) + zstream_usage(); - /* - * We need to recalculate the checksum, and it needs to be - * initially zero to do that. BEGIN records don't have - * a checksum. - */ - if (drr->drr_type != DRR_BEGIN) { - memset(&drr->drr_u.drr_checksum.drr_checksum, 0, - sizeof (drr->drr_u.drr_checksum.drr_checksum)); + compression_spec_t spec = { .cs_level = level }; + if (strcmp(argv[0], "off") == 0) { + spec.cs_type = ZIO_COMPRESS_OFF; + } else { + enum zio_compress ct; + for (ct = 0; ct < ZIO_COMPRESS_FUNCTIONS; ct++) { + const char *ci_name = zio_compress_table[ct].ci_name; + if (strcmp(argv[0], ci_name) == 0) + break; } - if (dump_record(drr, buf, payload_size, - &stream_cksum, STDOUT_FILENO) != 0) - break; - if (drr->drr_type == DRR_END) { - /* - * Typically the END record is either the last - * thing in the stream, or it is followed - * by a BEGIN record (which also zeros the checksum). - * However, a stream package ends with two END - * records. The last END record's checksum starts - * from zero. - */ - ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); + if (ct == ZIO_COMPRESS_FUNCTIONS || ctype_is_uncompressed(ct)) { + errx(2, "invalid compression type %s", argv[0]); } + spec.cs_type = ct; } - free(buf); - fletcher_4_fini(); - zio_fini(); - zstd_fini(); - abd_fini(); - zfs_refcount_fini(); + zstream_chain_t recompress_chain = { + STANDARD_INPUT_STACK(NULL), + serial_decompress_writes(&spec), + serial_compress_writes(&spec), + STANDARD_OUTPUT_STACK(NULL) + }; + + zstream_chain_exec(recompress_chain, &attrs); return (0); } diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_recompress.h b/sys/contrib/openzfs/cmd/zstream/zstream_recompress.h new file mode 100644 index 000000000000..d089f03c77eb --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_recompress.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#ifndef _ZSTREAM_RECOMPRESS_H +#define _ZSTREAM_RECOMPRESS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "zstream_util.h" +#include "zstream_io.h" + +chain_step_t +serial_decompress_writes(compression_spec_t *target); + +chain_step_t +serial_compress_writes(compression_spec_t *target); + +#ifdef __cplusplus +} +#endif + +#endif /* _ZSTREAM_RECOMPRESS_H */ diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_redup.c b/sys/contrib/openzfs/cmd/zstream/zstream_redup.c index 59bccd7b15ad..11e049a1a2d3 100644 --- a/sys/contrib/openzfs/cmd/zstream/zstream_redup.c +++ b/sys/contrib/openzfs/cmd/zstream/zstream_redup.c @@ -20,35 +20,34 @@ #include <assert.h> #include <cityhash.h> -#include <ctype.h> +#include <err.h> #include <errno.h> -#include <fcntl.h> -#include <libzfs.h> #include <libzutil.h> -#include <stddef.h> +#include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <sys/bitops.h> +#include <sys/param.h> +#include <sys/stdtypes.h> +#include <sys/sysmacros.h> +#include <sys/zfs_ioctl.h> #include <umem.h> #include <unistd.h> -#include <sys/debug.h> -#include <sys/stat.h> -#include <sys/zfs_ioctl.h> -#include <sys/zio_checksum.h> -#include "zfs_fletcher.h" + #include "zstream.h" +#include "zstream_modules.h" #include "zstream_util.h" - #define MAX_RDT_PHYSMEM_PERCENT 20 -#define SMALLEST_POSSIBLE_MAX_RDT_MB 128 +#define SMALLEST_POSSIBLE_MAX_RDT_MB 128 typedef struct redup_entry { struct redup_entry *rde_next; - uint64_t rde_guid; - uint64_t rde_object; - uint64_t rde_offset; - uint64_t rde_stream_offset; + uint64_t rde_guid; + uint64_t rde_object; + uint64_t rde_offset; + uint64_t rde_stream_offset; } redup_entry_t; typedef struct redup_table { @@ -58,24 +57,10 @@ typedef struct redup_table { int numhashbits; } redup_table_t; -/* - * Safe version of pread(), exits on error. - */ -static void -spread(int fd, void *buf, size_t count, off_t offset) -{ - ssize_t err = pread(fd, buf, count, offset); - if (err == -1) { - (void) fprintf(stderr, - "Error while reading file: %s\n", - strerror(errno)); - exit(1); - } else if (err != count) { - (void) fprintf(stderr, - "Error while reading file: short read\n"); - exit(1); - } -} +typedef struct { + redup_table_t rc_rdt; + FILE *rc_fp; +} redup_context_t; static void rdt_insert(redup_table_t *rdt, @@ -116,283 +101,114 @@ rdt_lookup(redup_table_t *rdt, assert(!"could not find expected redup table entry"); } -/* - * Convert a dedup stream (generated by "zfs send -D") to a - * non-deduplicated stream. The entire infd will be converted, including - * any substreams in a stream package (generated by "zfs send -RD"). The - * infd must be seekable. - */ -static void -zfs_redup_stream(int infd, int outfd, boolean_t verbose) +static disposition_t +chain_redup_writes(drr_packet_t *item, redup_context_t *context) { - int bufsz = SPA_MAXBLOCKSIZE; - dmu_replay_record_t thedrr; - dmu_replay_record_t *drr = &thedrr; - redup_table_t rdt; - zio_cksum_t stream_cksum; - uint64_t numbuckets; - uint64_t num_records = 0; - uint64_t num_write_byref_records = 0; - - memset(&thedrr, 0, sizeof (dmu_replay_record_t)); - -#ifdef _ILP32 - uint64_t max_rde_size = SMALLEST_POSSIBLE_MAX_RDT_MB << 20; -#else - uint64_t physbytes = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); - uint64_t max_rde_size = - MAX((physbytes * MAX_RDT_PHYSMEM_PERCENT) / 100, - SMALLEST_POSSIBLE_MAX_RDT_MB << 20); -#endif + if (item == NULL) { + return (D_OK); + } - numbuckets = max_rde_size / (sizeof (redup_entry_t)); + dmu_replay_record_t *drr = &item->dp_drr; + struct drr_write *drrw = &drr->drr_u.drr_write; + struct drr_begin *drrb = &drr->drr_u.drr_begin; - /* - * numbuckets must be a power of 2. Increase number to - * a power of 2 if necessary. - */ - if (!ISP2(numbuckets)) - numbuckets = 1ULL << highbit64(numbuckets); + switch (drr->drr_type) { - rdt.redup_hash_array = - safe_calloc(numbuckets * sizeof (redup_entry_t *)); - rdt.ddecache = umem_cache_create("rde", sizeof (redup_entry_t), 0, - NULL, NULL, NULL, NULL, NULL, 0); - rdt.numhashbits = highbit64(numbuckets) - 1; - rdt.ddt_count = 0; + case DRR_BEGIN: + { + uint64_t flags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo); + flags &= ~(DMU_BACKUP_FEATURE_DEDUP | + DMU_BACKUP_FEATURE_DEDUPPROPS); + DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, flags); + break; + } - char *buf = safe_calloc(bufsz); - FILE *ofp = fdopen(infd, "r"); - long offset = ftell(ofp); - int begin = 0; - boolean_t seen = B_FALSE; - while (sfread(drr, sizeof (*drr), ofp) != 0) { - num_records++; + case DRR_WRITE_BYREF: + { + struct drr_write_byref drrwb = drr->drr_u.drr_write_byref; /* - * We need to regenerate the checksum. + * Look up in hash table by drrwb->drr_refguid, + * drr_refobject, drr_refoffset. Replace this + * record with the found WRITE record, but with + * drr_object,drr_offset,drr_toguid replaced with ours. */ - if (drr->drr_type != DRR_BEGIN) { - memset(&drr->drr_u.drr_checksum.drr_checksum, 0, - sizeof (drr->drr_u.drr_checksum.drr_checksum)); - } - - uint64_t payload_size = 0; - switch (drr->drr_type) { - case DRR_BEGIN: - { - struct drr_begin *drrb = &drr->drr_u.drr_begin; - int fflags; - ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); - VERIFY0(begin++); - seen = B_TRUE; - - assert(drrb->drr_magic == DMU_BACKUP_MAGIC); - - /* clear the DEDUP feature flag for this stream */ - fflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo); - fflags &= ~(DMU_BACKUP_FEATURE_DEDUP | - DMU_BACKUP_FEATURE_DEDUPPROPS); - /* cppcheck-suppress syntaxError */ - DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags); - - uint32_t sz = drr->drr_payloadlen; - - VERIFY3U(sz, <=, 1U << 28); - - if (sz != 0) { - if (sz > bufsz) { - free(buf); - buf = safe_calloc(sz); - bufsz = sz; - } - (void) sfread(buf, sz, ofp); - } - payload_size = sz; - break; - } - - case DRR_END: - { - struct drr_end *drre = &drr->drr_u.drr_end; - /* - * We would prefer to just check --begin == 0, but - * replication streams have an end of stream END - * record, so we must avoid tripping it. - */ - VERIFY3B(seen, ==, B_TRUE); - begin--; - /* - * Use the recalculated checksum, unless this is - * the END record of a stream package, which has - * no checksum. - */ - if (!ZIO_CHECKSUM_IS_ZERO(&drre->drr_checksum)) - drre->drr_checksum = stream_cksum; - break; - } - - case DRR_OBJECT: - { - struct drr_object *drro = &drr->drr_u.drr_object; - VERIFY3S(begin, ==, 1); - - if (drro->drr_bonuslen > 0) { - payload_size = DRR_OBJECT_PAYLOAD_SIZE(drro); - (void) sfread(buf, payload_size, ofp); - } - break; - } - - case DRR_SPILL: - { - struct drr_spill *drrs = &drr->drr_u.drr_spill; - VERIFY3S(begin, ==, 1); - payload_size = DRR_SPILL_PAYLOAD_SIZE(drrs); - (void) sfread(buf, payload_size, ofp); - break; - } - - case DRR_WRITE_BYREF: - { - struct drr_write_byref drrwb = - drr->drr_u.drr_write_byref; - VERIFY3S(begin, ==, 1); + uint64_t stream_offset = 0; + rdt_lookup(&context->rc_rdt, drrwb.drr_refguid, + drrwb.drr_refobject, drrwb.drr_refoffset, + &stream_offset); - num_write_byref_records++; - - /* - * Look up in hash table by drrwb->drr_refguid, - * drr_refobject, drr_refoffset. Replace this - * record with the found WRITE record, but with - * drr_object,drr_offset,drr_toguid replaced with ours. - */ - uint64_t stream_offset = 0; - rdt_lookup(&rdt, drrwb.drr_refguid, - drrwb.drr_refobject, drrwb.drr_refoffset, - &stream_offset); - - spread(infd, drr, sizeof (*drr), stream_offset); - - assert(drr->drr_type == DRR_WRITE); - struct drr_write *drrw = &drr->drr_u.drr_write; - assert(drrw->drr_toguid == drrwb.drr_refguid); - assert(drrw->drr_object == drrwb.drr_refobject); - assert(drrw->drr_offset == drrwb.drr_refoffset); - - payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); - spread(infd, buf, payload_size, - stream_offset + sizeof (*drr)); - - drrw->drr_toguid = drrwb.drr_toguid; - drrw->drr_object = drrwb.drr_object; - drrw->drr_offset = drrwb.drr_offset; - break; + if (fseeko(context->rc_fp, stream_offset, SEEK_SET) != 0) { + err(1, "seek into source file failed, offset %llu", + (u_longlong_t)stream_offset); } - - case DRR_WRITE: - { - struct drr_write *drrw = &drr->drr_u.drr_write; - VERIFY3S(begin, ==, 1); - payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); - (void) sfread(buf, payload_size, ofp); - - rdt_insert(&rdt, drrw->drr_toguid, - drrw->drr_object, drrw->drr_offset, offset); - break; + if (fread(drr, sizeof (*drr), 1, context->rc_fp) != 1) { + err(1, "read of prior write failed"); } - - case DRR_WRITE_EMBEDDED: - { - struct drr_write_embedded *drrwe = - &drr->drr_u.drr_write_embedded; - VERIFY3S(begin, ==, 1); - payload_size = - P2ROUNDUP((uint64_t)drrwe->drr_psize, 8); - (void) sfread(buf, payload_size, ofp); - break; + if (ATTR_IS_SET(CA_BYTESWAPPED)) { + byteswap_record(drr, BSWAP_32(drr->drr_type)); } - case DRR_FREEOBJECTS: - case DRR_FREE: - case DRR_OBJECT_RANGE: - VERIFY3S(begin, ==, 1); - break; + VERIFY3U(drr->drr_type, ==, DRR_WRITE); + VERIFY3U(drrw->drr_toguid, ==, drrwb.drr_refguid); + VERIFY3U(drrw->drr_object, ==, drrwb.drr_refobject); + VERIFY3U(drrw->drr_offset, ==, drrwb.drr_refoffset); - default: - (void) fprintf(stderr, "INVALID record type 0x%x\n", - drr->drr_type); - /* should never happen, so assert */ - assert(B_FALSE); - } + item->dp_payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); + item->dp_payload = safe_malloc(item->dp_payload_size); - if (feof(ofp)) { - fprintf(stderr, "Error: unexpected end-of-file\n"); - exit(1); - } - if (ferror(ofp)) { - fprintf(stderr, "Error while reading file: %s\n", - strerror(errno)); - exit(1); - } + size_t n_read = fread(item->dp_payload, item->dp_payload_size, + 1, context->rc_fp); + if (n_read != 1) + err(1, "read of prior payload failed"); - /* - * We need to recalculate the checksum, and it needs to be - * initially zero to do that. BEGIN records don't have - * a checksum. - */ - if (drr->drr_type != DRR_BEGIN) { - memset(&drr->drr_u.drr_checksum.drr_checksum, 0, - sizeof (drr->drr_u.drr_checksum.drr_checksum)); - } - if (dump_record(drr, buf, payload_size, - &stream_cksum, outfd) != 0) - break; - if (drr->drr_type == DRR_END) { - /* - * Typically the END record is either the last - * thing in the stream, or it is followed - * by a BEGIN record (which also zeros the checksum). - * However, a stream package ends with two END - * records. The last END record's checksum starts - * from zero. - */ - ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); - } - offset = ftell(ofp); + drrw->drr_toguid = drrwb.drr_toguid; + drrw->drr_object = drrwb.drr_object; + drrw->drr_offset = drrwb.drr_offset; + break; } - if (verbose) { - char mem_str[16]; - zfs_nicenum(rdt.ddt_count * sizeof (redup_entry_t), - mem_str, sizeof (mem_str)); - fprintf(stderr, "converted stream with %llu total records, " - "including %llu dedup records, using %sB memory.\n", - (long long)num_records, - (long long)num_write_byref_records, - mem_str); + case DRR_WRITE: + rdt_insert(&context->rc_rdt, drrw->drr_toguid, drrw->drr_object, + drrw->drr_offset, item->dp_stream_offset); + break; + + default: + break; } + return (D_OK); +} - umem_cache_destroy(rdt.ddecache); - free(rdt.redup_hash_array); - free(buf); - (void) fclose(ofp); +static chain_step_t +serial_redup_writes(redup_context_t *context) +{ + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = sizeof (drr_packet_t), + .cs_context = context, + .cs_serial = { + .process = (zc_serial_process_f *)chain_redup_writes + } + }; + return (step); } int zstream_do_redup(int argc, char *argv[]) { - boolean_t verbose = B_FALSE; int c; + chain_attrs_t attrs = {0}; + redup_context_t context = {0}; + uint64_t numbuckets; while ((c = getopt(argc, argv, "v")) != -1) { switch (c) { case 'v': - verbose = B_TRUE; + ENABLE_OPTION(&attrs, CA_VERBOSE); break; case '?': - (void) fprintf(stderr, "invalid option '%c'\n", - optopt); + warnx("invalid option '%c'", optopt); zstream_usage(); break; } @@ -404,28 +220,51 @@ zstream_do_redup(int argc, char *argv[]) if (argc != 1) zstream_usage(); - const char *filename = argv[0]; - - if (isatty(STDOUT_FILENO)) { - (void) fprintf(stderr, - "Error: Stream can not be written to a terminal.\n" - "You must redirect standard output.\n"); - return (1); + context.rc_fp = fopen(argv[0], "rb"); + if (context.rc_fp == NULL) { + err(1, "unable to open %s", argv[0]); } - int fd = open(filename, O_RDONLY); - if (fd == -1) { - (void) fprintf(stderr, - "Error while opening file '%s': %s\n", - filename, strerror(errno)); - exit(1); - } +#ifdef _ILP32 + uint64_t max_rde_size = SMALLEST_POSSIBLE_MAX_RDT_MB << 20; +#else + uint64_t physbytes = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); + uint64_t max_rde_size = MAX((physbytes * MAX_RDT_PHYSMEM_PERCENT) / 100, + SMALLEST_POSSIBLE_MAX_RDT_MB << 20); +#endif + + numbuckets = max_rde_size / (sizeof (redup_entry_t)); + if (!ISP2(numbuckets)) + numbuckets = 1ULL << highbit64(numbuckets); + + context.rc_rdt.redup_hash_array = + safe_calloc(numbuckets * sizeof (redup_entry_t *)); + context.rc_rdt.ddecache = umem_cache_create("rde", + sizeof (redup_entry_t), 0, NULL, NULL, NULL, NULL, NULL, 0); + context.rc_rdt.numhashbits = highbit64(numbuckets) - 1; + context.rc_rdt.ddt_count = 0; - fletcher_4_init(); - zfs_redup_stream(fd, STDOUT_FILENO, verbose); - fletcher_4_fini(); + zstream_chain_t redup_chain = { + STANDARD_INPUT_STACK(argv[0]), + serial_redup_writes(&context), + STANDARD_OUTPUT_STACK(NULL) + }; + zstream_chain_exec(redup_chain, &attrs); - close(fd); + if (attrs.ca_command_opts & CA_VERBOSE) { + char mem_str[16]; + record_stats_t *acsi = attrs.ca_stats_in; + zfs_nicenum(context.rc_rdt.ddt_count * sizeof (redup_entry_t), + mem_str, sizeof (mem_str)); + fprintf(stderr, "Converted stream with %llu total records, " + "including %llu dedup records, using %sB memory.\n", + (u_longlong_t)attrs.ca_totals_in.rs_num_records, + (u_longlong_t)acsi[DRR_WRITE_BYREF].rs_num_records, + mem_str); + } + fclose(context.rc_fp); + umem_cache_destroy(context.rc_rdt.ddecache); + free(context.rc_rdt.redup_hash_array); return (0); } diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_token.c b/sys/contrib/openzfs/cmd/zstream/zstream_token.c index be6ae09f81e0..6bc0b364663a 100644 --- a/sys/contrib/openzfs/cmd/zstream/zstream_token.c +++ b/sys/contrib/openzfs/cmd/zstream/zstream_token.c @@ -31,19 +31,12 @@ * Copyright (c) 2020 by Datto Inc. All rights reserved. */ -#include <ctype.h> +#include <errno.h> #include <libnvpair.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> -#include <stddef.h> - #include <libzfs.h> -#include <libzfs_core.h> +#include <stdio.h> +#include <sys/nvpair.h> -#include <sys/dmu.h> -#include <sys/zfs_ioctl.h> #include "zstream.h" int diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_util.c b/sys/contrib/openzfs/cmd/zstream/zstream_util.c index b44175284bbd..5660c67cc015 100644 --- a/sys/contrib/openzfs/cmd/zstream/zstream_util.c +++ b/sys/contrib/openzfs/cmd/zstream/zstream_util.c @@ -28,52 +28,31 @@ * Copyright (c) 2024, Klara, Inc. */ -#include <sys/debug.h> -#include <stddef.h> +#include <assert.h> +#include <err.h> #include <errno.h> -#include <unistd.h> -#include <stdlib.h> +#include <stddef.h> #include <stdio.h> +#include <stdlib.h> #include <string.h> +#include <sys/abd.h> +#include <sys/fs/zfs.h> +#include <sys/stdtypes.h> +#include <sys/sysmacros.h> +#include <sys/zfs_ioctl.h> +#include <sys/zio.h> +#include <sys/zio_compress.h> +#include <unistd.h> #include <zfs_fletcher.h> -#include "zstream_util.h" -/* - * From libzfs_sendrecv.c - */ -int -dump_record(dmu_replay_record_t *drr, void *payload, size_t payload_len, - zio_cksum_t *zc, int outfd) -{ - ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), - ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t)); - fletcher_4_incremental_native(drr, - offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), zc); - if (drr->drr_type != DRR_BEGIN) { - ASSERT(ZIO_CHECKSUM_IS_ZERO(&drr->drr_u. - drr_checksum.drr_checksum)); - drr->drr_u.drr_checksum.drr_checksum = *zc; - } - fletcher_4_incremental_native(&drr->drr_u.drr_checksum.drr_checksum, - sizeof (zio_cksum_t), zc); - if (write(outfd, drr, sizeof (*drr)) == -1) - return (errno); - if (payload_len != 0) { - fletcher_4_incremental_native(payload, payload_len, zc); - if (write(outfd, payload, payload_len) == -1) - return (errno); - } - return (0); -} +#include "zstream_util.h" void * safe_malloc(size_t size) { void *rv = malloc(size); if (rv == NULL) { - (void) fprintf(stderr, "Error: failed to allocate %zu bytes\n", - size); - exit(1); + errx(1, "failed to allocate %zu bytes, aborting...", size); } return (rv); } @@ -83,24 +62,120 @@ safe_calloc(size_t size) { void *rv = calloc(1, size); if (rv == NULL) { - (void) fprintf(stderr, - "Error: failed to allocate %zu bytes\n", size); - exit(1); + errx(1, "failed to allocate %zu bytes, aborting...", size); } return (rv); } +char * +checksum_str(zio_cksum_t *cksum, char *buff, size_t buff_size) +{ + snprintf(buff, buff_size, "%.16llx / %.16llx / %.16llx / %.16llx", + (long long unsigned int) cksum->zc_word[0], + (long long unsigned int) cksum->zc_word[1], + (long long unsigned int) cksum->zc_word[2], + (long long unsigned int) cksum->zc_word[3]); + return (buff); +} + +boolean_t +validate_checksum(zio_cksum_t *expected, zio_cksum_t *actual, + boolean_t swap, const char *where, off_t stream_offset) +{ + static char buff[128]; + zio_cksum_t swapped_actual; + + if (swap) { + swapped_actual = *actual; + actual = &swapped_actual; + ZIO_CHECKSUM_BSWAP(actual); + } + /* cppcheck-suppress uninitvar */ + if (ZIO_CHECKSUM_EQUAL(*expected, *actual)) { + return (B_TRUE); + } + fflush(stdout); + fprintf(stderr, "Incorrect checksum %s (stream offset %lld)\n", where, + (longlong_t)stream_offset); + fprintf(stderr, "Expected = %s\n", checksum_str(expected, buff, + sizeof (buff))); + fprintf(stderr, " Actual = %s\n", checksum_str(actual, buff, + sizeof (buff))); + return (B_FALSE); +} + +boolean_t +write_is_encrypted(struct drr_write *drrw) +{ + for (int i = 0; i < ZIO_DATA_SALT_LEN; i++) { + if (drrw->drr_salt[i] != 0) { + return (B_TRUE); + } + } + return (B_FALSE); +} + /* - * Safe version of fread(), exits on error. + * The specified compress_type must reflect the buffer's actual compression. + * Returns an allocated buffer if decompression was successful, NULL + * otherwise. */ -int -sfread(void *buf, size_t size, FILE *fp) +uint8_t * +decompress_buffer(uint8_t *inbuff, size_t inbuff_size, size_t logical_size, + enum zio_compress compress_type) { - int rv = fread(buf, size, 1, fp); - if (rv == 0 && ferror(fp)) { - (void) fprintf(stderr, "Error while reading file: %s\n", - strerror(errno)); - exit(1); + uint8_t *outbuff = safe_malloc(logical_size); + abd_t sabd, dabd; + int ret; + + VERIFY3B(ctype_is_uncompressed(compress_type), ==, B_FALSE); + + abd_get_from_buf_struct(&sabd, inbuff, inbuff_size); + abd_get_from_buf_struct(&dabd, outbuff, logical_size); + ret = zio_decompress_data(compress_type, &sabd, &dabd, + inbuff_size, abd_get_size(&dabd), NULL); + + abd_free(&dabd); + abd_free(&sabd); + + if (ret != 0) { + free(outbuff); + return (NULL); } - return (rv); + + return (outbuff); +} + +/* + * Returns an allocated buffer if compression was successful, NULL + * otherwise. + */ +uint8_t * +compress_buffer(uint8_t *inbuff, size_t inbuff_size, + compression_spec_t compress_type, size_t *compressed_size) +{ + uint8_t *outbuff = safe_malloc(inbuff_size); + abd_t sabd, dabd; + size_t csize, rounded; + + VERIFY3B(ctype_is_uncompressed(compress_type.cs_type), ==, B_FALSE); + + abd_t *pabd = abd_get_from_buf_struct(&dabd, outbuff, inbuff_size); + abd_get_from_buf_struct(&sabd, inbuff, inbuff_size); + csize = zio_compress_data(compress_type.cs_type, &sabd, + &pabd, inbuff_size, inbuff_size, compress_type.cs_level); + + rounded = P2ROUNDUP(csize, SPA_MINBLOCKSIZE); + if (rounded < inbuff_size) { + abd_zero_off(pabd, csize, rounded - csize); + *compressed_size = rounded; + } else { + free(outbuff); + outbuff = NULL; + } + + abd_free(&sabd); + abd_free(&dabd); + + return (outbuff); } diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_util.h b/sys/contrib/openzfs/cmd/zstream/zstream_util.h index 50600fdd1811..8ae470a08e27 100644 --- a/sys/contrib/openzfs/cmd/zstream/zstream_util.h +++ b/sys/contrib/openzfs/cmd/zstream/zstream_util.h @@ -26,8 +26,15 @@ extern "C" { #endif #include <stddef.h> +#include <stdlib.h> #include <sys/zfs_ioctl.h> #include <sys/zio_checksum.h> +#include <sys/zio_compress.h> + +typedef struct { + enum zio_compress cs_type; + int cs_level; +} compression_spec_t; /* * The safe_ versions of the functions below terminate the process if the @@ -39,19 +46,46 @@ safe_malloc(size_t size); extern void * safe_calloc(size_t n); -extern int -sfread(void *buf, size_t size, FILE *fp); +extern char * +checksum_str(zio_cksum_t *cksum, char *buff, size_t buff_size); + +/* + * Prints an error message if checksums don't match. Returns B_TRUE for + * a match, B_FALSE otherwise. + */ +boolean_t +validate_checksum(zio_cksum_t *expect, zio_cksum_t *actual, boolean_t swap, + const char *where, off_t stream_offset); + +static inline void +validate_or_exit(zio_cksum_t *expect, zio_cksum_t *actual, boolean_t swap, + const char *where, off_t stream_offset) +{ + if (!validate_checksum(expect, actual, swap, where, stream_offset)) { + exit(1); + } +} /* - * 1) Update checksum with the record header up to drr_checksum. - * 2) Update checksum field in the record header. - * 3) Update checksum with the checksum field in the record header. - * 4) Update checksum with the contents of the payload. - * 5) Write header and payload to fd. + * Determine whether a compression type indicates no compression */ -extern int -dump_record(dmu_replay_record_t *drr, void *payload, size_t payload_len, - zio_cksum_t *zc, int outfd); +static inline boolean_t +ctype_is_uncompressed(enum zio_compress ct) +{ + VERIFY3U((int)ct, <, (int)ZIO_COMPRESS_FUNCTIONS); + return (zio_compress_table[(int)(ct)].ci_compress == NULL); +} + +boolean_t +write_is_encrypted(struct drr_write *drrw); + +uint8_t * +decompress_buffer(uint8_t *inbuff, size_t inbuff_size, size_t logical_size, + enum zio_compress compress_type); + +uint8_t * +compress_buffer(uint8_t *inbuff, size_t inbuff_size, + compression_spec_t compress_type, size_t *compressed_size); #ifdef __cplusplus } diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_validate.c b/sys/contrib/openzfs/cmd/zstream/zstream_validate.c new file mode 100644 index 000000000000..f70bef244cf3 --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_validate.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#include <assert.h> +#include <err.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/stdtypes.h> +#include <sys/zfs_ioctl.h> +#include <sys/zio_compress.h> + +#include "zstream_modules.h" + +/* + * Validate consistency and well-formedness of the actual DRR records. I + * have swept all the existing validation code into this module, but it's + * still pretty sparse. + */ + +#define MAX_VALIDATIONS 4 + +typedef struct { + int nesting; +} validate_context_t; + +static validate_context_t contexts[MAX_VALIDATIONS]; +static int next_context = 0; + +static disposition_t +chain_validate_records(drr_packet_t *item, validate_context_t *context) +{ + if (item == NULL) + return (D_OK); + + struct dmu_replay_record *drr = &item->dp_drr; + struct drr_write *drrw = &drr->drr_u.drr_write; + struct drr_object *drro = &drr->drr_u.drr_object; + + if (OPTION_ENABLED(CA_DO_NOT_VALIDATE)) + return (D_OK); + + if (item->dp_stream_offset == 0 && drr->drr_type != DRR_BEGIN) { + warnx("warning - first record is not DRR_BEGIN"); + } + + if (drr->drr_type == DRR_BEGIN) { + VERIFY0(context->nesting); + context->nesting++; + } else if (drr->drr_type == DRR_END) { + VERIFY3S(context->nesting, >=, 0); + if (context->nesting > 0) + context->nesting--; + } else if (drr->drr_type >= DRR_NUMTYPES) { + errx(1, "unknown record type: %d", drr->drr_type); + } else { + VERIFY3S(context->nesting, ==, 1); + } + + switch (drr->drr_type) { + + case DRR_BEGIN: + VERIFY3U(item->dp_payload_size, <=, 1UL << 28); + break; + + case DRR_OBJECT: + { + boolean_t is_raw = !!(chain_attrs->ca_feature_flags & + DMU_BACKUP_FEATURE_RAW); + boolean_t bonus_gt_raw = drro->drr_bonuslen > + drro->drr_raw_bonuslen; + if (is_raw && bonus_gt_raw) { + fprintf(stderr, + "Warning: object %llu has bonuslen = " + "%u > raw_bonuslen = %u\n\n", + (u_longlong_t)drro->drr_object, + drro->drr_bonuslen, + drro->drr_raw_bonuslen); + } + break; + } + + case DRR_WRITE: + if (drrw->drr_compressiontype >= ZIO_COMPRESS_FUNCTIONS) { + errx(1, "invalid compression type: %d", + drrw->drr_compressiontype); + } + break; + + default: + break; + } + + return (D_OK); +} + +chain_step_t +serial_validate_records(void) +{ + int context_ix = next_context++ % MAX_VALIDATIONS; + validate_context_t *context = &contexts[context_ix]; + context->nesting = 0; + + chain_step_t step = { + .cs_type = CS_SERIAL, + .cs_in_size = sizeof (drr_packet_t), + .cs_out_size = sizeof (drr_packet_t), + .cs_context = context, + .cs_serial = { + .process = (zc_serial_process_f *)chain_validate_records, + } + }; + return (step); +} diff --git a/sys/contrib/openzfs/cmd/zstream/zstream_validate.h b/sys/contrib/openzfs/cmd/zstream/zstream_validate.h new file mode 100644 index 000000000000..d8bde7e9c551 --- /dev/null +++ b/sys/contrib/openzfs/cmd/zstream/zstream_validate.h @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the Common + * Development and Distribution License ("CDDL"), version 1.0. You may only use + * this file in accordance with the terms of version 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this source. A + * copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2026 by Garth Snyder. All rights reserved. + */ + +#ifndef _ZSTREAM_VALIDATE_H +#define _ZSTREAM_VALIDATE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "zstream_io.h" + +chain_step_t +serial_validate_records(void); + +#ifdef __cplusplus +} +#endif + +#endif /* _ZSTREAM_VALIDATE_H */ diff --git a/sys/contrib/openzfs/contrib/initramfs/hooks/zfs.in b/sys/contrib/openzfs/contrib/initramfs/hooks/zfs.in index 6cd7e6f1ea3b..dc318933271d 100755 --- a/sys/contrib/openzfs/contrib/initramfs/hooks/zfs.in +++ b/sys/contrib/openzfs/contrib/initramfs/hooks/zfs.in @@ -31,7 +31,7 @@ done # shellcheck disable=SC2050 if [ @LIBFETCH_DYNAMIC@ -gt 0 ]; then - find /lib/ -name "@LIBFETCH_SONAME@" | while read -r libfetch; do + find /lib/ -type f -name "@LIBFETCH_SONAME@" | while read -r libfetch; do copy_exec "$libfetch" done fi diff --git a/sys/contrib/openzfs/include/os/freebsd/zfs/sys/zfs_vfsops_os.h b/sys/contrib/openzfs/include/os/freebsd/zfs/sys/zfs_vfsops_os.h index 62e78d82bcb9..99e147fb55b9 100644 --- a/sys/contrib/openzfs/include/os/freebsd/zfs/sys/zfs_vfsops_os.h +++ b/sys/contrib/openzfs/include/os/freebsd/zfs/sys/zfs_vfsops_os.h @@ -73,6 +73,7 @@ struct zfsvfs { boolean_t z_atime; /* enable atimes mount option */ boolean_t z_relatime; /* enable relatime mount option */ boolean_t z_unmounted; /* unmounted */ + boolean_t z_use_hold; /* held via dmu_objset_hold */ zfs_teardown_lock_t z_teardown_lock; zfs_teardown_inactive_lock_t z_teardown_inactive_lock; list_t z_all_znodes; /* all vnodes in the fs */ @@ -226,6 +227,7 @@ extern int zfs_resume_fs(zfsvfs_t *zfsvfs, struct dsl_dataset *ds); extern int zfs_end_fs(zfsvfs_t *zfsvfs, struct dsl_dataset *ds); extern int zfs_set_version(zfsvfs_t *zfsvfs, uint64_t newvers); extern int zfsvfs_create(const char *name, boolean_t readonly, zfsvfs_t **zfvp); +extern int zfsvfs_create_hold(const char *name, zfsvfs_t **zfvp); extern int zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os); extern void zfsvfs_free(zfsvfs_t *zfsvfs); extern int zfs_check_global_label(const char *dsname, const char *hexsl); diff --git a/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_acl.h b/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_acl.h index d88b4937ef08..a6d6ca5fe2bd 100644 --- a/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_acl.h +++ b/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_acl.h @@ -54,7 +54,7 @@ DECLARE_EVENT_CLASS(zfs_ace_class, __field(uint8_t, z_atime_dirty) __field(uint8_t, z_zn_prefetch) __field(uint_t, z_blksz) - __field(uint_t, z_seq) + __field(uint64_t, z_seq) __field(uint64_t, z_mapcnt) __field(uint64_t, z_size) __field(uint64_t, z_pflags) @@ -111,7 +111,7 @@ DECLARE_EVENT_CLASS(zfs_ace_class, __entry->mask_matched = mask_matched; ), TP_printk("zn { id %llu unlinked %u atime_dirty %u " - "zn_prefetch %u blksz %u seq %u " + "zn_prefetch %u blksz %u seq %llu " "mapcnt %llu size %llu pflags %llu " "sync_cnt %u " "mode 0x%x is_sa %d is_ctldir %d " diff --git a/sys/contrib/openzfs/include/os/linux/zfs/sys/zfs_vfsops_os.h b/sys/contrib/openzfs/include/os/linux/zfs/sys/zfs_vfsops_os.h index 123ea71b4de9..2058ef322945 100644 --- a/sys/contrib/openzfs/include/os/linux/zfs/sys/zfs_vfsops_os.h +++ b/sys/contrib/openzfs/include/os/linux/zfs/sys/zfs_vfsops_os.h @@ -100,6 +100,7 @@ struct zfsvfs { int z_norm; /* normalization flags */ boolean_t z_relatime; /* enable relatime mount option */ boolean_t z_unmounted; /* unmounted */ + boolean_t z_use_hold; /* held via dmu_objset_hold */ rrmlock_t z_teardown_lock; krwlock_t z_teardown_inactive_lock; list_t z_all_znodes; /* all znodes in the fs */ @@ -237,6 +238,7 @@ extern int zfs_end_fs(zfsvfs_t *zfsvfs, struct dsl_dataset *ds); extern void zfs_exit_fs(zfsvfs_t *zfsvfs); extern int zfs_set_version(zfsvfs_t *zfsvfs, uint64_t newvers); extern int zfsvfs_create(const char *name, boolean_t readony, zfsvfs_t **zfvp); +extern int zfsvfs_create_hold(const char *name, zfsvfs_t **zfvp); extern int zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os); extern void zfsvfs_free(zfsvfs_t *zfsvfs); extern int zfs_check_global_label(const char *dsname, const char *hexsl); diff --git a/sys/contrib/openzfs/include/sys/abd.h b/sys/contrib/openzfs/include/sys/abd.h index 41cdb2615427..62c8c0c4e539 100644 --- a/sys/contrib/openzfs/include/sys/abd.h +++ b/sys/contrib/openzfs/include/sys/abd.h @@ -81,8 +81,10 @@ __attribute__((malloc)) abd_t *abd_alloc(size_t, boolean_t); __attribute__((malloc)) abd_t *abd_alloc_linear(size_t, boolean_t); +abd_t *abd_alloc_linear_struct(abd_t *, size_t, boolean_t); __attribute__((malloc)) abd_t *abd_alloc_gang(void); +abd_t *abd_alloc_gang_struct(abd_t *); __attribute__((malloc)) abd_t *abd_alloc_for_io(size_t, boolean_t); __attribute__((malloc)) @@ -94,6 +96,7 @@ abd_t *abd_get_offset(abd_t *, size_t); abd_t *abd_get_offset_size(abd_t *, size_t, size_t); abd_t *abd_get_offset_struct(abd_t *, abd_t *, size_t, size_t); abd_t *abd_get_zeros(size_t); +abd_t *abd_get_zeros_struct(abd_t *, size_t); abd_t *abd_get_from_buf(void *, size_t); abd_t *abd_get_from_buf_struct(abd_t *, void *, size_t); void abd_cache_reap_now(void); diff --git a/sys/contrib/openzfs/include/sys/dsl_deleg.h b/sys/contrib/openzfs/include/sys/dsl_deleg.h index 36dd6211219d..b91b64731612 100644 --- a/sys/contrib/openzfs/include/sys/dsl_deleg.h +++ b/sys/contrib/openzfs/include/sys/dsl_deleg.h @@ -47,6 +47,7 @@ extern "C" { #define ZFS_DELEG_PERM_SHARE "share" #define ZFS_DELEG_PERM_SEND "send" #define ZFS_DELEG_PERM_SEND_RAW "send:raw" +#define ZFS_DELEG_PERM_SEND_ENCRYPTED "send:encrypted" #define ZFS_DELEG_PERM_RECEIVE "receive" #define ZFS_DELEG_PERM_RECEIVE_APPEND "receive:append" #define ZFS_DELEG_PERM_ALLOW "allow" diff --git a/sys/contrib/openzfs/include/sys/vdev.h b/sys/contrib/openzfs/include/sys/vdev.h index 131cfc9cd16b..f79a9fa4c572 100644 --- a/sys/contrib/openzfs/include/sys/vdev.h +++ b/sys/contrib/openzfs/include/sys/vdev.h @@ -243,8 +243,8 @@ typedef enum { extern int vdev_label_init(vdev_t *vd, uint64_t txg, vdev_labeltype_t reason); -extern int vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl); -extern int vdev_prop_get(vdev_t *vd, nvlist_t *nvprops, nvlist_t *outnvl); +extern int vdev_prop_set(spa_t *spa, nvlist_t *innvl, nvlist_t *outnvl); +extern int vdev_prop_get(spa_t *spa, nvlist_t *nvprops, nvlist_t *outnvl); #ifdef __cplusplus } diff --git a/sys/contrib/openzfs/include/sys/zap.h b/sys/contrib/openzfs/include/sys/zap.h index 50e7079e014a..85e8860e31de 100644 --- a/sys/contrib/openzfs/include/sys/zap.h +++ b/sys/contrib/openzfs/include/sys/zap.h @@ -349,6 +349,8 @@ int zap_remove(objset_t *os, uint64_t zapobj, const char *name, dmu_tx_t *tx); int zap_remove_by_dnode(dnode_t *dn, const char *name, dmu_tx_t *tx); int zap_remove_norm(objset_t *os, uint64_t zapobj, const char *name, matchtype_t mt, dmu_tx_t *tx); +int zap_remove_norm_by_dnode(dnode_t *dn, const char *name, + matchtype_t mt, dmu_tx_t *tx); /* Remove by uint64_t[] key. */ int zap_remove_uint64(objset_t *os, uint64_t zapobj, const uint64_t *key, diff --git a/sys/contrib/openzfs/include/sys/zfs_sa.h b/sys/contrib/openzfs/include/sys/zfs_sa.h index b40fbe469951..c83c6cf281c1 100644 --- a/sys/contrib/openzfs/include/sys/zfs_sa.h +++ b/sys/contrib/openzfs/include/sys/zfs_sa.h @@ -76,6 +76,7 @@ typedef enum zpl_attr { ZPL_DACL_ACES, ZPL_DXATTR, ZPL_PROJID, + ZPL_SEQ, ZPL_END } zpl_attr_t; diff --git a/sys/contrib/openzfs/include/sys/zfs_znode.h b/sys/contrib/openzfs/include/sys/zfs_znode.h index 79b845a672a8..3258b41bef6a 100644 --- a/sys/contrib/openzfs/include/sys/zfs_znode.h +++ b/sys/contrib/openzfs/include/sys/zfs_znode.h @@ -112,6 +112,30 @@ extern "C" { #define SA_ZPL_DXATTR(z) z->z_attr_table[ZPL_DXATTR] #define SA_ZPL_PAD(z) z->z_attr_table[ZPL_PAD] #define SA_ZPL_PROJID(z) z->z_attr_table[ZPL_PROJID] +#define SA_ZPL_SEQ(z) z->z_attr_table[ZPL_SEQ] + +/* + * may_grow for a dmu_tx_hold_sa() that may persist z_seq: the SA layout + * grows the first time SA_ZPL_SEQ is added, so grow until z_has_seq is + * set. z_has_seq is an in-core only marker (see znode_t) and is never + * persisted, so no global pflag bit is consumed. + */ +#define ZFS_SEQ_MAY_GROW(zp) \ + ((zp)->z_has_seq ? B_FALSE : B_TRUE) + +/* + * Persist zp->z_seq: mark z_has_seq and add SA_ZPL_SEQ to the caller's + * bulk. No-op for legacy (non-SA-native) znodes. Chunked writers add + * SA_ZPL_SEQ once before their loop and set z_has_seq per chunk instead. + */ +#define ZFS_PERSIST_SEQ(zp, bulk, count) \ +{ \ + if ((zp)->z_is_sa) { \ + (zp)->z_has_seq = B_TRUE; \ + SA_ADD_BULK_ATTR((bulk), (count), SA_ZPL_SEQ(ZTOZSB(zp)), \ + NULL, &(zp)->z_seq, 8); \ + } \ +} /* * Is ID ephemeral? @@ -195,8 +219,10 @@ typedef struct znode { boolean_t z_is_sa; /* are we native sa? */ boolean_t z_is_ctldir; /* are we .zfs entry */ boolean_t z_suspended; /* extra ref from a suspend? */ + boolean_t z_xattr_dir_absent; /* no xattr dir (cached) */ + boolean_t z_has_seq; /* SA_ZPL_SEQ present (in-core only) */ uint_t z_blksz; /* block size in bytes */ - uint_t z_seq; /* modification sequence number */ + uint64_t z_seq; /* modification sequence number */ uint64_t z_mapcnt; /* number of pages mapped to file */ uint64_t z_dnodesize; /* dnode size */ uint64_t z_size; /* file size (cached) */ diff --git a/sys/contrib/openzfs/include/zfs_crrd.h b/sys/contrib/openzfs/include/zfs_crrd.h index ba192a2062ea..669647aba6e4 100644 --- a/sys/contrib/openzfs/include/zfs_crrd.h +++ b/sys/contrib/openzfs/include/zfs_crrd.h @@ -60,16 +60,17 @@ typedef struct { rrd_t dbr_months; } dbrrd_t; -size_t rrd_len(rrd_t *rrd); +size_t rrd_len(const rrd_t *rrd); -const rrd_data_t *rrd_entry(rrd_t *r, size_t i); +const rrd_data_t *rrd_entry(const rrd_t *r, size_t i); rrd_data_t *rrd_tail_entry(rrd_t *rrd); uint64_t rrd_tail(rrd_t *rrd); -uint64_t rrd_get(rrd_t *rrd, size_t i); +uint64_t rrd_get(const rrd_t *rrd, size_t i); void rrd_add(rrd_t *rrd, hrtime_t time, uint64_t txg); void dbrrd_add(dbrrd_t *db, hrtime_t time, uint64_t txg); uint64_t dbrrd_query(dbrrd_t *r, hrtime_t tv, dbrrd_rounding_t rouding); +hrtime_t dbrrd_latest_time(dbrrd_t *r); #endif diff --git a/sys/contrib/openzfs/include/zfs_deleg.h b/sys/contrib/openzfs/include/zfs_deleg.h index a7bbf1620ad5..aecab3816895 100644 --- a/sys/contrib/openzfs/include/zfs_deleg.h +++ b/sys/contrib/openzfs/include/zfs_deleg.h @@ -56,6 +56,7 @@ typedef enum { ZFS_DELEG_NOTE_RENAME, ZFS_DELEG_NOTE_SEND, ZFS_DELEG_NOTE_SEND_RAW, + ZFS_DELEG_NOTE_SEND_ENCRYPTED, ZFS_DELEG_NOTE_RECEIVE, ZFS_DELEG_NOTE_ALLOW, ZFS_DELEG_NOTE_USERPROP, diff --git a/sys/contrib/openzfs/man/man1/ztest.1 b/sys/contrib/openzfs/man/man1/ztest.1 index ae857bfea29c..df54bbce5b6e 100644 --- a/sys/contrib/openzfs/man/man1/ztest.1 +++ b/sys/contrib/openzfs/man/man1/ztest.1 @@ -67,7 +67,7 @@ .Nm was written by the ZFS Developers as a ZFS unit test. The tool was developed in tandem with the ZFS functionality and was -executed nightly as one of the many regression test against the daily build. +executed nightly as one of the many regression tests against the daily build. As features were added to ZFS, unit tests were also added to .Nm . In addition, a separate test development team wrote and diff --git a/sys/contrib/openzfs/man/man4/zfs.4 b/sys/contrib/openzfs/man/man4/zfs.4 index 9967d9af7399..bb896ac9dc6a 100644 --- a/sys/contrib/openzfs/man/man4/zfs.4 +++ b/sys/contrib/openzfs/man/man4/zfs.4 @@ -1090,6 +1090,10 @@ and uses pages, unlike the Linux-specific .Sy zfs_arc_sys_free which is measured in bytes. . +.It Sy zfs_ccw_retry_interval Ns = Ns Sy 300 Ns s Pq int +Interval, in seconds, at which a failed write of the configuration cache file +is retried. +. .It Sy zfs_checksum_events_per_second Ns = Ns Sy 20 Ns /s Pq uint Rate limit checksum events to this many per second. Note that this should not be set below the ZED thresholds diff --git a/sys/contrib/openzfs/man/man8/zfs-allow.8 b/sys/contrib/openzfs/man/man8/zfs-allow.8 index e3b0e1ab3e12..76d7ddde660d 100644 --- a/sys/contrib/openzfs/man/man8/zfs-allow.8 +++ b/sys/contrib/openzfs/man/man8/zfs-allow.8 @@ -29,8 +29,9 @@ .\" Copyright 2019 Richard Laager. All rights reserved. .\" Copyright 2018 Nexenta Systems, Inc. .\" Copyright 2019 Joyent, Inc. +.\" Copyright (c) 2026, TrueNAS. .\" -.Dd September 8, 2025 +.Dd June 14, 2026 .Dt ZFS-ALLOW 8 .Os . @@ -214,6 +215,7 @@ rename subcommand Must also have the \fBmount\fR and \fBcreate\fR ability in the rollback subcommand Must also have the \fBmount\fR ability send subcommand Allows sending a replication stream of a dataset. send:raw subcommand Only allows sending raw replication streams, preventing encrypted datasets being sent in decrypted form. +send:encrypted subcommand Only allows sending raw replication streams of encrypted datasets, preventing encrypted datasets being sent in decrypted form and preventing decrypted datasets being sent at all. share subcommand Allows sharing file systems over NFS or SMB protocols snapshot subcommand Must also have the \fBmount\fR ability diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/sysctl_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/sysctl_os.c index a253280df0d6..c3eb3e6c779b 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/sysctl_os.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/sysctl_os.c @@ -555,15 +555,6 @@ param_set_multihost_interval(SYSCTL_HANDLER_ARGS) return (0); } -/* spa.c */ - -extern int zfs_ccw_retry_interval; - -SYSCTL_INT(_vfs_zfs, OID_AUTO, ccw_retry_interval, - CTLFLAG_RWTUN, &zfs_ccw_retry_interval, 0, - "Configuration cache file write, retry after failure, interval" - " (seconds)"); - /* spa_misc.c */ extern int zfs_flags; diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_acl.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_acl.c index c98ccd756405..b3f7a3d2fbc1 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_acl.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_acl.c @@ -1170,7 +1170,7 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx) dmu_object_type_t otype; zfs_acl_locator_cb_t locate = { 0 }; uint64_t mode; - sa_bulk_attr_t bulk[5]; + sa_bulk_attr_t bulk[6]; uint64_t ctime[2]; int count = 0; zfs_acl_phys_t acl_phys; @@ -1316,6 +1316,8 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx) zp->z_pflags |= ZFS_ACL_TRIVIAL; zfs_tstamp_update_setup(zp, STATE_CHANGED, NULL, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); return (sa_bulk_update(zp->z_sa_hdl, bulk, count, tx)); } diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_dir.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_dir.c index 75ba2ea0cb9e..7b3b02335c6c 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_dir.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_dir.c @@ -394,9 +394,9 @@ zfs_purgedir(znode_t *dzp) (ZTOV(xzp)->v_type == VLNK)); tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); dmu_tx_hold_zap(tx, dzp->z_id, FALSE, zap->za_name); - dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, xzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(xzp)); dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL); /* Is this really needed ? */ zfs_sa_upgrade_txholds(tx, xzp); @@ -584,7 +584,7 @@ zfs_link_create(znode_t *dzp, const char *name, znode_t *zp, dmu_tx_t *tx, vnode_t *vp = ZTOV(zp); uint64_t value; int zp_is_dir = (vp->v_type == VDIR); - sa_bulk_attr_t bulk[5]; + sa_bulk_attr_t bulk[6]; uint64_t mtime[2], ctime[2]; int count = 0; int error; @@ -647,7 +647,9 @@ zfs_link_create(znode_t *dzp, const char *name, znode_t *zp, dmu_tx_t *tx, ctime, sizeof (ctime)); zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); } + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); ASSERT0(error); @@ -665,6 +667,8 @@ zfs_link_create(znode_t *dzp, const char *name, znode_t *zp, dmu_tx_t *tx, SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, &dzp->z_pflags, sizeof (dzp->z_pflags)); zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime); + ZFS_PERSIST_SEQ(dzp, bulk, count); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx); ASSERT0(error); return (0); @@ -729,7 +733,7 @@ zfs_link_destroy(znode_t *dzp, const char *name, znode_t *zp, dmu_tx_t *tx, vnode_t *vp = ZTOV(zp); int zp_is_dir = (vp->v_type == VDIR); boolean_t unlinked = B_FALSE; - sa_bulk_attr_t bulk[5]; + sa_bulk_attr_t bulk[6]; uint64_t mtime[2], ctime[2]; int count = 0; int error; @@ -764,6 +768,13 @@ zfs_link_destroy(znode_t *dzp, const char *name, znode_t *zp, dmu_tx_t *tx, zp->z_unlinked = B_TRUE; zp->z_links = 0; unlinked = B_TRUE; + /* + * NFS observers see nlink=0; advance change_cookie. + * POSIX permits skipping the ctime stamp at nlink=0, + * and the znode is destined for reap so persistence + * would be wasted. + */ + atomic_inc_64(&zp->z_seq); } else { SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, sizeof (ctime)); @@ -771,9 +782,11 @@ zfs_link_destroy(znode_t *dzp, const char *name, znode_t *zp, dmu_tx_t *tx, NULL, &zp->z_pflags, sizeof (zp->z_pflags)); zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); } SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL, &zp->z_links, sizeof (zp->z_links)); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); count = 0; ASSERT0(error); @@ -797,6 +810,8 @@ zfs_link_destroy(znode_t *dzp, const char *name, znode_t *zp, dmu_tx_t *tx, SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, &dzp->z_pflags, sizeof (dzp->z_pflags)); zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime); + ZFS_PERSIST_SEQ(dzp, bulk, count); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx); ASSERT0(error); @@ -828,6 +843,9 @@ zfs_make_xattrdir(znode_t *zp, vattr_t *vap, znode_t **xvpp, cred_t *cr) boolean_t fuid_dirtied; uint64_t parent __maybe_unused; + if (zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_ELOCKED(ZTOV(zp), __func__); + *xvpp = NULL; if ((error = zfs_acl_ids_create(zp, IS_XATTR, vap, cr, NULL, @@ -875,6 +893,9 @@ zfs_make_xattrdir(znode_t *zp, vattr_t *vap, znode_t **xvpp, cred_t *cr) getnewvnode_drop_reserve(); + /* Record that the file now has an xattr directory. */ + zp->z_xattr_dir_absent = B_FALSE; + *xvpp = xzp; return (0); @@ -900,6 +921,19 @@ zfs_get_xattrdir(znode_t *zp, znode_t **xzpp, cred_t *cr, int flags) znode_t *xzp; vattr_t va; int error; + + if (zfsvfs->z_replay == B_FALSE) + ASSERT_VOP_LOCKED(ZTOV(zp), __func__); + + /* + * Fast path: a file already known to have no xattr directory, when not + * creating one, returns without taking the "" ZXATTR dirlock or doing + * the SA_ZPL_XATTR lookup below. z_xattr_dir_absent tracks this: it is + * set when the lookup finds no directory and cleared when one is found + * or created. + */ + if (!(flags & CREATE_XATTR_DIR) && zp->z_xattr_dir_absent) + return (SET_ERROR(ENOATTR)); top: error = zfs_dirent_lookup(zp, "", &xzp, ZXATTR); if (error) @@ -907,12 +941,19 @@ top: if (xzp != NULL) { *xzpp = xzp; + zp->z_xattr_dir_absent = B_FALSE; return (0); } - if (!(flags & CREATE_XATTR_DIR)) + if (!(flags & CREATE_XATTR_DIR)) { + /* + * z_xattr_dir_absent is serialized by the base vnode lock, + * which the creator holds exclusive and readers hold shared. + */ + zp->z_xattr_dir_absent = B_TRUE; return (SET_ERROR(ENOATTR)); + } if (zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) { return (SET_ERROR(EROFS)); diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vfsops.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vfsops.c index 7f9c02678229..4504f1ba1a4a 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vfsops.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vfsops.c @@ -51,7 +51,7 @@ #include <sys/dsl_prop.h> #include <sys/dsl_dataset.h> #include <sys/dsl_deleg.h> -#include <sys/spa.h> +#include <sys/spa_impl.h> #include <sys/zap.h> #include <sys/sa.h> #include <sys/sa_impl.h> @@ -69,6 +69,7 @@ #include <sys/zfs_quota.h> #include "zfs_comutil.h" +#include "zfs_crrd.h" #ifndef MNTK_VMSETSIZE_BUG #define MNTK_VMSETSIZE_BUG 0 @@ -1046,6 +1047,32 @@ zfsvfs_create(const char *osname, boolean_t readonly, zfsvfs_t **zfvp) return (error); } +int +zfsvfs_create_hold(const char *osname, zfsvfs_t **zfvp) +{ + objset_t *os; + zfsvfs_t *zfsvfs; + int error; + + zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP); + + error = dmu_objset_hold(osname, zfsvfs, &os); + if (error != 0) { + kmem_free(zfsvfs, sizeof (zfsvfs_t)); + return (error); + } + + if (dmu_objset_type(os) != DMU_OST_ZFS) { + dmu_objset_rele(os, zfsvfs); + kmem_free(zfsvfs, sizeof (zfsvfs_t)); + return (EINVAL); + } + + zfsvfs->z_use_hold = B_TRUE; + error = zfsvfs_create_impl(zfvp, zfsvfs, os); + + return (error); +} int zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os) @@ -1069,7 +1096,10 @@ zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os) error = zfsvfs_init(zfsvfs, os); if (error != 0) { - dmu_objset_disown(os, B_TRUE, zfsvfs); + if (zfsvfs->z_use_hold) + dmu_objset_rele(os, zfsvfs); + else + dmu_objset_disown(os, B_TRUE, zfsvfs); *zfvp = NULL; kmem_free(zfsvfs, sizeof (zfsvfs_t)); return (error); @@ -1323,7 +1353,12 @@ out: dmu_objset_disown(zfsvfs->z_os, B_TRUE, zfsvfs); zfsvfs_free(zfsvfs); } else { + spa_t *spa = zfsvfs->z_os->os_spa; + atomic_inc_32(&zfs_active_fs_count); + + vfsp->mnt_time = dbrrd_latest_time(&spa->spa_txg_log_time); + vfsp->mnt_time = MAX(vfsp->mnt_time, spa->spa_load_txg_ts); } return (error); diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c index 4dd9ed88d433..b4a9f5e3cca2 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c @@ -1138,7 +1138,7 @@ zfs_create(znode_t *dzp, const char *name, vattr_t *vap, int excl, int mode, if (fuid_dirtied) zfs_fuid_txhold(zfsvfs, tx); dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name); - dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) { dmu_tx_hold_write(tx, DMU_NEW_OBJECT, @@ -1269,7 +1269,8 @@ zfs_remove_(vnode_t *dvp, vnode_t *vp, const char *name, cred_t *cr) */ tx = dmu_tx_create(zfsvfs->z_os); dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); zfs_sa_upgrade_txholds(tx, zp); zfs_sa_upgrade_txholds(tx, dzp); @@ -1497,6 +1498,7 @@ zfs_mkdir(znode_t *dzp, const char *dirname, vattr_t *vap, znode_t **zpp, tx = dmu_tx_create(zfsvfs->z_os); dmu_tx_hold_zap(tx, dzp->z_id, TRUE, dirname); dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); fuid_dirtied = zfsvfs->z_fuid_dirty; if (fuid_dirtied) zfs_fuid_txhold(zfsvfs, tx); @@ -1605,7 +1607,8 @@ zfs_rmdir_(vnode_t *dvp, vnode_t *vp, const char *name, cred_t *cr) tx = dmu_tx_create(zfsvfs->z_os); dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL); zfs_sa_upgrade_txholds(tx, zp); zfs_sa_upgrade_txholds(tx, dzp); @@ -2010,7 +2013,7 @@ zfs_getattr(vnode_t *vp, vattr_t *vap, int flags, cred_t *cr) vap->va_rdev = NODEV; vap->va_gen = zp->z_gen; vap->va_flags = 0; /* FreeBSD: Reset chflags(2) flags. */ - vap->va_filerev = zp->z_seq; + vap->va_filerev = atomic_load_64(&zp->z_seq); /* * Add in any requested optional attributes and the create time. @@ -2316,7 +2319,7 @@ zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zidmap_t *mnt_ns) boolean_t skipaclchk = (flags & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE; boolean_t fuid_dirtied = B_FALSE; boolean_t handle_eadir = B_FALSE; - sa_bulk_attr_t bulk[7], xattr_bulk[7]; + sa_bulk_attr_t bulk[9], xattr_bulk[6]; int count = 0, xattr_count = 0; if (mask == 0) @@ -2756,14 +2759,15 @@ zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zidmap_t *mnt_ns) if (((mask & AT_XVATTR) && XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP)) || (projid != ZFS_INVALID_PROJID && - !(zp->z_pflags & ZFS_PROJID))) + !(zp->z_pflags & ZFS_PROJID)) || + !zp->z_has_seq) dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE); else dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); } if (attrzp) { - dmu_tx_hold_sa(tx, attrzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, attrzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(attrzp)); } fuid_dirtied = zfsvfs->z_fuid_dirty; @@ -2906,16 +2910,19 @@ zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zidmap_t *mnt_ns) SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, sizeof (ctime)); zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); } else if (mask != 0) { SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, sizeof (ctime)); zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); if (attrzp) { SA_ADD_BULK_ATTR(xattr_bulk, xattr_count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, sizeof (ctime)); zfs_tstamp_update_setup(attrzp, STATE_CHANGED, mtime, ctime); + ZFS_PERSIST_SEQ(attrzp, xattr_bulk, xattr_count); } } @@ -2976,6 +2983,7 @@ zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zidmap_t *mnt_ns) } out: if (err == 0 && attrzp) { + ASSERT3S(xattr_count, <=, ARRAY_SIZE(xattr_bulk)); err2 = sa_bulk_update(attrzp->z_sa_hdl, xattr_bulk, xattr_count, tx); ASSERT0(err2); @@ -2994,6 +3002,7 @@ out: if (attrzp) vput(ZTOV(attrzp)); } else { + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); err2 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); dmu_tx_commit(tx); if (attrzp) { @@ -3469,16 +3478,16 @@ zfs_do_rename_impl(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp, } tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE); - dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, szp->z_sa_hdl, ZFS_SEQ_MAY_GROW(szp)); + dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(sdzp)); dmu_tx_hold_zap(tx, sdzp->z_id, FALSE, snm); dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, tnm); if (sdzp != tdzp) { - dmu_tx_hold_sa(tx, tdzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, tdzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(tdzp)); zfs_sa_upgrade_txholds(tx, tdzp); } if (tzp) { - dmu_tx_hold_sa(tx, tzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, tzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(tzp)); zfs_sa_upgrade_txholds(tx, tzp); } @@ -3679,7 +3688,7 @@ zfs_symlink(znode_t *dzp, const char *name, vattr_t *vap, dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name); dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes + ZFS_SA_BASE_ATTR_SIZE + len); - dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) { dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, acl_ids.z_aclp->z_acl_bytes); @@ -3901,7 +3910,8 @@ zfs_link(znode_t *tdzp, znode_t *szp, const char *name, cred_t *cr, } tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, szp->z_sa_hdl, ZFS_SEQ_MAY_GROW(szp)); + dmu_tx_hold_sa(tx, tdzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(tdzp)); dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, name); zfs_sa_upgrade_txholds(tx, szp); zfs_sa_upgrade_txholds(tx, tdzp); @@ -4465,7 +4475,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags, tx = dmu_tx_create(zfsvfs->z_os); dmu_tx_hold_write(tx, zp->z_id, off, len); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); zfs_sa_upgrade_txholds(tx, zp); err = dmu_tx_assign(tx, DMU_TX_WAIT); if (err != 0) { @@ -4489,7 +4499,7 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags, if (err == 0) { uint64_t mtime[2], ctime[2]; - sa_bulk_attr_t bulk[3]; + sa_bulk_attr_t bulk[4]; int count = 0; SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, @@ -4499,6 +4509,8 @@ zfs_putpages(struct vnode *vp, vm_page_t *ma, size_t len, int flags, SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, &zp->z_pflags, 8); zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); err = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); ASSERT0(err); @@ -5770,7 +5782,8 @@ zfs_freebsd_pathconf(struct vop_pathconf_args *ap) { ulong_t val; int error; -#if defined(_PC_CLONE_BLKSIZE) || defined(_PC_CASE_INSENSITIVE) +#if defined(_PC_CLONE_BLKSIZE) || defined(_PC_CASE_INSENSITIVE) || \ + defined(_PC_HAS_HIDDENSYSTEM) zfsvfs_t *zfsvfs; #endif @@ -5816,7 +5829,11 @@ zfs_freebsd_pathconf(struct vop_pathconf_args *ap) #endif #ifdef _PC_HAS_HIDDENSYSTEM case _PC_HAS_HIDDENSYSTEM: - *ap->a_retval = 1; + zfsvfs = (zfsvfs_t *)ap->a_vp->v_mount->mnt_data; + if (zfsvfs->z_use_fuids == B_TRUE) + *ap->a_retval = 1; + else + *ap->a_retval = 0; return (0); #endif #ifdef _PC_CLONE_BLKSIZE @@ -6035,7 +6052,7 @@ zfs_getextattr_impl(struct vop_getextattr_args *ap, boolean_t compat) error = ENOENT; if (zfsvfs->z_use_sa && zp->z_is_sa) error = zfs_getextattr_sa(ap, attrname); - if (error == ENOENT) + if (error == ENOENT && !zp->z_xattr_dir_absent) error = zfs_getextattr_dir(ap, attrname); return (error); } diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_znode_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_znode_os.c index 6f74c924eb79..5b6589b8a551 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_znode_os.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_znode_os.c @@ -151,6 +151,7 @@ zfs_znode_cache_constructor(void *buf, void *arg, int kmflags) zp->z_xattr_cached = NULL; zp->z_xattr_parent = 0; zp->z_vnode = NULL; + zp->z_has_seq = B_FALSE; return (0); } @@ -287,6 +288,7 @@ zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx) ASSERT(!POINTER_IS_VALID(sharezp->z_zfsvfs)); sharezp->z_unlinked = 0; sharezp->z_atime_dirty = 0; + sharezp->z_xattr_dir_absent = B_FALSE; sharezp->z_zfsvfs = zfsvfs; sharezp->z_is_sa = zfsvfs->z_use_sa; sharezp->z_pflags = 0; @@ -447,6 +449,7 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz, zp->z_sa_hdl = NULL; zp->z_unlinked = 0; zp->z_atime_dirty = 0; + zp->z_xattr_dir_absent = B_FALSE; zp->z_mapcnt = 0; zp->z_id = db->db_object; zp->z_blksz = blksz; @@ -490,6 +493,18 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz, return (NULL); } + /* + * Restore z_seq from SA_ZPL_SEQ when present, marking the file migrated + * via the in-core z_has_seq (never persisted). Absence keeps the + * default z_seq; FreeBSD's va_filerev never folded ctime in, so no + * seed is needed across the upgrade. + */ + if (zp->z_is_sa && sa_lookup(zp->z_sa_hdl, SA_ZPL_SEQ(zfsvfs), + &zp->z_seq, sizeof (zp->z_seq)) == 0) + zp->z_has_seq = B_TRUE; + else + zp->z_has_seq = B_FALSE; + zp->z_projid = projid; zp->z_mode = mode; @@ -1130,6 +1145,7 @@ zfs_rezget(znode_t *zp) nvlist_free(zp->z_xattr_cached); zp->z_xattr_cached = NULL; } + zp->z_xattr_dir_absent = B_FALSE; rw_exit(&zp->z_xattr_lock); ASSERT0P(zp->z_sa_hdl); @@ -1187,6 +1203,17 @@ zfs_rezget(znode_t *zp) } zp->z_projid = projid; + + /* + * Reload z_has_seq and z_seq from disk so stale in-core state from + * before rollback/recv does not survive. A stale TRUE marker would + * make ZFS_SEQ_MAY_GROW() skip the grow reservation while SA_ZPL_SEQ + * is gone on disk. + */ + zp->z_has_seq = (zp->z_is_sa && + sa_lookup(zp->z_sa_hdl, SA_ZPL_SEQ(zfsvfs), + &zp->z_seq, sizeof (zp->z_seq)) == 0); + zp->z_mode = mode; if (gen != zp->z_gen) { @@ -1378,7 +1405,7 @@ zfs_tstamp_update_setup_ext(znode_t *zp, uint_t flag, uint64_t mtime[2], if (have_tx) { /* will sa_bulk_update happen really soon? */ zp->z_atime_dirty = 0; - zp->z_seq++; + atomic_inc_64(&zp->z_seq); } else { zp->z_atime_dirty = 1; } @@ -1668,7 +1695,7 @@ zfs_freesp(znode_t *zp, uint64_t off, uint64_t len, int flag, boolean_t log) zilog_t *zilog = zfsvfs->z_log; uint64_t mode; uint64_t mtime[2], ctime[2]; - sa_bulk_attr_t bulk[3]; + sa_bulk_attr_t bulk[4]; int count = 0; int error; @@ -1695,7 +1722,7 @@ zfs_freesp(znode_t *zp, uint64_t off, uint64_t len, int flag, boolean_t log) return (error); log: tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); zfs_sa_upgrade_txholds(tx, zp); error = dmu_tx_assign(tx, DMU_TX_WAIT); if (error) { @@ -1708,6 +1735,8 @@ log: SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, &zp->z_pflags, 8); zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); ASSERT0(error); @@ -1810,6 +1839,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) ASSERT(!POINTER_IS_VALID(rootzp->z_zfsvfs)); rootzp->z_unlinked = 0; rootzp->z_atime_dirty = 0; + rootzp->z_xattr_dir_absent = B_FALSE; rootzp->z_is_sa = USE_SA(version, os); rootzp->z_pflags = 0; diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_acl.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_acl.c index 4c929a4642b1..86eb58667df7 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_acl.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_acl.c @@ -1359,7 +1359,7 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx) dmu_object_type_t otype; zfs_acl_locator_cb_t locate = { 0 }; uint64_t mode; - sa_bulk_attr_t bulk[5]; + sa_bulk_attr_t bulk[6]; uint64_t ctime[2]; int count = 0; zfs_acl_phys_t acl_phys; @@ -1501,6 +1501,8 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx) zp->z_pflags |= ZFS_ACL_TRIVIAL; zfs_tstamp_update_setup(zp, STATE_CHANGED, NULL, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); return (sa_bulk_update(zp->z_sa_hdl, bulk, count, tx)); } diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_ctldir.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_ctldir.c index 767272177fb7..7e56cc476b67 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_ctldir.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_ctldir.c @@ -495,6 +495,7 @@ zfsctl_inode_alloc(zfsvfs_t *zfsvfs, uint64_t id, zp->z_zn_prefetch = B_FALSE; zp->z_is_sa = B_FALSE; zp->z_is_ctldir = B_TRUE; + zp->z_xattr_dir_absent = B_FALSE; zp->z_sa_hdl = NULL; zp->z_blksz = 0; zp->z_seq = 0; diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_dir.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_dir.c index 7edea05f94e6..7e9cafadd4d1 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_dir.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_dir.c @@ -617,9 +617,9 @@ zfs_purgedir(znode_t *dzp) S_ISLNK(ZTOI(xzp)->i_mode)); tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); dmu_tx_hold_zap(tx, dzp->z_id, FALSE, zap->za_name); - dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, xzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(xzp)); dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL); /* Is this really needed ? */ zfs_sa_upgrade_txholds(tx, xzp); @@ -809,7 +809,7 @@ zfs_link_create(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag) zfsvfs_t *zfsvfs = ZTOZSB(zp); uint64_t value; int zp_is_dir = S_ISDIR(ZTOI(zp)->i_mode); - sa_bulk_attr_t bulk[5]; + sa_bulk_attr_t bulk[6]; uint64_t mtime[2], ctime[2]; uint64_t links; int count = 0; @@ -871,7 +871,9 @@ zfs_link_create(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag) ctime, sizeof (ctime)); zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); } + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); ASSERT0(error); @@ -894,6 +896,8 @@ zfs_link_create(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag) SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, &dzp->z_pflags, sizeof (dzp->z_pflags)); zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime); + ZFS_PERSIST_SEQ(dzp, bulk, count); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx); ASSERT0(error); mutex_exit(&dzp->z_lock); @@ -955,7 +959,7 @@ zfs_drop_nlink_locked(znode_t *zp, dmu_tx_t *tx, boolean_t *unlinkedp) zfsvfs_t *zfsvfs = ZTOZSB(zp); int zp_is_dir = S_ISDIR(ZTOI(zp)->i_mode); boolean_t unlinked = B_FALSE; - sa_bulk_attr_t bulk[3]; + sa_bulk_attr_t bulk[4]; uint64_t mtime[2], ctime[2]; uint64_t links; int count = 0; @@ -975,6 +979,12 @@ zfs_drop_nlink_locked(znode_t *zp, dmu_tx_t *tx, boolean_t *unlinkedp) zp->z_unlinked = B_TRUE; clear_nlink(ZTOI(zp)); unlinked = B_TRUE; + /* + * NFS observers must see nlink=0; advance change_cookie. + * POSIX permits skipping the ctime stamp at nlink=0, and the + * znode is destined for reap so persistence would be wasted. + */ + atomic_inc_64(&zp->z_seq); } else { SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, sizeof (ctime)); @@ -982,10 +992,12 @@ zfs_drop_nlink_locked(znode_t *zp, dmu_tx_t *tx, boolean_t *unlinkedp) NULL, &zp->z_pflags, sizeof (zp->z_pflags)); zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); } links = ZTOI(zp)->i_nlink; SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL, &links, sizeof (links)); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); ASSERT0(error); @@ -1032,7 +1044,7 @@ zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag, zfsvfs_t *zfsvfs = ZTOZSB(dzp); int zp_is_dir = S_ISDIR(ZTOI(zp)->i_mode); boolean_t unlinked = B_FALSE; - sa_bulk_attr_t bulk[5]; + sa_bulk_attr_t bulk[6]; uint64_t mtime[2], ctime[2]; uint64_t links; int count = 0; @@ -1083,6 +1095,8 @@ zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag, SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, &dzp->z_pflags, sizeof (dzp->z_pflags)); zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime); + ZFS_PERSIST_SEQ(dzp, bulk, count); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx); ASSERT0(error); mutex_exit(&dzp->z_lock); @@ -1178,6 +1192,9 @@ zfs_make_xattrdir(znode_t *zp, vattr_t *vap, znode_t **xzpp, cred_t *cr) zfs_acl_ids_free(&acl_ids); dmu_tx_commit(tx); + /* Record that the file now has an xattr directory. */ + zp->z_xattr_dir_absent = B_FALSE; + *xzpp = xzp; return (0); @@ -1204,6 +1221,17 @@ zfs_get_xattrdir(znode_t *zp, znode_t **xzpp, cred_t *cr, int flags) zfs_dirlock_t *dl; vattr_t va; int error; + + /* + * Fast path for a file already known to have no xattr directory. When + * the caller is not creating one, return ENOENT without taking the "" + * ZXATTR dirlock or doing the SA_ZPL_XATTR lookup below, which is pure + * overhead when an absent xattr such as security.capability is looked + * up on every open. z_xattr_dir_absent tracks this: it is set when the + * lookup finds no directory and cleared when one is found or created. + */ + if (!(flags & CREATE_XATTR_DIR) && zp->z_xattr_dir_absent) + return (SET_ERROR(ENOENT)); top: error = zfs_dirent_lock(&dl, zp, "", &xzp, ZXATTR, NULL, NULL); if (error) @@ -1211,11 +1239,13 @@ top: if (xzp != NULL) { *xzpp = xzp; + zp->z_xattr_dir_absent = B_FALSE; zfs_dirent_unlock(dl); return (0); } if (!(flags & CREATE_XATTR_DIR)) { + zp->z_xattr_dir_absent = B_TRUE; zfs_dirent_unlock(dl); return (SET_ERROR(ENOENT)); } diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_vfsops.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_vfsops.c index 27f3bbb46f42..e7df1f6bcebf 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_vfsops.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_vfsops.c @@ -622,6 +622,32 @@ zfsvfs_init(zfsvfs_t *zfsvfs, objset_t *os) } int +zfsvfs_create_hold(const char *osname, zfsvfs_t **zfvp) +{ + objset_t *os; + zfsvfs_t *zfsvfs; + int error; + + zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP); + + error = dmu_objset_hold(osname, zfsvfs, &os); + if (error != 0) { + kmem_free(zfsvfs, sizeof (zfsvfs_t)); + return (error); + } + + if (dmu_objset_type(os) != DMU_OST_ZFS) { + dmu_objset_rele(os, zfsvfs); + kmem_free(zfsvfs, sizeof (zfsvfs_t)); + return (EINVAL); + } + + zfsvfs->z_use_hold = B_TRUE; + error = zfsvfs_create_impl(zfvp, zfsvfs, os); + return (error); +} + +int zfsvfs_create(const char *osname, boolean_t readonly, zfsvfs_t **zfvp) { objset_t *os; @@ -678,7 +704,10 @@ zfsvfs_create_impl(zfsvfs_t **zfvp, zfsvfs_t *zfsvfs, objset_t *os) error = zfsvfs_init(zfsvfs, os); if (error != 0) { - dmu_objset_disown(os, B_TRUE, zfsvfs); + if (zfsvfs->z_use_hold) + dmu_objset_rele(os, zfsvfs); + else + dmu_objset_disown(os, B_TRUE, zfsvfs); *zfvp = NULL; zfsvfs_free(zfsvfs); return (error); diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c index d6dad70ae09f..4ff9945c8956 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_vnops_os.c @@ -732,7 +732,7 @@ top: if (fuid_dirtied) zfs_fuid_txhold(zfsvfs, tx); dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name); - dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) { dmu_tx_hold_write(tx, DMU_NEW_OBJECT, @@ -810,9 +810,7 @@ top: goto out; } - mutex_enter(&dzp->z_lock); - dzp->z_seq++; - mutex_exit(&dzp->z_lock); + atomic_inc_64(&dzp->z_seq); /* * Truncate regular files if requested. @@ -1063,7 +1061,8 @@ top: obj = zp->z_id; tx = dmu_tx_create(zfsvfs->z_os); dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); zfs_sa_upgrade_txholds(tx, zp); zfs_sa_upgrade_txholds(tx, dzp); if (may_delete_now) { @@ -1080,7 +1079,7 @@ top: error = zfs_zget(zfsvfs, xattr_obj, &xzp); ASSERT0(error); dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE); - dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, xzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(xzp)); } mutex_enter(&zp->z_lock); @@ -1332,6 +1331,7 @@ top: tx = dmu_tx_create(zfsvfs->z_os); dmu_tx_hold_zap(tx, dzp->z_id, TRUE, dirname); dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); fuid_dirtied = zfsvfs->z_fuid_dirty; if (fuid_dirtied) zfs_fuid_txhold(zfsvfs, tx); @@ -1484,7 +1484,8 @@ top: tx = dmu_tx_create(zfsvfs->z_os); dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL); zfs_sa_upgrade_txholds(tx, zp); zfs_sa_upgrade_txholds(tx, dzp); @@ -1947,7 +1948,7 @@ zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zidmap_t *mnt_ns) boolean_t fuid_dirtied = B_FALSE; boolean_t handle_eadir = B_FALSE; sa_bulk_attr_t *bulk, *xattr_bulk; - int count = 0, xattr_count = 0, bulks = 8; + int count = 0, xattr_count = 0, bulks = 9; if (mask == 0) return (0); @@ -2376,7 +2377,8 @@ top: if (((mask & ATTR_XVATTR) && XVA_ISSET_REQ(xvap, XAT_AV_SCANSTAMP)) || (projid != ZFS_INVALID_PROJID && - !(zp->z_pflags & ZFS_PROJID))) + !(zp->z_pflags & ZFS_PROJID)) || + !zp->z_has_seq) dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE); else dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); @@ -2589,15 +2591,12 @@ top: if (mask != 0) { zfs_log_setattr(zilog, tx, TX_SETATTR, zp, vap, mask, fuidp); /* - * Ensure that the z_seq is always incremented on setattr - * operation. This is required for change accounting for - * NFS clients. - * - * ATTR_MODE already increments via zfs_acl_chmod_setattr. - * ATTR_SIZE already increments via zfs_freesp. + * ATTR_MODE bumps via zfs_aclset_common -> tstamp_update_setup; + * ATTR_SIZE goes through zfs_freesp(log=FALSE) which does not. */ - if (!(mask & (ATTR_MODE | ATTR_SIZE))) - zp->z_seq++; + if (!(mask & ATTR_MODE)) + atomic_inc_64(&zp->z_seq); + ZFS_PERSIST_SEQ(zp, bulk, count); } mutex_exit(&zp->z_lock); @@ -2611,6 +2610,7 @@ top: } out: if (err == 0 && xattr_count > 0) { + ASSERT3S(xattr_count, <=, bulks); err2 = sa_bulk_update(attrzp->z_sa_hdl, xattr_bulk, xattr_count, tx); ASSERT0(err2); @@ -2631,6 +2631,7 @@ out: if (err == ERESTART) goto top; } else { + ASSERT3S(count, <=, bulks); if (count > 0) err2 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); dmu_tx_commit(tx); @@ -3061,17 +3062,17 @@ top: } tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE); - dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, szp->z_sa_hdl, ZFS_SEQ_MAY_GROW(szp)); + dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(sdzp)); dmu_tx_hold_zap(tx, sdzp->z_id, (rflags & RENAME_EXCHANGE) ? TRUE : FALSE, snm); dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, tnm); if (sdzp != tdzp) { - dmu_tx_hold_sa(tx, tdzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, tdzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(tdzp)); zfs_sa_upgrade_txholds(tx, tdzp); } if (tzp) { - dmu_tx_hold_sa(tx, tzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, tzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(tzp)); zfs_sa_upgrade_txholds(tx, tzp); } if (rflags & RENAME_WHITEOUT) { @@ -3079,7 +3080,7 @@ top: ZFS_SA_BASE_ATTR_SIZE); dmu_tx_hold_zap(tx, sdzp->z_id, TRUE, snm); - dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, sdzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(sdzp)); if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) { dmu_tx_hold_write(tx, DMU_NEW_OBJECT, @@ -3380,7 +3381,7 @@ top: dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name); dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes + ZFS_SA_BASE_ATTR_SIZE + len); - dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, dzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(dzp)); if (!zfsvfs->z_use_sa && acl_ids.z_aclp->z_acl_bytes > ZFS_ACE_SPACE) { dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, acl_ids.z_aclp->z_acl_bytes); @@ -3632,7 +3633,8 @@ top: } tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, szp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, szp->z_sa_hdl, ZFS_SEQ_MAY_GROW(szp)); + dmu_tx_hold_sa(tx, tdzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(tdzp)); dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, name); if (is_tmpfile) dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL); @@ -3772,7 +3774,7 @@ zfs_putpage(struct inode *ip, struct page *pp, struct writeback_control *wbc, int err = 0; uint64_t mtime[2], ctime[2]; inode_timespec_t tmp_ts; - sa_bulk_attr_t bulk[3]; + sa_bulk_attr_t bulk[4]; int cnt = 0; struct address_space *mapping; @@ -3889,7 +3891,7 @@ zfs_putpage(struct inode *ip, struct page *pp, struct writeback_control *wbc, tx = dmu_tx_create(zfsvfs->z_os); dmu_tx_hold_write(tx, zp->z_id, pgoff, pglen); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); zfs_sa_upgrade_txholds(tx, zp); err = dmu_tx_assign(tx, DMU_TX_WAIT); @@ -3912,19 +3914,20 @@ zfs_putpage(struct inode *ip, struct page *pp, struct writeback_control *wbc, DMU_READ_PREFETCH); kunmap(pp); - SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16); - SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16); - SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_FLAGS(zfsvfs), NULL, - &zp->z_pflags, 8); - /* Preserve the mtime and ctime provided by the inode */ tmp_ts = zpl_inode_get_mtime(ip); ZFS_TIME_ENCODE(&tmp_ts, mtime); tmp_ts = zpl_inode_get_ctime(ip); ZFS_TIME_ENCODE(&tmp_ts, ctime); zp->z_atime_dirty = B_FALSE; - zp->z_seq++; + atomic_inc_64(&zp->z_seq); + SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16); + SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16); + SA_ADD_BULK_ATTR(bulk, cnt, SA_ZPL_FLAGS(zfsvfs), NULL, + &zp->z_pflags, 8); + ZFS_PERSIST_SEQ(zp, bulk, cnt); + ASSERT3S(cnt, <=, ARRAY_SIZE(bulk)); err = sa_bulk_update(zp->z_sa_hdl, bulk, cnt, tx); /* @@ -4010,7 +4013,7 @@ zfs_dirty_inode(struct inode *ip, int flags) dmu_tx_t *tx; uint64_t mode, atime[2], mtime[2], ctime[2]; inode_timespec_t tmp_ts; - sa_bulk_attr_t bulk[4]; + sa_bulk_attr_t bulk[5]; int error = 0; int cnt = 0; @@ -4036,7 +4039,7 @@ zfs_dirty_inode(struct inode *ip, int flags) tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); zfs_sa_upgrade_txholds(tx, zp); error = dmu_tx_assign(tx, DMU_TX_WAIT); @@ -4063,7 +4066,10 @@ zfs_dirty_inode(struct inode *ip, int flags) mode = ip->i_mode; zp->z_mode = mode; + /* persist z_seq; callers bump it before zfs_mark_inode_dirty */ + ZFS_PERSIST_SEQ(zp, bulk, cnt); + ASSERT3S(cnt, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(zp->z_sa_hdl, bulk, cnt, tx); mutex_exit(&zp->z_lock); diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode_os.c index 03976d01a0e3..be761cc6168a 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode_os.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_znode_os.c @@ -127,6 +127,7 @@ zfs_znode_cache_constructor(void *buf, void *arg, int kmflags) zp->z_acl_cached = NULL; zp->z_xattr_cached = NULL; zp->z_xattr_parent = 0; + zp->z_has_seq = B_FALSE; return (0); } @@ -538,11 +539,11 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz, zp->z_atime_dirty = B_FALSE; zp->z_is_ctldir = B_FALSE; zp->z_suspended = B_FALSE; + zp->z_xattr_dir_absent = B_FALSE; zp->z_sa_hdl = NULL; zp->z_mapcnt = 0; zp->z_id = db->db_object; zp->z_blksz = blksz; - zp->z_seq = 0x7A4653; zp->z_sync_cnt = 0; zfs_znode_sa_init(zfsvfs, zp, db, obj_type, hdl); @@ -573,6 +574,22 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz, goto error; } + /* + * Restore z_seq from SA_ZPL_SEQ. A successful lookup marks the file as + * migrated via the in-core z_has_seq (never persisted, so no pflag bit + * is consumed). Absence means the file predates persistence: seed z_seq + * above any cookie the pre-persistence code could have presented + * ((ctime << 32) | low) so it stays monotonic across the upgrade; the + * first modify migrates the file. + */ + if (zp->z_is_sa && sa_lookup(zp->z_sa_hdl, SA_ZPL_SEQ(zfsvfs), + &zp->z_seq, sizeof (zp->z_seq)) == 0) { + zp->z_has_seq = B_TRUE; + } else { + zp->z_has_seq = B_FALSE; + zp->z_seq = (ctime[0] + 1) << 32; + } + zp->z_projid = projid; zp->z_mode = ip->i_mode = mode; ip->i_generation = (uint32_t)tmp_gen; @@ -1201,6 +1218,8 @@ zfs_rezget(znode_t *zp) } rw_exit(&zp->z_xattr_lock); + zp->z_xattr_dir_absent = B_FALSE; + ASSERT0P(zp->z_sa_hdl); err = sa_buf_hold(zfsvfs->z_os, obj_num, NULL, &db); if (err) { @@ -1260,6 +1279,17 @@ zfs_rezget(znode_t *zp) } zp->z_projid = projid; + + /* + * Reload z_has_seq and z_seq from disk so stale in-core state from + * before rollback/recv does not survive. A stale TRUE marker would + * make ZFS_SEQ_MAY_GROW() skip the grow reservation while SA_ZPL_SEQ + * is gone on disk. + */ + zp->z_has_seq = (zp->z_is_sa && + sa_lookup(zp->z_sa_hdl, SA_ZPL_SEQ(zfsvfs), + &zp->z_seq, sizeof (zp->z_seq)) == 0); + zp->z_mode = ZTOI(zp)->i_mode = mode; zfs_uid_write(ZTOI(zp), z_uid); zfs_gid_write(ZTOI(zp), z_gid); @@ -1415,7 +1445,7 @@ zfs_tstamp_update_setup(znode_t *zp, uint_t flag, uint64_t mtime[2], gethrestime(&now); - zp->z_seq++; + atomic_inc_64(&zp->z_seq); if (flag & ATTR_MTIME) { ZFS_TIME_ENCODE(&now, mtime); @@ -1754,7 +1784,7 @@ zfs_freesp(znode_t *zp, uint64_t off, uint64_t len, int flag, boolean_t log) zilog_t *zilog = zfsvfs->z_log; uint64_t mode; uint64_t mtime[2], ctime[2]; - sa_bulk_attr_t bulk[3]; + sa_bulk_attr_t bulk[4]; int count = 0; int error; @@ -1780,7 +1810,7 @@ zfs_freesp(znode_t *zp, uint64_t off, uint64_t len, int flag, boolean_t log) goto out; log: tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); zfs_sa_upgrade_txholds(tx, zp); error = dmu_tx_assign(tx, DMU_TX_WAIT); if (error) { @@ -1793,6 +1823,8 @@ log: SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, &zp->z_pflags, 8); zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); ASSERT0(error); @@ -1904,6 +1936,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) rootzp = kmem_cache_alloc(znode_cache, KM_SLEEP); rootzp->z_unlinked = B_FALSE; rootzp->z_atime_dirty = B_FALSE; + rootzp->z_xattr_dir_absent = B_FALSE; rootzp->z_is_sa = USE_SA(version, os); rootzp->z_pflags = 0; diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c index 6d57bff56540..5e4009ac286b 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c @@ -372,6 +372,31 @@ zpl_llseek(struct file *filp, loff_t offset, int whence) * helpful to move the ARC buffers to a scatter-gather lists * rather than a vmalloc'ed region. */ +/* + * Bump z_seq when a clean page first transitions to dirty via an mmap store. + * The default generic_file_vm_ops.page_mkwrite (filemap_page_mkwrite) updates + * mtime/ctime via file_update_time -> __mark_inode_dirty, but never tells the + * filesystem that the change cookie should advance. Without this hook NFSv4 + * GETATTR between an mmap store and writeback returns a stale change_cookie + * alongside the newer mtime, violating monotonicity. zfs_dirty_inode persists + * the new value on the same dirty path. + */ +static vm_fault_t +zpl_page_mkwrite(struct vm_fault *vmf) +{ + znode_t *zp = ITOZ(file_inode(vmf->vma->vm_file)); + + atomic_inc_64(&zp->z_seq); + + return (filemap_page_mkwrite(vmf)); +} + +static const struct vm_operations_struct zpl_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = zpl_page_mkwrite, +}; + static int zpl_mmap(struct file *filp, struct vm_area_struct *vma) { @@ -391,6 +416,7 @@ zpl_mmap(struct file *filp, struct vm_area_struct *vma) if (error) return (error); + vma->vm_ops = &zpl_vm_ops; return (error); } @@ -741,7 +767,12 @@ zpl_fallocate_common(struct inode *ip, int mode, loff_t offset, loff_t len) if (error) goto out_unmark; - error = -zfs_freesp(zp, offset + len, 0, 0, FALSE); + /* + * extend file: log=TRUE drives z_seq bump, + * mtime/ctime advance, and TX_TRUNCATE ZIL + * record; matches zfs_space(). + */ + error = -zfs_freesp(zp, offset + len, 0, 0, TRUE); zfs_exit(zfsvfs, FTAG); } } diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c index e4e15c824f4b..a47fb1d7db7c 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c @@ -517,16 +517,18 @@ zpl_getattr_impl(const struct path *path, struct kstat *stat, u32 request_mask, * ZFS uses a coarse timer to set them, which may cause * clients to fail to detect changes and invalidate cache. * - * ZFS always increments znode z_seq number, but this is - * uint_t and so we mask in ctime to upper bits. + * z_seq is a per-file 64-bit counter bumped on every change + * and persisted across znode eviction, so it is presented + * directly as a monotonic change cookie. Files that predate + * persistence are seeded from ctime on load so the cookie + * never moves backward across the upgrade. * * STATX_ATTR_CHANGE_MONOTONIC is advertised * to prevent knfsd from generating the change cookie * based on ctime. C.f. nfsd4_change_attribute in * fs/nfsd/nfsfh.c. */ - stat->change_cookie = - ((u64)stat->ctime.tv_sec << 32) | zp->z_seq; + stat->change_cookie = atomic_load_64(&zp->z_seq); stat->attributes |= STATX_ATTR_CHANGE_MONOTONIC; stat->result_mask |= STATX_CHANGE_COOKIE; } diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_xattr.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_xattr.c index 68050c870de1..846735fb9a16 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_xattr.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_xattr.c @@ -381,6 +381,12 @@ __zpl_xattr_get(struct inode *ip, const char *name, void *value, size_t size, goto out; } + /* Known to have no xattr directory; skip the directory lookup. */ + if (zp->z_xattr_dir_absent) { + error = -ENOENT; + goto out; + } + error = zpl_xattr_get_dir(ip, name, value, size, cr); out: if (error == -ENOENT) @@ -517,6 +523,7 @@ zpl_xattr_set_dir(struct inode *ip, const char *name, const void *value, out: if (error == 0) { zpl_inode_set_ctime_to_ts(ip, current_time(ip)); + atomic_inc_64(&ITOZ(ip)->z_seq); zfs_mark_inode_dirty(ip); } @@ -1012,6 +1019,7 @@ zpl_set_acl_impl(struct inode *ip, struct posix_acl *acl, int type) ip->i_mode = ITOZ(ip)->z_mode = mode; zpl_inode_set_ctime_to_ts(ip, current_time(ip)); + atomic_inc_64(&ITOZ(ip)->z_seq); zfs_mark_inode_dirty(ip); } @@ -1152,6 +1160,7 @@ zpl_init_acl(struct inode *ip, struct inode *dir) if (!acl) { ITOZ(ip)->z_mode = (ip->i_mode &= ~current_umask()); zpl_inode_set_ctime_to_ts(ip, current_time(ip)); + atomic_inc_64(&ITOZ(ip)->z_seq); zfs_mark_inode_dirty(ip); return (0); } @@ -1170,6 +1179,7 @@ zpl_init_acl(struct inode *ip, struct inode *dir) error = __posix_acl_create(&acl, GFP_KERNEL, &mode); if (error >= 0) { ip->i_mode = ITOZ(ip)->z_mode = mode; + atomic_inc_64(&ITOZ(ip)->z_seq); zfs_mark_inode_dirty(ip); if (error > 0) { error = zpl_set_acl_impl(ip, acl, diff --git a/sys/contrib/openzfs/module/zcommon/zfs_deleg.c b/sys/contrib/openzfs/module/zcommon/zfs_deleg.c index 87596558c9a1..422c5544974d 100644 --- a/sys/contrib/openzfs/module/zcommon/zfs_deleg.c +++ b/sys/contrib/openzfs/module/zcommon/zfs_deleg.c @@ -60,6 +60,7 @@ const zfs_deleg_perm_tab_t zfs_deleg_perm_tab[] = { {ZFS_DELEG_PERM_SHARE}, {ZFS_DELEG_PERM_SEND}, {ZFS_DELEG_PERM_SEND_RAW}, + {ZFS_DELEG_PERM_SEND_ENCRYPTED}, {ZFS_DELEG_PERM_USERPROP}, {ZFS_DELEG_PERM_USERQUOTA}, {ZFS_DELEG_PERM_GROUPQUOTA}, diff --git a/sys/contrib/openzfs/module/zfs/abd.c b/sys/contrib/openzfs/module/zfs/abd.c index 7ea07c418300..cb575e21acd7 100644 --- a/sys/contrib/openzfs/module/zfs/abd.c +++ b/sys/contrib/openzfs/module/zfs/abd.c @@ -212,11 +212,9 @@ abd_alloc(size_t size, boolean_t is_metadata) * buffer. Only use this when it would be very annoying to write your ABD * consumer with a scattered ABD. */ -abd_t * -abd_alloc_linear(size_t size, boolean_t is_metadata) +static abd_t * +abd_alloc_linear_impl(abd_t *abd, size_t size, boolean_t is_metadata) { - abd_t *abd = abd_alloc_struct(0); - VERIFY3U(size, <=, SPA_MAXBLOCKSIZE); abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_OWNER; @@ -236,6 +234,19 @@ abd_alloc_linear(size_t size, boolean_t is_metadata) return (abd); } +abd_t * +abd_alloc_linear(size_t size, boolean_t is_metadata) +{ + return (abd_alloc_linear_impl(abd_alloc_struct(0), size, is_metadata)); +} + +abd_t * +abd_alloc_linear_struct(abd_t *abd, size_t size, boolean_t is_metadata) +{ + abd_init_struct(abd); + return (abd_alloc_linear_impl(abd, size, is_metadata)); +} + static void abd_free_linear(abd_t *abd) { @@ -349,16 +360,28 @@ abd_alloc_sametype(abd_t *sabd, size_t size) * to "chain" scatter/gather lists together when constructing aggregated * IO's. To free this abd, abd_free() must be called. */ -abd_t * -abd_alloc_gang(void) +static abd_t * +abd_alloc_gang_impl(abd_t *abd) { - abd_t *abd = abd_alloc_struct(0); abd->abd_flags |= ABD_FLAG_GANG | ABD_FLAG_OWNER; list_create(&ABD_GANG(abd).abd_gang_chain, sizeof (abd_t), offsetof(abd_t, abd_gang_link)); return (abd); } +abd_t * +abd_alloc_gang(void) +{ + return (abd_alloc_gang_impl(abd_alloc_struct(0))); +} + +abd_t * +abd_alloc_gang_struct(abd_t *abd) +{ + abd_init_struct(abd); + return (abd_alloc_gang_impl(abd)); +} + /* * Add a child gang ABD to a parent gang ABDs chained list. */ @@ -624,6 +647,14 @@ abd_get_zeros(size_t size) return (abd_get_offset_size(abd_zero_scatter, 0, size)); } +abd_t * +abd_get_zeros_struct(abd_t *abd, size_t size) +{ + ASSERT3P(abd_zero_scatter, !=, NULL); + ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); + return (abd_get_offset_struct(abd, abd_zero_scatter, 0, size)); +} + /* * Create a linear ABD for an existing buf. */ diff --git a/sys/contrib/openzfs/module/zfs/ddt_log.c b/sys/contrib/openzfs/module/zfs/ddt_log.c index 7e699a9b4252..f16731136c67 100644 --- a/sys/contrib/openzfs/module/zfs/ddt_log.c +++ b/sys/contrib/openzfs/module/zfs/ddt_log.c @@ -211,7 +211,7 @@ ddt_log_begin(ddt_t *ddt, size_t nentries, dmu_tx_t *tx, ddt_log_update_t *dlu) ASSERT3U(reclen, <=, UINT16_MAX); dlu->dlu_reclen = reclen; - VERIFY0(dnode_hold(ddt->ddt_os, ddt->ddt_log_active->ddl_object, FTAG, + VERIFY0(dnode_hold(ddt->ddt_os, ddt->ddt_log_active->ddl_object, dlu, &dlu->dlu_dn)); dnode_set_storage_type(dlu->dlu_dn, DMU_OT_DDT_ZAP); @@ -342,7 +342,7 @@ ddt_log_commit(ddt_t *ddt, ddt_log_update_t *dlu) ddt->ddt_log_active->ddl_length += dlu->dlu_ndbp * (uint64_t)dlu->dlu_dn->dn_datablksz; - dnode_rele(dlu->dlu_dn, FTAG); + dnode_rele(dlu->dlu_dn, dlu); ddt_log_update_header(ddt, ddt->ddt_log_active, dlu->dlu_tx); diff --git a/sys/contrib/openzfs/module/zfs/metaslab.c b/sys/contrib/openzfs/module/zfs/metaslab.c index 2be1f2812681..aff0940b978c 100644 --- a/sys/contrib/openzfs/module/zfs/metaslab.c +++ b/sys/contrib/openzfs/module/zfs/metaslab.c @@ -2854,6 +2854,9 @@ metaslab_set_selected_txg(metaslab_t *msp, uint64_t txg) { ASSERT(MUTEX_HELD(&msp->ms_lock)); metaslab_class_t *mc = msp->ms_group->mg_class; + if (msp->ms_selected_txg == txg && + multilist_link_active(&msp->ms_class_txg_node)) + return; multilist_sublist_t *mls = multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); if (multilist_link_active(&msp->ms_class_txg_node)) diff --git a/sys/contrib/openzfs/module/zfs/spa.c b/sys/contrib/openzfs/module/zfs/spa.c index c6ae91b8d9e9..810589b42f45 100644 --- a/sys/contrib/openzfs/module/zfs/spa.c +++ b/sys/contrib/openzfs/module/zfs/spa.c @@ -11866,3 +11866,7 @@ ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs_zio, zio_, taskq_free, ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_write_tpq, UINT, ZMOD_RW, "Number of CPUs per write issue taskq"); + +ZFS_MODULE_PARAM(zfs, zfs_, ccw_retry_interval, INT, ZMOD_RW, + "Configuration cache file write, retry after failure, interval " + "(seconds)"); diff --git a/sys/contrib/openzfs/module/zfs/vdev.c b/sys/contrib/openzfs/module/zfs/vdev.c index 821dfd6faffe..4a96d39eac1e 100644 --- a/sys/contrib/openzfs/module/zfs/vdev.c +++ b/sys/contrib/openzfs/module/zfs/vdev.c @@ -1739,11 +1739,12 @@ vdev_metaslab_init(vdev_t *vd, uint64_t txg) /* * The metaslab was marked as dirty at the end of * metaslab_init(). Remove it from the dirty list so that we - * can uninitialize and reinitialize it to the new class. + * can uninitialize and reinitialize it to the new class. It + * may be dirty in any txg slot, so clear them all. */ - if (txg != 0) { + for (int t = 0; t < TXG_SIZE; t++) { (void) txg_list_remove_this(&vd->vdev_ms_list, - slog_ms, txg); + slog_ms, t); } uint64_t sm_obj = space_map_object(slog_ms->ms_sm); metaslab_fini(slog_ms); @@ -4622,6 +4623,7 @@ vdev_offline_locked(spa_t *spa, uint64_t guid, uint64_t flags) int error = 0; uint64_t generation; metaslab_group_t *mg; + boolean_t dtl_required; top: spa_vdev_state_enter(spa, SCL_ALLOC); @@ -4643,13 +4645,14 @@ top: * If the device isn't already offline, try to offline it. */ if (!vd->vdev_offline) { + dtl_required = vdev_dtl_required(vd); + /* * If this device has the only valid copy of some data, * don't allow it to be offlined. Log devices are always * expendable. */ - if (!tvd->vdev_islog && vd->vdev_aux == NULL && - vdev_dtl_required(vd)) + if (!tvd->vdev_islog && vd->vdev_aux == NULL && dtl_required) return (spa_vdev_state_exit(spa, NULL, SET_ERROR(EBUSY))); @@ -4659,9 +4662,10 @@ top: * is not NULL since it's possible that we may have just * added this vdev but not yet initialized its metaslabs. */ - if (tvd->vdev_islog && mg != NULL) { + if (tvd->vdev_islog && mg != NULL && dtl_required) { /* - * Prevent any future allocations. + * Prevent future allocations unless the log device is + * redundant. */ ASSERT0P(tvd->vdev_log_mg); metaslab_group_passivate(mg); @@ -4717,7 +4721,7 @@ top: * Add the device back into the metaslab rotor so that * once we online the device it's open for business. */ - if (tvd->vdev_islog && mg != NULL) + if (tvd->vdev_islog && mg != NULL && dtl_required) metaslab_group_activate(mg); } @@ -6200,22 +6204,14 @@ vdev_props_set_sync(void *arg, dmu_tx_t *tx) } int -vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) +vdev_prop_set(spa_t *spa, nvlist_t *innvl, nvlist_t *outnvl) { - spa_t *spa = vd->vdev_spa; + vdev_t *vd; nvpair_t *elem = NULL; uint64_t vdev_guid; nvlist_t *nvprops; int error = 0; - ASSERT(vd != NULL); - - /* Check that vdev has a zap we can use */ - if (vd->vdev_root_zap == 0 && - vd->vdev_top_zap == 0 && - vd->vdev_leaf_zap == 0) - return (SET_ERROR(EINVAL)); - if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_SET_VDEV, &vdev_guid) != 0) return (SET_ERROR(EINVAL)); @@ -6224,8 +6220,31 @@ vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) &nvprops) != 0) return (SET_ERROR(EINVAL)); - if ((vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE)) == NULL) + /* + * Resolve the vdev by guid and hold SCL_CONFIG as a reader so the + * vdev tree can't change beneath us while we touch vd. The lock is + * dropped around the "path" and "allocating" handlers below: those + * descend into spa_vdev_enter() -> spa_config_enter(SCL_ALL, + * RW_WRITER), and taking SCL_CONFIG as a writer while this same + * thread already holds it as a reader is a self-deadlock (the writer + * waits for scl_count to drain to 0, but scl_count is this thread's + * own reader, which is never released). Those handlers re-resolve + * the vdev by guid under their own locking, so we re-resolve here + * after each one in case the tree changed. + */ + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + if ((vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE)) == NULL) { + spa_config_exit(spa, SCL_CONFIG, FTAG); + return (SET_ERROR(ENOENT)); + } + + /* Check that vdev has a zap we can use */ + if (vd->vdev_root_zap == 0 && + vd->vdev_top_zap == 0 && + vd->vdev_leaf_zap == 0) { + spa_config_exit(spa, SCL_CONFIG, FTAG); return (SET_ERROR(EINVAL)); + } while ((elem = nvlist_next_nvpair(nvprops, elem)) != NULL) { const char *propname = nvpair_name(elem); @@ -6259,7 +6278,17 @@ vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) error = EINVAL; break; } + /* + * spa_vdev_setpath() takes SCL_ALL as a writer, so we + * must not hold SCL_CONFIG across it (see above). Drop + * it, then re-resolve vd in case the tree changed. + */ + spa_config_exit(spa, SCL_CONFIG, FTAG); error = spa_vdev_setpath(spa, vdev_guid, strval); + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE); + if (vd == NULL && error == 0) + error = SET_ERROR(ENOENT); break; case VDEV_PROP_ALLOCATING: if (nvpair_value_uint64(elem, &intval) != 0) { @@ -6268,10 +6297,19 @@ vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) } if (intval != vd->vdev_noalloc) break; + /* + * spa_vdev_noalloc()/spa_vdev_alloc() take SCL_ALL as a + * writer; same locking dance as VDEV_PROP_PATH above. + */ + spa_config_exit(spa, SCL_CONFIG, FTAG); if (intval == 0) error = spa_vdev_noalloc(spa, vdev_guid); else error = spa_vdev_alloc(spa, vdev_guid); + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE); + if (vd == NULL && error == 0) + error = SET_ERROR(ENOENT); break; case VDEV_PROP_FAILFAST: if (nvpair_value_uint64(elem, &intval) != 0 || @@ -6444,10 +6482,15 @@ end: if (error != 0) { intval = error; vdev_prop_add_list(outnvl, propname, strval, intval, 0); - return (error); + break; } } + spa_config_exit(spa, SCL_CONFIG, FTAG); + + if (error != 0) + return (error); + return (dsl_sync_task(spa->spa_name, NULL, vdev_props_set_sync, innvl, 6, ZFS_SPACE_CHECK_EXTRA_RESERVED)); } @@ -6462,10 +6505,10 @@ vdev_get_child_idx(vdev_t *vd, uint64_t c_guid) } int -vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) +vdev_prop_get(spa_t *spa, nvlist_t *innvl, nvlist_t *outnvl) { - spa_t *spa = vd->vdev_spa; objset_t *mos = spa->spa_meta_objset; + vdev_t *vd; int err = 0; uint64_t objid = 0; uint64_t vdev_guid; @@ -6477,7 +6520,6 @@ vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) const char *propname = NULL; vdev_prop_t prop; - ASSERT(vd != NULL); ASSERT(mos != NULL); if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_GET_VDEV, @@ -6487,6 +6529,18 @@ vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) nvlist_lookup_nvlist(innvl, ZPOOL_VDEV_PROPS_GET_PROPS, &nvprops); /* + * Resolve the vdev by guid and hold SCL_CONFIG as a reader across the + * property fetch so the vdev tree can't change beneath us. This path + * is read-only and never takes SCL_CONFIG as a writer, so holding the + * reader throughout is safe. + */ + spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); + if ((vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE)) == NULL) { + spa_config_exit(spa, SCL_CONFIG, FTAG); + return (SET_ERROR(ENOENT)); + } + + /* * A missing ZAP is normal for spare and L2ARC vdevs, which are * not part of the main vdev tree and never get ZAPs allocated. * Many properties are sourced directly from vdev_t fields and @@ -6997,6 +7051,8 @@ vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) } mutex_exit(&spa->spa_props_lock); + spa_config_exit(spa, SCL_CONFIG, FTAG); + if (err && err != ENOENT) { return (err); } diff --git a/sys/contrib/openzfs/module/zfs/vdev_draid.c b/sys/contrib/openzfs/module/zfs/vdev_draid.c index 8f556b868784..019c20f3f99d 100644 --- a/sys/contrib/openzfs/module/zfs/vdev_draid.c +++ b/sys/contrib/openzfs/module/zfs/vdev_draid.c @@ -798,7 +798,8 @@ vdev_draid_map_alloc_write(zio_t *zio, uint64_t abd_offset, raidz_row_t *rr) if (rc->rc_size == 0) { /* empty data column (small write), add a skip sector */ ASSERT3U(skip_size, ==, parity_size); - rc->rc_abd = abd_get_zeros(skip_size); + rc->rc_abd = abd_get_zeros_struct(&rc->rc_abdstruct, + skip_size); } else if (rc->rc_size == parity_size) { /* this is a "big column" */ rc->rc_abd = abd_get_offset_struct(&rc->rc_abdstruct, @@ -806,7 +807,7 @@ vdev_draid_map_alloc_write(zio_t *zio, uint64_t abd_offset, raidz_row_t *rr) } else { /* short data column, add a skip sector */ ASSERT3U(rc->rc_size + skip_size, ==, parity_size); - rc->rc_abd = abd_alloc_gang(); + rc->rc_abd = abd_alloc_gang_struct(&rc->rc_abdstruct); abd_gang_add(rc->rc_abd, abd_get_offset_size( zio->io_abd, abd_off, rc->rc_size), B_TRUE); abd_gang_add(rc->rc_abd, abd_get_zeros(skip_size), @@ -863,7 +864,7 @@ vdev_draid_map_alloc_scrub(zio_t *zio, uint64_t abd_offset, raidz_row_t *rr) /* short data column, add a skip sector */ ASSERT3U(rc->rc_size + skip_size, ==, parity_size); ASSERT3U(rr->rr_nempty, !=, 0); - rc->rc_abd = abd_alloc_gang(); + rc->rc_abd = abd_alloc_gang_struct(&rc->rc_abdstruct); abd_gang_add(rc->rc_abd, abd_get_offset_size( zio->io_abd, abd_off, rc->rc_size), B_TRUE); abd_gang_add(rc->rc_abd, abd_get_offset_size( @@ -1240,7 +1241,8 @@ vdev_draid_map_alloc_row(zio_t *zio, raidz_row_t **rrp, uint64_t io_offset, /* Allocate buffers for the parity columns */ for (uint64_t c = 0; c < rr->rr_firstdatacol; c++) { raidz_col_t *rc = &rr->rr_col[c]; - rc->rc_abd = abd_alloc_linear(rc->rc_size, B_FALSE); + rc->rc_abd = abd_alloc_linear_struct(&rc->rc_abdstruct, + rc->rc_size, B_FALSE); } /* diff --git a/sys/contrib/openzfs/module/zfs/vdev_raidz.c b/sys/contrib/openzfs/module/zfs/vdev_raidz.c index 2db7422e772e..ab9f62c3c2d0 100644 --- a/sys/contrib/openzfs/module/zfs/vdev_raidz.c +++ b/sys/contrib/openzfs/module/zfs/vdev_raidz.c @@ -419,7 +419,16 @@ static int zfs_scrub_partial_writes = 1; static void vdev_raidz_row_free(raidz_row_t *rr) { - for (int c = 0; c < rr->rr_cols; c++) { + abd_t *dabd = rr->rr_col[rr->rr_firstdatacol].rc_abd; + for (int c = 0; c < rr->rr_firstdatacol; c++) { + raidz_col_t *rc = &rr->rr_col[c]; + + if (rc->rc_size != 0 && rc->rc_abd != dabd) + abd_free(rc->rc_abd); + if (rc->rc_orig_data != NULL) + abd_free(rc->rc_orig_data); + } + for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) { raidz_col_t *rc = &rr->rr_col[c]; if (rc->rc_size != 0) @@ -532,6 +541,22 @@ vdev_raidz_map_alloc_write(zio_t *zio, raidz_map_t *rm, uint64_t ashift) */ int skipped = rr->rr_scols - rr->rr_cols; + /* + * When there is only a single data column the parity is a copy of + * it, so point all parity columns at the data ABD directly to avoid + * allocating buffers and computing parity. + */ + if (rr->rr_cols == rr->rr_firstdatacol + 1) { + ASSERT0(nwrapped); + ASSERT0(rm->rm_nskip); + raidz_col_t *dc = &rr->rr_col[rr->rr_firstdatacol]; + dc->rc_abd = abd_get_offset_struct(&dc->rc_abdstruct, + zio->io_abd, 0, dc->rc_size); + for (c = 0; c < rr->rr_firstdatacol; c++) + rr->rr_col[c].rc_abd = dc->rc_abd; + return; + } + /* Allocate buffers for the parity columns */ for (c = 0; c < rr->rr_firstdatacol; c++) { raidz_col_t *rc = &rr->rr_col[c]; @@ -546,12 +571,13 @@ vdev_raidz_map_alloc_write(zio_t *zio, raidz_map_t *rm, uint64_t ashift) * VDEV queue locks (vq_lock). */ if (c < nwrapped) { - rc->rc_abd = abd_alloc_linear( + rc->rc_abd = abd_alloc_linear_struct(&rc->rc_abdstruct, rc->rc_size + (1ULL << ashift), B_FALSE); abd_zero_off(rc->rc_abd, rc->rc_size, 1ULL << ashift); skipped++; } else { - rc->rc_abd = abd_alloc_linear(rc->rc_size, B_FALSE); + rc->rc_abd = abd_alloc_linear_struct(&rc->rc_abdstruct, + rc->rc_size, B_FALSE); } } @@ -599,9 +625,11 @@ vdev_raidz_map_alloc_read(zio_t *zio, raidz_map_t *rm) ASSERT3U(rm->rm_nrows, ==, 1); /* Allocate buffers for the parity columns */ - for (c = 0; c < rr->rr_firstdatacol; c++) - rr->rr_col[c].rc_abd = - abd_alloc_linear(rr->rr_col[c].rc_size, B_FALSE); + for (c = 0; c < rr->rr_firstdatacol; c++) { + raidz_col_t *rc = &rr->rr_col[c]; + rc->rc_abd = abd_alloc_linear_struct(&rc->rc_abdstruct, + rc->rc_size, B_FALSE); + } for (uint64_t off = 0; c < rr->rr_cols; c++) { raidz_col_t *rc = &rr->rr_col[c]; @@ -1046,8 +1074,8 @@ vdev_raidz_map_alloc_expanded(zio_t *zio, continue; prc->rc_abd = - abd_alloc_linear(rm->rm_phys_col[i].rc_size, - B_FALSE); + abd_alloc_linear_struct(&prc->rc_abdstruct, + prc->rc_size, B_FALSE); } /* @@ -1075,8 +1103,8 @@ vdev_raidz_map_alloc_expanded(zio_t *zio, for (int c = 0; c < rr->rr_firstdatacol; c++) { raidz_col_t *rc = &rr->rr_col[c]; rc->rc_abd = - abd_alloc_linear(rc->rc_size, - B_TRUE); + abd_alloc_linear_struct(&rc->rc_abdstruct, + rc->rc_size, B_TRUE); } } } @@ -1272,6 +1300,13 @@ vdev_raidz_generate_parity_row(raidz_map_t *rm, raidz_row_t *rr) return; } + /* + * Single data column: parity is the data itself. + */ + if (rr->rr_col[VDEV_RAIDZ_P].rc_abd == + rr->rr_col[rr->rr_firstdatacol].rc_abd) + return; + /* Generate using the new math implementation */ if (vdev_raidz_math_generate(rm, rr) != RAIDZ_ORIGINAL_IMPL) return; diff --git a/sys/contrib/openzfs/module/zfs/zap.c b/sys/contrib/openzfs/module/zfs/zap.c index ca7598f489b0..9aeb18037a87 100644 --- a/sys/contrib/openzfs/module/zfs/zap.c +++ b/sys/contrib/openzfs/module/zfs/zap.c @@ -744,7 +744,7 @@ zap_length_uint64(objset_t *os, uint64_t zapobj, const uint64_t *key, /* zap_remove */ -static int +int zap_remove_norm_by_dnode(dnode_t *dn, const char *name, matchtype_t mt, dmu_tx_t *tx) { diff --git a/sys/contrib/openzfs/module/zfs/zfs_crrd.c b/sys/contrib/openzfs/module/zfs/zfs_crrd.c index 30d4c7c36897..8aee0727161f 100644 --- a/sys/contrib/openzfs/module/zfs/zfs_crrd.c +++ b/sys/contrib/openzfs/module/zfs/zfs_crrd.c @@ -99,14 +99,14 @@ rrd_tail(rrd_t *rrd) * rrd_get works from 0..rrd_len()-1. */ size_t -rrd_len(rrd_t *rrd) +rrd_len(const rrd_t *rrd) { return (rrd->rrd_length); } const rrd_data_t * -rrd_entry(rrd_t *rrd, size_t i) +rrd_entry(const rrd_t *rrd, size_t i) { size_t n; @@ -119,7 +119,7 @@ rrd_entry(rrd_t *rrd, size_t i) } uint64_t -rrd_get(rrd_t *rrd, size_t i) +rrd_get(const rrd_t *rrd, size_t i) { const rrd_data_t *data = rrd_entry(rrd, i); @@ -226,3 +226,19 @@ dbrrd_query(dbrrd_t *r, hrtime_t tv, dbrrd_rounding_t rounding) return (data == NULL ? 0 : data->rrdd_txg); } + +hrtime_t +dbrrd_latest_time(dbrrd_t *r) +{ + const rrd_data_t *head; + const rrd_t *curdb; + size_t dblen; + + curdb = &r->dbr_minutes; + dblen = rrd_len(curdb); + if (dblen == 0) + return (0); + + head = rrd_entry(curdb, dblen - 1); + return (head->rrdd_time); +} diff --git a/sys/contrib/openzfs/module/zfs/zfs_ioctl.c b/sys/contrib/openzfs/module/zfs/zfs_ioctl.c index a23f397e698e..3eb141d6e5aa 100644 --- a/sys/contrib/openzfs/module/zfs/zfs_ioctl.c +++ b/sys/contrib/openzfs/module/zfs/zfs_ioctl.c @@ -801,6 +801,47 @@ zfs_secpolicy_rollback(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) } static int +zfs_secpolicy_send_impl(const char *name, dsl_dataset_t *ds, cred_t *cr, + boolean_t rawok) +{ + /* Can't send from within a zone that can't see the dataset */ + int err = zfs_dozonecheck_ds(name, ds, cr); + if (err != 0) + return (err); + + /* ZFS global admin (root) can do anything. */ + err = secpolicy_zfs(cr); + if (err == 0) + return (0); + + /* 'send' permission on this dataset is allowed to send. */ + err = dsl_deleg_access_impl(ds, ZFS_DELEG_PERM_SEND, cr); + if (err == 0) + return (0); + + /* Raw sends have extra perms that might work. */ + if (rawok) { + /* 'send:raw' permission on this dataset can do raw sends. */ + err = dsl_deleg_access_impl(ds, ZFS_DELEG_PERM_SEND_RAW, cr); + if (err == 0) + return (0); + + if (ds->ds_dir->dd_crypto_obj != 0) { + /* + * Dataset is encrypted; 'send:encrypted' permission + * will allow a raw send. + */ + err = dsl_deleg_access_impl(ds, + ZFS_DELEG_PERM_SEND_ENCRYPTED, cr); + if (err == 0) + return (0); + } + } + + return (err); +} + +static int zfs_secpolicy_send(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) { (void) innvl; @@ -829,12 +870,8 @@ zfs_secpolicy_send(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) dsl_dataset_name(ds, zc->zc_name); - error = zfs_secpolicy_write_perms_ds(zc->zc_name, ds, - ZFS_DELEG_PERM_SEND, cr); - if (error != 0 && rawok) { - error = zfs_secpolicy_write_perms_ds(zc->zc_name, ds, - ZFS_DELEG_PERM_SEND_RAW, cr); - } + error = zfs_secpolicy_send_impl(zc->zc_name, ds, cr, rawok); + dsl_dataset_rele(ds, FTAG); dsl_pool_rele(dp, FTAG); @@ -844,16 +881,29 @@ zfs_secpolicy_send(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) static int zfs_secpolicy_send_new(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) { - boolean_t rawok = nvlist_exists(innvl, "rawok"); + dsl_pool_t *dp; + dsl_dataset_t *ds; int error; + boolean_t rawok = nvlist_exists(innvl, "rawok"); - (void) innvl; - error = zfs_secpolicy_write_perms(zc->zc_name, - ZFS_DELEG_PERM_SEND, cr); - if (error != 0 && rawok) { - error = zfs_secpolicy_write_perms(zc->zc_name, - ZFS_DELEG_PERM_SEND_RAW, cr); + if (INGLOBALZONE(curproc) && secpolicy_zfs(cr) == 0) + return (0); + + error = dsl_pool_hold(zc->zc_name, FTAG, &dp); + if (error != 0) + return (error); + + error = dsl_dataset_hold(dp, zc->zc_name, FTAG, &ds); + if (error != 0) { + dsl_pool_rele(dp, FTAG); + return (error); } + + error = zfs_secpolicy_send_impl(zc->zc_name, ds, cr, rawok); + + dsl_dataset_rele(ds, FTAG); + dsl_pool_rele(dp, FTAG); + return (error); } @@ -1635,8 +1685,17 @@ zfsvfs_hold(const char *name, const void *tag, zfsvfs_t **zfvp, int error = 0; if (getzfsvfs(name, zfvp) != 0) - error = zfsvfs_create(name, B_FALSE, zfvp); + error = zfsvfs_create_hold(name, zfvp); if (error == 0) { + /* + * dmu_objset_hold() keeps the pool config read lock held. + * Drop it before acquiring the teardown lock to avoid ABBA + * deadlock with zfs_resume_fs(), which holds teardown write + * then acquires the config lock. + */ + if ((*zfvp)->z_use_hold) + dsl_pool_config_exit( + dmu_objset_pool((*zfvp)->z_os), *zfvp); if (writer) ZFS_TEARDOWN_ENTER_WRITE(*zfvp, tag); else @@ -1663,7 +1722,18 @@ zfsvfs_rele(zfsvfs_t *zfsvfs, const void *tag) if (zfs_vfs_held(zfsvfs)) { zfs_vfs_rele(zfsvfs); } else { - dmu_objset_disown(zfsvfs->z_os, B_TRUE, zfsvfs); + objset_t *os = zfsvfs->z_os; + if (zfsvfs->z_use_hold) { + /* + * Opened via dmu_objset_hold(): re-acquire the pool + * config lock (released in zfsvfs_hold() before the + * teardown lock) so that dmu_objset_rele() can exit it. + */ + dsl_pool_config_enter(dmu_objset_pool(os), zfsvfs); + dmu_objset_rele(os, zfsvfs); + } else { + dmu_objset_disown(os, B_TRUE, zfsvfs); + } zfsvfs_free(zfsvfs); } } @@ -3457,13 +3527,6 @@ zfs_ioc_vdev_set_props(const char *poolname, nvlist_t *innvl, nvlist_t *outnvl) { spa_t *spa; int error; - vdev_t *vd; - uint64_t vdev_guid; - - /* Early validation */ - if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_SET_VDEV, - &vdev_guid) != 0) - return (SET_ERROR(EINVAL)); if (outnvl == NULL) return (SET_ERROR(EINVAL)); @@ -3473,15 +3536,7 @@ zfs_ioc_vdev_set_props(const char *poolname, nvlist_t *innvl, nvlist_t *outnvl) ASSERT(spa_writeable(spa)); - spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); - if ((vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE)) == NULL) { - spa_config_exit(spa, SCL_CONFIG, FTAG); - spa_close(spa, FTAG); - return (SET_ERROR(ENOENT)); - } - - error = vdev_prop_set(vd, innvl, outnvl); - spa_config_exit(spa, SCL_CONFIG, FTAG); + error = vdev_prop_set(spa, innvl, outnvl); spa_close(spa, FTAG); @@ -3506,13 +3561,6 @@ zfs_ioc_vdev_get_props(const char *poolname, nvlist_t *innvl, nvlist_t *outnvl) { spa_t *spa; int error; - vdev_t *vd; - uint64_t vdev_guid; - - /* Early validation */ - if (nvlist_lookup_uint64(innvl, ZPOOL_VDEV_PROPS_GET_VDEV, - &vdev_guid) != 0) - return (SET_ERROR(EINVAL)); if (outnvl == NULL) return (SET_ERROR(EINVAL)); @@ -3520,15 +3568,7 @@ zfs_ioc_vdev_get_props(const char *poolname, nvlist_t *innvl, nvlist_t *outnvl) if ((error = spa_open(poolname, &spa, FTAG)) != 0) return (error); - spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); - if ((vd = spa_lookup_by_guid(spa, vdev_guid, B_TRUE)) == NULL) { - spa_config_exit(spa, SCL_CONFIG, FTAG); - spa_close(spa, FTAG); - return (SET_ERROR(ENOENT)); - } - - error = vdev_prop_get(vd, innvl, outnvl); - spa_config_exit(spa, SCL_CONFIG, FTAG); + error = vdev_prop_get(spa, innvl, outnvl); spa_close(spa, FTAG); diff --git a/sys/contrib/openzfs/module/zfs/zfs_sa.c b/sys/contrib/openzfs/module/zfs/zfs_sa.c index 8b4fc6fd7fbd..ba0eeccac1d9 100644 --- a/sys/contrib/openzfs/module/zfs/zfs_sa.c +++ b/sys/contrib/openzfs/module/zfs/zfs_sa.c @@ -68,6 +68,7 @@ const sa_attr_reg_t zfs_attr_table[ZPL_END+1] = { {"ZPL_DACL_ACES", 0, SA_ACL, 0}, {"ZPL_DXATTR", 0, SA_UINT8_ARRAY, 0}, {"ZPL_PROJID", sizeof (uint64_t), SA_UINT64_ARRAY, 0}, + {"ZPL_SEQ", sizeof (uint64_t), SA_UINT64_ARRAY, 0}, {NULL, 0, 0, 0} }; @@ -270,7 +271,7 @@ zfs_sa_set_xattr(znode_t *zp, const char *name, const void *value, size_t vsize) dmu_tx_abort(tx); } else { int count = 0; - sa_bulk_attr_t bulk[2]; + sa_bulk_attr_t bulk[3]; uint64_t ctime[2]; if (logsaxattr) @@ -282,6 +283,8 @@ zfs_sa_set_xattr(znode_t *zp, const char *name, const void *value, size_t vsize) NULL, obj, size); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16); + ZFS_PERSIST_SEQ(zp, bulk, count); + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); VERIFY0(sa_bulk_update(zp->z_sa_hdl, bulk, count, tx)); dmu_tx_commit(tx); diff --git a/sys/contrib/openzfs/module/zfs/zfs_vnops.c b/sys/contrib/openzfs/module/zfs/zfs_vnops.c index 1ceedf28ed21..59e69dc389a2 100644 --- a/sys/contrib/openzfs/module/zfs/zfs_vnops.c +++ b/sys/contrib/openzfs/module/zfs/zfs_vnops.c @@ -647,7 +647,7 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) return (error); - sa_bulk_attr_t bulk[4]; + sa_bulk_attr_t bulk[5]; int count = 0; uint64_t mtime[2], ctime[2]; SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16); @@ -656,6 +656,9 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) &zp->z_size, 8); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, &zp->z_pflags, 8); + if (zp->z_is_sa) + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SEQ(zfsvfs), NULL, + &zp->z_seq, 8); /* * Callers might not be able to detect properly that we are read-only, @@ -870,7 +873,7 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) * Start a transaction. */ dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl); DB_DNODE_ENTER(db); dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff, nbytes); @@ -1010,6 +1013,8 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) &clear_setid_bits_txg, tx); zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); + if (zp->z_is_sa) + zp->z_has_seq = B_TRUE; /* * Update the file size (zp_size) if it has changed; @@ -1028,6 +1033,7 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0) zp->z_size = zfsvfs->z_replay_eof; + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error1 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); if (error1 != 0) /* Avoid clobbering EFAULT. */ @@ -1616,7 +1622,7 @@ zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp, uint64_t outsize, size; int error; int count = 0; - sa_bulk_attr_t bulk[3]; + sa_bulk_attr_t bulk[5]; uint64_t mtime[2], ctime[2]; uint64_t uid, gid, projid; blkptr_t *bps; @@ -1870,6 +1876,11 @@ zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp, &ctime, 16); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(outzfsvfs), NULL, &outzp->z_size, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(outzfsvfs), NULL, + &outzp->z_pflags, 8); + if (outzp->z_is_sa) + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SEQ(outzfsvfs), NULL, + &outzp->z_seq, 8); zilog = outzfsvfs->z_log; maxblocks = zil_max_log_data(zilog, sizeof (lr_clone_range_t)) / @@ -1933,7 +1944,7 @@ zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp, * Start a transaction. */ tx = dmu_tx_create(outos); - dmu_tx_hold_sa(tx, outzp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, outzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(outzp)); db = (dmu_buf_impl_t *)sa_get_db(outzp->z_sa_hdl); DB_DNODE_ENTER(db); dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), outoff, size, @@ -1988,6 +1999,8 @@ zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp, &clear_setid_bits_txg, tx); zfs_tstamp_update_setup(outzp, CONTENT_MODIFIED, mtime, ctime); + if (outzp->z_is_sa) + outzp->z_has_seq = B_TRUE; /* * Update the file size (zp_size) if it has changed; @@ -1998,6 +2011,7 @@ zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp, outoff + size); } + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(outzp->z_sa_hdl, bulk, count, tx); zfs_log_clone_range(zilog, tx, TX_CLONE_RANGE, outzp, outoff, @@ -2068,7 +2082,7 @@ zfs_clone_range_replay(znode_t *zp, uint64_t off, uint64_t len, uint64_t blksz, dmu_tx_t *tx; int error; int count = 0; - sa_bulk_attr_t bulk[3]; + sa_bulk_attr_t bulk[5]; uint64_t mtime[2], ctime[2]; ASSERT3U(off, <, MAXOFFSET_T); @@ -2091,17 +2105,12 @@ zfs_clone_range_replay(znode_t *zp, uint64_t off, uint64_t len, uint64_t blksz, return (SET_ERROR(EINVAL)); } - SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16); - SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16); - SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL, - &zp->z_size, 8); - /* * Start a transaction. */ tx = dmu_tx_create(zfsvfs->z_os); - dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); + dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp)); db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl); DB_DNODE_ENTER(db); dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), off, len, blksz); @@ -2119,11 +2128,19 @@ zfs_clone_range_replay(znode_t *zp, uint64_t off, uint64_t len, uint64_t blksz, dmu_brt_clone(zfsvfs->z_os, zp->z_id, off, len, tx, bps, nbps); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL, + &zp->z_size, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, + &zp->z_pflags, 8); zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); + ZFS_PERSIST_SEQ(zp, bulk, count); if (zp->z_size < off + len) zp->z_size = off + len; + ASSERT3S(count, <=, ARRAY_SIZE(bulk)); error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); /* diff --git a/sys/contrib/openzfs/module/zfs/zio.c b/sys/contrib/openzfs/module/zfs/zio.c index 4b7c13dd1e94..a45221d6544e 100644 --- a/sys/contrib/openzfs/module/zfs/zio.c +++ b/sys/contrib/openzfs/module/zfs/zio.c @@ -6002,6 +6002,67 @@ zbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2, uint8_t ibs2, zb1->zb_blkid == zb2->zb_blkid) return (0); + if (zb1->zb_level < 0 || zb2->zb_level < 0) { + /* + * "Negative" levels are ZB_ROOT_LEVEL, ZB_ZIL_LEVEL or + * ZB_DNODE_LEVEL, and represent some sort of auxiliary dataset + * block or object. In this case, we're usually being called + * from dsl_scan or dmu_traverse. + * + * These "levels" are more like a "type" signal, not directly + * comparable, but we have to do something. So we order them in + * the order we would see them during a typical scan or + * traverse: + * + * - ZB_ROOT_LEVEL: the "top" block carrying the dataset head + * - ZB_ZIL_LEVEL: the head ZIL block attached to the dataset + * - ZB_DNODE_LEVEL: "virtual" position representing an + * entire object. Sorts ahead of the true + * data blocks for the object. + * - level >= 0: data blocks + * + * We work through these cases from top to bottom, with + * appropriate tiebreaks for each kind. + */ + + /* + * Root level wins. It shouldn't be possible for both to be the + * root level in this per-dataset tree, and there's no obvious + * tiebreaker, but we handle it as a defensive measure. + */ + if (zb1->zb_level == ZB_ROOT_LEVEL && + zb2->zb_level == ZB_ROOT_LEVEL) + return (TREE_PCMP(zb1, zb2)); + if (zb1->zb_level == ZB_ROOT_LEVEL) + return (-1); + if (zb2->zb_level == ZB_ROOT_LEVEL) + return (1); + + /* ZIL bookmarks have valid blkid, so the earlier one wins. */ + if (zb1->zb_level == ZB_ZIL_LEVEL && + zb2->zb_level == ZB_ZIL_LEVEL) + return (TREE_CMP(zb1->zb_blkid, zb2->zb_blkid)); + if (zb1->zb_level == ZB_ZIL_LEVEL) + return (-1); + if (zb2->zb_level == ZB_ZIL_LEVEL) + return (1); + + /* + * If we get this far, then at least one is ZB_DNODE_LEVEL, and + * the other is either ZB_DNODE_LEVEL or a data block. + * Regardless, the one with the lower-numbered object wins - + * earler ZB_DNODE_LEVEL beats later, but data block on earlier + * objects beats the virtual marker on later objects. + */ + int cmp = TREE_CMP(zb1->zb_object, zb2->zb_object); + if (cmp != 0) + return (cmp); + + if (zb1->zb_level == ZB_DNODE_LEVEL) + return (-1); + return (1); + } + IMPLY(zb1->zb_level > 0, ibs1 >= SPA_MINBLOCKSHIFT); IMPLY(zb2->zb_level > 0, ibs2 >= SPA_MINBLOCKSHIFT); diff --git a/sys/contrib/openzfs/tests/runfiles/common.run b/sys/contrib/openzfs/tests/runfiles/common.run index 0dda8fdfa363..31b197e38dbf 100644 --- a/sys/contrib/openzfs/tests/runfiles/common.run +++ b/sys/contrib/openzfs/tests/runfiles/common.run @@ -335,10 +335,6 @@ tests = ['zfs_send_001_pos', 'zfs_send_002_pos', 'zfs_send_003_pos', 'zfs_send_raw', 'zfs_send_sparse', 'zfs_send-b', 'zfs_send_skip_missing'] tags = ['functional', 'cli_root', 'zfs_send'] -[tests/functional/cli_root/zfs_send_delegation] -tests = ['zfs_send_test'] -tags = ['functional', 'cli_root', 'zfs_send_delegation'] - [tests/functional/cli_root/zfs_set] tests = ['cache_001_pos', 'cache_002_neg', 'canmount_001_pos', 'canmount_002_pos', 'canmount_003_pos', 'canmount_004_pos', @@ -575,7 +571,8 @@ tags = ['functional', 'cli_root', 'zpool_scrub'] tests = ['zpool_set_001_pos', 'zpool_set_002_neg', 'zpool_set_003_neg', 'zpool_set_ashift', 'zpool_set_features', 'zpool_set_inherit', 'vdev_set_001_pos', 'user_property_001_pos', 'user_property_002_neg', - 'zpool_set_clear_userprop','vdev_set_scheduler'] + 'zpool_set_clear_userprop','vdev_set_scheduler', 'vdev_set_allocating', + 'vdev_set_path'] tags = ['functional', 'cli_root', 'zpool_set'] [tests/functional/cli_root/zpool_split] @@ -659,10 +656,6 @@ tests = ['zfs_list_001_pos', 'zfs_list_002_pos', 'zfs_list_003_pos', user = tags = ['functional', 'cli_user', 'zfs_list'] -[tests/functional/cli_user/zfs_send_delegation_user] -tests = ['zfs_send_usertest'] -tags = ['functional', 'cli_user', 'zfs_send_delegation_user'] - [tests/functional/cli_user/zpool_iostat] tests = ['zpool_iostat_001_neg', 'zpool_iostat_002_pos', 'zpool_iostat_003_neg', 'zpool_iostat_004_pos', @@ -731,9 +724,9 @@ tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos', 'zfs_allow_003_pos', 'zfs_allow_004_pos', 'zfs_allow_005_pos', 'zfs_allow_006_pos', 'zfs_allow_007_pos', 'zfs_allow_008_pos', 'zfs_allow_009_neg', 'zfs_allow_010_pos', 'zfs_allow_011_neg', 'zfs_allow_012_neg', - 'zfs_unallow_001_pos', 'zfs_unallow_002_pos', 'zfs_unallow_003_pos', - 'zfs_unallow_004_pos', 'zfs_unallow_005_pos', 'zfs_unallow_006_pos', - 'zfs_unallow_007_neg', 'zfs_unallow_008_neg'] + 'zfs_allow_send', 'zfs_unallow_001_pos', 'zfs_unallow_002_pos', + 'zfs_unallow_003_pos', 'zfs_unallow_004_pos', 'zfs_unallow_005_pos', + 'zfs_unallow_006_pos', 'zfs_unallow_007_neg', 'zfs_unallow_008_neg'] tags = ['functional', 'delegate'] [tests/functional/direct] @@ -888,7 +881,7 @@ tests = ['defaultprojectquota_002_pos', 'defaultprojectquota_003_neg', 'defaultprojectquota_007_pos', 'projectquota_002_pos', 'projectquota_004_neg', 'projectquota_005_pos', 'projectquota_007_pos', 'projectquota_008_pos', 'projectquota_009_pos', 'projecttree_002_pos', - 'projecttree_003_neg'] + 'projecttree_003_neg', 'projectspace_006_pos'] tags = ['functional', 'projectquota'] [tests/functional/poolversion] @@ -1004,7 +997,7 @@ tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos', 'rsend_030_pos', 'rsend_031_pos', 'rsend-exclude_001_pos', 'rsend-exclude_002_pos', 'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props', 'send-c_incremental', - 'send-c_volume', 'send-c_zstream_recompress', 'send-c_zstreamdump', + 'send-c_volume', 'send-c_lz4_disabled', 'send-c_recv_lz4_disabled', 'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize', @@ -1013,7 +1006,7 @@ tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos', 'send_encrypted_props', 'send_encrypted_truncated_files', 'send_freeobjects', 'send_realloc_files', 'send_realloc_encrypted_files', 'send_spill_block', 'send_holds', 'send_hole_birth', 'send_mixed_raw', - 'send-wR_encrypted_zvol', 'send-zstream_drop_record', + 'send-wR_encrypted_zvol', 'send_partial_dataset', 'send_invalid', 'send_large_blocks_incremental', 'send_large_blocks_initial', 'send_large_microzap_incremental', 'send_large_microzap_transitive', @@ -1097,8 +1090,9 @@ tests = [ 'userquota_004_pos', 'userquota_005_neg', 'userquota_006_pos', 'userquota_007_pos', 'userquota_008_pos', 'userquota_009_pos', 'userquota_010_pos', 'userquota_011_pos', 'userquota_012_neg', + 'groupspace_005_pos', 'userspace_001_pos', 'userspace_002_pos', 'userspace_004_pos', - 'userspace_encrypted', 'userspace_send_encrypted', + 'userspace_005_pos', 'userspace_encrypted', 'userspace_send_encrypted', 'userspace_encrypted_13709'] tags = ['functional', 'userquota'] @@ -1141,6 +1135,20 @@ tests = ['zoned_uid_001_pos', 'zoned_uid_002_pos', 'zoned_uid_003_pos', 'zoned_uid_029_neg', 'zoned_uid_031_pos'] tags = ['functional', 'zoned_uid'] +[tests/functional/zstream] +tests = ['zstream_checksum_001_pos', + 'zstream_decompress_001_pos', 'zstream_decompress_002_pos', + 'zstream_decompress_003_neg', 'zstream_decompress_004_pos', + 'zstream_decompress_005_pos', 'zstream_decompress_006_neg', + 'zstream_drop_record_001_pos', + 'zstream_dump_001_pos', 'zstream_dump_002_pos', + 'zstream_dump_003_pos', 'zstream_dump_004_neg', + 'zstream_recompress_001_pos', 'zstream_recompress_002_pos', + 'zstream_recompress_003_pos', 'zstream_recompress_004_pos', + 'zstream_recompress_005_pos', + 'zstream_redup_001_pos'] +tags = ['functional', 'zstream'] + [tests/functional/zvol/zvol_ENOSPC] tests = ['zvol_ENOSPC_001_pos'] tags = ['functional', 'zvol', 'zvol_ENOSPC'] diff --git a/sys/contrib/openzfs/tests/runfiles/linux.run b/sys/contrib/openzfs/tests/runfiles/linux.run index 009d984f2b99..8a34ff5e7641 100644 --- a/sys/contrib/openzfs/tests/runfiles/linux.run +++ b/sys/contrib/openzfs/tests/runfiles/linux.run @@ -111,7 +111,7 @@ tests = ['events_001_pos', 'events_002_pos', 'zed_rc_filter', 'zed_fd_spill', tags = ['functional', 'events'] [tests/functional/fallocate:Linux] -tests = ['fallocate_prealloc', 'fallocate_zero-range'] +tests = ['fallocate_extend_timestamps', 'fallocate_prealloc', 'fallocate_zero-range'] tags = ['functional', 'fallocate'] [tests/functional/fault:Linux] diff --git a/sys/contrib/openzfs/tests/runfiles/sanity.run b/sys/contrib/openzfs/tests/runfiles/sanity.run index 788c9b395316..4c62a5adbb99 100644 --- a/sys/contrib/openzfs/tests/runfiles/sanity.run +++ b/sys/contrib/openzfs/tests/runfiles/sanity.run @@ -546,7 +546,7 @@ tags = ['functional', 'reservation'] tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_016_neg', 'rsend-exclude_001_pos', - 'rsend-exclude_002_pos', 'send-c_volume', 'send-c_zstreamdump', + 'rsend-exclude_002_pos', 'send-c_volume', 'send-c_recv_dedup', 'send-L_toggle', 'send_encrypted_hierarchy', 'send_encrypted_props', 'send_encrypted_freeobjects', 'send_encrypted_truncated_files', 'send_freeobjects', 'send_holds', @@ -614,6 +614,11 @@ tests = ['xattr_001_pos', 'xattr_002_neg', 'xattr_003_neg', 'xattr_004_pos', 'xattr_011_pos', 'xattr_013_pos', 'xattr_014_pos', 'xattr_compat'] tags = ['functional', 'xattr'] +[tests/functional/zstream] +tests = ['zstream_dump_001_pos', 'zstream_dump_002_pos', + 'zstream_redup_001_pos'] +tags = ['functional', 'zstream'] + [tests/functional/zvol/zvol_ENOSPC] tests = ['zvol_ENOSPC_001_pos'] tags = ['functional', 'zvol', 'zvol_ENOSPC'] diff --git a/sys/contrib/openzfs/tests/unit/.gitignore b/sys/contrib/openzfs/tests/unit/.gitignore index 12a60a656668..5b5b903e9196 100644 --- a/sys/contrib/openzfs/tests/unit/.gitignore +++ b/sys/contrib/openzfs/tests/unit/.gitignore @@ -2,3 +2,4 @@ /test_*_coverage /test_zap +/test_namecheck diff --git a/sys/contrib/openzfs/tests/unit/Makefile.am b/sys/contrib/openzfs/tests/unit/Makefile.am index 80fe7311c468..c8b08fe53fc0 100644 --- a/sys/contrib/openzfs/tests/unit/Makefile.am +++ b/sys/contrib/openzfs/tests/unit/Makefile.am @@ -15,7 +15,8 @@ libunit_la_SOURCES = \ # all test binaries UNIT_TESTS = \ - %D%/test_zap + %D%/test_zap \ + %D%/test_namecheck noinst_PROGRAMS = $(UNIT_TESTS) @@ -38,6 +39,19 @@ nodist_%C%_test_zap_SOURCES = \ libunit.la +%C%_test_namecheck_CFLAGS = $(AM_CFLAGS) + +nodist_%C%_test_namecheck_SOURCES = \ + module/zcommon/zfs_namecheck.c + +%C%_test_namecheck_SOURCES = \ + %D%/test_namecheck.c + +%C%_test_namecheck_LDADD = \ + libspl.la \ + libunit.la + + # test run and coverage targets below PHONY += unit unit-coverage unit-coverage-html diff --git a/sys/contrib/openzfs/tests/unit/test_namecheck.c b/sys/contrib/openzfs/tests/unit/test_namecheck.c new file mode 100644 index 000000000000..e48a1e9feefb --- /dev/null +++ b/sys/contrib/openzfs/tests/unit/test_namecheck.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: CDDL-1.0 +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2026, Christos Longros. + */ + +#include <string.h> + +#include <sys/fs/zfs.h> +#include "zfs_namecheck.h" + +#include "unit.h" + +/* ========== */ + +/* + * The namecheck routines validate a name and report, via namecheck_err_t, + * exactly why it failed. We test them in two directions: + * + * - Validity path: randomly generated names from unit_rand_str(), which + * give only 'a'-'z' characters. + * + * - Invalidity path: explicit names, each tested against their specific + * error code since the rejection reason depends on the exact characters + * used. + */ +typedef int (*namecheck_f)(const char *, namecheck_err_t *, char *); + +/* Confirm 'name' is accepted by 'fn'. */ +static void +check_valid(namecheck_f fn, const char *name) +{ + namecheck_err_t why = (namecheck_err_t)-1; + char what = '\0'; + unit_ok(fn(name, &why, &what)); +} + +/* Confirm 'name' is rejected by 'fn' with the 'why' we expected. */ +static void +check_invalid(namecheck_f fn, const char *name, namecheck_err_t expected) +{ + namecheck_err_t why = (namecheck_err_t)-1; + char what = '\0'; + unit_err(fn(name, &why, &what), -1); + unit_eq(why, expected); +} + +/* Confirm 'fn' rejects a lengthy name and returns NAME_ERR_TOOLONG. */ +static void +check_longname_invalid(namecheck_f fn) +{ + char buf[ZFS_MAX_DATASET_NAME_LEN + 16]; + check_invalid(fn, unit_rand_str(buf, sizeof (buf)), NAME_ERR_TOOLONG); +} + +/* ========== */ + +/* pool_namecheck: dataset character set that must begin with a letter. */ +static MunitResult +test_pool_namecheck(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + /* A random array of letters is always a valid pool name. */ + char pool[16]; + check_valid(pool_namecheck, unit_rand_str(pool, sizeof (pool))); + + /* Fixed names cover the rest of the allowed character set. */ + check_valid(pool_namecheck, "tank_01"); + check_valid(pool_namecheck, "Pool-2.0:label"); + + /* A pool name has to start with a letter. */ + check_invalid(pool_namecheck, "0tank", NAME_ERR_NOLETTER); + check_invalid(pool_namecheck, "_tank", NAME_ERR_NOLETTER); + + /* These pool names are reserved. */ + check_invalid(pool_namecheck, "mirror", NAME_ERR_RESERVED); + check_invalid(pool_namecheck, "raidz", NAME_ERR_RESERVED); + check_invalid(pool_namecheck, "draid", NAME_ERR_RESERVED); + + /* A pool name carries no hierarchy or snapshot delimiter. */ + check_invalid(pool_namecheck, "tank/fs", NAME_ERR_INVALCHAR); + check_invalid(pool_namecheck, "tank@snap", NAME_ERR_INVALCHAR); + + check_longname_invalid(pool_namecheck); + + return (MUNIT_OK); +} + +/* dataset_namecheck: any entity except a bookmark. */ +static MunitResult +test_dataset_namecheck(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + /* A path of random, independently-valid components is accepted. */ + char path[64]; + unit_rand_str(path, sizeof (path)); + path[20] = path[40] = '/'; + check_valid(dataset_namecheck, path); + + /* A trailing snapshot is still a valid dataset name. */ + check_valid(dataset_namecheck, "tank/home@snap"); + + /* '%' is allowed, for temporary clone names (online recv). */ + check_valid(dataset_namecheck, "tank/%recv"); + + check_invalid(dataset_namecheck, "/tank", NAME_ERR_LEADING_SLASH); + check_invalid(dataset_namecheck, "tank/", NAME_ERR_TRAILING_SLASH); + check_invalid(dataset_namecheck, "tank//home", + NAME_ERR_EMPTY_COMPONENT); + check_invalid(dataset_namecheck, "", NAME_ERR_EMPTY_COMPONENT); + check_invalid(dataset_namecheck, "tank/./home", NAME_ERR_SELF_REF); + check_invalid(dataset_namecheck, "tank/../home", NAME_ERR_PARENT_REF); + check_invalid(dataset_namecheck, "tank/fs!", NAME_ERR_INVALCHAR); + + /* A bookmark delimiter does not belong in a dataset name. */ + check_invalid(dataset_namecheck, "tank/fs#bm", NAME_ERR_INVALCHAR); + + check_longname_invalid(dataset_namecheck); + + return (MUNIT_OK); +} + +/* snapshot_namecheck: a valid snapshot name (an entity with '@'). */ +static MunitResult +test_snapshot_namecheck(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + /* A random "filesystem@snapshot" pair is valid. */ + char path[64]; + unit_rand_str(path, sizeof (path)); + path[40] = '@'; + check_valid(snapshot_namecheck, path); + + /* Without an '@' it is not a snapshot. */ + check_invalid(snapshot_namecheck, "tank/home", NAME_ERR_NO_AT); + + /* Only one delimiter is allowed. */ + check_invalid(snapshot_namecheck, "tank@a@b", + NAME_ERR_MULTIPLE_DELIMITERS); + + /* Nothing may follow the snapshot name with a '/'. */ + check_invalid(snapshot_namecheck, "tank@snap/x", + NAME_ERR_TRAILING_SLASH); + + return (MUNIT_OK); +} + +/* bookmark_namecheck: a valid bookmark name (an entity with '#'). */ +static MunitResult +test_bookmark_namecheck(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + /* A random "filesystem#bookmark" pair is valid. */ + char path[64]; + unit_rand_str(path, sizeof (path)); + path[40] = '#'; + check_valid(bookmark_namecheck, path); + + /* Without a '#' it is not a bookmark. */ + check_invalid(bookmark_namecheck, "tank/home", NAME_ERR_NO_POUND); + + return (MUNIT_OK); +} + +/* zfs_component_namecheck: one component; alphanumeric plus [-_.: ]. */ +static MunitResult +test_component_namecheck(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + /* A bare random component is valid. */ + char comp[16]; + unit_rand_str(comp, sizeof (comp)); + check_valid(zfs_component_namecheck, comp); + + /* An empty component is not valid. */ + check_invalid(zfs_component_namecheck, "", NAME_ERR_EMPTY_COMPONENT); + + /* A single component cannot contain a path separator. */ + check_invalid(zfs_component_namecheck, "a/b", NAME_ERR_INVALCHAR); + + check_longname_invalid(zfs_component_namecheck); + + return (MUNIT_OK); +} + +/* permset_namecheck: a permission set name, starting with '@'. */ +static MunitResult +test_permset_namecheck(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + /* A random name behind a leading '@' is a valid permission set. */ + char set[16]; + set[0] = '@'; + unit_rand_str(set + 1, sizeof (set) - 1); + check_valid(permset_namecheck, set); + + /* It has to start with '@'. */ + check_invalid(permset_namecheck, "backup", NAME_ERR_NO_AT); + + /* The text after '@' follows the component rules. */ + check_invalid(permset_namecheck, "@bad/name", NAME_ERR_INVALCHAR); + + /* The length upper limit is checked ahead of everything else. */ + check_longname_invalid(permset_namecheck); + + return (MUNIT_OK); +} + +/* mountpoint_namecheck: a mountpoint path, /[component][/]*. */ +static MunitResult +test_mountpoint_namecheck(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + namecheck_err_t why = (namecheck_err_t)-1; + + /* An absolute path with a random component is accepted. */ + char path[64]; + unit_rand_str(path, sizeof (path)); + path[0] = '/'; + unit_ok(mountpoint_namecheck(path, &why)); + + /* The root mountpoint is valid. */ + unit_ok(mountpoint_namecheck("/", &why)); + + /* A mountpoint must be absolute. */ + unit_err(mountpoint_namecheck("relative/path", &why), -1); + unit_eq(why, NAME_ERR_LEADING_SLASH); + + /* A NULL path counts as missing the leading slash. */ + unit_err(mountpoint_namecheck(NULL, &why), -1); + unit_eq(why, NAME_ERR_LEADING_SLASH); + + /* A long path component is rejected. */ + char buf[ZFS_MAX_DATASET_NAME_LEN + 4]; + buf[0] = '/'; + (void) memset(buf + 1, 'a', sizeof (buf) - 2); + buf[sizeof (buf) - 1] = '\0'; + unit_err(mountpoint_namecheck(buf, &why), -1); + unit_eq(why, NAME_ERR_TOOLONG); + + return (MUNIT_OK); +} + +/* get_dataset_depth: a path's level of nesting (depth). */ +static MunitResult +test_dataset_depth(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + /* Depth is the number of '/' separators in the path. */ + unit_eq(get_dataset_depth("tank"), 0); + unit_eq(get_dataset_depth("tank/home"), 1); + unit_eq(get_dataset_depth("tank/home/user"), 2); + + /* Counting stops at the snapshot or bookmark delimiter. */ + unit_eq(get_dataset_depth("tank/home@snap"), 1); + unit_eq(get_dataset_depth("tank/home#bm"), 1); + + /* + * dataset_nestcheck() passes while the depth is under the limit and + * fails once it reaches it. zfs_max_dataset_nesting is a tunable that + * can be adjusted to the desired nesting. + */ + zfs_max_dataset_nesting = 2; + unit_ok(dataset_nestcheck("a/b")); /* depth 1, under 2 */ + unit_err(dataset_nestcheck("a/b/c"), -1); /* depth 2, at 2 */ + + return (MUNIT_OK); +} + +/* ========== */ + +static const MunitTest namecheck_tests[] = { + UNIT_TEST("pool", test_pool_namecheck), + UNIT_TEST("dataset", test_dataset_namecheck), + UNIT_TEST("snapshot", test_snapshot_namecheck), + UNIT_TEST("bookmark", test_bookmark_namecheck), + UNIT_TEST("component", test_component_namecheck), + UNIT_TEST("permset", test_permset_namecheck), + UNIT_TEST("mountpoint", test_mountpoint_namecheck), + UNIT_TEST("depth", test_dataset_depth), + { 0 }, +}; + +static const MunitSuite namecheck_test_suite = { + "namecheck.", + namecheck_tests, + NULL, + 1, + MUNIT_SUITE_OPTION_NONE, +}; + +int +main(int argc, char **argv) +{ + return (munit_suite_main(&namecheck_test_suite, NULL, argc, argv)); +} diff --git a/sys/contrib/openzfs/tests/unit/test_zap.c b/sys/contrib/openzfs/tests/unit/test_zap.c index c64de7d75c47..356b917723dc 100644 --- a/sys/contrib/openzfs/tests/unit/test_zap.c +++ b/sys/contrib/openzfs/tests/unit/test_zap.c @@ -53,7 +53,7 @@ mock_crc64_init(void) /* Create a microzap backed by a mock dnode. */ static dnode_t * -mock_zap_create_microzap(void) { +mock_zap_create_norm_microzap(int normflags) { /* * We use DMU_OTN_ZAP_DATA so that DMU_OT_BYTESWAP() returns * DMU_BSWAP_ZAP without consulting dmu_ot[], which is not currently @@ -62,20 +62,20 @@ mock_zap_create_microzap(void) { mock_dnode_t *mdn = mock_dnode_create(512, DMU_OTN_ZAP_DATA); dnode_t *dn = (dnode_t *)mdn; dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); - mzap_create_impl(dn, 0, 0, tx); + mzap_create_impl(dn, normflags, 0, tx); mock_tx_destroy((mock_dmu_tx_t *)tx); return (dn); } /* Create a fatzap backed by a mock dnode. */ static dnode_t * -mock_zap_create_fatzap(void) +mock_zap_create_norm_fatzap(int normflags) { /* * We can only create microzaps directly. They only take u64s as a * value, so we add a u16 to trigger an upgrade to fatzap. */ - dnode_t *dn = mock_zap_create_microzap(); + dnode_t *dn = mock_zap_create_norm_microzap(normflags); dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); uint16_t upgrade = 0; zap_add_by_dnode(dn, "_upgrade", sizeof (uint16_t), 1, &upgrade, tx); @@ -84,6 +84,10 @@ mock_zap_create_fatzap(void) return (dn); } +/* Shortcuts for ZAPs with no normalization flags set. */ +#define mock_zap_create_microzap() mock_zap_create_norm_microzap(0) +#define mock_zap_create_fatzap() mock_zap_create_norm_fatzap(0) + static bool mock_zap_is_microzap(dnode_t *dn) { @@ -110,19 +114,23 @@ mock_zap_destroy(dnode_t *dn) /* Create a ZAP of the type named in the given test params. */ static dnode_t * -mock_zap_create_params(const MunitParameter params[], const char *key) { +mock_zap_create_norm_params(const MunitParameter params[], const char *key, + int normflags) { const char *type = munit_parameters_get(params, key); if (type == NULL) munit_error("mock_zap_create_params: missing type param"); else if (strcmp(type, "micro") == 0) - return (mock_zap_create_microzap()); + return (mock_zap_create_norm_microzap(normflags)); else if (strcmp(type, "fat") == 0) - return (mock_zap_create_fatzap()); + return (mock_zap_create_norm_fatzap(normflags)); else munit_errorf("mock_zap_create_params: invalid type '%s'", type); __builtin_unreachable(); } +/* Shortcuts for ZAPs with no normalization flags set. */ +#define mock_zap_create_params(p, k) mock_zap_create_norm_params(p, k, 0) + /* * Confirm the stored ZAP is of the type named in the given test params. This * is useful for sanity checks within tests that a ZAP wasn't unexpectedly @@ -234,7 +242,7 @@ test_zap_basic(const MunitParameter params[], void *data) /* ========== */ /* - * "Core" ZAP API tests. Covers the most basic functionality upon which which + * "Core" ZAP API tests. Covers the most basic functionality upon which * everything else is built. * * Note that to avoid microzap upgrade here, we only short keys and @@ -687,7 +695,7 @@ test_cursor(const MunitParameter params[], void *data) unit_ok(zap_cursor_init_by_dnode(&zc, dn)); /* - * Cursors don't guarantee an order, so we run over them them all, + * Cursors don't guarantee an order, so we run over them all, * confirm the key matches the value, and then set a bit for each * one we've seen. By the end, we should have seen them all. */ @@ -1093,6 +1101,479 @@ test_zap_value_search_mask(const MunitParameter params[], void *data) /* ========== */ +/* Key case normalization. */ + +/* + * A TOUPPER ZAP hashes and matches keys by their uppercase form, ie a + * case-insensitive match. + */ +static MunitResult +test_norm_toupper(const MunitParameter params[], void *data) +{ + (void) data; + + dnode_t *dn = + mock_zap_create_norm_params(params, "type", U8_TEXTPREP_TOUPPER); + dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); + + uint64_t val = 42; + unit_ok(zap_add_by_dnode(dn, "Hello", + sizeof (uint64_t), 1, &val, tx)); + + /* MT_NORMALIZE finds any casing. */ + uint64_t result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, "hello", + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, "HELLO", + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, "HeLlO", + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + /* Exact lookup finds the stored form. */ + result = 0; + unit_ok(zap_lookup_by_dnode(dn, "Hello", + sizeof (uint64_t), 1, &result)); + unit_eq(result, 42); + + /* Exact lookup misses other casings. */ + unit_err(zap_lookup_by_dnode(dn, "hello", + sizeof (uint64_t), 1, &result), ENOENT); + unit_err(zap_lookup_by_dnode(dn, "HeLlO", + sizeof (uint64_t), 1, &result), ENOENT); + + mock_tx_destroy((mock_dmu_tx_t *)tx); + unit_true(mock_zap_is_params(dn, params, "type")); + mock_zap_destroy(dn); + + return (MUNIT_OK); +} + +/* + * MT_MATCH_CASE forces an exact-case match on a normalizing (TOUPPER) ZAP. + */ +static MunitResult +test_norm_match_case(const MunitParameter params[], void *data) +{ + (void) data; + + dnode_t *dn = + mock_zap_create_norm_params(params, "type", U8_TEXTPREP_TOUPPER); + dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); + + uint64_t val = 42; + unit_ok(zap_add_by_dnode(dn, "Hello", + sizeof (uint64_t), 1, &val, tx)); + + /* MT_NORMALIZE alone matches any casing. */ + uint64_t result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, "hello", + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + /* Adding MT_MATCH_CASE rejects a case variant name. */ + unit_err(zap_lookup_norm_by_dnode(dn, "hello", + sizeof (uint64_t), 1, &result, MT_NORMALIZE | MT_MATCH_CASE, + NULL, 0, NULL), ENOENT); + + /* The exact-case name still matches under MT_MATCH_CASE. */ + result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, "Hello", + sizeof (uint64_t), 1, &result, MT_NORMALIZE | MT_MATCH_CASE, + NULL, 0, NULL)); + unit_eq(result, 42); + + mock_tx_destroy((mock_dmu_tx_t *)tx); + unit_true(mock_zap_is_params(dn, params, "type")); + mock_zap_destroy(dn); + + return (MUNIT_OK); +} + +/* + * NFC: canonical decomposition + canonical recomposition. + * + * NFC-normalized keys are stored in their original encoding but match but + * match by canonical composed form. + */ +static MunitResult +test_norm_nfc(const MunitParameter params[], void *data) +{ + (void) data; + + /* + * "café" in NFC: "caf\xc3\xa9" (U+00E9, e-acute precomposed) + * "café" in NFD: "cafe\xcc\x81" (U+0065 + U+0301, e + combining acute) + */ + static const char nfc_key[] = "caf\xc3\xa9"; + static const char nfd_key[] = "cafe\xcc\x81"; + + dnode_t *dn = + mock_zap_create_norm_params(params, "type", U8_TEXTPREP_NFC); + dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); + + /* Insert the NFD form. */ + uint64_t val = 42; + unit_ok(zap_add_by_dnode(dn, nfd_key, sizeof (uint64_t), 1, &val, tx)); + + /* Normalized lookup with NFC form finds it. */ + uint64_t result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, nfc_key, + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + /* Normalized lookup with NFD form also finds it. */ + result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, nfd_key, + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + mock_tx_destroy((mock_dmu_tx_t *)tx); + unit_true(mock_zap_is_params(dn, params, "type")); + mock_zap_destroy(dn); + + return (MUNIT_OK); +} + +/* + * NFD: canonical decomposition without recomposition. + * + * NFD and NFC define the same canonical equivalence class, so this test uses + * the same pair as test_norm_nfc (NFC) and would pass under NFC too. Together + * the two tests verify the flags are independently wired. + */ +static MunitResult +test_norm_nfd(const MunitParameter params[], void *data) +{ + (void) data; + + /* + * "café" in NFC: "caf\xc3\xa9" (U+00E9, e-acute precomposed) + * "café" in NFD: "cafe\xcc\x81" (U+0065 + U+0301, e + combining acute) + */ + static const char nfc_key[] = "caf\xc3\xa9"; + static const char nfd_key[] = "cafe\xcc\x81"; + + dnode_t *dn = + mock_zap_create_norm_params(params, "type", U8_TEXTPREP_NFD); + dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); + + /* Insert the NFC form. */ + uint64_t val = 42; + unit_ok(zap_add_by_dnode(dn, nfc_key, sizeof (uint64_t), 1, &val, tx)); + + /* Normalized lookup with NFC form finds it. */ + uint64_t result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, nfc_key, + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + /* Normalized lookup with NFD form also finds it. */ + result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, nfd_key, + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + mock_tx_destroy((mock_dmu_tx_t *)tx); + unit_true(mock_zap_is_params(dn, params, "type")); + mock_zap_destroy(dn); + + return (MUNIT_OK); +} + +/* + * NFKC: compatibility decomposition + canonical recomposition. + * + * Compatibility characters (e.g. the fi ligature U+FB01 ~~ "fi") are only + * recognised as equivalent under the K forms (NFKC/NFKD). They are NOT + * folded by NFC, NFD, or TOUPPER — so this pair is wrong under all of those. + */ +static MunitResult +test_norm_nfkc(const MunitParameter params[], void *data) +{ + (void) data; + + /* + * "file": fi ligature (U+FB01 = \xef\xac\x81) + "le". + * Normalizes to plain "file". + */ + static const char ligature_key[] = "\xef\xac\x81le"; + static const char plain_key[] = "file"; + + dnode_t *dn = + mock_zap_create_norm_params(params, "type", U8_TEXTPREP_NFKC); + dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); + + /* Insert the ligature. */ + uint64_t val = 42; + unit_ok(zap_add_by_dnode(dn, ligature_key, + sizeof (uint64_t), 1, &val, tx)); + + /* NFKC lookup with the plain form finds it. */ + uint64_t result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, plain_key, + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + /* NFKC lookup with the ligature also finds it */ + result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, ligature_key, + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + mock_tx_destroy((mock_dmu_tx_t *)tx); + unit_true(mock_zap_is_params(dn, params, "type")); + mock_zap_destroy(dn); + + return (MUNIT_OK); +} + +/* + * NFKD: compatibility decomposition without recomposition. + * + * NFKC and NFKD define the same compatibility equivalence class, so this test + * uses the same pair as test_norm_nfkc. Together they verify the flags are + * independently wired. + */ +static MunitResult +test_norm_nfkd(const MunitParameter params[], void *data) +{ + (void) data; + + /* + * "file": fi ligature (U+FB01 = \xef\xac\x81) + "le". + * Normalizes to plain "file". + */ + static const char ligature_key[] = "\xef\xac\x81le"; + static const char plain_key[] = "file"; + + dnode_t *dn = + mock_zap_create_norm_params(params, "type", U8_TEXTPREP_NFKC); + dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); + + /* Insert the plain form. */ + uint64_t val = 42; + unit_ok(zap_add_by_dnode(dn, plain_key, + sizeof (uint64_t), 1, &val, tx)); + + /* NFKD lookup with the ligature finds it. */ + uint64_t result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, ligature_key, + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + /* NFKD lookup with plain form also finds it. */ + result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, plain_key, + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, NULL)); + unit_eq(result, 42); + + mock_tx_destroy((mock_dmu_tx_t *)tx); + unit_true(mock_zap_is_params(dn, params, "type")); + mock_zap_destroy(dn); + + return (MUNIT_OK); +} + +/* + * A ZAP does not enforce normalization uniqueness on insert - two keys that + * normalize to the same form can coexist. A normalized lookup detects the + * conflict and flags it for the caller on request. + */ +static MunitResult +test_norm_conflict(const MunitParameter params[], void *data) +{ + (void) data; + + dnode_t *dn = + mock_zap_create_norm_params(params, "type", U8_TEXTPREP_TOUPPER); + dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); + + /* Insert two distinct keys that normalize to the same form. */ + uint64_t v1 = 1, v2 = 2; + unit_ok(zap_add_by_dnode(dn, "Hello", sizeof (uint64_t), 1, &v1, tx)); + unit_ok(zap_add_by_dnode(dn, "hello", sizeof (uint64_t), 1, &v2, tx)); + + /* Confirm both stored. */ + uint64_t count = 0; + unit_ok(zap_count_by_dnode(dn, &count)); + unit_eq(count, 2); + + /* + * Three different normalized lookups all return the first match for + * the normalized form, but signal that a normalization conflict + * occurred. + */ + uint64_t result = 0; + boolean_t conflict = B_FALSE; + unit_ok(zap_lookup_norm_by_dnode(dn, "Hello", + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, &conflict)); + unit_true(conflict); + unit_eq(result, 1); + + result = 0; + conflict = B_FALSE; + unit_ok(zap_lookup_norm_by_dnode(dn, "hello", + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, &conflict)); + unit_true(conflict); + unit_eq(result, 1); + + result = 0; + conflict = B_FALSE; + unit_ok(zap_lookup_norm_by_dnode(dn, "HELLO", + sizeof (uint64_t), 1, &result, MT_NORMALIZE, NULL, 0, &conflict)); + unit_true(conflict); + unit_eq(result, 1); + + mock_tx_destroy((mock_dmu_tx_t *)tx); + unit_true(mock_zap_is_params(dn, params, "type")); + mock_zap_destroy(dn); + + return (MUNIT_OK); +} + +/* A normalized lookup can return the original name form on request. */ +static MunitResult +test_norm_realname(const MunitParameter params[], void *data) +{ + (void) data; + + dnode_t *dn = + mock_zap_create_norm_params(params, "type", U8_TEXTPREP_TOUPPER); + dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); + + uint64_t v = 1; + unit_ok(zap_add_by_dnode(dn, "HeLlO", sizeof (uint64_t), 1, &v, tx)); + + char realname[ZAP_MAXNAMELEN]; + uint64_t result = 0; + unit_ok(zap_lookup_norm_by_dnode(dn, "hello", + sizeof (uint64_t), 1, &result, MT_NORMALIZE, + realname, sizeof (realname), NULL)); + unit_eq(result, 1); + unit_str_eq(realname, "HeLlO"); + + mock_tx_destroy((mock_dmu_tx_t *)tx); + unit_true(mock_zap_is_params(dn, params, "type")); + mock_zap_destroy(dn); + + return (MUNIT_OK); +} + +/* zap_remove_norm: remove by normalized name. */ +static MunitResult +test_norm_remove(const MunitParameter params[], void *data) +{ + (void) data; + + dnode_t *dn = + mock_zap_create_norm_params(params, "type", U8_TEXTPREP_TOUPPER); + dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); + + /* Add two items with mixed-case keys. */ + uint64_t v = 1; + unit_ok(zap_add_by_dnode(dn, "Hello", sizeof (uint64_t), 1, &v, tx)); + unit_ok(zap_add_by_dnode(dn, "World", sizeof (uint64_t), 1, &v, tx)); + + /* + * Removing with MT_NORMALIZE normalizes both the requested and stored + * keys before comparison. + */ + unit_ok(zap_remove_norm_by_dnode(dn, "HELLO", MT_NORMALIZE, tx)); + unit_err(zap_lookup_by_dnode(dn, "Hello", + sizeof (uint64_t), 1, &v), ENOENT); + + /* Must use exact case to remove without MT_NORMALIZE. */ + unit_eq(zap_remove_norm_by_dnode(dn, "world", 0, tx), ENOENT); + unit_ok(zap_lookup_by_dnode(dn, "World", sizeof (uint64_t), 1, &v)); + unit_ok(zap_remove_norm_by_dnode(dn, "World", 0, tx)); + unit_eq(zap_lookup_by_dnode(dn, "World", + sizeof (uint64_t), 1, &v), ENOENT); + + mock_tx_destroy((mock_dmu_tx_t *)tx); + unit_true(mock_zap_is_params(dn, params, "type")); + mock_zap_destroy(dn); + + return (MUNIT_OK); +} + +/* ========== */ + +/* + * Binary uint64-array keys (ZAP_FLAG_UINT64_KEY), as used by the dedup table + * and the block reference table. Such a ZAP is always a fatzap. Covers the + * uint64-key by-dnode operations: add, lookup, length, lookup_length, update + * and remove. + */ +static MunitResult +test_zap_uint64_keys(const MunitParameter params[], void *data) +{ + (void) params, (void) data; + + mock_dnode_t *mdn = mock_dnode_create(512, DMU_OTN_ZAP_DATA); + dnode_t *dn = (dnode_t *)mdn; + dmu_tx_t *tx = (dmu_tx_t *)mock_tx_create(); + mzap_create_impl(dn, 0, ZAP_FLAG_HASH64 | ZAP_FLAG_UINT64_KEY, tx); + unit_true(mock_zap_is_fatzap(dn)); + + /* A two-word binary key, as used by the dedup and clone tables. */ + uint64_t key[2] = { unit_rand_uint64(), unit_rand_uint64() }; + uint64_t val = 1; + + /* Store a value, then read it back by the same key. */ + unit_ok(zap_add_uint64_by_dnode(dn, key, 2, sizeof (uint64_t), 1, + &val, tx)); + uint64_t out = 0; + unit_ok(zap_lookup_uint64_by_dnode(dn, key, 2, sizeof (uint64_t), 1, + &out)); + unit_eq(out, 1); + + /* zap_length reports a value's size without reading it. */ + uint64_t isz = 0, nint = 0; + unit_ok(zap_length_uint64_by_dnode(dn, key, 2, &isz, &nint)); + unit_eq(isz, sizeof (uint64_t)); + unit_eq(nint, 1); + + /* zap_lookup_length: the value plus its real element count. */ + out = 0; + uint64_t actual = 0; + unit_ok(zap_lookup_length_uint64_by_dnode(dn, key, 2, + sizeof (uint64_t), 1, &out, &actual)); + unit_eq(out, 1); + unit_eq(actual, 1); + + /* zap_update replaces the value for an existing key. */ + val = 2; + unit_ok(zap_update_uint64_by_dnode(dn, key, 2, sizeof (uint64_t), 1, + &val, tx)); + unit_ok(zap_lookup_uint64_by_dnode(dn, key, 2, sizeof (uint64_t), 1, + &out)); + unit_eq(out, 2); + + /* A key that was never added is not found. */ + uint64_t key2[2] = { unit_rand_uint64(), unit_rand_uint64() }; + unit_err(zap_lookup_uint64_by_dnode(dn, key2, 2, sizeof (uint64_t), 1, + &out), ENOENT); + + /* After removal, the key can no longer be looked up. */ + unit_ok(zap_remove_uint64_by_dnode(dn, key, 2, tx)); + unit_err(zap_lookup_uint64_by_dnode(dn, key, 2, sizeof (uint64_t), 1, + &out), ENOENT); + + mock_tx_destroy((mock_dmu_tx_t *)tx); + mock_zap_destroy(dn); + + return (MUNIT_OK); +} + +/* ========== */ + /* Test suite definition and boilerplate. */ #define UNIT_PARAM_ZAP_TYPES(p) \ @@ -1127,6 +1608,8 @@ static const MunitTest zap_tests[] = { UNIT_TEST("microzap_stats", test_microzap_stats), UNIT_TEST("fatzap_stats", test_fatzap_stats), + UNIT_TEST("uint64_keys", test_zap_uint64_keys), + UNIT_TEST_ZAP_TYPES("cursor", test_cursor), UNIT_TEST_ZAP_TYPES("cursor_serialize", test_cursor_serialize), @@ -1144,6 +1627,17 @@ static const MunitTest zap_tests[] = { UNIT_TEST_ZAP_TYPES( "zap_value_search_mask", test_zap_value_search_mask), + UNIT_TEST("norm_toupper", test_norm_toupper, zap_type_params), + UNIT_TEST("norm_match_case", test_norm_match_case, zap_type_params), + UNIT_TEST("norm_nfc", test_norm_nfc, zap_type_params), + UNIT_TEST("norm_nfd", test_norm_nfd, zap_type_params), + UNIT_TEST("norm_nfkc", test_norm_nfkc, zap_type_params), + UNIT_TEST("norm_nfkd", test_norm_nfkd, zap_type_params), + + UNIT_TEST("norm_conflict", test_norm_conflict, zap_type_params), + UNIT_TEST("norm_realname", test_norm_realname, zap_type_params), + UNIT_TEST("norm_remove", test_norm_remove, zap_type_params), + { 0 }, }; diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am index c7931ca95e29..2662bb2bfd02 100644 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am @@ -403,6 +403,42 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \ functional/zvol/zvol_ENOSPC/zvol_ENOSPC.cfg \ functional/zvol/zvol_misc/zvol_misc_common.kshlib \ functional/zvol/zvol_swap/zvol_swap.cfg \ + functional/zstream/zstream.cfg \ + functional/zstream/zstream.kshlib \ + functional/zstream/big-endian-all-drr-types-base-NATIVE.zsend.bz2 \ + functional/zstream/big-endian-all-drr-types-base-XDR.zsend.bz2 \ + functional/zstream/big-endian-all-drr-types-incr-NATIVE.zsend.bz2 \ + functional/zstream/big-endian-all-drr-types-incr-XDR.zsend.bz2 \ + functional/zstream/little-endian-all-drr-types-base-NATIVE.zsend.bz2 \ + functional/zstream/little-endian-all-drr-types-base-XDR.zsend.bz2 \ + functional/zstream/little-endian-all-drr-types-incr-NATIVE.zsend.bz2 \ + functional/zstream/little-endian-all-drr-types-incr-XDR.zsend.bz2 \ + functional/zstream/decompress.zsend.bz2 \ + functional/zstream/decompress-crypt.zsend.bz2 \ + functional/zstream/big-endian-long-payloads.zsend.bz2 \ + functional/zstream/little-endian-long-payloads.zsend.bz2 \ + functional/zstream/beadtbn-new.dump.bz2 \ + functional/zstream/beadtbn-old.dump.bz2 \ + functional/zstream/beadtbx-new.dump.bz2 \ + functional/zstream/beadtbx-old.dump.bz2 \ + functional/zstream/beadtin-new.dump.bz2 \ + functional/zstream/beadtin-old.dump.bz2 \ + functional/zstream/beadtix-new.dump.bz2 \ + functional/zstream/beadtix-old.dump.bz2 \ + functional/zstream/belp-new.dump.bz2 \ + functional/zstream/belp-old.dump.bz2 \ + functional/zstream/d-new.dump.bz2 \ + functional/zstream/dc-new.dump.bz2 \ + functional/zstream/leadtbn-new.dump.bz2 \ + functional/zstream/leadtbn-old.dump.bz2 \ + functional/zstream/leadtbx-new.dump.bz2 \ + functional/zstream/leadtbx-old.dump.bz2 \ + functional/zstream/leadtin-new.dump.bz2 \ + functional/zstream/leadtin-old.dump.bz2 \ + functional/zstream/leadtix-new.dump.bz2 \ + functional/zstream/leadtix-old.dump.bz2 \ + functional/zstream/lelp-new.dump.bz2 \ + functional/zstream/lelp-old.dump.bz2 \ functional/idmap_mount/idmap_mount.cfg \ functional/idmap_mount/idmap_mount_common.kshlib @@ -918,9 +954,6 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zfs_send/zfs_send_raw.ksh \ functional/cli_root/zfs_send/zfs_send_skip_missing.ksh \ functional/cli_root/zfs_send/zfs_send_sparse.ksh \ - functional/cli_root/zfs_send_delegation/cleanup.ksh \ - functional/cli_root/zfs_send_delegation/setup.ksh \ - functional/cli_root/zfs_send_delegation/zfs_send_test.ksh \ functional/cli_root/zfs_set/cache_001_pos.ksh \ functional/cli_root/zfs_set/cache_002_neg.ksh \ functional/cli_root/zfs_set/canmount_001_pos.ksh \ @@ -1299,6 +1332,8 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zpool/setup.ksh \ functional/cli_root/zpool_set/vdev_set_001_pos.ksh \ functional/cli_root/zpool_set/vdev_set_scheduler.ksh \ + functional/cli_root/zpool_set/vdev_set_allocating.ksh \ + functional/cli_root/zpool_set/vdev_set_path.ksh \ functional/cli_root/zpool_set/zpool_set_common.kshlib \ functional/cli_root/zpool_set/zpool_set_001_pos.ksh \ functional/cli_root/zpool_set/zpool_set_002_neg.ksh \ @@ -1453,9 +1488,6 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_user/zfs_list/zfs_list_005_neg.ksh \ functional/cli_user/zfs_list/zfs_list_007_pos.ksh \ functional/cli_user/zfs_list/zfs_list_008_neg.ksh \ - functional/cli_user/zfs_send_delegation_user/cleanup.ksh \ - functional/cli_user/zfs_send_delegation_user/setup.ksh \ - functional/cli_user/zfs_send_delegation_user/zfs_send_usertest.ksh \ functional/cli_user/zpool_iostat/cleanup.ksh \ functional/cli_user/zpool_iostat/setup.ksh \ functional/cli_user/zpool_iostat/zpool_iostat_001_neg.ksh \ @@ -1532,6 +1564,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/delegate/zfs_allow_010_pos.ksh \ functional/delegate/zfs_allow_011_neg.ksh \ functional/delegate/zfs_allow_012_neg.ksh \ + functional/delegate/zfs_allow_send.ksh \ functional/delegate/zfs_unallow_001_pos.ksh \ functional/delegate/zfs_unallow_002_pos.ksh \ functional/delegate/zfs_unallow_003_pos.ksh \ @@ -1608,6 +1641,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/failmode/failmode_syncalways_continue.ksh \ functional/failmode/setup.ksh \ functional/fallocate/cleanup.ksh \ + functional/fallocate/fallocate_extend_timestamps.ksh \ functional/fallocate/fallocate_prealloc.ksh \ functional/fallocate/fallocate_punch-hole.ksh \ functional/fallocate/fallocate_zero-range.ksh \ @@ -1878,6 +1912,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/projectquota/projectspace_003_pos.ksh \ functional/projectquota/projectspace_004_pos.ksh \ functional/projectquota/projectspace_005_pos.ksh \ + functional/projectquota/projectspace_006_pos.ksh \ functional/projectquota/projecttree_001_pos.ksh \ functional/projectquota/projecttree_002_pos.ksh \ functional/projectquota/projecttree_003_neg.ksh \ @@ -2098,8 +2133,6 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/rsend/send-c_verify_contents.ksh \ functional/rsend/send-c_verify_ratio.ksh \ functional/rsend/send-c_volume.ksh \ - functional/rsend/send-c_zstream_recompress.ksh \ - functional/rsend/send-c_zstreamdump.ksh \ functional/rsend/send-cpL_varied_recsize.ksh \ functional/rsend/send_doall.ksh \ functional/rsend/send_encrypted_incremental.ksh \ @@ -2128,7 +2161,6 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/rsend/send_realloc_files.ksh \ functional/rsend/send_spill_block.ksh \ functional/rsend/send-wR_encrypted_zvol.ksh \ - functional/rsend/send-zstream_drop_record.ksh \ functional/rsend/setup.ksh \ functional/scrub_mirror/cleanup.ksh \ functional/scrub_mirror/scrub_mirror_001_pos.ksh \ @@ -2250,6 +2282,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/userquota/groupspace_002_pos.ksh \ functional/userquota/groupspace_003_pos.ksh \ functional/userquota/groupspace_004_pos.ksh \ + functional/userquota/groupspace_005_pos.ksh \ functional/userquota/setup.ksh \ functional/userquota/defaultuserquota_001_pos.ksh \ functional/userquota/defaultuserquota_002_pos.ksh \ @@ -2281,6 +2314,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/userquota/userspace_002_pos.ksh \ functional/userquota/userspace_003_pos.ksh \ functional/userquota/userspace_004_pos.ksh \ + functional/userquota/userspace_005_pos.ksh \ functional/userquota/userspace_encrypted.ksh \ functional/userquota/userspace_send_encrypted.ksh \ functional/userquota/userspace_encrypted_13709.ksh \ @@ -2354,6 +2388,26 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/zpool_influxdb/cleanup.ksh \ functional/zpool_influxdb/setup.ksh \ functional/zpool_influxdb/zpool_influxdb.ksh \ + functional/zstream/setup.ksh \ + functional/zstream/cleanup.ksh \ + functional/zstream/zstream_checksum_001_pos.ksh \ + functional/zstream/zstream_decompress_001_pos.ksh \ + functional/zstream/zstream_decompress_002_pos.ksh \ + functional/zstream/zstream_decompress_003_neg.ksh \ + functional/zstream/zstream_decompress_004_pos.ksh \ + functional/zstream/zstream_decompress_005_pos.ksh \ + functional/zstream/zstream_decompress_006_neg.ksh \ + functional/zstream/zstream_drop_record_001_pos.ksh \ + functional/zstream/zstream_dump_001_pos.ksh \ + functional/zstream/zstream_dump_002_pos.ksh \ + functional/zstream/zstream_dump_003_pos.ksh \ + functional/zstream/zstream_dump_004_neg.ksh \ + functional/zstream/zstream_recompress_001_pos.ksh \ + functional/zstream/zstream_recompress_002_pos.ksh \ + functional/zstream/zstream_recompress_003_pos.ksh \ + functional/zstream/zstream_recompress_004_pos.ksh \ + functional/zstream/zstream_recompress_005_pos.ksh \ + functional/zstream/zstream_redup_001_pos.ksh \ functional/zvol/zvol_cli/cleanup.ksh \ functional/zvol/zvol_cli/setup.ksh \ functional/zvol/zvol_cli/zvol_cli_001_pos.ksh \ diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/zfs_send_test.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/zfs_send_test.ksh deleted file mode 100755 index d018f313fae1..000000000000 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/zfs_send_test.ksh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/ksh -p -# SPDX-License-Identifier: CDDL-1.0 -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or https://opensource.org/licenses/CDDL-1.0. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# - -# -# Copyright (c) 2025, Klara Inc. -# - -# STRATEGY: -# 1. Create a pool (this is done by the test framework) -# 2. Create an encrypted dataset -# 3. Write random data to the encrypted dataset -# 4. Snapshot the dataset -# 5. As root: attempt a send and raw send (both should succeed) -# 6. Create a delegation (zfs allow -u user send testpool/encrypted_dataset) -# 7. As root: attempt a send and raw send (both should succeed) -# 8. Create a delegation (zfs allow -u user send:raw testpool/encrypted_dataset) -# 9. As root: attempt a send and raw send (both should succeed) -# 10. Disable delegation (zfs unallow) -# 11. As root: attempt a send and raw send (both should succeed) -# 12. Clean up (handled by framework) -# -# Tested as a user under ../cli_user/zfs_send_delegation_user/ - -. $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/cli_root/zfs_create/zfs_create_common.kshlib -. $STF_SUITE/tests/functional/cli_root/zfs_create/properties.kshlib -. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib -. $STF_SUITE/tests/functional/delegate/delegate.cfg - -# create encrypted dataset - -log_must eval "echo $PASSPHRASE | zfs create -o encryption=on -o keyformat=passphrase $TESTPOOL/$TESTFS1" - -# create target dataset for receives -if ! zfs list | grep testfs2 >/dev/null 2>&1; then - dataset_created="TRUE" - log_must zfs create $TESTPOOL/$TESTFS2 -fi - -# create user and group -typeset perms="snapshot,reservation,compression,checksum,userprop,receive" - -log_note "Added permissions to the $OTHER1 user." -log_must zfs allow $OTHER1 $perms $TESTPOOL/$TESTFS1 -log_must zfs allow $OTHER1 $perms $TESTPOOL/$TESTFS2 - -# create random data -log_must fill_fs $TESTPOOL/$TESTFS1/child 1 2047 1024 1 R - -# snapshot -log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1 - - -# check baseline send abilities (should pass) -log_must eval "zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv0_datastream.$$" -log_must eval "zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv0raw_datastream.$$" - - -# create delegation -log_must zfs allow $OTHER1 send $TESTPOOL/$TESTFS1 - -# attempt send with full allow - -log_must eval "zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv1_datastream.$$" -log_must eval "zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv1raw_datastream.$$" - -# create raw delegation -log_must zfs allow $OTHER1 send:raw $TESTPOOL/$TESTFS1 -log_must zfs unallow $OTHER1 send $TESTPOOL/$TESTFS1 - -# test new send abilities (should pass) -log_must eval "zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv2_datastream.$$" -log_must eval "zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv2raw_datastream.$$" - - -# disable raw delegation -zfs unallow $OTHER1 send:raw $TESTPOOL/$TESTFS1 -zfs allow $OTHER1 send $TESTPOOL/$TESTFS1 - -# verify original send abilities (should pass) -log_must eval "zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv3_datastream.$$" -log_must eval "zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive $TESTPOOL/$TESTFS2/zfsrecv3raw_datastream.$$" - - -function cleanup -{ - datasetexists $TESTPOOL/$TESTFS1 && \ - destroy_dataset $TESTPOOL/$TESTFS1 -r \ - destroy_dataset $TESTPOOL/$TESTFS2 -r - -} diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_set/vdev_set_allocating.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_set/vdev_set_allocating.ksh new file mode 100755 index 000000000000..cfed80b43363 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_set/vdev_set_allocating.ksh @@ -0,0 +1,80 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# Toggling the "allocating" vdev property completes and takes effect. +# +# When zfs_ioc_vdev_set_props() holds SCL_CONFIG as a reader across +# vdev_prop_set(), setting "allocating" re-enters +# spa_config_enter(spa, SCL_ALL, RW_WRITER) via +# spa_vdev_noalloc()/spa_vdev_alloc(); taking SCL_CONFIG as a writer +# while the same thread already holds it as a reader self-deadlocks the +# calling thread, so "zpool set allocating=off" hangs in +# spa_config_enter(). +# +# STRATEGY: +# 1. Create a pool with two top-level vdevs (so one can stop allocating +# while a normal vdev remains). +# 2. Set allocating=off on the first vdev; verify the command returns and +# the property reads back "off". +# 3. Set allocating=on; verify it reads back "on". +# + +verify_runnable "global" + +typeset POOL=alloc_testpool +typeset VDEV0=$TEST_BASE_DIR/vdev_alloc0.$$ +typeset VDEV1=$TEST_BASE_DIR/vdev_alloc1.$$ + +function cleanup +{ + poolexists $POOL && destroy_pool $POOL + rm -f $VDEV0 $VDEV1 +} + +log_onexit cleanup + +log_assert "toggling the allocating vdev property completes and takes effect" + +log_must truncate -s $MINVDEVSIZE $VDEV0 $VDEV1 +log_must zpool create $POOL $VDEV0 $VDEV1 + +# Both top-level vdevs allocate by default. +log_must test "$(zpool get -H -o value allocating $POOL $VDEV0)" = "on" + +# The operation that previously deadlocked. +log_must zpool set allocating=off $POOL $VDEV0 +log_must test "$(zpool get -H -o value allocating $POOL $VDEV0)" = "off" + +# And back on. +log_must zpool set allocating=on $POOL $VDEV0 +log_must test "$(zpool get -H -o value allocating $POOL $VDEV0)" = "on" + +log_pass "toggling the allocating vdev property completes and takes effect" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_set/vdev_set_path.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_set/vdev_set_path.ksh new file mode 100755 index 000000000000..e1fbfcef065c --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_set/vdev_set_path.ksh @@ -0,0 +1,90 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# Setting the "path" vdev property completes and takes effect. +# +# Like "allocating", the "path" property re-takes SCL_CONFIG as a +# writer underneath vdev_prop_set(): it calls spa_vdev_setpath() -> +# spa_vdev_set_common() -> spa_vdev_state_enter(spa, SCL_ALL) -> +# spa_config_enter(spa, SCL_ALL, RW_WRITER). When +# zfs_ioc_vdev_set_props() holds SCL_CONFIG as a reader across +# vdev_prop_set(), taking it as a writer while the same thread already +# holds it as a reader self-deadlocks the calling thread, so "zpool set +# path=..." hangs in spa_config_enter(). +# +# STRATEGY: +# 1. Create a pool on our own file vdev (so we don't collide with the +# pool the zpool_set setup creates on $DISKS). +# 2. Point a /dev symlink at the leaf and set path= to it; verify the +# command returns and the property reads back the new path. +# +# "path" only records the string and requires a /dev/ prefix; it does +# not reopen the device, so a /dev symlink standing in for the file +# vdev is sufficient to exercise the set path. +# + +verify_runnable "global" + +typeset POOL=path_testpool +typeset VDEV=$TEST_BASE_DIR/vdev_path.$$ +typeset SYMLINK=/dev/vdev_set_path_$$ + +function cleanup +{ + poolexists $POOL && destroy_pool $POOL + rm -f $VDEV $SYMLINK +} + +log_onexit cleanup + +log_assert "setting the path vdev property completes and takes effect" + +log_must truncate -s $MINVDEVSIZE $VDEV +log_must zpool create $POOL $VDEV + +# The stored leaf path is the file vdev we created. +typeset OLDPATH=$(zpool get -H -o value path $POOL $VDEV) +log_must test -n "$OLDPATH" + +# A new /dev path for the leaf; "path" must start with /dev/. +log_must ln -s $OLDPATH $SYMLINK + +# The operation that previously deadlocked. +log_must zpool set path=$SYMLINK $POOL $OLDPATH + +# The single leaf is the only vdev with a path; confirm it now reads back +# the new value. (Address by "all-vdevs" rather than by the new path, to +# avoid depending on how the CLI re-resolves a just-renamed vdev.) +log_must test \ + "$(zpool get -H -o value path $POOL all-vdevs | grep '^/dev/')" = \ + "$SYMLINK" + +log_pass "setting the path vdev property completes and takes effect" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/setup.ksh deleted file mode 100755 index 0978193eddc4..000000000000 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/setup.ksh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/ksh -p -# SPDX-License-Identifier: CDDL-1.0 -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or https://opensource.org/licenses/CDDL-1.0. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# - -# -# Copyright (c) 2025, Klara Inc. -# - -. $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/delegate/delegate_common.kshlib - -# Create staff group and add two user to it -log_must add_group $STAFF_GROUP -if ! id $STAFF1 > /dev/null 2>&1; then - log_must add_user $STAFF_GROUP $STAFF1 -fi -if ! id $STAFF2 > /dev/null 2>&1; then - log_must add_user $STAFF_GROUP $STAFF2 -fi - -# Create other group and add two user to it -log_must add_group $OTHER_GROUP -if ! id $OTHER1 > /dev/null 2>&1; then - log_must add_user $OTHER_GROUP $OTHER1 -fi -if ! id $OTHER2 > /dev/null 2>&1; then - log_must add_user $OTHER_GROUP $OTHER2 -fi -DISK=${DISKS%% *} - -default_raidz_setup $DISKS diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/zfs_send_usertest.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/zfs_send_usertest.ksh deleted file mode 100755 index ea54a962fb0a..000000000000 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/zfs_send_usertest.ksh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/ksh -p -# SPDX-License-Identifier: CDDL-1.0 -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or https://opensource.org/licenses/CDDL-1.0. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# - -# -# Copyright (c) 2025, Klara Inc. -# - -# STRATEGY: -# 1. Create a pool (this is done by the test framework) -# 2. Create a user -# 3. Create an encrypted dataset -# 4. Write random data to the encrypted dataset -# 5. Snapshot the dataset -# 6. As root: attempt a send and raw send (both should succeed) -# 7. As user: attempt a send and raw send (both should fail, no permission) -# 8. Create a delegation (zfs allow -u user send testpool/encrypted_dataset) -# 9. As root: attempt a send and raw send (both should succeed) -# 10. As user: attempt a send and raw send (both should succeed) -# 11. Create a delegation (zfs allow -u user sendraw testpool/encrypted_dataset) -# 12. As root: attempt a send and raw send (both should succeed) -# 13. As user: attempt a send and raw send (send should fail, raw send should succeed) -# 14. Disable delegation (zfs unallow) -# 15. As root: attempt a send and raw send (both should succeed) -# 16. As user: attempt a send and raw send (both should fail, no permission) -# 17. Clean up (handled by framework) -# root tests to verify this doesnt affect root user under ../cli_root/zfs_send_delegation/ -. $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/cli_root/zfs_create/zfs_create_common.kshlib -. $STF_SUITE/tests/functional/cli_root/zfs_create/properties.kshlib -. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib -. $STF_SUITE/tests/functional/delegate/delegate.cfg - -# create encrypted dataset - -log_must eval "echo $PASSPHRASE | zfs create -o encryption=on -o keyformat=passphrase $TESTPOOL/$TESTFS1" - -# create target dataset for receives -log_must zfs create $TESTPOOL/$TESTFS2 - -# set user perms -# need to run chown for fs permissions for $OTHER1 -typeset perms="snapshot,reservation,compression,checksum,userprop,receive,mount,create" - -log_must zfs allow $OTHER1 $perms $TESTPOOL/$TESTFS1 -log_must zfs allow $OTHER1 $perms $TESTPOOL/$TESTFS2 -log_must chown ${OTHER1}:${OTHER_GROUP} /$TESTPOOL/$TESTFS2 - -# create random data -log_must fill_fs $TESTPOOL/$TESTFS1/child 1 2047 1024 1 R - -# snapshot -log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1 - -# note -# we need to use `sh -c` here becuase the quoting on <<<"$*" in the user_run wrapper is broken once pipes and redirects get involved - -# check baseline send abilities (should fail) -log_mustnot user_run $OTHER1 sh -c "'zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv0_user_datastream.$$'" -# verify nothing went through -if [ -s $TESTPOOL/$TESTFS2/zfsrecv0_user_datastream.$$ ] -then - log_fail "A zfs recieve was completed in $TESTPOOL/$TESTFS2/zfsrecv0_user_datastream !" -fi -log_mustnot user_run $OTHER1 sh -c "'zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv0raw_user_datastream.$$'" -# verify nothing went through -if [ -s $TESTPOOL/$TESTFS2/zfsrecv0raw_user_datastream.$$ ] -then - log_fail "A zfs recieve was completed in $TESTPOOL/$TESTFS2/zfsrecv0raw_user_datastream !" -fi - -# create delegation -log_must zfs allow $OTHER1 send $TESTPOOL/$TESTFS1 - -# attempt send with full allow (should pass) -log_must user_run $OTHER1 sh -c "'zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv1_user_datastream.$$'" -log_must user_run $OTHER1 sh -c "'zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv1raw_user_datastream.$$'" - - -# create raw delegation -log_must zfs allow $OTHER1 send:raw $TESTPOOL/$TESTFS1 -# We have to remove 'send' to confirm 'send raw' only allows what we want -log_must zfs unallow -u $OTHER1 send $TESTPOOL/$TESTFS1 - -# test new sendraw abilities (send should fail, sendraw should pass) -log_mustnot user_run $OTHER1 sh -c "'zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv2_user_datastream.$$'" -# verify nothing went through -if [ -s $TESTPOOL/$TESTFS2/zfsrecv2_user_datastream.$$ ] -then - log_fail "A zfs recieve was completed in $TESTPOOL/$TESTFS2/zfsrecv2_user_datastream !" -fi -log_must user_run $OTHER1 sh -c "'zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv2raw_user_datastream.$$'" - -# test incremental send with intermediates (should pass) -log_must zfs allow $OTHER1 hold $TESTPOOL/$TESTFS1 -log_must zfs snapshot $TESTPOOL/$TESTFS1@snap2 -log_must user_run $OTHER1 sh -c "'zfs send -w -I $TESTPOOL/$TESTFS1@snap1 $TESTPOOL/$TESTFS1@snap2 > /dev/null'" - -# disable raw delegation -log_must zfs unallow -u $OTHER1 send:raw $TESTPOOL/$TESTFS1 -log_must zfs allow $OTHER1 send $TESTPOOL/$TESTFS1 - -# test with raw taken away (should pass) -log_must user_run $OTHER1 sh -c "'zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv3_user_datastream.$$'" -log_must user_run $OTHER1 sh -c "'zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv3raw_user_datastream.$$'" - -# disable send abilities -log_must zfs unallow -u $OTHER1 send $TESTPOOL/$TESTFS1 - -# verify original send abilities (should fail) -log_mustnot user_run $OTHER1 sh -c "'zfs send $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv4_user_datastream.$$'" -# verify nothing went through -if [ -s $TESTPOOL/$TESTFS2/zfsrecv4_user_datastream.$$ ] -then - log_fail "A zfs recieve was completed in $TESTPOOL/$TESTFS2/zfsrecv4_user_datastream !" -fi -log_mustnot user_run $OTHER1 sh -c "'zfs send -w $TESTPOOL/$TESTFS1@snap1 | zfs receive -u $TESTPOOL/$TESTFS2/zfsrecv4raw_user_datastream.$$'" -# verify nothing went through -if [ -s $TESTPOOL/$TESTFS2/zfsrecv4raw_user_datastream.$$ ] -then - log_fail "A zfs recieve was completed in $TESTPOOL/$TESTFS2/zfsrecv4raw_user_datastream !" -fi - - -function cleanup -{ - datasetexists $TESTPOOL/$TESTFS1 && \ - destroy_dataset $TESTPOOL/$TESTFS1 -r \ - destroy_dataset $TESTPOOL/$TESTFS2 -r - -} diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib index 345239b88680..f8c6a937adb6 100644 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib @@ -53,14 +53,23 @@ function cleanup_user_group # # Restore test file system to the original status. # -function restore_root_datasets +function restore_root_datasets # do_encrypt { + typeset do_encrypt=${1:-} + + typeset enc_args="" + typeset password="" + if [[ -n "$do_encrypt" ]] ; then + enc_args="-o encryption=on -o keyformat=passphrase -o keylocation=prompt" + password="password" + fi + destroy_dataset "$ROOT_TESTFS" "-Rf" - log_must zfs create $ROOT_TESTFS + log_must eval "echo $password | zfs create $enc_args $ROOT_TESTFS" if is_global_zone ; then destroy_dataset "$ROOT_TESTVOL" "-Rf" - log_must zfs create -V $VOLSIZE $ROOT_TESTVOL + log_must eval "echo $password | zfs create -V $VOLSIZE $enc_args $ROOT_TESTVOL" block_device_wait fi @@ -166,6 +175,9 @@ function common_perm send) verify_send $user $perm $dtst ;; + send_raw) # not a true permission + verify_send_raw $user $perm $dtst + ;; allow) verify_allow $user $perm $dtst ;; @@ -357,6 +369,40 @@ function verify_send return $ret } +# +# Verify permission send for specified user on the dataset +# $1 user +# $2 permission +# $3 dataset +# +function verify_send_raw +{ + typeset user=$1 + typeset perm=$2 + typeset dtst=$3 + + typeset oldval + typeset stamp=${perm}.${user}.$RANDOM + typeset snap=$dtst@snap.$stamp + + typeset -i ret=1 + + log_must zfs snapshot $snap + typeset bak_user=$TEST_BASE_DIR/bak.$user.$stamp + typeset bak_root=$TEST_BASE_DIR/bak.root.$stamp + + user_run $user eval "zfs send -w $snap > $bak_user" + log_must eval "zfs send -w $snap > $bak_root" + + if [ "$(cksum < $bak_user)" = "$(cksum < $bak_root)" ]; then + ret=0 + fi + + rm -rf $bak_user $bak_root + + return $ret +} + function verify_fs_receive { typeset user=$1 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/zfs_allow_send.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/zfs_allow_send.ksh new file mode 100755 index 000000000000..6e9e78ad7c09 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/zfs_allow_send.ksh @@ -0,0 +1,57 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026, TrueNAS. +# + +. $STF_SUITE/tests/functional/delegate/delegate_common.kshlib + +verify_runnable "both" + +log_assert "Verify send sub-permissions correctly limit send options." +log_onexit restore_root_datasets + +for dtst in $DATASETS; do + + for enc in '' 'y' ; do + # full 'send' permission can do it all + log_must restore_root_datasets $enc + log_must zfs allow $STAFF1 "send" $dtst + log_must verify_perm $dtst "send" $STAFF1 + log_must verify_perm $dtst "send_raw" $STAFF1 + + # 'send:raw' can only do raw send + log_must restore_root_datasets $enc + log_must zfs allow $STAFF1 "send:raw" $dtst + log_must verify_noperm $dtst "send" $STAFF1 + log_must verify_perm $dtst "send_raw" $STAFF1 + done + + # 'send:encrypted' cannot do anything on unencrypted datasets + log_must restore_root_datasets + log_must zfs allow $STAFF1 "send:encrypted" $dtst + log_must verify_noperm $dtst "send" $STAFF1 + log_must verify_noperm $dtst "send_raw" $STAFF1 + + # 'send:encrypted' can do raw send only on encrypted datasets + log_must restore_root_datasets 'y' + log_must zfs allow $STAFF1 "send:encrypted" $dtst + log_must verify_noperm $dtst "send" $STAFF1 + log_must verify_perm $dtst "send_raw" $STAFF1 + +done + +log_must restore_root_datasets + +log_pass "Verify send-permissions correctly limit send options." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/events_common.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/events_common.kshlib index 79813e33bc77..89b25c8f654a 100644 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/events_common.kshlib +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/events/events_common.kshlib @@ -40,7 +40,7 @@ function file_wait_event # file event timeout SECONDS=0 - until grep -q "^ZEVENT_CLASS=$event" $ZED_DEBUG_LOG ; do + until grep -q "^ZEVENT_CLASS=$event" "$file" ; do if [[ $SECONDS -gt $timeout ]]; then echo file_wait_event exceeded $SECONDS seconds return 1 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fallocate/fallocate_extend_timestamps.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fallocate/fallocate_extend_timestamps.ksh new file mode 100755 index 000000000000..e96e3259d141 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/fallocate/fallocate_extend_timestamps.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by iXsystems, Inc. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/include/math.shlib + +# +# DESCRIPTION: +# A fallocate(2) that extends a file past EOF must update mtime/ctime; a +# FALLOC_FL_KEEP_SIZE preallocation must not change the size or timestamps. +# +# STRATEGY: +# 1. Create a small file and record its mtime/ctime +# 2. fallocate -l to a larger size (extend) and verify the size grew and both +# mtime and ctime advanced +# 3. fallocate --keep-size to a larger length and verify the size and both +# timestamps are unchanged +# + +verify_runnable "global" + +FILE=$TESTDIR/$TESTFILE0 +STARTSIZE=$((128 * 1024)) + +function cleanup +{ + [[ -f $FILE ]] && rm -f $FILE +} + +# extend past EOF: size must grow and mtime/ctime must advance +function verify_extend # <newsize> +{ + typeset -i size="$1" + + log_must mkfile $STARTSIZE $FILE + typeset -i tm="$(stat -c %Y $FILE)" + typeset -i tc="$(stat -c %Z $FILE)" + log_must sleep 1 + log_must fallocate -l $size $FILE + verify_eq $size "$(stat_size $FILE)" "size" + verify_ne $tm "$(stat -c %Y $FILE)" "mtime" + verify_ne $tc "$(stat -c %Z $FILE)" "ctime" + log_must rm -f $FILE +} + +log_assert "Ensure fallocate extend updates mtime/ctime and --keep-size does not" +log_onexit cleanup + +verify_extend $((256 * 1024)) +verify_extend $((1024 * 1024)) + +# --keep-size preallocation must not change size or timestamps +log_must mkfile $STARTSIZE $FILE +typeset -i tm="$(stat -c %Y $FILE)" +typeset -i tc="$(stat -c %Z $FILE)" +log_must sleep 1 +log_must fallocate --keep-size -l $((1024 * 1024)) $FILE +verify_eq $STARTSIZE "$(stat_size $FILE)" "size" +verify_eq $tm "$(stat -c %Y $FILE)" "mtime" +verify_eq $tc "$(stat -c %Z $FILE)" "ctime" +log_must rm -f $FILE + +log_pass "fallocate extend correctly updates timestamps" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/projectquota/projectspace_006_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/projectquota/projectspace_006_pos.ksh new file mode 100755 index 000000000000..29309c68e1bc --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/projectquota/projectspace_006_pos.ksh @@ -0,0 +1,97 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026, Gluesys. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/projectquota/projectquota_common.kshlib + +# +# DESCRIPTION: +# Verify 'zfs projectspace' reports correct quota values whether the +# dataset is mounted or not, and does not fail with EBUSY when +# raced against 'zfs mount'. +# +# STRATEGY: +# 1. Create a dataset with a projectquota and write 50M of data. +# 2. Check quota and used values while mounted. +# 3. Unmount and verify the same values are still readable. +# 4. Remount and re-verify values are consistent. +# 5. Race 'zfs projectspace' against 'zfs mount' 50 times; expect no EBUSY. +# + +function cleanup +{ + zfs unmount $RACE_FS 2>/dev/null + datasetexists $RACE_FS && destroy_dataset $RACE_FS + rm -f "$ERRFILE" "$DATAFILE" +} + +log_onexit cleanup + +typeset RACE_FS=$QFS/projectspace_race +typeset ERRFILE=/tmp/projectspace_006_err.$$ +typeset DATAFILE +typeset -i ITERS=50 + +log_assert "zfs projectspace shows correct quota values and does not EBUSY on mount race" + +log_must zfs create $RACE_FS +log_must zfs set projectquota@$PRJID1=100m $RACE_FS +mkmount_writable $RACE_FS +DATAFILE=$(get_prop mountpoint $RACE_FS)/projectspace_006_data.$$ +log_must mkfile 50m $DATAFILE +sync_all_pools + +log_note "check projectspace values while mounted" +log_must eval "zfs projectspace $RACE_FS | grep $PRJID1 | grep 100M" + +log_must zfs unmount $RACE_FS +log_note "check projectspace values while unmounted" +log_must eval "zfs projectspace $RACE_FS | grep $PRJID1 | grep 100M" + +log_must zfs mount $RACE_FS +log_note "check projectspace values after remount" +log_must eval "zfs projectspace $RACE_FS | grep $PRJID1 | grep 100M" + +typeset -i i=0 ebusy=0 +while (( i < ITERS )); do + log_must zfs unmount $RACE_FS + zfs mount $RACE_FS & + typeset mpid=$! + zfs projectspace $RACE_FS >"$ERRFILE" 2>&1 + typeset rc=$? + wait $mpid + if (( rc != 0 )) && grep -qi "busy" "$ERRFILE" 2>/dev/null; then + (( ebusy++ )) + log_note "Iteration $i: EBUSY: $(cat $ERRFILE)" + fi + rm -f "$ERRFILE" + (( i++ )) +done + +(( ebusy > 0 )) && log_fail "EBUSY seen $ebusy/$ITERS times" + +log_pass "zfs projectspace shows correct quota values and does not EBUSY on mount race" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/userquota/groupspace_005_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/userquota/groupspace_005_pos.ksh new file mode 100755 index 000000000000..1955ae051237 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/userquota/groupspace_005_pos.ksh @@ -0,0 +1,100 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026, Gluesys. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib + +# +# DESCRIPTION: +# Verify 'zfs groupspace' reports correct quota values whether the +# dataset is mounted or not, and does not fail with EBUSY when +# raced against 'zfs mount'. +# +# STRATEGY: +# 1. Create a dataset with a groupquota and write 50M of data. +# 2. Check quota and used values while mounted. +# 3. Unmount and verify the same values are still readable. +# 4. Remount and re-verify values are consistent. +# 5. Race 'zfs groupspace' against 'zfs mount' 50 times; expect no EBUSY. +# + +function cleanup +{ + zfs unmount $RACE_FS 2>/dev/null + datasetexists $RACE_FS && destroy_dataset $RACE_FS + rm -f "$ERRFILE" "$DATAFILE" +} + +log_onexit cleanup + +typeset RACE_FS=$QFS/groupspace_race +typeset ERRFILE=/tmp/groupspace_005_err.$$ +typeset DATAFILE +typeset -i ITERS=50 + +log_assert "zfs groupspace shows correct quota values and does not EBUSY on mount race" + +log_must zfs create $RACE_FS +log_must zfs set groupquota@$QGROUP=100m $RACE_FS +mkmount_writable $RACE_FS +DATAFILE=$(get_prop mountpoint $RACE_FS)/groupspace_005_data.$$ +log_must user_run $QUSER1 mkfile 50m $DATAFILE +sync_all_pools + +log_note "check groupspace values while mounted" +log_must eval "zfs groupspace $RACE_FS | grep $QGROUP | grep 100M" +log_must eval "zfs groupspace $RACE_FS | grep $QGROUP | grep '50\\..*M'" + +log_must zfs unmount $RACE_FS +log_note "check groupspace values while unmounted" +log_must eval "zfs groupspace $RACE_FS | grep $QGROUP | grep 100M" +log_must eval "zfs groupspace $RACE_FS | grep $QGROUP | grep '50\\..*M'" + +log_must zfs mount $RACE_FS +log_note "check groupspace values after remount" +log_must eval "zfs groupspace $RACE_FS | grep $QGROUP | grep 100M" +log_must eval "zfs groupspace $RACE_FS | grep $QGROUP | grep '50\\..*M'" + +typeset -i i=0 ebusy=0 +while (( i < ITERS )); do + log_must zfs unmount $RACE_FS + zfs mount $RACE_FS & + typeset mpid=$! + zfs groupspace $RACE_FS >"$ERRFILE" 2>&1 + typeset rc=$? + wait $mpid + if (( rc != 0 )) && grep -qi "busy" "$ERRFILE" 2>/dev/null; then + (( ebusy++ )) + log_note "Iteration $i: EBUSY: $(cat $ERRFILE)" + fi + rm -f "$ERRFILE" + (( i++ )) +done + +(( ebusy > 0 )) && log_fail "EBUSY seen $ebusy/$ITERS times" + +log_pass "zfs groupspace shows correct quota values and does not EBUSY on mount race" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/userquota/userspace_005_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/userquota/userspace_005_pos.ksh new file mode 100755 index 000000000000..fd98ded923a5 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/userquota/userspace_005_pos.ksh @@ -0,0 +1,100 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026, Gluesys. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib + +# +# DESCRIPTION: +# Verify 'zfs userspace' reports correct quota values whether the +# dataset is mounted or not, and does not fail with EBUSY when +# raced against 'zfs mount'. +# +# STRATEGY: +# 1. Create a dataset with a userquota and write 50M of data. +# 2. Check quota and used values while mounted. +# 3. Unmount and verify the same values are still readable. +# 4. Remount and re-verify values are consistent. +# 5. Race 'zfs userspace' against 'zfs mount' 50 times; expect no EBUSY. +# + +function cleanup +{ + zfs unmount $RACE_FS 2>/dev/null + datasetexists $RACE_FS && destroy_dataset $RACE_FS + rm -f "$ERRFILE" "$DATAFILE" +} + +log_onexit cleanup + +typeset RACE_FS=$QFS/userspace_race +typeset ERRFILE=/tmp/userspace_005_err.$$ +typeset DATAFILE +typeset -i ITERS=50 + +log_assert "zfs userspace shows correct quota values and does not EBUSY on mount race" + +log_must zfs create $RACE_FS +log_must zfs set userquota@$QUSER1=100m $RACE_FS +mkmount_writable $RACE_FS +DATAFILE=$(get_prop mountpoint $RACE_FS)/userspace_005_data.$$ +log_must user_run $QUSER1 mkfile 50m $DATAFILE +sync_all_pools + +log_note "check userspace values while mounted" +log_must eval "zfs userspace $RACE_FS | grep $QUSER1 | grep 100M" +log_must eval "zfs userspace $RACE_FS | grep $QUSER1 | grep '50\\..*M'" + +log_must zfs unmount $RACE_FS +log_note "check userspace values while unmounted" +log_must eval "zfs userspace $RACE_FS | grep $QUSER1 | grep 100M" +log_must eval "zfs userspace $RACE_FS | grep $QUSER1 | grep '50\\..*M'" + +log_must zfs mount $RACE_FS +log_note "check userspace values after remount" +log_must eval "zfs userspace $RACE_FS | grep $QUSER1 | grep 100M" +log_must eval "zfs userspace $RACE_FS | grep $QUSER1 | grep '50\\..*M'" + +typeset -i i=0 ebusy=0 +while (( i < ITERS )); do + log_must zfs unmount $RACE_FS + zfs mount $RACE_FS & + typeset mpid=$! + zfs userspace $RACE_FS >"$ERRFILE" 2>&1 + typeset rc=$? + wait $mpid + if (( rc != 0 )) && grep -qi "busy" "$ERRFILE" 2>/dev/null; then + (( ebusy++ )) + log_note "Iteration $i: EBUSY: $(cat $ERRFILE)" + fi + rm -f "$ERRFILE" + (( i++ )) +done + +(( ebusy > 0 )) && log_fail "EBUSY seen $ebusy/$ITERS times" + +log_pass "zfs userspace shows correct quota values and does not EBUSY on mount race" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbn-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbn-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..947d615c7462 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbn-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbn-old.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbn-old.dump.bz2 Binary files differnew file mode 100644 index 000000000000..c80330c6e736 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbn-old.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbx-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbx-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..884bbde5fcad --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbx-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbx-old.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbx-old.dump.bz2 Binary files differnew file mode 100644 index 000000000000..b52fc091e705 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtbx-old.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtin-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtin-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..22b4ed986a18 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtin-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtin-old.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtin-old.dump.bz2 Binary files differnew file mode 100644 index 000000000000..e467af8ce1bc --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtin-old.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtix-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtix-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..456d6da63547 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtix-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtix-old.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtix-old.dump.bz2 Binary files differnew file mode 100644 index 000000000000..8a677e3d030b --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/beadtix-old.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/belp-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/belp-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..49a98d9c62af --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/belp-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/belp-old.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/belp-old.dump.bz2 Binary files differnew file mode 100644 index 000000000000..83a20115a842 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/belp-old.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-NATIVE.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-NATIVE.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..5ebe9e40c1cd --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-NATIVE.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-XDR.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-XDR.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..a9c6c0445346 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-XDR.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-NATIVE.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-NATIVE.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..385aecd46aa2 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-NATIVE.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-XDR.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-XDR.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..162dde833c91 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-XDR.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-long-payloads.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-long-payloads.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..3fae565450b6 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/big-endian-long-payloads.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/cleanup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/cleanup.ksh index 4a59e15cc693..1bae927d29c5 100755 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/cleanup.ksh +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/cleanup.ksh @@ -22,22 +22,12 @@ # # -# Copyright (c) 2025, Klara Inc. +# Copyright (c) 2026 by ConnectWise. All rights reserved. # -. $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/delegate/delegate_common.kshlib +. $STF_SUITE/tests/functional/zstream/zstream.kshlib +verify_runnable "both" -poolexists $TESTPOOL1 && \ - destroy_pool $TESTPOOL1 - -del_user $STAFF1 -del_user $STAFF2 -del_group $STAFF_GROUP - -del_user $OTHER1 -del_user $OTHER2 -del_group $OTHER_GROUP - -default_cleanup +log_must rm -rf $BACKDIR +default_cleanup $POOL diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/d-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/d-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..a378b6c2aef9 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/d-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/dc-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/dc-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..613ffbd3b3bc --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/dc-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/decompress-crypt.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/decompress-crypt.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..de1931ff6bc7 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/decompress-crypt.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/decompress.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/decompress.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..7f2ef6141cdc --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/decompress.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbn-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbn-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..753082a111b2 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbn-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbn-old.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbn-old.dump.bz2 Binary files differnew file mode 100644 index 000000000000..7838243c9cf3 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbn-old.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbx-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbx-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..70862d375d9b --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbx-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbx-old.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbx-old.dump.bz2 Binary files differnew file mode 100644 index 000000000000..48c4892c181a --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtbx-old.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtin-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtin-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..b25ed3897ada --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtin-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtin-old.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtin-old.dump.bz2 Binary files differnew file mode 100644 index 000000000000..fd9acbf8507c --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtin-old.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtix-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtix-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..b3de0d7bda00 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtix-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtix-old.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtix-old.dump.bz2 Binary files differnew file mode 100644 index 000000000000..04d0c41b9978 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/leadtix-old.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/lelp-new.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/lelp-new.dump.bz2 Binary files differnew file mode 100644 index 000000000000..67a3eb068806 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/lelp-new.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/lelp-old.dump.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/lelp-old.dump.bz2 Binary files differnew file mode 100644 index 000000000000..fce108ee3e0e --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/lelp-old.dump.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-NATIVE.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-NATIVE.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..de08809d7945 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-NATIVE.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-XDR.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-XDR.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..bb8ab79fcf60 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-XDR.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-NATIVE.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-NATIVE.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..ec781bc96ec8 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-NATIVE.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-XDR.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-XDR.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..73cea6f1ed6b --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-XDR.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-long-payloads.zsend.bz2 b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-long-payloads.zsend.bz2 Binary files differnew file mode 100644 index 000000000000..16ca84407051 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/little-endian-long-payloads.zsend.bz2 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/cleanup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/setup.ksh index 4a59e15cc693..fda341f5165e 100755 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_user/zfs_send_delegation_user/cleanup.ksh +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/setup.ksh @@ -22,22 +22,12 @@ # # -# Copyright (c) 2025, Klara Inc. +# Copyright (c) 2026 by ConnectWise. All rights reserved. # -. $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/delegate/delegate_common.kshlib +. $STF_SUITE/tests/functional/zstream/zstream.kshlib +verify_runnable "both" -poolexists $TESTPOOL1 && \ - destroy_pool $TESTPOOL1 - -del_user $STAFF1 -del_user $STAFF2 -del_group $STAFF_GROUP - -del_user $OTHER1 -del_user $OTHER2 -del_group $OTHER_GROUP - -default_cleanup +log_must mkdir -p $BACKDIR +default_setup $DISK1 diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/README b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/README new file mode 100644 index 000000000000..1033ced8d3d9 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/README @@ -0,0 +1,19 @@ +The scripts in this directory were used to generate most of the static +test streams used by the zstream test suite. They're not really +intended to be run again and are here more as documentation than as +code. + +Some of the python scripts use the 'lorem' pip module, so you need to +run the make-venv.sh to set up before running them. The venv is created +in /tmp and the scripts hard-code that path for python. + +The "all-drr-types" streams have to be generated in two pieces because +of the way redaction works. A *base* stream and an *incr* stream +collectively include every possible type of record, but neither +substream is complete on its own. + +Many test streams are big-endian, so given hardware's convergence on +little-endian order, you will probably need to create big-endian +streams under emulation with, e.g., QEMU. As of mid-2026, the easiest +target for big-endian emulation is an IBM s390x CPU, which is supported +by both QEMU and Ubuntu. diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/add-xattrs.py b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/add-xattrs.py new file mode 100755 index 000000000000..7b7102ea188a --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/add-xattrs.py @@ -0,0 +1,112 @@ +#!/tmp/zstream-venv/bin/python3 +"""Add at least 1024 bytes of random extended attributes to files.""" + +# +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the Common +# Development and Distribution License ("CDDL"), version 1.0. You may only use +# this file in accordance with the terms of version 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this source. A +# copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +import argparse +import os +import random +import sys +from lorem_text import lorem + +ADJECTIVES = [ + "boogie", "funky", "wobbly", "snazzy", "jazzy", "groovy", "zippy", + "bouncy", "fluffy", "crunchy", "sparkly", "fuzzy", "spiffy", "dandy", + "peppy", "snappy", "sassy", "zesty", "swanky", "nifty", "plucky", + "quirky", "wacky", "goofy", "dizzy", "breezy", "cheery", "perky", + "frisky", "chirpy", "feisty", "jolly", "lively", "merry", "spunky", + "zippy", "vivid", "brisk", "sunny", "witty", "kinky", +] + +NOUNS = [ + "woogie", "monkey", "noodle", "pickle", "muffin", "waffle", "pebble", + "wobble", "doodle", "tangle", "giggle", "wiggle", "jiggle", "sparkle", + "crinkle", "twinkle", "frizzle", "drizzle", "sizzle", "fizzle", + "puddle", "bubble", "muddle", "huddle", "cuddle", "juggle", "muggle", + "snuggle", "tuggle", "buggle", "nugget", "widget", "gadget", "gibbet", + "trinket", "bracket", "racket", "jacket", "ticket", "cricket", "thicket", + "biscuit", "circuit", "summit", "muppet", "trumpet", "basket", "casket", +] + +TARGET_BYTES = 1024 + + +def random_attr_name(used: set) -> str: + for _ in range(1000): + name = f"user.{random.choice(ADJECTIVES)}-{random.choice(NOUNS)}" + if name not in used: + return name + base = f"user.{random.choice(ADJECTIVES)}-{random.choice(NOUNS)}" + i = 2 + while f"{base}-{i}" in used: + i += 1 + return f"{base}-{i}" + + +def random_value(length: int) -> bytes: + # Pull words from lorem sentences and trim/pad to exact length + text = "" + while len(text) < length: + text += lorem.sentence() + " " + return text[:length].encode() + + +def add_xattrs(path: str) -> int: + """Add xattrs to path until TARGET_BYTES added. Returns bytes added.""" + used_names = set() + total = 0 + while total < TARGET_BYTES: + remaining = TARGET_BYTES - total + if remaining < 40: + length = remaining + else: + length = random.randint(40, min(200, remaining)) + name = random_attr_name(used_names) + used_names.add(name) + value = random_value(length) + os.setxattr(path, name, value) + total += len(value) + return total + + +def main(): + parser = argparse.ArgumentParser( + description=f"Add random xattrs to files until {TARGET_BYTES} bytes " + "of xattr values are added." + ) + parser.add_argument("files", nargs="+", help="Files to add xattrs to") + args = parser.parse_args() + + errors = 0 + for path in args.files: + try: + added = add_xattrs(path) + print(f" {path} ({added:,} bytes in xattrs)") + except OSError as e: + print(f" {path} error: {e}", file=sys.stderr) + errors += 1 + + if errors: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/gen-lorem-files.py b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/gen-lorem-files.py new file mode 100755 index 000000000000..5b54953d2ea4 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/gen-lorem-files.py @@ -0,0 +1,109 @@ +#!/tmp/zstream-venv/bin/python3 +"""Generate randomly-named files with lorem ipsum paragraphs.""" + +# +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the Common +# Development and Distribution License ("CDDL"), version 1.0. You may only use +# this file in accordance with the terms of version 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this source. A +# copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +import argparse +import random +import sys +from pathlib import Path +from lorem_text import lorem + +ADJECTIVES = [ + "boogie", "funky", "wobbly", "snazzy", "jazzy", "groovy", "zippy", + "bouncy", "fluffy", "crunchy", "sparkly", "fuzzy", "spiffy", "dandy", + "peppy", "snappy", "sassy", "zesty", "swanky", "nifty", "plucky", + "quirky", "wacky", "goofy", "dizzy", "breezy", "cheery", "perky", + "frisky", "chirpy", "feisty", "jolly", "lively", "merry", "spunky", + "frisky", "zippy", "vivid", "brisk", "sunny", "witty", "kinky", +] + +NOUNS = [ + "woogie", "monkey", "noodle", "pickle", "muffin", "waffle", "pebble", + "wobble", "doodle", "tangle", "giggle", "wiggle", "jiggle", "sparkle", + "crinkle", "twinkle", "frizzle", "drizzle", "sizzle", "fizzle", + "puddle", "bubble", "muddle", "huddle", "cuddle", "juggle", "muggle", + "snuggle", "tuggle", "buggle", "nugget", "widget", "gadget", "gibbet", + "trinket", "bracket", "racket", "jacket", "ticket", "cricket", "thicket", + "biscuit", "circuit", "summit", "muppet", "trumpet", "basket", "casket", +] + + +def random_name(used: set) -> str: + for _ in range(1000): + name = f"{random.choice(ADJECTIVES)}-{random.choice(NOUNS)}" + if name not in used: + return name + # Fallback: append a number + base = f"{random.choice(ADJECTIVES)}-{random.choice(NOUNS)}" + i = 2 + while f"{base}-{i}" in used: + i += 1 + return f"{base}-{i}" + + +def fill_file(path: Path, target_size: int, repeat=False) -> None: + content_parts = [] + total = 0 + para = lorem.paragraph() + while total < target_size: + content_parts.append(para) + total += len(para) + 1 # +1 for newline + if not repeat: + para = lorem.paragraph() + path.write_text("\n\n".join(content_parts) + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description="Generate files with random names and lorem ipsum content." + ) + parser.add_argument("count", type=int, help="Number of files to create") + parser.add_argument("-d", "--directory", default=".", + help="Target directory (default: .)") + parser.add_argument("-r", "--repeat", action="store_true", + help="Fill files with reps of a single paragraph") + parser.add_argument("--min-size", type=int, default=16384, + help="Minimum file size in bytes (default: 16384)") + parser.add_argument("--max-size", type=int, default=128000, + help="Maximum file size in bytes (default: 128000)") + args = parser.parse_args() + + if args.min_size >= args.max_size: + print(f"error: min-size ({args.min_size}) must be less than max-size " + f" ({args.max_size})", file=sys.stderr) + sys.exit(1) + + directory = Path(args.directory) + directory.mkdir(parents=True, exist_ok=True) + + used_names = set() + for i in range(args.count): + name = random_name(used_names) + used_names.add(name) + target_size = random.randint(args.min_size, args.max_size) + path = directory / name + fill_file(path, target_size, args.repeat) + print(f" {path} ({path.stat().st_size:,} bytes)") + + +if __name__ == "__main__": + main() diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-all-records-streams.sh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-all-records-streams.sh new file mode 100755 index 000000000000..6264f6a08063 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-all-records-streams.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +# +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the Common +# Development and Distribution License ("CDDL"), version 1.0. You may only use +# this file in accordance with the terms of version 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this source. A +# copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +if [ $# -ne 1 ]; then + echo "Usage: $0 <device>" >&2 + exit 1 +fi + +DEVICE="$1" +SCRIPTDIR="$(cd "$(dirname "$0")" && pwd)" + +zpool create -o ashift=12 test "$DEVICE" +zfs set compression=on xattr=sa test +zfs create test/source + +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/source --min-size 2048 \ + --max-size 32000 3 +"$SCRIPTDIR/add-xattrs.py" /test/source/* +echo "very small" > /test/source/small +echo "password" > /test/source/to-be-redacted +chmod 400 /test/source/to-be-redacted + +zfs snapshot -r test/source@baseline +zfs clone test/source@baseline test/redacted +rm /test/redacted/to-be-redacted +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/redacted --min-size 4096 \ + --max-size 32000 3 +"$SCRIPTDIR/add-xattrs.py" /test/redacted/* +cd /test/redacted || exit 1 +tar cf /tmp/dups.tar . +mkdir copies +cd copies || exit 1 +tar xvf /tmp/dups.tar + +echo "password" > /test/redacted/new-key +zfs create -o encryption=on -o keylocation=file:///test/redacted/new-key \ + -o keyformat=passphrase test/redacted/encrypted +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/redacted/encrypted 3 +echo "very small" > /test/redacted/encrypted/small-encrypted +# "$SCRIPTDIR/add-xattrs.py" /test/redacted/encrypted/* + +zfs snapshot -r test/redacted@clean + +zfs redact test/source@baseline redaction-bookmark test/redacted@clean +zfs send -ce --redact redaction-bookmark test/source@baseline \ + > /tmp/all-record-types-base.zsend +zfs send -Rcew -i test/source@baseline test/redacted@clean \ + > /tmp/all-record-types-incr.zsend diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-decompression-streams.sh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-decompression-streams.sh new file mode 100755 index 000000000000..2b7e3407d389 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-decompression-streams.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +# +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the Common +# Development and Distribution License ("CDDL"), version 1.0. You may only use +# this file in accordance with the terms of version 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this source. A +# copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +if [ $# -ne 1 ]; then + echo "Usage: $0 <device>" >&2 + exit 1 +fi + +set -e + +DEVICE="$1" +SCRIPTDIR="$(cd "$(dirname "$0")" && pwd)" + +zpool create -o ashift=12 test "$DEVICE" +echo "password" > /test/password + +zfs create -o compression=zstd-5 test/unencrypted +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/unencrypted --min-size 12000 \ + --max-size 40000 2 +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/unencrypted --min-size 140000 \ + --max-size 160000 1 + +zfs create -o compression=lz4 -o encryption=on \ + -o keylocation=file:///test/password -o keyformat=passphrase test/encrypted +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/encrypted --min-size 12000 \ + --max-size 40000 3 + +zfs snapshot -r test@decompression +zfs send -cw test/unencrypted@decompression > /tmp/decompression.zsend +zfs send -cw test/encrypted@decompression > /tmp/decompression-crypt.zsend diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-dump-files.py b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-dump-files.py new file mode 100755 index 000000000000..79bb22f90dcf --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-dump-files.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Run old and new zstream dump -v on streams, producing dump outputs.""" + +# +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the Common +# Development and Distribution License ("CDDL"), version 1.0. You may only use +# this file in accordance with the terms of version 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this source. A +# copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +import argparse +import subprocess +import sys +from pathlib import Path + + +def abbreviate(filename: str) -> str: + """Split filename at dashes, take first letter of each segment, lc.""" + stem = Path(filename).stem + # Strip common compression suffixes to get the logical stem + for ext in (".zfs", ".gz", ".bz2", ".xz", ".zst", ".lz4"): + if stem.endswith(ext): + stem = stem[: -len(ext)] + return "".join(seg[0] for seg in stem.split("-") if seg).lower() + + +def run_dump(zstream: Path, stream: Path, output: Path) -> bool: + """Run `zstream dump -v < stream > output`. Returns True on success.""" + try: + with open(stream, "rb") as inf, open(output, "w") as outf: + proc = subprocess.run( + [str(zstream), "dump", "-v"], + stdin=inf, + stdout=outf, + stderr=outf, + ) + if proc.returncode != 0: + print( + f" WARNING: {zstream} exited {proc.returncode} for " + f"{stream.name}", file=sys.stderr + ) + return True + except Exception as e: + print(f" ERROR: {e}", file=sys.stderr) + return False + + +def main(): + parser = argparse.ArgumentParser( + description="Run old and new zstream dump -v on stream files." + ) + parser.add_argument("old_zstream", type=Path, help="Path to old zstream") + parser.add_argument("new_zstream", type=Path, help="Path to new zstream") + parser.add_argument( + "streams", nargs="+", type=Path, help="Streams to process" + ) + args = parser.parse_args() + + for zs in (args.old_zstream, args.new_zstream): + if not zs.is_file(): + parser.error(f"zstream binary not found: {zs}") + + for stream in args.streams: + if not stream.is_file(): + print(f"Skipping missing file: {stream}", file=sys.stderr) + continue + + abbrev = abbreviate(stream.name) + out_dir = stream.parent + + old_out = out_dir / f"{abbrev}-old.dump" + new_out = out_dir / f"{abbrev}-new.dump" + + print(f"{stream.name} -> {abbrev}") + + print(f" old: {old_out}") + run_dump(args.old_zstream, stream, old_out) + + print(f" new: {new_out}") + run_dump(args.new_zstream, stream, new_out) + + +if __name__ == "__main__": + main() diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-long-payloads.sh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-long-payloads.sh new file mode 100755 index 000000000000..3b29a8226fdc --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-long-payloads.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +# +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the Common +# Development and Distribution License ("CDDL"), version 1.0. You may only use +# this file in accordance with the terms of version 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this source. A +# copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +if [ $# -ne 1 ]; then + echo "Usage: $0 <device>" >&2 + exit 1 +fi + +set -e + +DEVICE="$1" +SCRIPTDIR="$(cd "$(dirname "$0")" && pwd)" + +zpool create -o ashift=12 test "$DEVICE" + +zfs set compression=off recordsize=16MiB test + +# We are testing 8MB blocks, so write one short file, 8.5MB +# file, and one 24.5MB file. + +"$SCRIPTDIR/gen-lorem-files.py" -d /test -r --min-size 20000 \ + --max-size 24000 1 +"$SCRIPTDIR/gen-lorem-files.py" -d /test -r --min-size 8500000 \ + --max-size 8510000 1 +"$SCRIPTDIR/gen-lorem-files.py" -d /test -r --min-size 24500000 \ + --max-size 24510000 1 + +zfs snapshot test@long-payloads +zfs send -L test@long-payloads > /tmp/long-payloads.zsend diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-venv.sh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-venv.sh new file mode 100755 index 000000000000..eddfb858d697 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/test-stream-creation-scripts/make-venv.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the Common +# Development and Distribution License ("CDDL"), version 1.0. You may only use +# this file in accordance with the terms of version 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this source. A +# copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +set -e + +python3 -m venv /tmp/zstream-venv +. /tmp/zstream-venv/bin/activate +pip install lorem_text diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream.cfg index 0978193eddc4..8efbc18cb7c5 100755..100644 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zfs_send_delegation/setup.ksh +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream.cfg @@ -1,4 +1,3 @@ -#!/bin/ksh -p # SPDX-License-Identifier: CDDL-1.0 # # CDDL HEADER START @@ -22,29 +21,13 @@ # # -# Copyright (c) 2025, Klara Inc. +# Copyright (c) 2026 by ConnectWise. All rights reserved. # -. $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/delegate/delegate_common.kshlib +export BACKDIR=${TEST_BASE_DIR%%/}/backdir-zstream +export PATH=${PATH}:/usr/local/sbin:/usr/local/bin:/usr/bin -# Create staff group and add two user to it -log_must add_group $STAFF_GROUP -if ! id $STAFF1 > /dev/null 2>&1; then - log_must add_user $STAFF_GROUP $STAFF1 -fi -if ! id $STAFF2 > /dev/null 2>&1; then - log_must add_user $STAFF_GROUP $STAFF2 -fi +read -r DISK1 _ <<<"$DISKS" +export DISK1 -# Create other group and add two user to it -log_must add_group $OTHER_GROUP -if ! id $OTHER1 > /dev/null 2>&1; then - log_must add_user $OTHER_GROUP $OTHER1 -fi -if ! id $OTHER2 > /dev/null 2>&1; then - log_must add_user $OTHER_GROUP $OTHER2 -fi -DISK=${DISKS%% *} - -default_raidz_setup $DISKS +export POOL=$TESTPOOL diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream.kshlib new file mode 100644 index 000000000000..e4c6bcd5bc43 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream.kshlib @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by ConnectWise. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/include/math.shlib +. $STF_SUITE/tests/functional/zstream/zstream.cfg + +# +# Several zstream tests need functions defined in rsend.kshlib +# (stream_has_features, get_resume_token, cleanup_pool). Rather than +# duplicate them, source the rsend library. Note that rsend.kshlib +# also sources rsend.cfg, which will define its own $BACKDIR, $POOL, +# $POOL2, etc. Our zstream.cfg is sourced first, and we reassert our +# values afterward so that our tests use the zstream-specific paths +# and pool names. +# +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# Reassert our config over rsend's +. $STF_SUITE/tests/functional/zstream/zstream.cfg + +ZSTREAM_DATADIR=$STF_SUITE/tests/functional/zstream + +# +# Return "little" or "big" depending on the system's byte order. +# +function get_system_endian +{ + python3 -c "import sys; print(sys.byteorder)" +} + +# +# Map a stream filename stem to its dump-file abbreviation. +# e.g. "big-endian-all-drr-types-base-NATIVE" -> "beadtbn" +# +function get_stream_abbrev +{ + typeset stem=$1 + typeset -l abbrev="" + typeset IFS="-" + + for part in $stem; do + abbrev="${abbrev}${part%"${part#?}"}" + done + + echo "$abbrev" +} + +# +# Receive a stream into $POOL/recv, compute sorted xxhsum -H2 of all +# regular files under the mountpoint, and write the result to $1. +# Destroys $POOL/recv afterward if $3 is set to "cleanup". +# +function recv_and_hash +{ + typeset hashfile=$1 + typeset stream=$2 + typeset cleanup=${3:-""} + + log_must zfs receive -F $POOL/recv < "$stream" + typeset mnt=$(get_prop mountpoint $POOL/recv) + ( cd "$mnt" && find . -type f -print0 | sort -z | \ + xargs -0 xxh128sum ) > "$hashfile" + if [[ -n $cleanup ]]; then + log_must zfs destroy -r $POOL/recv + fi +} diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_checksum_001_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_checksum_001_pos.ksh new file mode 100755 index 000000000000..344fd768278e --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_checksum_001_pos.ksh @@ -0,0 +1,64 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that very long payload records are checksummed correctly by +# running them through zstream redup (which recalculates checksums but is +# otherwise a no-op). This exercises the 8MiB Fletcher4 chunk boundary +# handling. There are test streams of both endiannesses because opposite- +# endian checksumming is a slightly separate path. +# +# Strategy: +# 1. Decompress the long-payloads test streams +# 2. Pipe through zstream redup +# 3. Verify the output is byte-identical to the input +# + +verify_runnable "both" + +log_assert "Verify long payload records are checksummed correctly." + +typeset -a streams=( + little-endian-long-payloads + big-endian-long-payloads +) + +typeset failed="" + +for stem in "${streams[@]}"; do + + typeset src="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + typeset orig="$BACKDIR/${stem}.zsend.orig" + typeset redup_out="$BACKDIR/${stem}.zsend.redup" + + bzcat "$src" > "$orig" + log_must eval "zstream redup '$orig' > '$redup_out'" + + if ! cmp -s "$orig" "$redup_out" > /dev/null 2>&1; then + log_note "MISMATCH: $stem" + failed="$failed $stem" + fi + +done + +[[ -z $failed ]] || log_fail "Round-trip mismatch for: $failed" + +log_pass "Long-payload records are checksummed correctly." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_001_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_001_pos.ksh new file mode 100755 index 000000000000..9029b4880619 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_001_pos.ksh @@ -0,0 +1,98 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream decompress actually decompresses selected WRITE +# records. The input stream contains zstd-compressed writes. +# +# Strategy: +# 1. Decompress selected records (2,0 3,0 128,131072) from the stream +# 2. Run zstream dump -v on both original and decompressed streams +# 3. Verify the selected records now show compression type = 0, +# compressed_size = 0, and payload_size = logical_size +# + +verify_runnable "both" + +log_assert "Verify zstream decompress decompresses selected WRITE records." + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/decompress.orig.zsend" +typeset decompressed="$BACKDIR/decompress.out.zsend" +typeset orig_dump="$BACKDIR/decompress.orig.dump" +typeset decomp_dump="$BACKDIR/decompress.out.dump" + +# Selected records: object,offset +typeset -a records=(2,0 3,0 128,131072) + +# Decompress the bz2 and run zstream decompress on selected records +bzcat "$src" > "$orig" +log_must eval "zstream decompress ${records[*]} < '$orig' > '$decompressed'" + +# Dump both streams +log_must eval "zstream dump -v < '$orig' > '$orig_dump' 2>&1" +log_must eval "zstream dump -v < '$decompressed' > '$decomp_dump' 2>&1" + +# For each selected record, verify it was decompressed in the output +typeset failed="" +for rec in "${records[@]}"; do + typeset obj=${rec%,*} + typeset off=${rec#*,} + + # Find the WRITE line for this object/offset in the original dump + # and extract the logical_size + typeset orig_line=$(awk \ + "/^WRITE object = $obj .* offset = $off /" \ + "$orig_dump") + typeset lsize=$(echo "$orig_line" | \ + sed 's/.*logical_size = \([0-9]*\).*/\1/') + + # Find the same record in the decompressed dump + typeset decomp_line=$(awk \ + "/^WRITE object = $obj .* offset = $off /" \ + "$decomp_dump") + + # Verify compression type = 0 + if ! echo "$decomp_line" | grep -q 'compression type = 0'; then + log_note "Record $rec: compression type not 0" + log_note " got: $decomp_line" + failed="$failed ${rec}(comp)" + fi + + # Verify compressed_size = 0 (indicates uncompressed) + if ! echo "$decomp_line" | grep -q 'compressed_size = 0'; then + log_note "Record $rec: compressed_size not 0" + failed="$failed ${rec}(csize)" + fi + + # Verify payload_size = logical_size + typeset psize=$(echo "$decomp_line" | \ + sed 's/.*payload_size = \([0-9]*\).*/\1/') + if [[ "$psize" != "$lsize" ]]; then + log_note "Record $rec: payload_size ($psize) != logical_size ($lsize)" + failed="$failed ${rec}(psize)" + fi +done + +[[ -z $failed ]] || \ + log_fail "Decompression verification failed for:$failed" + +log_pass "zstream decompress decompresses selected WRITE records." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_002_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_002_pos.ksh new file mode 100755 index 000000000000..1c7d9b7dcfc0 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_002_pos.ksh @@ -0,0 +1,57 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that a decompressed stream produces identical filesystem contents +# when received. +# +# Strategy: +# 1. Receive the original decompress.zsend into a test pool and hash files +# 2. Receive the decompressed stream (with records 2,0 3,0 +# and 128,131072 decompressed) +# 3. Verify file hashes are identical +# + +verify_runnable "both" + +log_assert "Verify decompressed stream receives with identical file contents." +log_onexit cleanup_pool $POOL + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/decompress.orig.zsend" +typeset decompressed="$BACKDIR/decompress.out.zsend" + +typeset -a records=(2,0 3,0 128,131072) + +# Prepare streams +bzcat "$src" > "$orig" +log_must eval "zstream decompress ${records[*]} < '$orig' > '$decompressed'" + +# Receive original and hash +recv_and_hash "$BACKDIR/hash-orig.txt" "$orig" cleanup + +# Receive decompressed and hash +recv_and_hash "$BACKDIR/hash-decomp.txt" "$decompressed" cleanup + +# Compare +log_must diff "$BACKDIR/hash-orig.txt" "$BACKDIR/hash-decomp.txt" + +log_pass "Decompressed stream receives with identical file contents." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_003_neg.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_003_neg.ksh new file mode 100755 index 000000000000..9a394d14f972 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_003_neg.ksh @@ -0,0 +1,59 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that specifying the wrong compression type (lz4 for a zstd +# stream) causes decompression to fail gracefully, leaving the output +# stream identical to the input. +# +# Strategy: +# 1. Run zstream decompress with lz4 type on zstd-compressed records +# 2. Verify the output stream is byte-identical to the input +# 3. Check stderr for failure messages +# + +verify_runnable "both" + +log_assert "Verify wrong compression type leaves stream unchanged." + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/decompress.orig" +typeset output="$BACKDIR/decompress-lz4.out" +typeset errfile="$BACKDIR/decompress-lz4.err" + +typeset -a records=(2,0,lz4 3,0,lz4 128,131072,lz4) + +bzcat "$src" > "$orig" + +# Attempt to decompress zstd records as lz4 — should fail for each +zstream decompress ${records[*]} < "$orig" > "$output" 2>"$errfile" + +# Output stream must be identical to input (nothing decompressed) +log_must cmp -s "$orig" "$output" + +# Stderr should contain messages about the failed decompressions +typeset errcount=$(wc -l < "$errfile") +if [[ $errcount -ne 3 ]]; then + log_fail "Did not receive 3 error messages on stderr, got $errcount" +fi +log_note "Got $errcount lines of error output (expected)" + +log_pass "Wrong compression type leaves stream unchanged." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_004_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_004_pos.ksh new file mode 100755 index 000000000000..9b16d89cc17e --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_004_pos.ksh @@ -0,0 +1,55 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that specifying the correct compression type (zstd) produces +# the same output as omitting the type. +# +# Strategy: +# 1. Decompress records without specifying type +# 2. Decompress records specifying zstd as type +# 3. Verify both outputs are identical +# + +verify_runnable "both" + +log_assert "Verify explicit correct compression type matches default." + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/decompress.orig" +typeset out_default="$BACKDIR/decompress-default.out" +typeset out_zstd="$BACKDIR/decompress-zstd.out" + +typeset -a records=(2,0 3,0 128,131072) +typeset -a zstd_records=(2,0,zstd 3,0,zstd 128,131072,zstd) + +bzcat "$src" > "$orig" + +# Decompress without specifying type +log_must eval "zstream decompress ${records[*]} < '$orig' > '$out_default'" + +# Decompress specifying zstd +log_must eval "zstream decompress ${zstd_records[*]} < '$orig' > '$out_zstd'" + +# Both outputs must be identical +log_must cmp -s "$out_default" "$out_zstd" + +log_pass "Explicit correct compression type matches default." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_005_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_005_pos.ksh new file mode 100755 index 000000000000..89e1a64d1d4a --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_005_pos.ksh @@ -0,0 +1,115 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream decompress with "off" as the compression type +# changes record headers to mark them as uncompressed but leaves the +# actual data payload untouched. +# +# Strategy: +# 1. Decompress selected records with type "off" +# 2. Verify via zstream dump that selected records now show +# compression type = 0 and logical_size equals the original +# compressed_size (i.e., the header now claims the record is +# uncompressed at the smaller, originally-compressed size) +# +# Note: we intentionally do not attempt to zfs receive the resulting +# stream. The data payloads are still compressed despite the header's +# claims otherwise, so the affected WRITE records are now inconsistent +# with the dnodes in the corresponding OBJECT records. zfs receive will +# fail with EINVAL. +# +# zstream decompress off is intended to correct a specific error case +# in which these header adjustments bring the WRITE records into alignment +# with their dnodes rather than disrupting that relationship. +# + +verify_runnable "both" + +log_assert "Verify decompress with 'off' changes headers but not data." + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/decompress.orig" +typeset off_out="$BACKDIR/decompress-off.out" +typeset orig_dump="$BACKDIR/decompress-orig.dump" +typeset off_dump="$BACKDIR/decompress-off.dump" + +typeset -a records=(2,0 3,0 128,131072) +typeset -a off_records=(2,0,off 3,0,off 128,131072,off) + +bzcat "$src" > "$orig" + +log_must eval "zstream decompress ${off_records[*]} < '$orig' > '$off_out'" + +# Dump both streams +log_must eval "zstream dump -v < '$orig' > '$orig_dump' 2>&1" +log_must eval "zstream dump -v < '$off_out' > '$off_dump' 2>&1" + +# Verify selected records show compression type = 0 with same logical_size +typeset failed="" +for rec in "${records[@]}"; do + typeset obj=${rec%,*} + typeset off=${rec#*,} + + typeset orig_line=$(awk \ + "/^WRITE object = $obj .* offset = $off /" \ + "$orig_dump") + typeset orig_lsize=$(echo "$orig_line" | \ + sed 's/.*logical_size = \([0-9]*\).*/\1/') + typeset orig_csize=$(echo "$orig_line" | \ + sed 's/.*compressed_size = \([0-9]*\).*/\1/') + typeset orig_ctype=$(echo "$orig_line" | \ + sed 's/.*compression type = \([0-9]*\).*/\1/') + + typeset off_line=$(awk \ + "/^WRITE object = $obj .* offset = $off /" \ + "$off_dump") + typeset off_lsize=$(echo "$off_line" | \ + sed 's/.*logical_size = \([0-9]*\).*/\1/') + typeset off_csize=$(echo "$off_line" | \ + sed 's/.*compressed_size = \([0-9]*\).*/\1/') + typeset off_ctype=$(echo "$off_line" | \ + sed 's/.*compression type = \([0-9]*\).*/\1/') + + if [[ "$orig_ctype" == "0" ]]; then + log_note "Record $rec: original compression type is 0" + failed="$failed ${rec}(orig ctype)" + fi + if [[ "$off_ctype" != "0" ]]; then + log_note "Record $rec: modified compression type is not 0" + failed="$failed ${rec}(modified ctype)" + fi + if [[ "$off_lsize" != "$orig_csize" ]]; then + log_note "Record $rec: modified logical_size != original " \ + "compressed_size ($orig_lsize -> $off_lsize)" + failed="$failed ${rec}(lsize)" + fi + if [[ "$off_csize" != "0" ]]; then + log_note "Record $rec: modified compressed_size != 0 " \ + "($orig_csize -> $off_csize)" + failed="$failed ${rec}(csize)" + fi +done + +[[ -z $failed ]] || \ + log_fail "Header verification failed for:$failed" + +log_pass "Decompress with 'off' changes headers correctly." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_006_neg.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_006_neg.ksh new file mode 100755 index 000000000000..32ce347e4afa --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_decompress_006_neg.ksh @@ -0,0 +1,61 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream decompress prints a warning to stderr for encrypted +# WRITES, attempts to decompress anyway, fails, and leaves stream unchanged. +# +# Strategy: +# 1. Select compressed encrypted records from decompress-crypt.zsend +# 2. Attempt to decompress them +# 3. Verify stderr contains warnings for each record +# 4. Verify output stream is byte-identical to input +# + +verify_runnable "both" + +log_assert "Verify that zstream decompress handles encrypted records correctly." + +typeset src="$ZSTREAM_DATADIR/decompress-crypt.zsend.bz2" +typeset orig="$BACKDIR/decompress-crypt.orig" +typeset output="$BACKDIR/decompress-crypt.out" +typeset errfile="$BACKDIR/decompress-crypt.err" + + + +typeset -a records=(2,0 3,0 36,0) + +bzcat "$src" > "$orig" + +# Attempt to decompress encrypted records +zstream decompress ${records[*]} < "$orig" > "$output" 2>"$errfile" + +# Output stream must be identical to input +log_must cmp -s "$orig" "$output" + +# Stderr should contain warnings about each record +typeset errcount=$(wc -l < "$errfile") +if [[ $errcount -ne 6 ]]; then + log_fail "Expected 6 messages on stderr, got $errcount" +fi +log_note "Got $errcount lines of warning output (expected)" + +log_pass "Encrypted records refuse to decompress." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-zstream_drop_record.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_drop_record_001_pos.ksh index a2e810fd40dc..25ffcb3015c7 100755 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-zstream_drop_record.ksh +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_drop_record_001_pos.ksh @@ -16,8 +16,7 @@ # Copyright (c) 2026 by ConnectWise. All rights reserved. # -. $STF_SUITE/tests/functional/rsend/rsend.kshlib -. $STF_SUITE/include/math.shlib +. $STF_SUITE/tests/functional/zstream/zstream.kshlib # # Description: @@ -32,10 +31,10 @@ verify_runnable "both" log_assert "Verify zstream drop_record correctly drops records." -log_onexit cleanup_pool $POOL2 +log_onexit cleanup_pool $POOL -typeset sendfs=$POOL2/fs -typeset recvfs=$POOL2/fs2 +typeset sendfs=$POOL/fs +typeset recvfs=$POOL/fs2 typeset stream=$BACKDIR/stream typeset filtered=$BACKDIR/filtered typeset dump=$BACKDIR/dump diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_001_pos.ksh index 5ff2cbf0c6a9..c604f36c52fa 100755 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_001_pos.ksh @@ -17,12 +17,11 @@ # Copyright (c) 2020 by Datto, Inc. All rights reserved. # -. $STF_SUITE/tests/functional/rsend/rsend.kshlib -. $STF_SUITE/include/math.shlib +. $STF_SUITE/tests/functional/zstream/zstream.kshlib # # Description: -# Verify compression features show up in zstream dump +# Verify that compression features show up in zstream dump # # Strategy: # 1. Create a full compressed send stream @@ -36,11 +35,11 @@ verify_runnable "both" log_assert "Verify zstream dump correctly interprets compressed send streams." -log_onexit cleanup_pool $POOL2 +log_onexit cleanup_pool $POOL -typeset sendfs=$POOL2/fs -typeset streamfs=$POOL2/fs2 -typeset recvfs=$POOL2/fs3 +typeset sendfs=$POOL/fs +typeset streamfs=$POOL/fs2 +typeset recvfs=$POOL/fs3 log_must zfs create -o compress=lz4 $sendfs log_must zfs create -o compress=lz4 $streamfs diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_002_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_002_pos.ksh new file mode 100755 index 000000000000..558a79a29659 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_002_pos.ksh @@ -0,0 +1,84 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream dump -v output is as expected for pregenerated +# same-endian, neutral (no BEGIN nvlists), and XDR-encoded test streams. +# NV_ENCODE_NATIVE-encoded BEGIN records aren't readable on opposite-endian +# systems, so for these the output of zstream dump varies according to +# host endianness. +# +# Strategy: +# 1. For each of the test streams, run zstream dump -v +# 2. Compare stdout+stderr with the corresponding reference dump +# + +verify_runnable "both" + +log_assert "Verify zstream dump -v output matches reference dump files." + +typeset sys_endian=$(get_system_endian) + +typeset -a streams=( + decompress + decompress-crypt + little-endian-long-payloads + big-endian-long-payloads + big-endian-all-drr-types-base-XDR + big-endian-all-drr-types-incr-XDR + little-endian-all-drr-types-base-XDR + little-endian-all-drr-types-incr-XDR +) + +if [[ $sys_endian == "little" ]]; then + streams+=( + little-endian-all-drr-types-base-NATIVE + little-endian-all-drr-types-incr-NATIVE + ) +else + streams+=( + big-endian-all-drr-types-base-NATIVE + big-endian-all-drr-types-incr-NATIVE + ) +fi + +typeset failed="" + +for stem in "${streams[@]}"; do + typeset abbrev=$(get_stream_abbrev "$stem") + typeset ref_src="$ZSTREAM_DATADIR/${abbrev}-new.dump.bz2" + typeset send_src="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + typeset ref="$BACKDIR/${abbrev}-new.dump" + typeset out="$BACKDIR/${abbrev}-out.dump" + + bzcat "$send_src" | zstream dump -v > "$out" 2>&1 + bzcat "$ref_src" > "$ref" + + if ! diff -q "$ref" "$out" > /dev/null 2>&1; then + log_note "MISMATCH: $stem (abbrev $abbrev)" + log_note "$(diff "$ref" "$out")" + failed="$failed $stem" + fi +done + +[[ -z $failed ]] || log_fail "Dump output mismatch for:$failed" + +log_pass "zstream dump -v output matches reference dump files." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_003_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_003_pos.ksh new file mode 100755 index 000000000000..7bc001f5c96d --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_003_pos.ksh @@ -0,0 +1,92 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream dump -v output for all same-endian and XDR-encoded +# test streams matches that of the previous version of zstream, with the +# following exceptions: +# +# 1. Add a line that describes the nvlist packing format for BEGIN records +# 2. Include DRR_OBJECT_RANGE and DRR_REDACT records in end summary +# +# The previous version of zstream does not dump opposite-endian streams +# correctly, so these don't have a comparison basis. +# + +verify_runnable "both" + +log_assert "Verify old-vs-new dump diff contains only expected additions." + +typeset sys_endian=$(get_system_endian) + +typeset -a streams=( + little-endian-long-payloads + big-endian-long-payloads + little-endian-all-drr-types-base-XDR + little-endian-all-drr-types-incr-XDR + big-endian-all-drr-types-base-XDR + big-endian-all-drr-types-incr-XDR +) + +if [[ $sys_endian == "little" ]]; then + streams+=( + little-endian-all-drr-types-base-NATIVE + little-endian-all-drr-types-incr-NATIVE + ) +else + streams+=( + big-endian-all-drr-types-base-NATIVE + big-endian-all-drr-types-incr-NATIVE + ) +fi + +typeset failed="" + +for stem in "${streams[@]}"; do + + typeset abbrev=$(get_stream_abbrev "$stem") + typeset send_src="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + typeset old_src="$ZSTREAM_DATADIR/${abbrev}-old.dump.bz2" + typeset new_dump="$BACKDIR/${abbrev}-new.dump" + typeset old_dump="$BACKDIR/${abbrev}-old.dump" + typeset filtered="$BACKDIR/${abbrev}-filtered.dump" + + bzcat "$send_src" | zstream dump -v > "$new_dump" 2>&1 + bzcat "$old_src" > "$old_dump" + + # Remove the lines that are new additions: + # 1. "nvlist encoding = ..." lines + # 2. Summary lines for DRR_OBJECT_RANGE and DRR_REDACT + grep -v '^nvlist encoding = ' "$new_dump" | \ + grep -v 'Total DRR_OBJECT_RANGE records' | \ + grep -v 'Total DRR_REDACT records' > "$filtered" + + if ! diff -q "$old_dump" "$filtered" > /dev/null 2>&1; then + log_note "MISMATCH after filtering: $stem (abbrev $abbrev)" + log_note "$(diff "$old_dump" "$filtered")" + failed="$failed $stem" + fi +done + +[[ -z $failed ]] || \ + log_fail "Filtered new dump did not match old dump for:$failed" + +log_pass "Old-vs-new dump diff contains only expected additions." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_004_neg.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_004_neg.ksh new file mode 100755 index 000000000000..f98cefce5b65 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_dump_004_neg.ksh @@ -0,0 +1,88 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream dump on non-native-endian NATIVE-encoded streams +# eventually exits with code ENOTSUP (95 on Linux, 45 on FreeBSD) but +# still dumps the complete stream (minus nondecodable nvlists) and +# prints the rollup summary. +# +# Strategy: +# 1. Determine system endianness to identify non-native NATIVE streams +# 2. Run zstream dump -v on each non-native NATIVE stream +# 3. Verify exit code is 95 (ENOTSUP) +# 4. Verify that SUMMARY section is present and complete +# + +verify_runnable "both" + +log_assert "Non-native NATIVE-encoded streams exit with code 45 or 95." + +typeset sys_endian=$(get_system_endian) + +if [[ $sys_endian == "little" ]]; then + typeset -a streams=( + big-endian-all-drr-types-base-NATIVE + big-endian-all-drr-types-incr-NATIVE + ) +else + typeset -a streams=( + little-endian-all-drr-types-base-NATIVE + little-endian-all-drr-types-incr-NATIVE + ) +fi + +typeset failed="" + +for stem in "${streams[@]}"; do + typeset out="$BACKDIR/${stem}-out.dump" + typeset stream="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + + bzcat "$stream" | zstream dump -v > "$out" 2>&1 + typeset rc=$? + + if [[ $rc -ne 45 && $rc -ne 95 ]]; then + log_note "$stem: expected exit code 45 or 95, got $rc" + failed="$failed ${stem}(rc=$rc)" + fi + + # Verify the SUMMARY section is present + if ! grep -q '^SUMMARY:' "$out"; then + log_note "$stem: missing SUMMARY section" + failed="$failed ${stem}(no-summary)" + fi + + # Verify key summary lines are present + if ! grep -q 'Total DRR_BEGIN records' "$out"; then + log_note "$stem: missing DRR_BEGIN in summary" + failed="$failed ${stem}(incomplete-summary)" + fi + + if ! grep -q 'Total stream length' "$out"; then + log_note "$stem: missing total stream length in summary" + failed="$failed ${stem}(no-stream-length)" + fi +done + +[[ -z $failed ]] || \ + log_fail "Non-native NATIVE stream check failed:$failed" + +log_pass "Non-native NATIVE-encoded streams exit with code 45 or 95." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_zstream_recompress.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_001_pos.ksh index 81c7e24a2204..ea332a32b833 100755 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send-c_zstream_recompress.ksh +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_001_pos.ksh @@ -16,8 +16,7 @@ # Copyright (c) 2022 by Delphix. All rights reserved. # -. $STF_SUITE/tests/functional/rsend/rsend.kshlib -. $STF_SUITE/include/math.shlib +. $STF_SUITE/tests/functional/zstream/zstream.kshlib # # Description: @@ -36,10 +35,10 @@ verify_runnable "both" log_assert "Verify zstream recompress correctly modifies send streams." -log_onexit cleanup_pool $POOL2 +log_onexit cleanup_pool $POOL -typeset sendfs=$POOL2/fs -typeset recvfs=$POOL2/fs2 +typeset sendfs=$POOL/fs +typeset recvfs=$POOL/fs2 log_must zfs create -o compress=lz4 $sendfs typeset dir=$(get_prop mountpoint $sendfs) diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_002_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_002_pos.ksh new file mode 100755 index 000000000000..0bfbc0fe56a4 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_002_pos.ksh @@ -0,0 +1,51 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2024 by the OpenZFS project. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that recompressing a send stream and then decompressing it +# with zstream produces a stream identical to the original. +# +# Strategy: +# 1. Create a filesystem with compressible data +# 2. Generate a replication send stream +# 3. Pipe the stream through zstream recompress lz4 | zstream recompress off +# 4. Verify the result is byte-identical to the original stream +# + +verify_runnable "both" + +log_assert "Verify zstream recompress round-trip produces identical stream." +log_onexit cleanup_pool $POOL + +typeset sendfs=$POOL/fs + +log_must zfs create $sendfs +typeset dir=$(get_prop mountpoint $sendfs) +write_compressible $dir 16m +log_must zfs snapshot $sendfs@snap + +log_must eval "zfs send -R $sendfs@snap >$BACKDIR/original" +log_must eval "zstream recompress lz4 <$BACKDIR/original | \ + zstream recompress off >$BACKDIR/roundtrip" + +log_must cmp $BACKDIR/original $BACKDIR/roundtrip + +log_pass "zstream recompress round-trip produces identical stream." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_003_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_003_pos.ksh new file mode 100755 index 000000000000..2d04f38e8504 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_003_pos.ksh @@ -0,0 +1,64 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream recompress with zstd at level 10 produces a smaller +# stream that receives with identical file contents. +# +# Strategy: +# 1. Receive the original stream and compute file hashes as baseline +# 2. Recompress the stream with zstd-10 +# 3. Verify the recompressed stream is smaller than the original +# 4. Receive the recompressed stream and verify file hashes match +# + +verify_runnable "both" + +log_assert "Verify zstream recompress with zstd-10 produces smaller stream." +log_onexit cleanup_pool $POOL + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/recompress.orig" +typeset recompressed="$BACKDIR/recompress-zstd10.out" +typeset orig_hash="$BACKDIR/hash-baseline.txt" +typeset rc_hash="$BACKDIR/hash-rc.txt" + +bzcat "$src" > "$orig" + +# Baseline: receive original and hash +recv_and_hash "$orig_hash" "$orig" cleanup + +# Recompress with zstd at level 10 +log_must eval "zstream recompress -l 10 zstd \ + < '$orig' > '$recompressed'" + +# Verify size is smaller +typeset orig_size=$(wc -c < "$orig") +typeset recomp_size=$(wc -c < "$recompressed") +log_note "Original size: $orig_size, recompressed size: $recomp_size" +[[ $recomp_size -lt $orig_size ]] || \ + log_fail "Recompressed stream ($recomp_size) not smaller than original ($orig_size)" + +# Receive recompressed and verify +recv_and_hash "$rc_hash" "$recompressed" cleanup +log_must diff "$orig_hash" "$rc_hash" + +log_pass "zstream recompress with zstd-10 produces smaller stream." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_004_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_004_pos.ksh new file mode 100755 index 000000000000..c72eff14086f --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_004_pos.ksh @@ -0,0 +1,61 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream recompress with "off" produces a larger (uncompressed) +# stream that still receives with identical file contents. +# +# Strategy: +# 1. Receive the original stream and compute file hashes as baseline +# 2. Recompress with "off" (decompress) +# 3. Verify the output stream is larger than the original +# 4. Receive and verify file hashes match +# + +verify_runnable "both" + +log_assert "Verify zstream recompress with 'off' produces larger stream." +log_onexit cleanup_pool $POOL + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/recompress.orig" +typeset uncompressed="$BACKDIR/recompress-off.out" + +bzcat "$src" > "$orig" + +# Baseline +recv_and_hash "$BACKDIR/hash-baseline.txt" "$orig" cleanup + +# Recompress with off +log_must eval "zstream recompress off < '$orig' > '$uncompressed'" + +# Verify size is larger +typeset orig_size=$(wc -c < "$orig") +typeset uncomp_size=$(wc -c < "$uncompressed") +log_note "Original size: $orig_size, uncompressed size: $uncomp_size" +[[ $uncomp_size -gt $orig_size ]] || \ + log_fail "Uncompressed stream ($uncomp_size) not larger than original ($orig_size)" + +# Receive and verify +recv_and_hash "$BACKDIR/hash-off.txt" "$uncompressed" cleanup +log_must diff "$BACKDIR/hash-baseline.txt" "$BACKDIR/hash-off.txt" + +log_pass "zstream recompress with 'off' produces larger stream." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_005_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_005_pos.ksh new file mode 100755 index 000000000000..374ab2841a39 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_recompress_005_pos.ksh @@ -0,0 +1,62 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream recompress lz4 of a zstd-5-compressed input stream +# yields a stream with a nonidentical size that zfs receives with identical +# file contents. +# +# Strategy: +# 1. Receive the original stream and compute file hashes as baseline +# 2. Recompress with lz4 +# 3. Verify the output stream size differs from the original +# 4. Receive and verify file hashes match +# + +verify_runnable "both" + +log_assert "Verify zstream recompress with lz4 preserves data." +log_onexit cleanup_pool $POOL + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/recompress.orig" +typeset lz4_out="$BACKDIR/recompress-lz4.out" + +bzcat "$src" > "$orig" + +# Baseline +recv_and_hash "$BACKDIR/hash-baseline.txt" "$orig" cleanup + +# Recompress with lz4 +log_must eval "zstream recompress lz4 < '$orig' > '$lz4_out'" + +# Verify size is different +typeset orig_size=$(wc -c < "$orig") +typeset lz4_size=$(wc -c < "$lz4_out") +log_note "Original size: $orig_size, lz4 size: $lz4_size" +[[ $lz4_size -ne $orig_size ]] || \ + log_fail "LZ4 stream size ($lz4_size) same as original ($orig_size)" + +# Receive and verify +recv_and_hash "$BACKDIR/hash-lz4.txt" "$lz4_out" cleanup +log_must diff "$BACKDIR/hash-baseline.txt" "$BACKDIR/hash-lz4.txt" + +log_pass "zstream recompress with lz4 preserves data." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_redup_001_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_redup_001_pos.ksh new file mode 100755 index 000000000000..422a9fd0e5fd --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zstream/zstream_redup_001_pos.ksh @@ -0,0 +1,79 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# +# Verify that zstream redup produces output identical to input for same-endian +# test streams. These input files contain no dedup records. However, the round +# trip does involve the full pipeline as well as validating and regenerating all +# checksums, so this is a useful check. +# +# Strategy: +# 1. For each of the same-endian test streams, decompress with bzcat +# 2. Pipe through zstream redup +# 3. Compare with cmp against the original decompressed stream +# + +verify_runnable "both" + +log_assert "Verify zstream redup is an identity transform on non-dedup streams." + +typeset sys_endian=$(get_system_endian) + +if [[ $sys_endian == "little" ]]; then + typeset -a streams=( + decompress + decompress-crypt + little-endian-all-drr-types-base-NATIVE + little-endian-all-drr-types-base-XDR + little-endian-all-drr-types-incr-NATIVE + little-endian-all-drr-types-incr-XDR + ) +else + typeset -a streams=( + big-endian-all-drr-types-base-NATIVE + big-endian-all-drr-types-base-XDR + big-endian-all-drr-types-incr-NATIVE + big-endian-all-drr-types-incr-XDR + ) +fi + +typeset failed="" + +for stem in "${streams[@]}"; do + typeset src="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + typeset orig="$BACKDIR/${stem}.orig" + typeset redup_out="$BACKDIR/${stem}.redup" + + bzcat "$src" > "$orig" + zstream redup "$orig" > "$redup_out" + + if ! cmp -s "$orig" "$redup_out"; then + log_note "MISMATCH: zstream redup output differs for $stem" + failed="$failed $stem" + fi + + rm -f "$orig" "$redup_out" +done + +[[ -z $failed ]] || log_fail "Redup identity check failed for:$failed" + +log_pass "zstream redup is an identity transform on non-dedup streams." |
