diff options
Diffstat (limited to 'include/sys/vdev_raidz_impl.h')
| -rw-r--r-- | include/sys/vdev_raidz_impl.h | 413 |
1 files changed, 0 insertions, 413 deletions
diff --git a/include/sys/vdev_raidz_impl.h b/include/sys/vdev_raidz_impl.h deleted file mode 100644 index 8c8dcfb077f6..000000000000 --- a/include/sys/vdev_raidz_impl.h +++ /dev/null @@ -1,413 +0,0 @@ -// SPDX-License-Identifier: CDDL-1.0 -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or https://opensource.org/licenses/CDDL-1.0. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright (C) 2016 Gvozden Nešković. All rights reserved. - */ - -#ifndef _VDEV_RAIDZ_H -#define _VDEV_RAIDZ_H - -#include <sys/types.h> -#include <sys/debug.h> -#include <sys/kstat.h> -#include <sys/abd.h> -#include <sys/vdev_impl.h> -#include <sys/abd_impl.h> -#include <sys/zfs_rlock.h> - -#ifdef __cplusplus -extern "C" { -#endif - -#define CODE_P (0U) -#define CODE_Q (1U) -#define CODE_R (2U) - -#define PARITY_P (1U) -#define PARITY_PQ (2U) -#define PARITY_PQR (3U) - -#define TARGET_X (0U) -#define TARGET_Y (1U) -#define TARGET_Z (2U) - -/* - * Parity generation methods indexes - */ -enum raidz_math_gen_op { - RAIDZ_GEN_P = 0, - RAIDZ_GEN_PQ, - RAIDZ_GEN_PQR, - RAIDZ_GEN_NUM = 3 -}; -/* - * Data reconstruction methods indexes - */ -enum raidz_rec_op { - RAIDZ_REC_P = 0, - RAIDZ_REC_Q, - RAIDZ_REC_R, - RAIDZ_REC_PQ, - RAIDZ_REC_PR, - RAIDZ_REC_QR, - RAIDZ_REC_PQR, - RAIDZ_REC_NUM = 7 -}; - -extern const char *const raidz_gen_name[RAIDZ_GEN_NUM]; -extern const char *const raidz_rec_name[RAIDZ_REC_NUM]; - -/* - * Methods used to define raidz implementation - * - * @raidz_gen_f Parity generation function - * @par1 pointer to raidz_map - * @raidz_rec_f Data reconstruction function - * @par1 pointer to raidz_map - * @par2 array of reconstruction targets - * @will_work_f Function returns TRUE if impl. is supported on the system - * @init_impl_f Function is called once on init - * @fini_impl_f Function is called once on fini - */ -typedef void (*raidz_gen_f)(void *); -typedef int (*raidz_rec_f)(void *, const int *); -typedef boolean_t (*will_work_f)(void); -typedef void (*init_impl_f)(void); -typedef void (*fini_impl_f)(void); - -#define RAIDZ_IMPL_NAME_MAX (20) - -typedef struct raidz_impl_ops { - init_impl_f init; - fini_impl_f fini; - raidz_gen_f gen[RAIDZ_GEN_NUM]; /* Parity generate functions */ - raidz_rec_f rec[RAIDZ_REC_NUM]; /* Data reconstruction functions */ - will_work_f is_supported; /* Support check function */ - char name[RAIDZ_IMPL_NAME_MAX]; /* Name of the implementation */ -} raidz_impl_ops_t; - - -typedef struct raidz_col { - int rc_devidx; /* child device index for I/O */ - uint32_t rc_size; /* I/O size */ - uint64_t rc_offset; /* device offset */ - abd_t rc_abdstruct; /* rc_abd probably points here */ - abd_t *rc_abd; /* I/O data */ - abd_t *rc_orig_data; /* pre-reconstruction */ - int rc_error; /* I/O error for this device */ - uint8_t rc_tried:1; /* Did we attempt this I/O column? */ - uint8_t rc_skipped:1; /* Did we skip this I/O column? */ - uint8_t rc_need_orig_restore:1; /* need to restore from orig_data? */ - uint8_t rc_force_repair:1; /* Write good data to this column */ - uint8_t rc_allow_repair:1; /* Allow repair I/O to this column */ - uint8_t rc_latency_outlier:1; /* Latency outlier for this device */ - int rc_shadow_devidx; /* for double write during expansion */ - int rc_shadow_error; /* for double write during expansion */ - uint64_t rc_shadow_offset; /* for double write during expansion */ -} raidz_col_t; - -typedef struct raidz_row { - int rr_cols; /* Regular column count */ - int rr_scols; /* Count including skipped columns */ - int rr_bigcols; /* Remainder data column count */ - int rr_missingdata; /* Count of missing data devices */ - int rr_missingparity; /* Count of missing parity devices */ - int rr_firstdatacol; /* First data column/parity count */ - abd_t *rr_abd_empty; /* dRAID empty sector buffer */ - int rr_nempty; /* empty sectors included in parity */ - int rr_outlier_cnt; /* Count of latency outlier devices */ -#ifdef ZFS_DEBUG - uint64_t rr_offset; /* Logical offset for *_io_verify() */ - uint64_t rr_size; /* Physical size for *_io_verify() */ -#endif - raidz_col_t rr_col[]; /* Flexible array of I/O columns */ -} raidz_row_t; - -typedef struct raidz_map { - boolean_t rm_ecksuminjected; /* checksum error was injected */ - int rm_nrows; /* Regular row count */ - int rm_nskip; /* RAIDZ sectors skipped for padding */ - int rm_skipstart; /* Column index of padding start */ - int rm_original_width; /* pre-expansion width of raidz vdev */ - int rm_nphys_cols; /* num entries in rm_phys_col[] */ - zfs_locked_range_t *rm_lr; - const raidz_impl_ops_t *rm_ops; /* RAIDZ math operations */ - raidz_col_t *rm_phys_col; /* if non-NULL, read i/o aggregation */ - raidz_row_t *rm_row[]; /* flexible array of rows */ -} raidz_map_t; - -/* - * Nodes in vdev_raidz_t:vd_expand_txgs. - * Blocks with physical birth time of re_txg or later have the specified - * logical width (until the next node). - */ -typedef struct reflow_node { - uint64_t re_txg; - uint64_t re_logical_width; - avl_node_t re_link; -} reflow_node_t; - - -#define RAIDZ_ORIGINAL_IMPL (INT_MAX) - -extern const raidz_impl_ops_t vdev_raidz_scalar_impl; -extern boolean_t raidz_will_scalar_work(void); - -#if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */ -extern const raidz_impl_ops_t vdev_raidz_sse2_impl; -#endif -#if defined(__x86_64) && defined(HAVE_SSSE3) /* only x86_64 for now */ -extern const raidz_impl_ops_t vdev_raidz_ssse3_impl; -#endif -#if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */ -extern const raidz_impl_ops_t vdev_raidz_avx2_impl; -#endif -#if defined(__x86_64) && defined(HAVE_AVX512F) /* only x86_64 for now */ -extern const raidz_impl_ops_t vdev_raidz_avx512f_impl; -#endif -#if defined(__x86_64) && defined(HAVE_AVX512BW) /* only x86_64 for now */ -extern const raidz_impl_ops_t vdev_raidz_avx512bw_impl; -#endif -#if defined(__aarch64__) -extern const raidz_impl_ops_t vdev_raidz_aarch64_neon_impl; -extern const raidz_impl_ops_t vdev_raidz_aarch64_neonx2_impl; -#endif -#if defined(__powerpc__) -extern const raidz_impl_ops_t vdev_raidz_powerpc_altivec_impl; -#endif - -/* - * Commonly used raidz_map helpers - * - * raidz_parity Returns parity of the RAIDZ block - * raidz_ncols Returns number of columns the block spans - * Note, all rows have the same number of columns. - * raidz_nbigcols Returns number of big columns - * raidz_col_p Returns pointer to a column - * raidz_col_size Returns size of a column - * raidz_big_size Returns size of big columns - * raidz_short_size Returns size of short columns - */ -#define raidz_parity(rm) ((rm)->rm_row[0]->rr_firstdatacol) -#define raidz_ncols(rm) ((rm)->rm_row[0]->rr_cols) -#define raidz_nbigcols(rm) ((rm)->rm_bigcols) -#define raidz_col_p(rm, c) ((rm)->rm_col + (c)) -#define raidz_col_size(rm, c) ((rm)->rm_col[c].rc_size) -#define raidz_big_size(rm) (raidz_col_size(rm, CODE_P)) -#define raidz_short_size(rm) (raidz_col_size(rm, raidz_ncols(rm)-1)) - -/* - * Macro defines an RAIDZ parity generation method - * - * @code parity the function produce - * @impl name of the implementation - */ -#define _RAIDZ_GEN_WRAP(code, impl) \ -static void \ -impl ## _gen_ ## code(void *rrp) \ -{ \ - raidz_row_t *rr = (raidz_row_t *)rrp; \ - raidz_generate_## code ## _impl(rr); \ -} - -/* - * Macro defines an RAIDZ data reconstruction method - * - * @code parity the function produce - * @impl name of the implementation - */ -#define _RAIDZ_REC_WRAP(code, impl) \ -static int \ -impl ## _rec_ ## code(void *rrp, const int *tgtidx) \ -{ \ - raidz_row_t *rr = (raidz_row_t *)rrp; \ - return (raidz_reconstruct_## code ## _impl(rr, tgtidx)); \ -} - -/* - * Define all gen methods for an implementation - * - * @impl name of the implementation - */ -#define DEFINE_GEN_METHODS(impl) \ - _RAIDZ_GEN_WRAP(p, impl); \ - _RAIDZ_GEN_WRAP(pq, impl); \ - _RAIDZ_GEN_WRAP(pqr, impl) - -/* - * Define all rec functions for an implementation - * - * @impl name of the implementation - */ -#define DEFINE_REC_METHODS(impl) \ - _RAIDZ_REC_WRAP(p, impl); \ - _RAIDZ_REC_WRAP(q, impl); \ - _RAIDZ_REC_WRAP(r, impl); \ - _RAIDZ_REC_WRAP(pq, impl); \ - _RAIDZ_REC_WRAP(pr, impl); \ - _RAIDZ_REC_WRAP(qr, impl); \ - _RAIDZ_REC_WRAP(pqr, impl) - -#define RAIDZ_GEN_METHODS(impl) \ -{ \ - [RAIDZ_GEN_P] = & impl ## _gen_p, \ - [RAIDZ_GEN_PQ] = & impl ## _gen_pq, \ - [RAIDZ_GEN_PQR] = & impl ## _gen_pqr \ -} - -#define RAIDZ_REC_METHODS(impl) \ -{ \ - [RAIDZ_REC_P] = & impl ## _rec_p, \ - [RAIDZ_REC_Q] = & impl ## _rec_q, \ - [RAIDZ_REC_R] = & impl ## _rec_r, \ - [RAIDZ_REC_PQ] = & impl ## _rec_pq, \ - [RAIDZ_REC_PR] = & impl ## _rec_pr, \ - [RAIDZ_REC_QR] = & impl ## _rec_qr, \ - [RAIDZ_REC_PQR] = & impl ## _rec_pqr \ -} - - -typedef struct raidz_impl_kstat { - uint64_t gen[RAIDZ_GEN_NUM]; /* gen method speed B/s */ - uint64_t rec[RAIDZ_REC_NUM]; /* rec method speed B/s */ -} raidz_impl_kstat_t; - -/* - * Enumerate various multiplication constants - * used in reconstruction methods - */ -typedef enum raidz_mul_info { - /* Reconstruct Q */ - MUL_Q_X = 0, - /* Reconstruct R */ - MUL_R_X = 0, - /* Reconstruct PQ */ - MUL_PQ_X = 0, - MUL_PQ_Y = 1, - /* Reconstruct PR */ - MUL_PR_X = 0, - MUL_PR_Y = 1, - /* Reconstruct QR */ - MUL_QR_XQ = 0, - MUL_QR_X = 1, - MUL_QR_YQ = 2, - MUL_QR_Y = 3, - /* Reconstruct PQR */ - MUL_PQR_XP = 0, - MUL_PQR_XQ = 1, - MUL_PQR_XR = 2, - MUL_PQR_YU = 3, - MUL_PQR_YP = 4, - MUL_PQR_YQ = 5, - - MUL_CNT = 6 -} raidz_mul_info_t; - -/* - * Powers of 2 in the Galois field. - */ -extern const uint8_t vdev_raidz_pow2[256] __attribute__((aligned(256))); -/* Logs of 2 in the Galois field defined above. */ -extern const uint8_t vdev_raidz_log2[256] __attribute__((aligned(256))); - -/* - * Multiply a given number by 2 raised to the given power. - */ -static inline uint8_t -vdev_raidz_exp2(const uint8_t a, const unsigned exp) -{ - if (a == 0) - return (0); - - return (vdev_raidz_pow2[(exp + (unsigned)vdev_raidz_log2[a]) % 255]); -} - -/* - * Galois Field operations. - * - * gf_exp2 - computes 2 raised to the given power - * gf_exp4 - computes 4 raised to the given power - * gf_mul - multiplication - * gf_div - division - * gf_inv - multiplicative inverse - */ -typedef unsigned gf_t; -typedef unsigned gf_log_t; - -static inline gf_t -gf_mul(const gf_t a, const gf_t b) -{ - gf_log_t logsum; - - if (a == 0 || b == 0) - return (0); - - logsum = (gf_log_t)vdev_raidz_log2[a] + (gf_log_t)vdev_raidz_log2[b]; - - return ((gf_t)vdev_raidz_pow2[logsum % 255]); -} - -static inline gf_t -gf_div(const gf_t a, const gf_t b) -{ - gf_log_t logsum; - - ASSERT3U(b, >, 0); - if (a == 0) - return (0); - - logsum = (gf_log_t)255 + (gf_log_t)vdev_raidz_log2[a] - - (gf_log_t)vdev_raidz_log2[b]; - - return ((gf_t)vdev_raidz_pow2[logsum % 255]); -} - -static inline gf_t -gf_inv(const gf_t a) -{ - gf_log_t logsum; - - ASSERT3U(a, >, 0); - - logsum = (gf_log_t)255 - (gf_log_t)vdev_raidz_log2[a]; - - return ((gf_t)vdev_raidz_pow2[logsum]); -} - -static inline gf_t -gf_exp2(gf_log_t exp) -{ - return (vdev_raidz_pow2[exp % 255]); -} - -static inline gf_t -gf_exp4(gf_log_t exp) -{ - ASSERT3U(exp, <=, 255); - return ((gf_t)vdev_raidz_pow2[(2 * exp) % 255]); -} - -#ifdef __cplusplus -} -#endif - -#endif /* _VDEV_RAIDZ_H */ |