diff options
92 files changed, 3070 insertions, 1126 deletions
diff --git a/Makefile.inc1 b/Makefile.inc1 index a75e5c760cdc..6ace6a6b5eaf 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -523,25 +523,6 @@ BUILDENV_SHELL?=/bin/sh .endif .if !defined(_MKSHOWCONFIG) -.if !defined(VCS_REVISION) || empty(VCS_REVISION) -.if !defined(SVNVERSION_CMD) || empty(SVNVERSION_CMD) -. for _D in ${PATH:S,:, ,g} -. if exists(${_D}/svnversion) -SVNVERSION_CMD?=${_D}/svnversion -. endif -. if exists(${_D}/svnliteversion) -SVNVERSION_CMD?=${_D}/svnliteversion -. endif -. endfor -.endif -.if defined(SVNVERSION_CMD) && !empty(SVNVERSION_CMD) -_VCS_REVISION?= $$(eval ${SVNVERSION_CMD} ${SRCDIR}) -. if !empty(_VCS_REVISION) -VCS_REVISION= $$(echo r${_VCS_REVISION}) -.export VCS_REVISION -. endif -.endif -.endif .if !defined(GIT_CMD) || empty(GIT_CMD) . for _P in /usr/bin /usr/local/bin @@ -603,6 +584,7 @@ EXTRA_REVISION= p${_BRANCH:C/.*-p([0-9]+$)/\1/} .if !defined(PKG_VERSION) PKG_VERSION:= ${_PKG_REVISION}${EXTRA_REVISION:C/[[:space:]]//g} .endif + .endif # !defined(_MKSHOWCONFIG) .if make(*package*) @@ -2225,9 +2207,6 @@ create-world-package-${pkgname}: .PHONY /^name/ { printf("===> Creating %s-", $$2); next } \ /^version/ { print $$2; next } \ ' ${WSTAGEDIR}/${pkgname}.ucl - @if [ "${pkgname}" == "runtime" ]; then \ - sed -i '' -e "s/%VCS_REVISION%/${VCS_REVISION}/" ${WSTAGEDIR}/${pkgname}.ucl ; \ - fi ${PKG_CMD} -o ABI=${PKG_ABI} -o ALLOW_BASE_SHLIBS=yes \ -o OSVERSION="${SRCRELDATE}" \ create -f ${PKG_FORMAT} ${PKG_CLEVEL} \ diff --git a/crypto/heimdal/kadmin/dump.c b/crypto/heimdal/kadmin/dump.c index 91a5ada86607..2c302eced210 100644 --- a/crypto/heimdal/kadmin/dump.c +++ b/crypto/heimdal/kadmin/dump.c @@ -42,32 +42,51 @@ dump(struct dump_options *opt, int argc, char **argv) { krb5_error_code ret; FILE *f; + struct hdb_print_entry_arg parg; HDB *db = NULL; - if(!local_flag) { + if (!local_flag) { krb5_warnx(context, "dump is only available in local (-l) mode"); return 0; } db = _kadm5_s_get_db(kadm_handle); - if(argc == 0) + if (argc == 0) f = stdout; else f = fopen(argv[0], "w"); - if(f == NULL) { + if (f == NULL) { krb5_warn(context, errno, "open: %s", argv[0]); goto out; } ret = db->hdb_open(context, db, O_RDONLY, 0600); - if(ret) { + if (ret) { krb5_warn(context, ret, "hdb_open"); goto out; } + if (!opt->format_string || strcmp(opt->format_string, "Heimdal") == 0) { + parg.fmt = HDB_DUMP_HEIMDAL; + } else if (opt->format_string && strcmp(opt->format_string, "MIT") == 0) { + parg.fmt = HDB_DUMP_MIT; + fprintf(f, "kdb5_util load_dump version 5\n"); /* 5||6, either way */ + } else if (opt->format_string) { + /* Open the format string as a MIT mkey file. */ + ret = hdb_read_master_key(context, opt->format_string, &db->hdb_mit_key); + if (ret) + krb5_errx(context, 1, "Cannot open MIT mkey file"); + db->hdb_mit_key_set = 1; + parg.fmt = HDB_DUMP_MIT; + opt->decrypt_flag = 1; + fprintf(f, "kdb5_util load_dump version 5\n"); /* 5||6, either way */ + } else { + krb5_errx(context, 1, "Supported dump formats: Heimdal and MIT"); + } + parg.out = f; hdb_foreach(context, db, opt->decrypt_flag ? HDB_F_DECRYPT : 0, - hdb_print_entry, f); + hdb_print_entry, &parg); db->hdb_close(context, db); out: diff --git a/crypto/heimdal/kadmin/kadmin-commands.in b/crypto/heimdal/kadmin/kadmin-commands.in index 4396ff800441..dc36db4e0a00 100644 --- a/crypto/heimdal/kadmin/kadmin-commands.in +++ b/crypto/heimdal/kadmin/kadmin-commands.in @@ -76,6 +76,12 @@ command = { type = "flag" help = "decrypt keys" } + option = { + long = "format" + short = "f" + type = "string" + help = "dump format, mit or heimdal (default: heimdal)" + } argument = "[dump-file]" min_args = "0" max_args = "1" diff --git a/crypto/heimdal/kdc/hpropd.c b/crypto/heimdal/kdc/hpropd.c index 1cfc688b2a6c..c76be0446705 100644 --- a/crypto/heimdal/kdc/hpropd.c +++ b/crypto/heimdal/kdc/hpropd.c @@ -85,23 +85,23 @@ main(int argc, char **argv) setprogname(argv[0]); ret = krb5_init_context(&context); - if(ret) + if (ret) exit(1); ret = krb5_openlog(context, "hpropd", &fac); - if(ret) + if (ret) errx(1, "krb5_openlog"); krb5_set_warn_dest(context, fac); - if(getarg(args, num_args, argc, argv, &optidx)) + if (getarg(args, num_args, argc, argv, &optidx)) usage(1); - if(local_realm != NULL) + if (local_realm != NULL) krb5_set_default_realm(context, local_realm); - if(help_flag) + if (help_flag) usage(0); - if(version_flag) { + if (version_flag) { print_version(NULL); exit(0); } @@ -117,7 +117,7 @@ main(int argc, char **argv) if (database == NULL) database = hdb_default_db(context); - if(from_stdin) { + if (from_stdin) { sock = STDIN_FILENO; } else { struct sockaddr_storage ss; @@ -145,7 +145,7 @@ main(int argc, char **argv) HPROP_PORT), &sock); } sin_len = sizeof(ss); - if(getpeername(sock, sa, &sin_len) < 0) + if (getpeername(sock, sa, &sin_len) < 0) krb5_err(context, 1, errno, "getpeername"); if (inet_ntop(sa->sa_family, @@ -158,7 +158,7 @@ main(int argc, char **argv) krb5_log(context, fac, 0, "Connection from %s", addr_name); ret = krb5_kt_register(context, &hdb_kt_ops); - if(ret) + if (ret) krb5_err(context, 1, ret, "krb5_kt_register"); if (ktname != NULL) { @@ -173,7 +173,7 @@ main(int argc, char **argv) ret = krb5_recvauth(context, &ac, &sock, HPROP_VERSION, NULL, 0, keytab, &ticket); - if(ret) + if (ret) krb5_err(context, 1, ret, "krb5_recvauth"); ret = krb5_unparse_name(context, ticket->server, &server); @@ -186,15 +186,15 @@ main(int argc, char **argv) krb5_free_ticket (context, ticket); ret = krb5_auth_con_getauthenticator(context, ac, &authent); - if(ret) + if (ret) krb5_err(context, 1, ret, "krb5_auth_con_getauthenticator"); ret = krb5_make_principal(context, &c1, NULL, "kadmin", "hprop", NULL); - if(ret) + if (ret) krb5_err(context, 1, ret, "krb5_make_principal"); _krb5_principalname2krb5_principal(context, &c2, authent->cname, authent->crealm); - if(!krb5_principal_compare(context, c1, c2)) { + if (!krb5_principal_compare(context, c1, c2)) { char *s; ret = krb5_unparse_name(context, c2, &s); if (ret) @@ -205,48 +205,48 @@ main(int argc, char **argv) krb5_free_principal(context, c2); ret = krb5_kt_close(context, keytab); - if(ret) + if (ret) krb5_err(context, 1, ret, "krb5_kt_close"); } - if(!print_dump) { + if (!print_dump) { asprintf(&tmp_db, "%s~", database); ret = hdb_create(context, &db, tmp_db); - if(ret) + if (ret) krb5_err(context, 1, ret, "hdb_create(%s)", tmp_db); ret = db->hdb_open(context, db, O_RDWR | O_CREAT | O_TRUNC, 0600); - if(ret) + if (ret) krb5_err(context, 1, ret, "hdb_open(%s)", tmp_db); } nprincs = 0; - while(1){ + while (1){ krb5_data data; hdb_entry_ex entry; - if(from_stdin) { + if (from_stdin) { ret = krb5_read_message(context, &sock, &data); - if(ret != 0 && ret != HEIM_ERR_EOF) + if (ret != 0 && ret != HEIM_ERR_EOF) krb5_err(context, 1, ret, "krb5_read_message"); } else { ret = krb5_read_priv_message(context, ac, &sock, &data); - if(ret) + if (ret) krb5_err(context, 1, ret, "krb5_read_priv_message"); } - if(ret == HEIM_ERR_EOF || data.length == 0) { - if(!from_stdin) { + if (ret == HEIM_ERR_EOF || data.length == 0) { + if (!from_stdin) { data.data = NULL; data.length = 0; krb5_write_priv_message(context, ac, &sock, &data); } - if(!print_dump) { + if (!print_dump) { ret = db->hdb_close(context, db); - if(ret) + if (ret) krb5_err(context, 1, ret, "db_close"); ret = db->hdb_rename(context, db, database); - if(ret) + if (ret) krb5_err(context, 1, ret, "db_rename"); } break; @@ -254,20 +254,24 @@ main(int argc, char **argv) memset(&entry, 0, sizeof(entry)); ret = hdb_value2entry(context, &data, &entry.entry); krb5_data_free(&data); - if(ret) + if (ret) krb5_err(context, 1, ret, "hdb_value2entry"); - if(print_dump) - hdb_print_entry(context, db, &entry, stdout); - else { + if (print_dump) { + struct hdb_print_entry_arg parg; + + parg.out = stdout; + parg.fmt = HDB_DUMP_HEIMDAL; + hdb_print_entry(context, db, &entry, &parg); + } else { ret = db->hdb_store(context, db, 0, &entry); - if(ret == HDB_ERR_EXISTS) { + if (ret == HDB_ERR_EXISTS) { char *s; ret = krb5_unparse_name(context, entry.entry.principal, &s); if (ret) s = strdup(unparseable_name); krb5_warnx(context, "Entry exists: %s", s); free(s); - } else if(ret) + } else if (ret) krb5_err(context, 1, ret, "db_store"); else nprincs++; diff --git a/crypto/heimdal/kdc/mit_dump.c b/crypto/heimdal/kdc/mit_dump.c index f28e932b15b4..4397d1ad897d 100644 --- a/crypto/heimdal/kdc/mit_dump.c +++ b/crypto/heimdal/kdc/mit_dump.c @@ -33,6 +33,17 @@ #include "hprop.h" +extern krb5_error_code _hdb_mdb_value2entry(krb5_context context, + krb5_data *data, + krb5_kvno target_kvno, + hdb_entry *entry); + +extern int _hdb_mit_dump2mitdb_entry(krb5_context context, + char *line, + krb5_storage *sp); + + + /* can have any number of princ stanzas. format is as follows (only \n indicates newlines) @@ -74,19 +85,6 @@ unless no extra data */ -static int -hex_to_octet_string(const char *ptr, krb5_data *data) -{ - size_t i; - unsigned int v; - for(i = 0; i < data->length; i++) { - if(sscanf(ptr + 2 * i, "%02x", &v) != 1) - return -1; - ((unsigned char*)data->data)[i] = v; - } - return 2 * i; -} - static char * nexttoken(char **p) { @@ -97,321 +95,116 @@ nexttoken(char **p) return q; } -static size_t -getdata(char **p, unsigned char *buf, size_t len) -{ - size_t i; - int v; - char *q = nexttoken(p); - i = 0; - while(*q && i < len) { - if(sscanf(q, "%02x", &v) != 1) - break; - buf[i++] = v; - q += 2; - } - return i; -} - -static int -getint(char **p) -{ - int val; - char *q = nexttoken(p); - sscanf(q, "%d", &val); - return val; -} - #include <kadm5/admin.h> -static void -attr_to_flags(unsigned attr, HDBFlags *flags) -{ - flags->postdate = !(attr & KRB5_KDB_DISALLOW_POSTDATED); - flags->forwardable = !(attr & KRB5_KDB_DISALLOW_FORWARDABLE); - flags->initial = !!(attr & KRB5_KDB_DISALLOW_TGT_BASED); - flags->renewable = !(attr & KRB5_KDB_DISALLOW_RENEWABLE); - flags->proxiable = !(attr & KRB5_KDB_DISALLOW_PROXIABLE); - /* DUP_SKEY */ - flags->invalid = !!(attr & KRB5_KDB_DISALLOW_ALL_TIX); - flags->require_preauth = !!(attr & KRB5_KDB_REQUIRES_PRE_AUTH); - flags->require_hwauth = !!(attr & KRB5_KDB_REQUIRES_HW_AUTH); - flags->server = !(attr & KRB5_KDB_DISALLOW_SVR); - flags->change_pw = !!(attr & KRB5_KDB_PWCHANGE_SERVICE); - flags->client = 1; /* XXX */ -} - -#define KRB5_KDB_SALTTYPE_NORMAL 0 -#define KRB5_KDB_SALTTYPE_V4 1 -#define KRB5_KDB_SALTTYPE_NOREALM 2 -#define KRB5_KDB_SALTTYPE_ONLYREALM 3 -#define KRB5_KDB_SALTTYPE_SPECIAL 4 -#define KRB5_KDB_SALTTYPE_AFS3 5 - -static krb5_error_code -fix_salt(krb5_context context, hdb_entry *ent, int key_num) +static int +my_fgetln(FILE *f, char **buf, size_t *sz, size_t *len) { - krb5_error_code ret; - Salt *salt = ent->keys.val[key_num].salt; - /* fix salt type */ - switch((int)salt->type) { - case KRB5_KDB_SALTTYPE_NORMAL: - salt->type = KRB5_PADATA_PW_SALT; - break; - case KRB5_KDB_SALTTYPE_V4: - krb5_data_free(&salt->salt); - salt->type = KRB5_PADATA_PW_SALT; - break; - case KRB5_KDB_SALTTYPE_NOREALM: - { - size_t len; - size_t i; - char *p; - - len = 0; - for (i = 0; i < ent->principal->name.name_string.len; ++i) - len += strlen(ent->principal->name.name_string.val[i]); - ret = krb5_data_alloc (&salt->salt, len); - if (ret) - return ret; - p = salt->salt.data; - for (i = 0; i < ent->principal->name.name_string.len; ++i) { - memcpy (p, - ent->principal->name.name_string.val[i], - strlen(ent->principal->name.name_string.val[i])); - p += strlen(ent->principal->name.name_string.val[i]); - } - - salt->type = KRB5_PADATA_PW_SALT; - break; + char *p, *n; + + if (!*buf) { + *buf = malloc(*sz ? *sz : 2048); + if (!*buf) + return ENOMEM; + if (!*sz) + *sz = 2048; } - case KRB5_KDB_SALTTYPE_ONLYREALM: - krb5_data_free(&salt->salt); - ret = krb5_data_copy(&salt->salt, - ent->principal->realm, - strlen(ent->principal->realm)); - if(ret) - return ret; - salt->type = KRB5_PADATA_PW_SALT; - break; - case KRB5_KDB_SALTTYPE_SPECIAL: - salt->type = KRB5_PADATA_PW_SALT; - break; - case KRB5_KDB_SALTTYPE_AFS3: - krb5_data_free(&salt->salt); - ret = krb5_data_copy(&salt->salt, - ent->principal->realm, - strlen(ent->principal->realm)); - if(ret) - return ret; - salt->type = KRB5_PADATA_AFS3_SALT; - break; - default: - abort(); + *len = 0; + while ((p = fgets(&(*buf)[*len], *sz, f))) { + if (strcspn(*buf, "\r\n") || feof(f)) { + *len = strlen(*buf); + return 0; + } + *len += strlen(&(*buf)[*len]); /* *len should be == *sz */ + n = realloc(buf, *sz + (*sz >> 1)); + if (!n) { + free(*buf); + *buf = NULL; + *sz = 0; + *len = 0; + return ENOMEM; + } + *buf = n; + *sz += *sz >> 1; } - return 0; + return 0; /* *len == 0 || no EOL -> EOF */ } int mit_prop_dump(void *arg, const char *file) { krb5_error_code ret; - char line [2048]; - FILE *f; + size_t line_bufsz = 0; + size_t line_len = 0; + char *line = NULL; int lineno = 0; + FILE *f; struct hdb_entry_ex ent; - struct prop_data *pd = arg; + krb5_storage *sp = NULL; + krb5_data kdb_ent; + memset(&ent, 0, sizeof (ent)); f = fopen(file, "r"); - if(f == NULL) + if (f == NULL) return errno; - while(fgets(line, sizeof(line), f)) { - char *p = line, *q; - - int i; - - int num_tl_data; - int num_key_data; - int high_kvno; - int attributes; + ret = ENOMEM; + sp = krb5_storage_emem(); + if (!sp) + goto out; + while ((ret = my_fgetln(f, &line, &line_bufsz, &line_len)) == 0) { + char *p = line; + char *q; + lineno++; - int tmp; - - lineno++; - - memset(&ent, 0, sizeof(ent)); - - q = nexttoken(&p); - if(strcmp(q, "kdb5_util") == 0) { + if(strncmp(line, "kdb5_util", strlen("kdb5_util")) == 0) { int major; + q = nexttoken(&p); + if (strcmp(q, "kdb5_util")) + errx(1, "line %d: unknown version", lineno); q = nexttoken(&p); /* load_dump */ - if(strcmp(q, "load_dump")) + if (strcmp(q, "load_dump")) errx(1, "line %d: unknown version", lineno); q = nexttoken(&p); /* load_dump */ - if(strcmp(q, "version")) + if (strcmp(q, "version")) errx(1, "line %d: unknown version", lineno); q = nexttoken(&p); /* x.0 */ - if(sscanf(q, "%d", &major) != 1) + if (sscanf(q, "%d", &major) != 1) errx(1, "line %d: unknown version", lineno); - if(major != 4 && major != 5 && major != 6) + if (major != 4 && major != 5 && major != 6) errx(1, "unknown dump file format, got %d, expected 4-6", major); continue; - } else if(strcmp(q, "policy") == 0) { + } else if(strncmp(p, "policy", strlen("policy")) == 0) { + warnx("line: %d: ignoring policy (not supported)", lineno); continue; - } else if(strcmp(q, "princ") != 0) { + } else if(strncmp(p, "princ", strlen("princ")) != 0) { warnx("line %d: not a principal", lineno); continue; } - tmp = getint(&p); - if(tmp != 38) { - warnx("line %d: bad base length %d != 38", lineno, tmp); - continue; - } - nexttoken(&p); /* length of principal */ - num_tl_data = getint(&p); /* number of tl-data */ - num_key_data = getint(&p); /* number of key-data */ - getint(&p); /* length of extra data */ - q = nexttoken(&p); /* principal name */ - krb5_parse_name(pd->context, q, &ent.entry.principal); - attributes = getint(&p); /* attributes */ - attr_to_flags(attributes, &ent.entry.flags); - tmp = getint(&p); /* max life */ - if(tmp != 0) { - ALLOC(ent.entry.max_life); - *ent.entry.max_life = tmp; - } - tmp = getint(&p); /* max renewable life */ - if(tmp != 0) { - ALLOC(ent.entry.max_renew); - *ent.entry.max_renew = tmp; - } - tmp = getint(&p); /* expiration */ - if(tmp != 0 && tmp != 2145830400) { - ALLOC(ent.entry.valid_end); - *ent.entry.valid_end = tmp; - } - tmp = getint(&p); /* pw expiration */ - if(tmp != 0) { - ALLOC(ent.entry.pw_end); - *ent.entry.pw_end = tmp; - } - nexttoken(&p); /* last auth */ - nexttoken(&p); /* last failed auth */ - nexttoken(&p); /* fail auth count */ - for(i = 0; i < num_tl_data; i++) { - unsigned long val; - int tl_type, tl_length; - unsigned char *buf; - krb5_principal princ; - - tl_type = getint(&p); /* data type */ - tl_length = getint(&p); /* data length */ - -#define mit_KRB5_TL_LAST_PWD_CHANGE 1 -#define mit_KRB5_TL_MOD_PRINC 2 - switch(tl_type) { - case mit_KRB5_TL_LAST_PWD_CHANGE: - buf = malloc(tl_length); - if (buf == NULL) - errx(ENOMEM, "malloc"); - getdata(&p, buf, tl_length); /* data itself */ - val = buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); - free(buf); - ALLOC(ent.entry.extensions); - ALLOC_SEQ(ent.entry.extensions, 1); - ent.entry.extensions->val[0].mandatory = 0; - ent.entry.extensions->val[0].data.element - = choice_HDB_extension_data_last_pw_change; - ent.entry.extensions->val[0].data.u.last_pw_change = val; - break; - case mit_KRB5_TL_MOD_PRINC: - buf = malloc(tl_length); - if (buf == NULL) - errx(ENOMEM, "malloc"); - getdata(&p, buf, tl_length); /* data itself */ - val = buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); - ret = krb5_parse_name(pd->context, (char *)buf + 4, &princ); - if (ret) - krb5_err(pd->context, 1, ret, - "parse_name: %s", (char *)buf + 4); - free(buf); - ALLOC(ent.entry.modified_by); - ent.entry.modified_by->time = val; - ent.entry.modified_by->principal = princ; - break; - default: - nexttoken(&p); - break; - } - } - ALLOC_SEQ(&ent.entry.keys, num_key_data); - high_kvno = -1; - for(i = 0; i < num_key_data; i++) { - int key_versions; - int kvno; - key_versions = getint(&p); /* key data version */ - kvno = getint(&p); - - /* - * An MIT dump file may contain multiple sets of keys with - * different kvnos. Since the Heimdal database can only represent - * one kvno per principal, we only want the highest set. Assume - * that set will be given first, and discard all keys with lower - * kvnos. - */ - if (kvno > high_kvno && high_kvno != -1) - errx(1, "line %d: high kvno keys given after low kvno keys", - lineno); - else if (kvno < high_kvno) { - nexttoken(&p); /* key type */ - nexttoken(&p); /* key length */ - nexttoken(&p); /* key */ - if (key_versions > 1) { - nexttoken(&p); /* salt type */ - nexttoken(&p); /* salt length */ - nexttoken(&p); /* salt */ - } - ent.entry.keys.len--; - continue; - } - ent.entry.kvno = kvno; - high_kvno = kvno; - ALLOC(ent.entry.keys.val[i].mkvno); - *ent.entry.keys.val[i].mkvno = 1; - - /* key version 0 -- actual key */ - ent.entry.keys.val[i].key.keytype = getint(&p); /* key type */ - tmp = getint(&p); /* key length */ - /* the first two bytes of the key is the key length -- - skip it */ - krb5_data_alloc(&ent.entry.keys.val[i].key.keyvalue, tmp - 2); - q = nexttoken(&p); /* key itself */ - hex_to_octet_string(q + 4, &ent.entry.keys.val[i].key.keyvalue); - - if(key_versions > 1) { - /* key version 1 -- optional salt */ - ALLOC(ent.entry.keys.val[i].salt); - ent.entry.keys.val[i].salt->type = getint(&p); /* salt type */ - tmp = getint(&p); /* salt length */ - if(tmp > 0) { - krb5_data_alloc(&ent.entry.keys.val[i].salt->salt, tmp - 2); - q = nexttoken(&p); /* salt itself */ - hex_to_octet_string(q + 4, - &ent.entry.keys.val[i].salt->salt); - } else { - ent.entry.keys.val[i].salt->salt.length = 0; - ent.entry.keys.val[i].salt->salt.data = NULL; - getint(&p); /* -1, if no data. */ - } - fix_salt(pd->context, &ent.entry, i); - } - } - nexttoken(&p); /* extra data */ - v5_prop(pd->context, NULL, &ent, arg); + krb5_storage_truncate(sp, 0); + ret = _hdb_mit_dump2mitdb_entry(pd->context, line, sp); + if (ret) break; + ret = krb5_storage_to_data(sp, &kdb_ent); + if (ret) break; + ret = _hdb_mdb_value2entry(pd->context, &kdb_ent, 0, &ent.entry); + krb5_data_free(&kdb_ent); + if (ret) break; + ret = v5_prop(pd->context, NULL, &ent, arg); + hdb_free_entry(pd->context, &ent); + if (ret) break; } + +out: fclose(f); - return 0; + free(line); + if (sp) + krb5_storage_free(sp); + if (ret && ret == ENOMEM) + errx(1, "out of memory"); + if (ret) + errx(1, "line %d: problem parsing dump line", lineno); + return ret; } + diff --git a/crypto/heimdal/lib/hdb/Makefile.am b/crypto/heimdal/lib/hdb/Makefile.am index b629f56258d2..fd009bd26867 100644 --- a/crypto/heimdal/lib/hdb/Makefile.am +++ b/crypto/heimdal/lib/hdb/Makefile.am @@ -29,6 +29,7 @@ gen_files_hdb = \ asn1_HDB_Ext_Lan_Manager_OWF.x \ asn1_HDB_Ext_Password.x \ asn1_HDB_Ext_Aliases.x \ + asn1_HDB_Ext_KeySet.x \ asn1_HDB_extension.x \ asn1_HDB_extensions.x \ asn1_hdb_entry.x \ diff --git a/crypto/heimdal/lib/hdb/common.c b/crypto/heimdal/lib/hdb/common.c index 2715adf63dca..80482e7a4c1c 100644 --- a/crypto/heimdal/lib/hdb/common.c +++ b/crypto/heimdal/lib/hdb/common.c @@ -105,7 +105,6 @@ _hdb_fetch_kvno(krb5_context context, HDB *db, krb5_const_principal principal, krb5_principal enterprise_principal = NULL; krb5_data key, value; krb5_error_code ret; - int code; if (principal->name.name_type == KRB5_NT_ENTERPRISE_PRINCIPAL) { if (principal->name.name_string.len != 1) { @@ -125,43 +124,74 @@ _hdb_fetch_kvno(krb5_context context, HDB *db, krb5_const_principal principal, hdb_principal2key(context, principal, &key); if (enterprise_principal) krb5_free_principal(context, enterprise_principal); - code = db->hdb__get(context, db, key, &value); + ret = db->hdb__get(context, db, key, &value); krb5_data_free(&key); - if(code) - return code; - code = hdb_value2entry(context, &value, &entry->entry); - if (code == ASN1_BAD_ID && (flags & HDB_F_CANON) == 0) { + if(ret) + return ret; + ret = hdb_value2entry(context, &value, &entry->entry); + if (ret == ASN1_BAD_ID && (flags & HDB_F_CANON) == 0) { krb5_data_free(&value); return HDB_ERR_NOENTRY; - } else if (code == ASN1_BAD_ID) { + } else if (ret == ASN1_BAD_ID) { hdb_entry_alias alias; - code = hdb_value2entry_alias(context, &value, &alias); - if (code) { + ret = hdb_value2entry_alias(context, &value, &alias); + if (ret) { krb5_data_free(&value); - return code; + return ret; } hdb_principal2key(context, alias.principal, &key); krb5_data_free(&value); free_hdb_entry_alias(&alias); - code = db->hdb__get(context, db, key, &value); + ret = db->hdb__get(context, db, key, &value); krb5_data_free(&key); - if (code) - return code; - code = hdb_value2entry(context, &value, &entry->entry); - if (code) { + if (ret) + return ret; + ret = hdb_value2entry(context, &value, &entry->entry); + if (ret) { krb5_data_free(&value); - return code; + return ret; } } krb5_data_free(&value); if (db->hdb_master_key_set && (flags & HDB_F_DECRYPT)) { - code = hdb_unseal_keys (context, db, &entry->entry); - if (code) +#ifdef notnow + if ((flags & HDB_F_KVNO_SPECIFIED) == 0 && + (flags & HDB_F_CURRENT_KVNO) == 0) { + + /* + * Decrypt all the old keys too, since we don't know which + * the caller will need. + */ + ret = hdb_unseal_keys_kvno(context, db, 0, &entry->entry); + if (ret) { + hdb_free_entry(context, entry); + return ret; + } + } else if ((flags & HDB_F_KVNO_SPECIFIED) != 0 && + kvno != entry->entry.kvno && + kvno < entry->entry.kvno && + kvno > 0) { + + /* Decrypt the keys we were asked for, if not the current ones */ + ret = hdb_unseal_keys_kvno(context, db, kvno, &entry->entry); + if (ret) { + hdb_free_entry(context, entry); + return ret; + } + } +#endif + + /* Always decrypt the current keys too */ + ret = hdb_unseal_keys(context, db, &entry->entry); + if (ret) { hdb_free_entry(context, entry); + return ret; + } } - return code; + + return ret; } static krb5_error_code diff --git a/crypto/heimdal/lib/hdb/ext.c b/crypto/heimdal/lib/hdb/ext.c index d2a4373b9b38..5f7a19a55e04 100644 --- a/crypto/heimdal/lib/hdb/ext.c +++ b/crypto/heimdal/lib/hdb/ext.c @@ -432,3 +432,34 @@ hdb_entry_get_aliases(const hdb_entry *entry, const HDB_Ext_Aliases **a) return 0; } + +krb5_error_code +hdb_set_last_modified_by(krb5_context context, hdb_entry *entry, + krb5_principal modby, time_t modtime) +{ + krb5_error_code ret; + Event *old_ev; + Event *ev; + + old_ev = entry->modified_by; + + ev = calloc(1, sizeof (*ev)); + if (!ev) + return ENOMEM; + if (modby) + ret = krb5_copy_principal(context, modby, &ev->principal); + else + ret = krb5_parse_name(context, "root/admin", &ev->principal); + if (ret) { + free(ev); + return ret; + } + ev->time = modtime; + if (!ev->time) + time(&ev->time); + + entry->modified_by = ev; + if (old_ev) + free_Event(old_ev); + return 0; +} diff --git a/crypto/heimdal/lib/hdb/hdb-mitdb.c b/crypto/heimdal/lib/hdb/hdb-mitdb.c index 02c575050fe2..1dfe7835cb4d 100644 --- a/crypto/heimdal/lib/hdb/hdb-mitdb.c +++ b/crypto/heimdal/lib/hdb/hdb-mitdb.c @@ -91,18 +91,28 @@ salt: #include "hdb_locl.h" -#define KDB_V1_BASE_LENGTH 38 - -#if HAVE_DB1 +static void +attr_to_flags(unsigned attr, HDBFlags *flags) +{ + flags->postdate = !(attr & KRB5_KDB_DISALLOW_POSTDATED); + flags->forwardable = !(attr & KRB5_KDB_DISALLOW_FORWARDABLE); + flags->initial = !!(attr & KRB5_KDB_DISALLOW_TGT_BASED); + flags->renewable = !(attr & KRB5_KDB_DISALLOW_RENEWABLE); + flags->proxiable = !(attr & KRB5_KDB_DISALLOW_PROXIABLE); + /* DUP_SKEY */ + flags->invalid = !!(attr & KRB5_KDB_DISALLOW_ALL_TIX); + flags->require_preauth = !!(attr & KRB5_KDB_REQUIRES_PRE_AUTH); + flags->require_hwauth = !!(attr & KRB5_KDB_REQUIRES_HW_AUTH); + flags->server = !(attr & KRB5_KDB_DISALLOW_SVR); + flags->change_pw = !!(attr & KRB5_KDB_PWCHANGE_SERVICE); + flags->client = 1; /* XXX */ +} -#if defined(HAVE_DB_185_H) -#include <db_185.h> -#elif defined(HAVE_DB_H) -#include <db.h> -#endif +#define KDB_V1_BASE_LENGTH 38 #define CHECK(x) do { if ((x)) goto out; } while(0) +#ifdef HAVE_DB1 static krb5_error_code mdb_principal2key(krb5_context context, krb5_const_principal principal, @@ -118,6 +128,7 @@ mdb_principal2key(krb5_context context, key->length = strlen(str) + 1; return 0; } +#endif /* HAVE_DB1 */ #define KRB5_KDB_SALTTYPE_NORMAL 0 #define KRB5_KDB_SALTTYPE_V4 1 @@ -197,13 +208,15 @@ fix_salt(krb5_context context, hdb_entry *ent, int key_num) } -static krb5_error_code -mdb_value2entry(krb5_context context, krb5_data *data, krb5_kvno kvno, hdb_entry *entry) +krb5_error_code +_hdb_mdb_value2entry(krb5_context context, krb5_data *data, + krb5_kvno kvno, hdb_entry *entry) { krb5_error_code ret; krb5_storage *sp; uint32_t u32; uint16_t u16, num_keys, num_tl; + ssize_t sz; size_t i, j; char *p; @@ -234,18 +247,7 @@ mdb_value2entry(krb5_context context, krb5_data *data, krb5_kvno kvno, hdb_entry if (u16 != KDB_V1_BASE_LENGTH) { ret = EINVAL; goto out; } /* 32: attributes */ CHECK(ret = krb5_ret_uint32(sp, &u32)); - entry->flags.postdate = !(u32 & KRB5_KDB_DISALLOW_POSTDATED); - entry->flags.forwardable = !(u32 & KRB5_KDB_DISALLOW_FORWARDABLE); - entry->flags.initial = !!(u32 & KRB5_KDB_DISALLOW_TGT_BASED); - entry->flags.renewable = !(u32 & KRB5_KDB_DISALLOW_RENEWABLE); - entry->flags.proxiable = !(u32 & KRB5_KDB_DISALLOW_PROXIABLE); - /* DUP_SKEY */ - entry->flags.invalid = !!(u32 & KRB5_KDB_DISALLOW_ALL_TIX); - entry->flags.require_preauth =!!(u32 & KRB5_KDB_REQUIRES_PRE_AUTH); - entry->flags.require_hwauth =!!(u32 & KRB5_KDB_REQUIRES_HW_AUTH); - entry->flags.server = !(u32 & KRB5_KDB_DISALLOW_SVR); - entry->flags.change_pw = !!(u32 & KRB5_KDB_PWCHANGE_SERVICE); - entry->flags.client = 1; /* XXX */ + attr_to_flags(u32, &entry->flags); /* 32: max time */ CHECK(ret = krb5_ret_uint32(sp, &u32)); @@ -296,7 +298,11 @@ mdb_value2entry(krb5_context context, krb5_data *data, krb5_kvno kvno, hdb_entry ret = ENOMEM; goto out; } - krb5_storage_read(sp, p, u16); + sz = krb5_storage_read(sp, p, u16); + if (sz != u16) { + ret = EINVAL; /* XXX */ + goto out; + } p[u16] = '\0'; CHECK(ret = krb5_parse_name(context, p, &entry->principal)); free(p); @@ -305,12 +311,53 @@ mdb_value2entry(krb5_context context, krb5_data *data, krb5_kvno kvno, hdb_entry 16: tl data type 16: tl data length length: length */ +#define mit_KRB5_TL_LAST_PWD_CHANGE 1 +#define mit_KRB5_TL_MOD_PRINC 2 for (i = 0; i < num_tl; i++) { + int tl_type; + krb5_principal modby; /* 16: TL data type */ CHECK(ret = krb5_ret_uint16(sp, &u16)); + tl_type = u16; /* 16: TL data length */ CHECK(ret = krb5_ret_uint16(sp, &u16)); - krb5_storage_seek(sp, u16, SEEK_CUR); + /* + * For rollback to MIT purposes we really must understand some + * TL data! + * + * XXX Move all this to separate functions, one per-TL type. + */ + switch (tl_type) { + case mit_KRB5_TL_LAST_PWD_CHANGE: + CHECK(ret = krb5_ret_uint32(sp, &u32)); + CHECK(ret = hdb_entry_set_pw_change_time(context, entry, u32)); + break; + case mit_KRB5_TL_MOD_PRINC: + if (u16 < 5) { + ret = EINVAL; /* XXX */ + goto out; + } + CHECK(ret = krb5_ret_uint32(sp, &u32)); /* mod time */ + p = malloc(u16 - 4 + 1); + if (!p) { + ret = ENOMEM; + goto out; + } + p[u16 - 4] = '\0'; + sz = krb5_storage_read(sp, p, u16 - 4); + if (sz != u16 - 4) { + ret = EINVAL; /* XXX */ + goto out; + } + CHECK(ret = krb5_parse_name(context, p, &modby)); + ret = hdb_set_last_modified_by(context, entry, modby, u32); + krb5_free_principal(context, modby); + free(p); + break; + default: + krb5_storage_seek(sp, u16, SEEK_CUR); + break; + } } /* * for num key data times @@ -471,6 +518,14 @@ mdb_entry2value(krb5_context context, hdb_entry *entry, krb5_data *data) } #endif +#if HAVE_DB1 + +#if defined(HAVE_DB_185_H) +#include <db_185.h> +#elif defined(HAVE_DB_H) +#include <db.h> +#endif + static krb5_error_code mdb_close(krb5_context context, HDB *db) @@ -551,7 +606,7 @@ mdb_seq(krb5_context context, HDB *db, data.length = value.size; memset(entry, 0, sizeof(*entry)); - if (mdb_value2entry(context, &data, 0, &entry->entry)) + if (_hdb_mdb_value2entry(context, &data, 0, &entry->entry)) return mdb_seq(context, db, flags, entry, R_NEXT); if (db->hdb_master_key_set && (flags & HDB_F_DECRYPT)) { @@ -684,24 +739,26 @@ mdb_fetch_kvno(krb5_context context, HDB *db, krb5_const_principal principal, unsigned flags, krb5_kvno kvno, hdb_entry_ex *entry) { krb5_data key, value; - krb5_error_code code; + krb5_error_code ret; - code = mdb_principal2key(context, principal, &key); - if (code) - return code; - code = db->hdb__get(context, db, key, &value); + ret = mdb_principal2key(context, principal, &key); + if (ret) + return ret; + ret = db->hdb__get(context, db, key, &value); krb5_data_free(&key); - if(code) - return code; - code = mdb_value2entry(context, &value, kvno, &entry->entry); + if(ret) + return ret; + ret = _hdb_mdb_value2entry(context, &value, kvno, &entry->entry); krb5_data_free(&value); - if (code) - return code; + if (ret) + return ret; if (db->hdb_master_key_set && (flags & HDB_F_DECRYPT)) { - code = hdb_unseal_keys (context, db, &entry->entry); - if (code) + ret = hdb_unseal_keys (context, db, &entry->entry); + if (ret) { hdb_free_entry(context, entry); + return ret; + } } return 0; @@ -710,8 +767,48 @@ mdb_fetch_kvno(krb5_context context, HDB *db, krb5_const_principal principal, static krb5_error_code mdb_store(krb5_context context, HDB *db, unsigned flags, hdb_entry_ex *entry) { - krb5_set_error_message(context, EINVAL, "can't set principal in mdb"); - return EINVAL; + krb5_error_code ret; + krb5_storage *sp = NULL; + krb5_storage *spent = NULL; + krb5_data line = { 0, 0 }; + krb5_data kdb_ent = { 0, 0 }; + krb5_data key = { 0, 0 }; + ssize_t sz; + + sp = krb5_storage_emem(); + if (!sp) return ENOMEM; + ret = _hdb_set_master_key_usage(context, db, 0); /* MIT KDB uses KU 0 */ + ret = hdb_seal_keys(context, db, &entry->entry); + if (ret) return ret; + ret = entry2mit_string_int(context, sp, &entry->entry); + if (ret) goto out; + sz = krb5_storage_write(sp, "\n", 2); /* NUL-terminate */ + ret = ENOMEM; + if (sz == -1) goto out; + ret = krb5_storage_to_data(sp, &line); + if (ret) goto out; + + ret = ENOMEM; + spent = krb5_storage_emem(); + if (!spent) goto out; + ret = _hdb_mit_dump2mitdb_entry(context, line.data, spent); + if (ret) goto out; + ret = krb5_storage_to_data(spent, &kdb_ent); + if (ret) goto out; + ret = mdb_principal2key(context, entry->entry.principal, &key); + if (ret) goto out; + ret = mdb__put(context, db, 1, key, kdb_ent); + +out: + if (sp) + krb5_storage_free(sp); + if (spent) + krb5_storage_free(spent); + krb5_data_free(&line); + krb5_data_free(&kdb_ent); + krb5_data_free(&key); + + return ret; } static krb5_error_code @@ -729,25 +826,31 @@ static krb5_error_code mdb_open(krb5_context context, HDB *db, int flags, mode_t mode) { char *fn; + char *actual_fn; krb5_error_code ret; + struct stat st; asprintf(&fn, "%s.db", db->hdb_name); if (fn == NULL) { krb5_set_error_message(context, ENOMEM, "malloc: out of memory"); return ENOMEM; } - db->hdb_db = dbopen(fn, flags, mode, DB_BTREE, NULL); - free(fn); + if (stat(fn, &st) == 0) + actual_fn = fn; + else + actual_fn = db->hdb_name; + db->hdb_db = dbopen(actual_fn, flags, mode, DB_BTREE, NULL); if (db->hdb_db == NULL) { switch (errno) { #ifdef EFTYPE case EFTYPE: #endif case EINVAL: - db->hdb_db = dbopen(fn, flags, mode, DB_BTREE, NULL); + db->hdb_db = dbopen(actual_fn, flags, mode, DB_BTREE, NULL); } } + free(fn); /* try to open without .db extension */ if(db->hdb_db == NULL && errno == ENOENT) @@ -758,11 +861,16 @@ mdb_open(krb5_context context, HDB *db, int flags, mode_t mode) db->hdb_name, strerror(ret)); return ret; } - if((flags & O_ACCMODE) == O_RDONLY) - ret = hdb_check_db_format(context, db); - else +#if 0 + /* + * Don't do this -- MIT won't be able to handle the + * HDB_DB_FORMAT_ENTRY key. + */ + if ((flags & O_ACCMODE) != O_RDONLY) ret = hdb_init_db(context, db); - if(ret == HDB_ERR_NOENTRY) { +#endif + ret = hdb_check_db_format(context, db); + if (ret == HDB_ERR_NOENTRY) { krb5_clear_error_message(context); return 0; } @@ -815,3 +923,280 @@ hdb_mdb_create(krb5_context context, HDB **db, } #endif /* HAVE_DB1 */ + +/* +can have any number of princ stanzas. +format is as follows (only \n indicates newlines) +princ\t%d\t (%d is KRB5_KDB_V1_BASE_LENGTH, always 38) +%d\t (strlen of principal e.g. shadow/foo@ANDREW.CMU.EDU) +%d\t (number of tl_data) +%d\t (number of key data, e.g. how many keys for this user) +%d\t (extra data length) +%s\t (principal name) +%d\t (attributes) +%d\t (max lifetime, seconds) +%d\t (max renewable life, seconds) +%d\t (expiration, seconds since epoch or 2145830400 for never) +%d\t (password expiration, seconds, 0 for never) +%d\t (last successful auth, seconds since epoch) +%d\t (last failed auth, per above) +%d\t (failed auth count) +foreach tl_data 0 to number of tl_data - 1 as above + %d\t%d\t (data type, data length) + foreach tl_data 0 to length-1 + %02x (tl data contents[element n]) + except if tl_data length is 0 + %d (always -1) + \t +foreach key 0 to number of keys - 1 as above + %d\t%d\t (key data version, kvno) + foreach version 0 to key data version - 1 (a key or a salt) + %d\t%d\t(data type for this key, data length for this key) + foreach key data length 0 to length-1 + %02x (key data contents[element n]) + except if key_data length is 0 + %d (always -1) + \t +foreach extra data length 0 to length - 1 + %02x (extra data part) +unless no extra data + %d (always -1) +;\n + +*/ + +static char * +nexttoken(char **p) +{ + char *q; + do { + q = strsep(p, " \t"); + } while(q && *q == '\0'); + return q; +} + +static size_t +getdata(char **p, unsigned char *buf, size_t len) +{ + size_t i; + int v; + char *q = nexttoken(p); + i = 0; + while(*q && i < len) { + if(sscanf(q, "%02x", &v) != 1) + break; + buf[i++] = v; + q += 2; + } + return i; +} + +static int +getint(char **p) +{ + int val; + char *q = nexttoken(p); + sscanf(q, "%d", &val); + return val; +} + +static unsigned int +getuint(char **p) +{ + int val; + char *q = nexttoken(p); + sscanf(q, "%u", &val); + return val; +} + +#define KRB5_KDB_SALTTYPE_NORMAL 0 +#define KRB5_KDB_SALTTYPE_V4 1 +#define KRB5_KDB_SALTTYPE_NOREALM 2 +#define KRB5_KDB_SALTTYPE_ONLYREALM 3 +#define KRB5_KDB_SALTTYPE_SPECIAL 4 +#define KRB5_KDB_SALTTYPE_AFS3 5 + +#define CHECK_UINT(num) \ + if ((num) < 0 || (num) > INT_MAX) return EINVAL +#define CHECK_UINT16(num) \ + if ((num) < 0 || (num) > 1<<15) return EINVAL +#define CHECK_NUM(num, maxv) \ + if ((num) > (maxv)) return EINVAL + +/* + * This utility function converts an MIT dump entry to an MIT on-disk + * encoded entry, which can then be decoded with _hdb_mdb_value2entry(). + * This allows us to have a single decoding function (_hdb_mdb_value2entry), + * which makes the code cleaner (less code duplication), if a bit less + * efficient. It also will allow us to have a function to dump an HDB + * entry in MIT format so we can dump HDB into MIT format for rollback + * purposes. And that will allow us to write to MIT KDBs, again + * somewhat inefficiently, also for migration/rollback purposes. + */ +int +_hdb_mit_dump2mitdb_entry(krb5_context context, char *line, krb5_storage *sp) +{ + krb5_error_code ret = EINVAL; + char *p = line, *q; + char *princ; + ssize_t sz; + size_t i; + size_t princ_len; + unsigned int num_tl_data; + size_t num_key_data; + unsigned int attributes; + int tmp; + + krb5_storage_set_byteorder(sp, KRB5_STORAGE_BYTEORDER_LE); + + q = nexttoken(&p); + if (strcmp(q, "kdb5_util") == 0 || strcmp(q, "policy") == 0 || + strcmp(q, "princ") != 0) { + return -1; + } + if (getint(&p) != 38) + return EINVAL; +#define KDB_V1_BASE_LENGTH 38 + ret = krb5_store_int16(sp, KDB_V1_BASE_LENGTH); + if (ret) return ret; + + nexttoken(&p); /* length of principal */ + num_tl_data = getuint(&p); /* number of tl-data */ + num_key_data = getuint(&p); /* number of key-data */ + getint(&p); /* length of extra data */ + princ = nexttoken(&p); /* principal name */ + + attributes = getuint(&p); /* attributes */ + ret = krb5_store_uint32(sp, attributes); + if (ret) return ret; + + tmp = getint(&p); /* max life */ + CHECK_UINT(tmp); + ret = krb5_store_uint32(sp, tmp); + if (ret) return ret; + + tmp = getint(&p); /* max renewable life */ + CHECK_UINT(tmp); + ret = krb5_store_uint32(sp, tmp); + if (ret) return ret; + + tmp = getint(&p); /* expiration */ + CHECK_UINT(tmp); + ret = krb5_store_uint32(sp, tmp); + if (ret) return ret; + + tmp = getint(&p); /* pw expiration */ + CHECK_UINT(tmp); + ret = krb5_store_uint32(sp, tmp); + if (ret) return ret; + + tmp = getint(&p); /* last auth */ + CHECK_UINT(tmp); + ret = krb5_store_uint32(sp, tmp); + if (ret) return ret; + + tmp = getint(&p); /* last failed auth */ + CHECK_UINT(tmp); + ret = krb5_store_uint32(sp, tmp); + if (ret) return ret; + + tmp = getint(&p); /* fail auth count */ + CHECK_UINT(tmp); + ret = krb5_store_uint32(sp, tmp); + if (ret) return ret; + + /* add TL data count */ + CHECK_NUM(num_tl_data, 1023); + ret = krb5_store_uint16(sp, num_tl_data); + if (ret) return ret; + + /* add key count */ + CHECK_NUM(num_key_data, 1023); + ret = krb5_store_uint16(sp, num_key_data); + if (ret) return ret; + + /* add principal unparsed name length and unparsed name */ + princ_len = strlen(princ); + if (princ_len > (1<<15) - 1) return EINVAL; + princ_len++; /* must count and write the NUL in the on-disk encoding */ + ret = krb5_store_uint16(sp, princ_len); + if (ret) return ret; + sz = krb5_storage_write(sp, princ, princ_len); + if (sz == -1) return ENOMEM; + + /* scan and write TL data */ + for (i = 0; i < num_tl_data; i++) { + int tl_type, tl_length; + unsigned char *buf; + + tl_type = getint(&p); /* data type */ + tl_length = getint(&p); /* data length */ + + CHECK_UINT16(tl_type); + ret = krb5_store_uint16(sp, tl_type); + if (ret) return ret; + CHECK_UINT16(tl_length); + ret = krb5_store_uint16(sp, tl_length); + if (ret) return ret; + + if (tl_length) { + buf = malloc(tl_length); + if (!buf) return ENOMEM; + if (getdata(&p, buf, tl_length) != tl_length) return EINVAL; + sz = krb5_storage_write(sp, buf, tl_length); + free(buf); + if (sz == -1) return ENOMEM; + } else { + if (strcmp(nexttoken(&p), "-1") != 0) return EINVAL; + } + } + + for (i = 0; i < num_key_data; i++) { + unsigned char *buf; + int key_versions; + int kvno; + int keytype; + int keylen; + size_t k; + + key_versions = getint(&p); /* key data version */ + CHECK_UINT16(key_versions); + ret = krb5_store_int16(sp, key_versions); + if (ret) return ret; + + kvno = getint(&p); + CHECK_UINT16(kvno); + ret = krb5_store_int16(sp, kvno); + if (ret) return ret; + + for (k = 0; k < key_versions; k++) { + keytype = getint(&p); + CHECK_UINT16(keytype); + ret = krb5_store_int16(sp, keytype); + if (ret) return ret; + + keylen = getint(&p); + CHECK_UINT16(keylen); + ret = krb5_store_int16(sp, keylen); + if (ret) return ret; + + if (keylen) { + buf = malloc(keylen); + if (!buf) return ENOMEM; + if (getdata(&p, buf, keylen) != keylen) return EINVAL; + sz = krb5_storage_write(sp, buf, keylen); + free(buf); + if (sz == -1) return ENOMEM; + } else { + if (strcmp(nexttoken(&p), "-1") != 0) return EINVAL; + } + } + } + /* + * The rest is "extra data", but there's never any and we wouldn't + * know what to do with it. + */ + /* nexttoken(&p); */ + return 0; +} + diff --git a/crypto/heimdal/lib/hdb/hdb-protos.h b/crypto/heimdal/lib/hdb/hdb-protos.h index 44a1bddc7625..2b692855c405 100644 --- a/crypto/heimdal/lib/hdb/hdb-protos.h +++ b/crypto/heimdal/lib/hdb/hdb-protos.h @@ -9,6 +9,17 @@ extern "C" { #endif krb5_error_code +entry2mit_string_int ( + krb5_context /*context*/, + krb5_storage */*sp*/, + hdb_entry */*ent*/); + +krb5_error_code +hdb_add_current_keys_to_history ( + krb5_context /*context*/, + hdb_entry */*entry*/); + +krb5_error_code hdb_add_master_key ( krb5_context /*context*/, krb5_keyblock */*key*/, @@ -347,6 +358,13 @@ hdb_seal_keys_mkey ( hdb_master_key /*mkey*/); krb5_error_code +hdb_set_last_modified_by ( + krb5_context /*context*/, + hdb_entry */*entry*/, + krb5_principal /*modby*/, + time_t /*modtime*/); + +krb5_error_code hdb_set_master_key ( krb5_context /*context*/, HDB */*db*/, @@ -386,6 +404,13 @@ hdb_unseal_keys ( hdb_entry */*ent*/); krb5_error_code +hdb_unseal_keys_kvno ( + krb5_context /*context*/, + HDB */*db*/, + krb5_kvno /*kvno*/, + hdb_entry */*ent*/); + +krb5_error_code hdb_unseal_keys_mkey ( krb5_context /*context*/, hdb_entry */*ent*/, diff --git a/crypto/heimdal/lib/hdb/hdb.asn1 b/crypto/heimdal/lib/hdb/hdb.asn1 index a72851c9f201..d24737fbf6e8 100644 --- a/crypto/heimdal/lib/hdb/hdb.asn1 +++ b/crypto/heimdal/lib/hdb/hdb.asn1 @@ -87,6 +87,14 @@ HDB-Ext-Aliases ::= SEQUENCE { aliases[1] SEQUENCE OF Principal -- all names, inc primary } +hdb_keyset ::= SEQUENCE { + kvno[0] INTEGER (0..4294967295), + replace-time[1] KerberosTime, -- time this key was replaced + keys[2] SEQUENCE OF Key +} + +HDB-Ext-KeySet ::= SEQUENCE OF hdb_keyset + HDB-extension ::= SEQUENCE { mandatory[0] BOOLEAN, -- kdc MUST understand this extension, @@ -102,6 +110,7 @@ HDB-extension ::= SEQUENCE { aliases[6] HDB-Ext-Aliases, last-pw-change[7] KerberosTime, pkinit-cert[8] HDB-Ext-PKINIT-cert, + hist-keys[9] HDB-Ext-KeySet, ... }, ... @@ -109,11 +118,6 @@ HDB-extension ::= SEQUENCE { HDB-extensions ::= SEQUENCE OF HDB-extension -hdb_keyset ::= SEQUENCE { - kvno[1] INTEGER (0..4294967295), - keys[0] SEQUENCE OF Key -} - hdb_entry ::= SEQUENCE { principal[0] Principal OPTIONAL, -- this is optional only -- for compatibility with libkrb5 diff --git a/crypto/heimdal/lib/hdb/hdb.c b/crypto/heimdal/lib/hdb/hdb.c index ca05cc4a1785..aff576a30d37 100644 --- a/crypto/heimdal/lib/hdb/hdb.c +++ b/crypto/heimdal/lib/hdb/hdb.c @@ -168,13 +168,14 @@ hdb_unlock(int fd) void hdb_free_entry(krb5_context context, hdb_entry_ex *ent) { - size_t i; + Key *k; + int i; if (ent->free_entry) (*ent->free_entry)(context, ent); - for(i = 0; i < ent->entry.keys.len; ++i) { - Key *k = &ent->entry.keys.val[i]; + for(i = 0; i < ent->entry.keys.len; i++) { + k = &ent->entry.keys.val[i]; memset (k->key.keyvalue.data, 0, k->key.keyvalue.length); } diff --git a/crypto/heimdal/lib/hdb/hdb.h b/crypto/heimdal/lib/hdb/hdb.h index a1692ce82ca2..4c4c2c2b1a5c 100644 --- a/crypto/heimdal/lib/hdb/hdb.h +++ b/crypto/heimdal/lib/hdb/hdb.h @@ -99,7 +99,7 @@ typedef struct hdb_entry_ex { * query the backend database when talking about principals. */ -typedef struct HDB{ +typedef struct HDB { void *hdb_db; void *hdb_dbc; /** don't use, only for DB3 */ char *hdb_name; @@ -256,6 +256,8 @@ typedef struct HDB{ * Check if s4u2self is allowed from this client to this server */ krb5_error_code (*hdb_check_s4u2self)(krb5_context, struct HDB *, hdb_entry_ex *, krb5_const_principal); + int hdb_mit_key_set; + hdb_master_key hdb_mit_key; }HDB; #define HDB_INTERFACE_VERSION 7 @@ -266,6 +268,17 @@ struct hdb_so_method { krb5_error_code (*create)(krb5_context, HDB **, const char *filename); }; +/* dump entry format, for hdb_print_entry() */ +typedef enum hdb_dump_format { + HDB_DUMP_HEIMDAL = 0, + HDB_DUMP_MIT = 1, +} hdb_dump_format_t; + +struct hdb_print_entry_arg { + FILE *out; + hdb_dump_format_t fmt; +}; + typedef krb5_error_code (*hdb_foreach_func_t)(krb5_context, HDB*, hdb_entry_ex*, void*); extern krb5_kt_ops hdb_kt_ops; diff --git a/crypto/heimdal/lib/hdb/hdb_locl.h b/crypto/heimdal/lib/hdb/hdb_locl.h index e896b5802575..5aad504589df 100644 --- a/crypto/heimdal/lib/hdb/hdb_locl.h +++ b/crypto/heimdal/lib/hdb/hdb_locl.h @@ -38,6 +38,8 @@ #include <config.h> +#include <heimbase.h> + #include <stdio.h> #include <string.h> #include <stdlib.h> @@ -67,4 +69,9 @@ #define HDB_DEFAULT_DB HDB_DB_DIR "/heimdal" #define HDB_DB_FORMAT_ENTRY "hdb/db-format" +/* Test for strong key etypes accepted by MIT's KDC. */ +#define mit_strong_etype(t) \ + ((t) == ETYPE_AES128_CTS_HMAC_SHA1_96 || \ + (t) == ETYPE_AES256_CTS_HMAC_SHA1_96) + #endif /* __HDB_LOCL_H__ */ diff --git a/crypto/heimdal/lib/hdb/keys.c b/crypto/heimdal/lib/hdb/keys.c index 3d0b9d7c1b31..9c0af5c47f23 100644 --- a/crypto/heimdal/lib/hdb/keys.c +++ b/crypto/heimdal/lib/hdb/keys.c @@ -39,7 +39,7 @@ */ void -hdb_free_keys (krb5_context context, int len, Key *keys) +hdb_free_keys(krb5_context context, int len, Key *keys) { int i; @@ -56,6 +56,19 @@ hdb_free_keys (krb5_context context, int len, Key *keys) free (keys); } +void +hdb_free_keysets(krb5_context context, int len, hdb_keyset *keysets) +{ + int i; + + for (i = 0; i < len; i++) { + hdb_free_keys(context, keysets[i].keys.len, keysets[i].keys.val); + keysets[i].keys.val = NULL; + keysets[i].keys.len = 0; + } + free (keysets); +} + /* * for each entry in `default_keys' try to parse it as a sequence * of etype:salttype:salt, syntax of this if something like: @@ -196,6 +209,60 @@ parse_key_set(krb5_context context, const char *key, return 0; } + +krb5_error_code +hdb_add_current_keys_to_history(krb5_context context, hdb_entry *entry) +{ + krb5_error_code ret; + HDB_extension *ext; + HDB_Ext_KeySet *hist_keys; + hdb_keyset *tmp_keysets; + int add = 0; + + ext = hdb_find_extension(entry, choice_HDB_extension_data_hist_keys); + if (ext != NULL) { + hist_keys = &ext->data.u.hist_keys; + tmp_keysets = realloc(hist_keys->val, + sizeof (*hist_keys->val) * (hist_keys->len + 1)); + if (tmp_keysets == NULL) + return ENOMEM; + hist_keys->val = tmp_keysets; + memmove(&hist_keys->val[1], hist_keys->val, + sizeof (*hist_keys->val) * hist_keys->len++); + } else { + add = 1; + ext = calloc(1, sizeof (*ext)); + if (ext == NULL) + return ENOMEM; + ext->data.element = choice_HDB_extension_data_hist_keys; + hist_keys = &ext->data.u.hist_keys; + hist_keys->val = calloc(1, sizeof (*hist_keys->val)); + if (hist_keys->val == NULL) { + free(hist_keys); + return ENOMEM; + } + hist_keys->len = 1; + } + + hist_keys->val[0].keys.val = entry->keys.val; + hist_keys->val[0].keys.len = entry->keys.len; + hist_keys->val[0].kvno = entry->kvno; + hist_keys->val[0].replace_time = time(NULL); + + if (add) { + ret = hdb_replace_extension(context, entry, ext); + if (ret) { + free_HDB_extension(ext); + return ret; + } + } + + /* hdb_replace_extension() copies ext, so we have to free it */ + free_HDB_extension(ext); + return 0; +} + + static krb5_error_code add_enctype_to_key_set(Key **key_set, size_t *nkeyset, krb5_enctype enctype, krb5_salt *salt) diff --git a/crypto/heimdal/lib/hdb/mkey.c b/crypto/heimdal/lib/hdb/mkey.c index 9eb98fca32c0..78a9e51a132c 100644 --- a/crypto/heimdal/lib/hdb/mkey.c +++ b/crypto/heimdal/lib/hdb/mkey.c @@ -32,6 +32,7 @@ */ #include "hdb_locl.h" +#include <assert.h> #ifndef O_BINARY #define O_BINARY 0 #endif @@ -40,6 +41,7 @@ struct hdb_master_key_data { krb5_keytab_entry keytab; krb5_crypto crypto; struct hdb_master_key_data *next; + unsigned int key_usage; }; void @@ -68,6 +70,7 @@ hdb_process_master_key(krb5_context context, krb5_set_error_message(context, ENOMEM, "malloc: out of memory"); return ENOMEM; } + (*mkey)->key_usage = HDB_KU_MKEY; (*mkey)->keytab.vno = kvno; ret = krb5_parse_name(context, "K/M", &(*mkey)->keytab.principal); if(ret) @@ -362,6 +365,15 @@ hdb_write_master_key(krb5_context context, const char *filename, return ret; } +krb5_error_code +_hdb_set_master_key_usage(krb5_context context, HDB *db, unsigned int key_usage) +{ + if (db->hdb_master_key_set == 0) + return HDB_ERR_NO_MKEY; + db->hdb_master_key->key_usage = key_usage; + return 0; +} + hdb_master_key _hdb_find_master_key(uint32_t *mkvno, hdb_master_key mkey) { @@ -403,15 +415,20 @@ _hdb_mkey_encrypt(krb5_context context, hdb_master_key key, ptr, size, res); } -krb5_error_code -hdb_unseal_key_mkey(krb5_context context, Key *k, hdb_master_key mkey) +/* + * Unseal and optionally reseal the key in the MIT KDC master key. + * If mit_key != NULL, the key is sealed using this key. + */ +static krb5_error_code +_hdb_reseal_key_mkey(krb5_context context, Key *k, hdb_master_key mkey, + hdb_master_key mit_key) { krb5_error_code ret; - krb5_data res; + krb5_data mitres, res; size_t keysize; - hdb_master_key key; + hdb_master_key key, mitkey; if(k->mkvno == NULL) return 0; @@ -428,9 +445,9 @@ hdb_unseal_key_mkey(krb5_context context, Key *k, hdb_master_key mkey) if(ret == KRB5KRB_AP_ERR_BAD_INTEGRITY) { /* try to decrypt with MIT key usage */ ret = _hdb_mkey_decrypt(context, key, 0, - k->key.keyvalue.data, - k->key.keyvalue.length, - &res); + k->key.keyvalue.data, + k->key.keyvalue.length, + &res); } if (ret) return ret; @@ -446,25 +463,81 @@ hdb_unseal_key_mkey(krb5_context context, Key *k, hdb_master_key mkey) return KRB5_BAD_KEYSIZE; } - memset(k->key.keyvalue.data, 0, k->key.keyvalue.length); - free(k->key.keyvalue.data); - k->key.keyvalue = res; - k->key.keyvalue.length = keysize; - free(k->mkvno); - k->mkvno = NULL; + /* For mit_key != NULL, re-encrypt the key using the mitkey. */ + if (mit_key != NULL) { + mitkey = _hdb_find_master_key(NULL, mit_key); + if (mitkey == NULL) { + krb5_data_free(&res); + return HDB_ERR_NO_MKEY; + } + + ret = _hdb_mkey_encrypt(context, mitkey, 0, + res.data, + keysize, + &mitres); + krb5_data_free(&res); + if (ret) + return ret; + } + + krb5_data_free(&k->key.keyvalue); + if (mit_key == NULL) { + k->key.keyvalue = res; + k->key.keyvalue.length = keysize; + free(k->mkvno); + k->mkvno = NULL; + } else { + k->key.keyvalue = mitres; + *k->mkvno = mitkey->keytab.vno; + } return 0; } krb5_error_code -hdb_unseal_keys_mkey(krb5_context context, hdb_entry *ent, hdb_master_key mkey) +hdb_unseal_key_mkey(krb5_context context, Key *k, hdb_master_key mkey) +{ + + krb5_error_code ret; + + ret = _hdb_reseal_key_mkey(context, k, mkey, NULL); + return ret; +} + +static krb5_error_code +_hdb_unseal_keys_mkey(krb5_context context, hdb_entry *ent, hdb_master_key mkey, + hdb_master_key mitkey) { + krb5_error_code ret; size_t i; + int got_one = 0; for(i = 0; i < ent->keys.len; i++){ - krb5_error_code ret; + if (mitkey == NULL || mit_strong_etype(ent->keys.val[i].key.keytype)) { + ret = _hdb_reseal_key_mkey(context, &ent->keys.val[i], mkey, + mitkey); + if (ret) + return ret; + got_one = 1; + } + } - ret = hdb_unseal_key_mkey(context, &ent->keys.val[i], mkey); + /* + * If none of the keys were string enough, create a strong key, + * but one that is not encrypted in the MIT master key. As such, + * it will require a "change_password" once in the MIT KDC to + * make it work. + */ + if (got_one == 0 && mitkey != NULL && ent->keys.len > 0) { + krb5_keyblock key; + krb5_salt salt; + + krb5_free_keyblock_contents(context, &ent->keys.val[0].key); + salt.salttype = KRB5_PW_SALT; + salt.saltvalue.data = NULL; + salt.saltvalue.length = 0; + ret = krb5_string_to_key_salt(context, ETYPE_AES256_CTS_HMAC_SHA1_96, + "XXXX", salt, &ent->keys.val[0].key); if (ret) return ret; } @@ -472,19 +545,114 @@ hdb_unseal_keys_mkey(krb5_context context, hdb_entry *ent, hdb_master_key mkey) } krb5_error_code +hdb_unseal_keys_mkey(krb5_context context, hdb_entry *ent, hdb_master_key mkey) +{ + krb5_error_code ret; + + ret = _hdb_unseal_keys_mkey(context, ent, mkey, NULL); + return ret; +} + +krb5_error_code hdb_unseal_keys(krb5_context context, HDB *db, hdb_entry *ent) { if (db->hdb_master_key_set == 0) return 0; - return hdb_unseal_keys_mkey(context, ent, db->hdb_master_key); + if (db->hdb_mit_key_set != 0) + return _hdb_unseal_keys_mkey(context, ent, db->hdb_master_key, + db->hdb_mit_key); + else + return _hdb_unseal_keys_mkey(context, ent, db->hdb_master_key, + NULL); +} + +#ifdef notnow +krb5_error_code +hdb_unseal_keys_kvno(krb5_context context, HDB *db, krb5_kvno kvno, + hdb_entry *ent) +{ + krb5_error_code ret = KRB5KRB_AP_ERR_NOKEY; /* XXX need a better code? */ + HDB_extension *tmp; + HDB_Ext_KeySet *hist_keys; + hdb_keyset *tmp_keys; + Key *tmp_val; + unsigned int tmp_len; + krb5_kvno tmp_kvno; + int i, k; + + assert(kvno == 0 || kvno < ent->kvno); + + tmp = hdb_find_extension(ent, choice_HDB_extension_data_hist_keys); + if (tmp == NULL) + return ret; + + tmp_len = ent->keys.len; + tmp_val = ent->keys.val; + tmp_kvno = ent->kvno; + + hist_keys = &tmp->data.u.hist_keys; + + for (i = hist_keys->len - 1; i >= 0; i++) { + if (kvno != 0 && hist_keys->val[i].kvno != kvno) + continue; + for (k = 0; k < hist_keys->val[i].keys.len; k++) { + ret = _hdb_reseal_key_mkey(context, + &hist_keys->val[i].keys.val[k], + db->hdb_master_key, NULL); + if (ret) + return (ret); + } + + if (kvno == 0) + continue; + + /* + * NOTE: What follows is a bit of an ugly hack. + * + * This is the keyset we're being asked for, so we add the + * current keyset to the history, leave the one we were asked + * for in the history, and pretend the one we were asked for is + * also the current keyset. + * + * This is a bit of a defensive hack in case an entry fetched + * this way ever gets modified then stored: if the keyset is not + * changed we can detect this and put things back, else we won't + * drop any keysets from history by accident. + * + * Note too that we only ever get called with a non-zero kvno + * either in the KDC or in cases where we aren't changing the + * HDB entry anyways, which is why this is just a defensive + * hack. We also don't fetch specific kvnos in the dump case, + * so there's no danger that we'll dump this entry and load it + * again, repeatedly causing the history to grow boundelessly. + */ + tmp_keys = realloc(hist_keys->val, + sizeof (*hist_keys->val) * (hist_keys->len + 1)); + if (tmp_keys == NULL) + return ENOMEM; + + memmove(&tmp_keys[1], tmp_keys, + sizeof (*hist_keys->val) * hist_keys->len++); + tmp_keys[0].keys.len = ent->keys.len; + tmp_keys[0].keys.val = ent->keys.val; + tmp_keys[0].kvno = ent->kvno; + tmp_keys[0].replace_time = time(NULL); + i++; + ent->keys.len = hist_keys->val[i].keys.len; + ent->keys.val = hist_keys->val[i].keys.val; + ent->kvno = kvno; + } + + return (ret); } +#endif krb5_error_code hdb_unseal_key(krb5_context context, HDB *db, Key *k) { if (db->hdb_master_key_set == 0) return 0; - return hdb_unseal_key_mkey(context, k, db->hdb_master_key); + return _hdb_reseal_key_mkey(context, k, db->hdb_master_key, NULL); } krb5_error_code @@ -556,9 +724,9 @@ hdb_seal_key(krb5_context context, HDB *db, Key *k) } krb5_error_code -hdb_set_master_key (krb5_context context, - HDB *db, - krb5_keyblock *key) +hdb_set_master_key(krb5_context context, + HDB *db, + krb5_keyblock *key) { krb5_error_code ret; hdb_master_key mkey; @@ -571,6 +739,7 @@ hdb_set_master_key (krb5_context context, des_set_random_generator_seed(key.keyvalue.data); #endif db->hdb_master_key_set = 1; + db->hdb_master_key->key_usage = HDB_KU_MKEY; return 0; } diff --git a/crypto/heimdal/lib/hdb/print.c b/crypto/heimdal/lib/hdb/print.c index 697d32d2909c..d5359f538056 100644 --- a/crypto/heimdal/lib/hdb/print.c +++ b/crypto/heimdal/lib/hdb/print.c @@ -57,44 +57,57 @@ generation number */ -static krb5_error_code +/* + * These utility functions return the number of bytes written or -1, and + * they set an error in the context. + */ +static ssize_t append_string(krb5_context context, krb5_storage *sp, const char *fmt, ...) { - krb5_error_code ret; + ssize_t sz; char *s; + int rc; va_list ap; va_start(ap, fmt); - vasprintf(&s, fmt, ap); + rc = vasprintf(&s, fmt, ap); va_end(ap); - if(s == NULL) { + if(rc < 0) { krb5_set_error_message(context, ENOMEM, "malloc: out of memory"); - return ENOMEM; + return -1; } - ret = krb5_storage_write(sp, s, strlen(s)); + sz = krb5_storage_write(sp, s, strlen(s)); free(s); - return ret; + return sz; } static krb5_error_code -append_hex(krb5_context context, krb5_storage *sp, krb5_data *data) +append_hex(krb5_context context, krb5_storage *sp, + int always_encode, int lower, krb5_data *data) { + ssize_t sz; int printable = 1; size_t i; char *p; p = data->data; - for(i = 0; i < data->length; i++) - if(!isalnum((unsigned char)p[i]) && p[i] != '.'){ - printable = 0; - break; - } - if(printable) + if (!always_encode) { + for (i = 0; i < data->length; i++) { + if (!isalnum((unsigned char)p[i]) && p[i] != '.'){ + printable = 0; + break; + } + } + } + if (printable && !always_encode) return append_string(context, sp, "\"%.*s\"", data->length, data->data); - hex_encode(data->data, data->length, &p); - append_string(context, sp, "%s", p); + sz = hex_encode(data->data, data->length, &p); + if (sz == -1) return sz; + if (lower) + strlwr(p); + sz = append_string(context, sp, "%s", p); free(p); - return 0; + return sz; } static char * @@ -105,29 +118,97 @@ time2str(time_t t) return buf; } -static krb5_error_code +static ssize_t append_event(krb5_context context, krb5_storage *sp, Event *ev) { - char *pr = NULL; krb5_error_code ret; + ssize_t sz; + char *pr = NULL; if(ev == NULL) return append_string(context, sp, "- "); if (ev->principal != NULL) { ret = krb5_unparse_name(context, ev->principal, &pr); - if(ret) - return ret; + if (ret) return -1; /* krb5_unparse_name() sets error info */ } - ret = append_string(context, sp, "%s:%s ", - time2str(ev->time), pr ? pr : "UNKNOWN"); + sz = append_string(context, sp, "%s:%s ", time2str(ev->time), + pr ? pr : "UNKNOWN"); free(pr); - return ret; + return sz; +} + +#define KRB5_KDB_SALTTYPE_NORMAL 0 +#define KRB5_KDB_SALTTYPE_V4 1 +#define KRB5_KDB_SALTTYPE_NOREALM 2 +#define KRB5_KDB_SALTTYPE_ONLYREALM 3 +#define KRB5_KDB_SALTTYPE_SPECIAL 4 +#define KRB5_KDB_SALTTYPE_AFS3 5 + +static ssize_t +append_mit_key(krb5_context context, krb5_storage *sp, + krb5_const_principal princ, + unsigned int kvno, Key *key) +{ + krb5_error_code ret; + ssize_t sz; + size_t key_versions = key->salt ? 2 : 1; + size_t decrypted_key_length; + char buf[2]; + krb5_data keylenbytes; + unsigned int salttype; + + sz = append_string(context, sp, "\t%u\t%u\t%d\t%d\t", key_versions, kvno, + key->key.keytype, key->key.keyvalue.length + 2); + if (sz == -1) return sz; + ret = krb5_enctype_keysize(context, key->key.keytype, &decrypted_key_length); + if (ret) return -1; /* XXX we lose the error code */ + buf[0] = decrypted_key_length & 0xff; + buf[1] = (decrypted_key_length & 0xff00) >> 8; + keylenbytes.data = buf; + keylenbytes.length = sizeof (buf); + sz = append_hex(context, sp, 1, 1, &keylenbytes); + if (sz == -1) return sz; + sz = append_hex(context, sp, 1, 1, &key->key.keyvalue); + if (!key->salt) + return sz; + + /* Map salt to MIT KDB style */ + if (key->salt->type == KRB5_PADATA_PW_SALT) { + krb5_salt k5salt; + + /* + * Compute normal salt and then see whether it matches the stored one + */ + ret = krb5_get_pw_salt(context, princ, &k5salt); + if (ret) return -1; + if (k5salt.saltvalue.length == key->salt->salt.length && + memcmp(k5salt.saltvalue.data, key->salt->salt.data, + k5salt.saltvalue.length) == 0) + salttype = KRB5_KDB_SALTTYPE_NORMAL; /* matches */ + else if (key->salt->salt.length == strlen(princ->realm) && + memcmp(key->salt->salt.data, princ->realm, + key->salt->salt.length) == 0) + salttype = KRB5_KDB_SALTTYPE_ONLYREALM; /* matches realm */ + else if (key->salt->salt.length == k5salt.saltvalue.length - strlen(princ->realm) && + memcmp((char *)k5salt.saltvalue.data + strlen(princ->realm), + key->salt->salt.data, key->salt->salt.length) == 0) + salttype = KRB5_KDB_SALTTYPE_NOREALM; /* matches w/o realm */ + else + salttype = KRB5_KDB_SALTTYPE_NORMAL; /* hope for best */ + + } else if (key->salt->type == KRB5_PADATA_AFS3_SALT) { + salttype = KRB5_KDB_SALTTYPE_AFS3; + } + sz = append_string(context, sp, "\t%u\t%u\t", salttype, + key->salt->salt.length); + if (sz == -1) return sz; + return append_hex(context, sp, 1, 1, &key->salt->salt); } static krb5_error_code entry2string_int (krb5_context context, krb5_storage *sp, hdb_entry *ent) { char *p; - size_t i; + int i; krb5_error_code ret; /* --- principal */ @@ -149,12 +230,12 @@ entry2string_int (krb5_context context, krb5_storage *sp, hdb_entry *ent) append_string(context, sp, "::%d:", ent->keys.val[i].key.keytype); /* --- keydata */ - append_hex(context, sp, &ent->keys.val[i].key.keyvalue); + append_hex(context, sp, 0, 0, &ent->keys.val[i].key.keyvalue); append_string(context, sp, ":"); /* --- salt */ if(ent->keys.val[i].salt){ append_string(context, sp, "%u/", ent->keys.val[i].salt->type); - append_hex(context, sp, &ent->keys.val[i].salt->salt); + append_hex(context, sp, 0, 0, &ent->keys.val[i].salt->salt); }else append_string(context, sp, "-"); } @@ -234,25 +315,266 @@ entry2string_int (krb5_context context, krb5_storage *sp, hdb_entry *ent) } else append_string(context, sp, "-"); + return 0; +} + +#define KRB5_KDB_DISALLOW_POSTDATED 0x00000001 +#define KRB5_KDB_DISALLOW_FORWARDABLE 0x00000002 +#define KRB5_KDB_DISALLOW_TGT_BASED 0x00000004 +#define KRB5_KDB_DISALLOW_RENEWABLE 0x00000008 +#define KRB5_KDB_DISALLOW_PROXIABLE 0x00000010 +#define KRB5_KDB_DISALLOW_DUP_SKEY 0x00000020 +#define KRB5_KDB_DISALLOW_ALL_TIX 0x00000040 +#define KRB5_KDB_REQUIRES_PRE_AUTH 0x00000080 +#define KRB5_KDB_REQUIRES_HW_AUTH 0x00000100 +#define KRB5_KDB_REQUIRES_PWCHANGE 0x00000200 +#define KRB5_KDB_DISALLOW_SVR 0x00001000 +#define KRB5_KDB_PWCHANGE_SERVICE 0x00002000 +#define KRB5_KDB_SUPPORT_DESMD5 0x00004000 +#define KRB5_KDB_NEW_PRINC 0x00008000 + +static int +flags_to_attr(HDBFlags flags) +{ + int a = 0; + + if (!flags.postdate) + a |= KRB5_KDB_DISALLOW_POSTDATED; + if (!flags.forwardable) + a |= KRB5_KDB_DISALLOW_FORWARDABLE; + if (flags.initial) + a |= KRB5_KDB_DISALLOW_TGT_BASED; + if (!flags.renewable) + a |= KRB5_KDB_DISALLOW_RENEWABLE; + if (!flags.proxiable) + a |= KRB5_KDB_DISALLOW_PROXIABLE; + if (flags.invalid) + a |= KRB5_KDB_DISALLOW_ALL_TIX; + if (flags.require_preauth) + a |= KRB5_KDB_REQUIRES_PRE_AUTH; + if (flags.require_hwauth) + a |= KRB5_KDB_REQUIRES_HW_AUTH; + if (!flags.server) + a |= KRB5_KDB_DISALLOW_SVR; + if (flags.change_pw) + a |= KRB5_KDB_PWCHANGE_SERVICE; + return a; +} +krb5_error_code +entry2mit_string_int(krb5_context context, krb5_storage *sp, hdb_entry *ent) +{ + krb5_error_code ret; + ssize_t sz; + size_t i, k; + size_t num_tl_data = 0; + size_t num_key_data = 0; + char *p; + HDB_Ext_KeySet *hist_keys = NULL; + HDB_extension *extp; + time_t last_pw_chg = 0; + time_t exp = 0; + time_t pwexp = 0; + unsigned int max_life = 0; + unsigned int max_renew = 0; + + /* Always create a modified_by entry. */ + num_tl_data++; + + ret = hdb_entry_get_pw_change_time(ent, &last_pw_chg); + if (ret) return ret; + if (last_pw_chg) + num_tl_data++; + + extp = hdb_find_extension(ent, choice_HDB_extension_data_hist_keys); + if (extp) + hist_keys = &extp->data.u.hist_keys; + + for (i = 0; i < ent->keys.len;i++) { + if (!mit_strong_etype(ent->keys.val[i].key.keytype)) + continue; + num_key_data++; + } + if (hist_keys) { + for (i = 0; i < hist_keys->len; i++) { + /* + * MIT uses the highest kvno as the current kvno instead of + * tracking kvno separately, so we can't dump keysets with kvno + * higher than the entry's kvno. + */ + if (hist_keys->val[i].kvno >= ent->kvno) + continue; + for (k = 0; k < hist_keys->val[i].keys.len; k++) { + if (ent->keys.val[k].key.keytype == ETYPE_DES_CBC_MD4 || + ent->keys.val[k].key.keytype == ETYPE_DES_CBC_MD5) + continue; + num_key_data++; + } + } + } + + ret = krb5_unparse_name(context, ent->principal, &p); + if (ret) return ret; + sz = append_string(context, sp, "princ\t38\t%u\t%u\t%u\t0\t%s\t%d", + strlen(p), num_tl_data, num_key_data, p, + flags_to_attr(ent->flags)); + if (sz == -1) { + free(p); + return ENOMEM; + } + + if (ent->max_life) + max_life = *ent->max_life; + if (ent->max_renew) + max_renew = *ent->max_renew; + if (ent->valid_end) + exp = *ent->valid_end; + if (ent->pw_end) + pwexp = *ent->pw_end; + + sz = append_string(context, sp, "\t%u\t%u\t%u\t%u\t0\t0\t0", + max_life, max_renew, exp, pwexp); + if (sz == -1) { + free(p); + return ENOMEM; + } + + /* Dump TL data we know: last pw chg and modified_by */ +#define mit_KRB5_TL_LAST_PWD_CHANGE 1 +#define mit_KRB5_TL_MOD_PRINC 2 + if (last_pw_chg) { + krb5_data d; + time_t val; + unsigned char *ptr; + + ptr = (unsigned char *)&last_pw_chg; + val = ptr[0] | (ptr[1] << 8) | (ptr[2] << 16) | (ptr[3] << 24); + d.data = &val; + d.length = sizeof (last_pw_chg); + sz = append_string(context, sp, "\t%u\t%u\t", + mit_KRB5_TL_LAST_PWD_CHANGE, d.length); + if (sz == -1) { + free(p); + return ENOMEM; + } + sz = append_hex(context, sp, 1, 1, &d); + if (sz == -1) { + free(p); + return ENOMEM; + } + } + if (ent->modified_by) { + krb5_data d; + unsigned int val; + size_t plen; + unsigned char *ptr; + char *modby_p; + + free(p); + ptr = (unsigned char *)&ent->modified_by->time; + val = ptr[0] | (ptr[1] << 8) | (ptr[2] << 16) | (ptr[3] << 24); + d.data = &val; + d.length = sizeof (ent->modified_by->time); + ret = krb5_unparse_name(context, ent->modified_by->principal, &modby_p); + if (ret) return ret; + plen = strlen(modby_p); + sz = append_string(context, sp, "\t%u\t%u\t", + mit_KRB5_TL_MOD_PRINC, + d.length + plen + 1 /* NULL counted */); + if (sz == -1) { + free(modby_p); + return ENOMEM; + } + sz = append_hex(context, sp, 1, 1, &d); + if (sz == -1) { + free(modby_p); + return ENOMEM; + } + d.data = modby_p; + d.length = plen + 1; + sz = append_hex(context, sp, 1, 1, &d); + free(modby_p); + if (sz == -1) return ENOMEM; + } else { + krb5_data d; + unsigned int val; + size_t plen; + unsigned char *ptr; + + /* Fake the entry to make MIT happy. */ + ptr = (unsigned char *)&last_pw_chg; + val = ptr[0] | (ptr[1] << 8) | (ptr[2] << 16) | (ptr[3] << 24); + d.data = &val; + d.length = sizeof (last_pw_chg); + plen = strlen(p); + sz = append_string(context, sp, "\t%u\t%u\t", + mit_KRB5_TL_MOD_PRINC, + d.length + plen + 1 /* NULL counted */); + if (sz == -1) { + free(p); + return ENOMEM; + } + sz = append_hex(context, sp, 1, 1, &d); + if (sz == -1) { + free(p); + return ENOMEM; + } + d.data = p; + d.length = plen + 1; + sz = append_hex(context, sp, 1, 1, &d); + free(p); + if (sz == -1) return ENOMEM; + } + /* + * Dump keys (remembering to not include any with kvno higher than + * the entry's because MIT doesn't track entry kvno separately from + * the entry's keys -- max kvno is it) + */ + for (i = 0; i < ent->keys.len; i++) { + if (!mit_strong_etype(ent->keys.val[i].key.keytype)) + continue; + sz = append_mit_key(context, sp, ent->principal, ent->kvno, + &ent->keys.val[i]); + if (sz == -1) return ENOMEM; + } + for (i = 0; hist_keys && i < ent->kvno; i++) { + size_t m; + + /* dump historical keys */ + for (k = 0; k < hist_keys->len; k++) { + if (hist_keys->val[k].kvno != ent->kvno - i) + continue; + for (m = 0; m < hist_keys->val[k].keys.len; m++) { + if (ent->keys.val[k].key.keytype == ETYPE_DES_CBC_MD4 || + ent->keys.val[k].key.keytype == ETYPE_DES_CBC_MD5) + continue; + sz = append_mit_key(context, sp, ent->principal, + hist_keys->val[k].kvno, + &hist_keys->val[k].keys.val[m]); + if (sz == -1) return ENOMEM; + } + } + } + sz = append_string(context, sp, "\t-1;"); /* "extra data" */ + if (sz == -1) return ENOMEM; return 0; } krb5_error_code -hdb_entry2string (krb5_context context, hdb_entry *ent, char **str) +hdb_entry2string(krb5_context context, hdb_entry *ent, char **str) { krb5_error_code ret; krb5_data data; krb5_storage *sp; sp = krb5_storage_emem(); - if(sp == NULL) { + if (sp == NULL) { krb5_set_error_message(context, ENOMEM, "malloc: out of memory"); return ENOMEM; } ret = entry2string_int(context, sp, ent); - if(ret) { + if (ret) { krb5_storage_free(sp); return ret; } @@ -267,22 +589,31 @@ hdb_entry2string (krb5_context context, hdb_entry *ent, char **str) /* print a hdb_entry to (FILE*)data; suitable for hdb_foreach */ krb5_error_code -hdb_print_entry(krb5_context context, HDB *db, hdb_entry_ex *entry, void *data) +hdb_print_entry(krb5_context context, HDB *db, hdb_entry_ex *entry, + void *data) { + struct hdb_print_entry_arg *parg = data; krb5_error_code ret; krb5_storage *sp; - FILE *f = data; - - fflush(f); - sp = krb5_storage_from_fd(fileno(f)); - if(sp == NULL) { + fflush(parg->out); + sp = krb5_storage_from_fd(fileno(parg->out)); + if (sp == NULL) { krb5_set_error_message(context, ENOMEM, "malloc: out of memory"); return ENOMEM; } - ret = entry2string_int(context, sp, &entry->entry); - if(ret) { + switch (parg->fmt) { + case HDB_DUMP_HEIMDAL: + ret = entry2string_int(context, sp, &entry->entry); + break; + case HDB_DUMP_MIT: + ret = entry2mit_string_int(context, sp, &entry->entry); + break; + default: + heim_abort("Only two dump formats supported: Heimdal and MIT"); + } + if (ret) { krb5_storage_free(sp); return ret; } diff --git a/crypto/heimdal/lib/hdb/version-script.map b/crypto/heimdal/lib/hdb/version-script.map index 50a36cec0aa9..42e2043b4b47 100644 --- a/crypto/heimdal/lib/hdb/version-script.map +++ b/crypto/heimdal/lib/hdb/version-script.map @@ -4,6 +4,7 @@ HEIMDAL_HDB_1.0 { global: encode_hdb_keyset; hdb_add_master_key; + hdb_add_current_keys_to_history; hdb_check_db_format; hdb_clear_extension; hdb_clear_master_key; @@ -57,6 +58,7 @@ HEIMDAL_HDB_1.0 { hdb_seal_key_mkey; hdb_seal_keys; hdb_seal_keys_mkey; + hdb_set_last_modified_by; hdb_set_master_key; hdb_set_master_keyfile; hdb_unlock; @@ -71,6 +73,10 @@ HEIMDAL_HDB_1.0 { hdb_interface_version; initialize_hdb_error_table_r; + # MIT KDB related entries + _hdb_mdb_value2entry; + _hdb_mit_dump2mitdb_entry; + hdb_kt_ops; # some random bits needed for libkadm diff --git a/crypto/heimdal/lib/kadm5/chpass_s.c b/crypto/heimdal/lib/kadm5/chpass_s.c index 624293e5c3a3..22f65517fcc5 100644 --- a/crypto/heimdal/lib/kadm5/chpass_s.c +++ b/crypto/heimdal/lib/kadm5/chpass_s.c @@ -58,6 +58,10 @@ change(void *server_handle, if(ret) goto out; + ret = hdb_add_current_keys_to_history(context->context, &ent.entry); + if (ret) + goto out; + if (context->db->hdb_capability_flags & HDB_CAP_F_HANDLE_PASSWORDS) { ret = context->db->hdb_password(context->context, context->db, &ent, password, cond); @@ -170,6 +174,9 @@ kadm5_s_chpass_principal_with_key(void *server_handle, HDB_F_GET_ANY|HDB_F_ADMIN_DATA, &ent); if(ret) goto out; + ret = hdb_add_current_keys_to_history(context->context, &ent.entry); + if (ret) + goto out2; ret = _kadm5_set_keys2(context, &ent.entry, n_key_data, key_data); if(ret) goto out2; diff --git a/crypto/heimdal/lib/kadm5/randkey_s.c b/crypto/heimdal/lib/kadm5/randkey_s.c index dcb179aac40f..adb3564be719 100644 --- a/crypto/heimdal/lib/kadm5/randkey_s.c +++ b/crypto/heimdal/lib/kadm5/randkey_s.c @@ -59,6 +59,10 @@ kadm5_s_randkey_principal(void *server_handle, if(ret) goto out; + ret = hdb_add_current_keys_to_history(context->context, &ent.entry); + if (ret) + goto out2; + ret = _kadm5_set_keys_randomly (context, &ent.entry, new_keys, diff --git a/kerberos5/lib/libhdb/Makefile b/kerberos5/lib/libhdb/Makefile index bcc3f75206bc..14fa242bc9e5 100644 --- a/kerberos5/lib/libhdb/Makefile +++ b/kerberos5/lib/libhdb/Makefile @@ -1,7 +1,9 @@ +PACKAGE= kerberos-lib + LIB= hdb LDFLAGS+= -Wl,--no-undefined ${LDAPLDFLAGS} VERSION_MAP= ${KRB5DIR}/lib/hdb/version-script.map -LIBADD= asn1 com_err krb5 roken sqlite3 +LIBADD= asn1 com_err krb5 roken sqlite3 heimbase LDADD= ${LDAPLDADD} DPADD= ${LDAPDPADD} @@ -74,6 +76,7 @@ GEN= asn1_Salt.x \ asn1_HDB_Ext_Lan_Manager_OWF.x \ asn1_HDB_Ext_Password.x \ asn1_HDB_Ext_Aliases.x \ + asn1_HDB_Ext_KeySet.x \ asn1_HDB_extension.x \ asn1_HDB_extensions.x \ asn1_hdb_entry.x \ diff --git a/lib/libc/nls/Makefile.inc b/lib/libc/nls/Makefile.inc index f26e04c187a5..c211026aba72 100644 --- a/lib/libc/nls/Makefile.inc +++ b/lib/libc/nls/Makefile.inc @@ -12,6 +12,11 @@ MAN+= catclose.3 catgets.3 catopen.3 # for translators. NLSNAME= libc +# We don't want libc's NLS catalogues to be installed in the clibs package. +# Put them in locales instead, since anyone interested in NLS will have +# that installed. +NLSPACKAGE= locales + NLS+= be_BY.UTF-8 NLS+= ca_ES.ISO8859-1 NLS+= de_DE.ISO8859-1 diff --git a/share/man/man4/ice.4 b/share/man/man4/ice.4 index 01e3f0daa65a..13ad304a2d5a 100644 --- a/share/man/man4/ice.4 +++ b/share/man/man4/ice.4 @@ -1,4 +1,4 @@ -.\"- +.\" .\" SPDX-License-Identifier: BSD-3-Clause .\" .\" Copyright (c) 2019-2020, Intel Corporation @@ -37,7 +37,7 @@ .Os .Sh NAME .Nm ice -.Nd "Intel\(rg Ethernet 800 Series Driver" +.Nd Intel Ethernet 800 Series Driver .Sh SYNOPSIS To compile this driver into the kernel, place the following lines in your kernel configuration file: diff --git a/share/mk/bsd.nls.mk b/share/mk/bsd.nls.mk index bd13fe56679d..5a28e1cdd026 100644 --- a/share/mk/bsd.nls.mk +++ b/share/mk/bsd.nls.mk @@ -1,6 +1,8 @@ # -# This include file <bsd.nls.mk> handles building and installing Native -# Language Support (NLS) catalogs +# Handle building and installing Native Language Support (NLS) catalogs. +# This is implemented using a <bsd.files.mk> files group called "NLS", +# so any per-group options that bsd.files.mk supports can be used here +# with the prefix "NLS". # # +++ variables +++ # @@ -17,6 +19,9 @@ # NLSMODE National Language Support files mode. [${NOBINMODE}] # # NLSOWN National Language Support files owner. [${SHAREOWN}] +# +# NLSPACKAGE Package to install the NLS files in. +# [${PACKAGE}, or "utilities" if not set] .if !target(__<bsd.init.mk>__) .error bsd.nls.mk cannot be included directly. diff --git a/share/mk/src.libnames.mk b/share/mk/src.libnames.mk index 95cd3b02d8e4..3e108591251a 100644 --- a/share/mk/src.libnames.mk +++ b/share/mk/src.libnames.mk @@ -425,7 +425,7 @@ _DP_kadm5clnt= com_err krb5 roken _DP_kadm5srv= com_err hdb krb5 roken _DP_heimntlm= crypto com_err krb5 roken _DP_hx509= asn1 com_err crypto roken wind -_DP_hdb= asn1 com_err krb5 roken sqlite3 +_DP_hdb= asn1 com_err krb5 roken sqlite3 heimbase _DP_asn1= com_err roken _DP_kdc= roken hdb hx509 krb5 heimntlm asn1 crypto _DP_wind= com_err roken diff --git a/sys/contrib/openzfs/.github/ISSUE_TEMPLATE/feature_request.md b/sys/contrib/openzfs/.github/ISSUE_TEMPLATE/feature_request.md index 9b50a4a3d96e..f3d4316f6f67 100644 --- a/sys/contrib/openzfs/.github/ISSUE_TEMPLATE/feature_request.md +++ b/sys/contrib/openzfs/.github/ISSUE_TEMPLATE/feature_request.md @@ -14,7 +14,7 @@ Please check our issue tracker before opening a new feature request. Filling out the following template will help other contributors better understand your proposed feature. --> -### Describe the feature would like to see added to OpenZFS +### Describe the feature you would like to see added to OpenZFS <!-- Provide a clear and concise description of the feature. diff --git a/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md b/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md index 79809179cf13..47edc8174603 100644 --- a/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md +++ b/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md @@ -2,11 +2,6 @@ <!--- Provide a general summary of your changes in the Title above --> -<!--- -Documentation on ZFS Buildbot options can be found at -https://openzfs.github.io/openzfs-docs/Developer%20Resources/Buildbot%20Options.html ---> - ### Motivation and Context <!--- Why is this change required? What problem does it solve? --> <!--- If it fixes an open issue, please link to the issue here. --> diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh index 8439942c5a41..1c608348ffcd 100755 --- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh +++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh @@ -121,7 +121,7 @@ case "$OS" in KSRC="$FREEBSD_SNAP/../amd64/$FreeBSD/src.txz" ;; freebsd15-0c) - FreeBSD="15.0-ALPHA2" + FreeBSD="15.0-ALPHA3" OSNAME="FreeBSD $FreeBSD" OSv="freebsd14.0" URLxz="$FREEBSD_SNAP/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI-ufs.raw.xz" diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh index ee058b488088..f67bb2f68e94 100755 --- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh +++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh @@ -20,7 +20,7 @@ function archlinux() { sudo pacman -Sy --noconfirm base-devel bc cpio cryptsetup dhclient dkms \ fakeroot fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils \ parted pax perf python-packaging python-setuptools qemu-guest-agent ksh \ - samba sysstat rng-tools rsync wget xxhash + samba strace sysstat rng-tools rsync wget xxhash echo "##[endgroup]" } @@ -43,7 +43,8 @@ function debian() { lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \ python3-cffi python3-dev python3-distlib python3-packaging libtirpc-dev \ python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \ - rsync samba sysstat uuid-dev watchdog wget xfslibs-dev xxhash zlib1g-dev + rsync samba strace sysstat uuid-dev watchdog wget xfslibs-dev xxhash \ + zlib1g-dev echo "##[endgroup]" } @@ -87,8 +88,8 @@ function rhel() { libuuid-devel lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester \ parted perf python3 python3-cffi python3-devel python3-packaging \ kernel-devel python3-setuptools qemu-guest-agent rng-tools rpcgen \ - rpm-build rsync samba sysstat systemd watchdog wget xfsprogs-devel xxhash \ - zlib-devel + rpm-build rsync samba strace sysstat systemd watchdog wget xfsprogs-devel \ + xxhash zlib-devel echo "##[endgroup]" } @@ -104,7 +105,7 @@ function install_fedora_experimental_kernel { our_version="$1" sudo dnf -y copr enable @kernel-vanilla/stable sudo dnf -y copr enable @kernel-vanilla/mainline - all="$(sudo dnf list --showduplicates kernel-*)" + all="$(sudo dnf list --showduplicates kernel-* python3-perf* perf* bpftool*)" echo "Available versions:" echo "$all" diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh index 0adcad2a99bc..4869c1003e48 100755 --- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh +++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh @@ -108,19 +108,30 @@ echo '*/5 * * * * /root/cronjob.sh' > crontab.txt sudo crontab crontab.txt rm crontab.txt -# check if the machines are okay -echo "Waiting for vm's to come up... (${VMs}x CPU=$CPU RAM=$RAM)" -for ((i=1; i<=VMs; i++)); do - .github/workflows/scripts/qemu-wait-for-vm.sh vm$i -done -echo "All $VMs VMs are up now." - # Save the VM's serial output (ttyS0) to /var/tmp/console.txt # - ttyS0 on the VM corresponds to a local /dev/pty/N entry # - use 'virsh ttyconsole' to lookup the /dev/pty/N entry for ((i=1; i<=VMs; i++)); do mkdir -p $RESPATH/vm$i read "pty" <<< $(sudo virsh ttyconsole vm$i) + + # Create the file so we can tail it, even if there's no output. + touch $RESPATH/vm$i/console.txt + sudo nohup bash -c "cat $pty > $RESPATH/vm$i/console.txt" & + + # Write all VM boot lines to the console to aid in debugging failed boots. + # The boot lines from all the VMs will be munged together, so prepend each + # line with the vm hostname (like 'vm1:'). + (while IFS=$'\n' read -r line; do echo "vm$i: $line" ; done < <(sudo tail -f $RESPATH/vm$i/console.txt)) & + done echo "Console logging for ${VMs}x $OS started." + + +# check if the machines are okay +echo "Waiting for vm's to come up... (${VMs}x CPU=$CPU RAM=$RAM)" +for ((i=1; i<=VMs; i++)); do + .github/workflows/scripts/qemu-wait-for-vm.sh vm$i +done +echo "All $VMs VMs are up now." diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh index 5ab822f4f076..ca6ac77f146d 100755 --- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh +++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh @@ -111,7 +111,7 @@ fi sudo dmesg -c > dmesg-prerun.txt mount > mount.txt df -h > df-prerun.txt -$TDIR/zfs-tests.sh -vK -s 3GB -T $TAGS +$TDIR/zfs-tests.sh -vKO -s 3GB -T $TAGS RV=$? df -h > df-postrun.txt echo $RV > tests-exitcode.txt diff --git a/sys/contrib/openzfs/META b/sys/contrib/openzfs/META index 5704b5c6de8a..bdb7aee48041 100644 --- a/sys/contrib/openzfs/META +++ b/sys/contrib/openzfs/META @@ -6,5 +6,5 @@ Release: 1 Release-Tags: relext License: CDDL Author: OpenZFS -Linux-Maximum: 6.16 +Linux-Maximum: 6.17 Linux-Minimum: 4.18 diff --git a/sys/contrib/openzfs/cmd/zdb/zdb.c b/sys/contrib/openzfs/cmd/zdb/zdb.c index d655fa715e15..70a4ed46f263 100644 --- a/sys/contrib/openzfs/cmd/zdb/zdb.c +++ b/sys/contrib/openzfs/cmd/zdb/zdb.c @@ -3301,6 +3301,7 @@ zdb_derive_key(dsl_dir_t *dd, uint8_t *key_out) uint64_t keyformat, salt, iters; int i; unsigned char c; + FILE *f; VERIFY0(zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, zfs_prop_to_name(ZFS_PROP_KEYFORMAT), sizeof (uint64_t), @@ -3333,6 +3334,25 @@ zdb_derive_key(dsl_dir_t *dd, uint8_t *key_out) break; + case ZFS_KEYFORMAT_RAW: + if ((f = fopen(key_material, "r")) == NULL) + return (B_FALSE); + + if (fread(key_out, 1, WRAPPING_KEY_LEN, f) != + WRAPPING_KEY_LEN) { + (void) fclose(f); + return (B_FALSE); + } + + /* Check the key length */ + if (fgetc(f) != EOF) { + (void) fclose(f); + return (B_FALSE); + } + + (void) fclose(f); + break; + default: fatal("no support for key format %u\n", (unsigned int) keyformat); diff --git a/sys/contrib/openzfs/cmd/zfs/zfs_main.c b/sys/contrib/openzfs/cmd/zfs/zfs_main.c index 484986bde719..ccdd5ffef8e6 100644 --- a/sys/contrib/openzfs/cmd/zfs/zfs_main.c +++ b/sys/contrib/openzfs/cmd/zfs/zfs_main.c @@ -914,7 +914,11 @@ zfs_do_clone(int argc, char **argv) log_history = B_FALSE; } - ret = zfs_mount_and_share(g_zfs, argv[1], ZFS_TYPE_DATASET); + /* + * Dataset cloned successfully, mount/share failures are + * non-fatal. + */ + (void) zfs_mount_and_share(g_zfs, argv[1], ZFS_TYPE_DATASET); } zfs_close(zhp); @@ -930,19 +934,15 @@ usage: } /* - * Return a default volblocksize for the pool which always uses more than - * half of the data sectors. This primarily applies to dRAID which always - * writes full stripe widths. + * Calculate the minimum allocation size based on the top-level vdevs. */ static uint64_t -default_volblocksize(zpool_handle_t *zhp, nvlist_t *props) +calculate_volblocksize(nvlist_t *config) { - uint64_t volblocksize, asize = SPA_MINBLOCKSIZE; + uint64_t asize = SPA_MINBLOCKSIZE; nvlist_t *tree, **vdevs; uint_t nvdevs; - nvlist_t *config = zpool_get_config(zhp, NULL); - if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &tree) != 0 || nvlist_lookup_nvlist_array(tree, ZPOOL_CONFIG_CHILDREN, &vdevs, &nvdevs) != 0) { @@ -973,6 +973,24 @@ default_volblocksize(zpool_handle_t *zhp, nvlist_t *props) } } + return (asize); +} + +/* + * Return a default volblocksize for the pool which always uses more than + * half of the data sectors. This primarily applies to dRAID which always + * writes full stripe widths. + */ +static uint64_t +default_volblocksize(zpool_handle_t *zhp, nvlist_t *props) +{ + uint64_t volblocksize, asize = SPA_MINBLOCKSIZE; + + nvlist_t *config = zpool_get_config(zhp, NULL); + + if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_MAX_ALLOC, &asize) != 0) + asize = calculate_volblocksize(config); + /* * Calculate the target volblocksize such that more than half * of the asize is used. The following table is for 4k sectors. @@ -1319,7 +1337,9 @@ zfs_do_create(int argc, char **argv) goto error; } - ret = zfs_mount_and_share(g_zfs, argv[0], ZFS_TYPE_DATASET); + /* Dataset created successfully, mount/share failures are non-fatal */ + ret = 0; + (void) zfs_mount_and_share(g_zfs, argv[0], ZFS_TYPE_DATASET); error: nvlist_free(props); return (ret); diff --git a/sys/contrib/openzfs/cmd/zinject/zinject.c b/sys/contrib/openzfs/cmd/zinject/zinject.c index 113797c878b9..c2f646f2567d 100644 --- a/sys/contrib/openzfs/cmd/zinject/zinject.c +++ b/sys/contrib/openzfs/cmd/zinject/zinject.c @@ -107,6 +107,8 @@ * zinject * zinject <-a | -u pool> * zinject -c <id|all> + * zinject -E <delay> [-a] [-m] [-f freq] [-l level] [-r range] + * [-T iotype] [-t type object | -b bookmark pool] * zinject [-q] <-t type> [-f freq] [-u] [-a] [-m] [-e errno] [-l level] * [-r range] <object> * zinject [-f freq] [-a] [-m] [-u] -b objset:object:level:start:end pool @@ -132,14 +134,18 @@ * The '-f' flag controls the frequency of errors injected, expressed as a * real number percentage between 0.0001 and 100. The default is 100. * - * The this form is responsible for actually injecting the handler into the + * The <object> form is responsible for actually injecting the handler into the * framework. It takes the arguments described above, translates them to the * internal tuple using libzpool, and then issues an ioctl() to register the * handler. * - * The final form can target a specific bookmark, regardless of whether a + * The '-b' option can target a specific bookmark, regardless of whether a * human-readable interface has been designed. It allows developers to specify * a particular block by number. + * + * The '-E' option injects pipeline ready stage delays for the given object or + * bookmark. The delay is specified in milliseconds, and it supports I/O type + * and range filters. */ #include <errno.h> @@ -346,6 +352,13 @@ usage(void) "\t\tsuch that the operation takes a minimum of supplied seconds\n" "\t\tto complete.\n" "\n" + "\tzinject -E <delay> [-a] [-m] [-f freq] [-l level] [-r range]\n" + "\t\t[-T iotype] [-t type object | -b bookmark pool]\n" + "\n" + "\t\tInject pipeline ready stage delays for the given object path\n" + "\t\t(data or dnode) or raw bookmark. The delay is specified in\n" + "\t\tmilliseconds.\n" + "\n" "\tzinject -I [-s <seconds> | -g <txgs>] pool\n" "\t\tCause the pool to stop writing blocks yet not\n" "\t\treport errors for a duration. Simulates buggy hardware\n" @@ -724,12 +737,15 @@ register_handler(const char *pool, int flags, zinject_record_t *record, if (quiet) { (void) printf("%llu\n", (u_longlong_t)zc.zc_guid); } else { + boolean_t show_object = B_FALSE; + boolean_t show_iotype = B_FALSE; (void) printf("Added handler %llu with the following " "properties:\n", (u_longlong_t)zc.zc_guid); (void) printf(" pool: %s\n", pool); if (record->zi_guid) { (void) printf(" vdev: %llx\n", (u_longlong_t)record->zi_guid); + show_iotype = B_TRUE; } else if (record->zi_func[0] != '\0') { (void) printf(" panic function: %s\n", record->zi_func); @@ -742,7 +758,18 @@ register_handler(const char *pool, int flags, zinject_record_t *record, } else if (record->zi_timer > 0) { (void) printf(" timer: %lld ms\n", (u_longlong_t)NSEC2MSEC(record->zi_timer)); + if (record->zi_cmd == ZINJECT_DELAY_READY) { + show_object = B_TRUE; + show_iotype = B_TRUE; + } } else { + show_object = B_TRUE; + } + if (show_iotype) { + (void) printf("iotype: %s\n", + iotype_to_str(record->zi_iotype)); + } + if (show_object) { (void) printf("objset: %llu\n", (u_longlong_t)record->zi_objset); (void) printf("object: %llu\n", @@ -910,6 +937,7 @@ main(int argc, char **argv) int ret; int flags = 0; uint32_t dvas = 0; + hrtime_t ready_delay = -1; if ((g_zfs = libzfs_init()) == NULL) { (void) fprintf(stderr, "%s\n", libzfs_error_init(errno)); @@ -940,7 +968,7 @@ main(int argc, char **argv) } while ((c = getopt(argc, argv, - ":aA:b:C:d:D:f:Fg:qhIc:t:T:l:mr:s:e:uL:p:P:")) != -1) { + ":aA:b:C:d:D:E:f:Fg:qhIc:t:T:l:mr:s:e:uL:p:P:")) != -1) { switch (c) { case 'a': flags |= ZINJECT_FLUSH_ARC; @@ -1113,6 +1141,18 @@ main(int argc, char **argv) case 'u': flags |= ZINJECT_UNLOAD_SPA; break; + case 'E': + ready_delay = MSEC2NSEC(strtol(optarg, &end, 10)); + if (ready_delay <= 0 || *end != '\0') { + (void) fprintf(stderr, "invalid delay '%s': " + "must be a positive duration\n", optarg); + usage(); + libzfs_fini(g_zfs); + return (1); + } + record.zi_cmd = ZINJECT_DELAY_READY; + record.zi_timer = ready_delay; + break; case 'L': if ((label = name_to_type(optarg)) == TYPE_INVAL && !LABEL_TYPE(type)) { @@ -1150,7 +1190,7 @@ main(int argc, char **argv) */ if (raw != NULL || range != NULL || type != TYPE_INVAL || level != 0 || record.zi_cmd != ZINJECT_UNINITIALIZED || - record.zi_freq > 0 || dvas != 0) { + record.zi_freq > 0 || dvas != 0 || ready_delay >= 0) { (void) fprintf(stderr, "cancel (-c) incompatible with " "any other options\n"); usage(); @@ -1186,7 +1226,7 @@ main(int argc, char **argv) */ if (raw != NULL || range != NULL || type != TYPE_INVAL || level != 0 || record.zi_cmd != ZINJECT_UNINITIALIZED || - dvas != 0) { + dvas != 0 || ready_delay >= 0) { (void) fprintf(stderr, "device (-d) incompatible with " "data error injection\n"); usage(); @@ -1276,13 +1316,23 @@ main(int argc, char **argv) return (1); } - record.zi_cmd = ZINJECT_DATA_FAULT; + if (record.zi_cmd == ZINJECT_UNINITIALIZED) { + record.zi_cmd = ZINJECT_DATA_FAULT; + if (!error) + error = EIO; + } else if (error != 0) { + (void) fprintf(stderr, "error type -e incompatible " + "with delay injection\n"); + libzfs_fini(g_zfs); + return (1); + } else { + record.zi_iotype = io_type; + } + if (translate_raw(raw, &record) != 0) { libzfs_fini(g_zfs); return (1); } - if (!error) - error = EIO; } else if (record.zi_cmd == ZINJECT_PANIC) { if (raw != NULL || range != NULL || type != TYPE_INVAL || level != 0 || device != NULL || record.zi_freq > 0 || @@ -1410,6 +1460,13 @@ main(int argc, char **argv) record.zi_dvas = dvas; } + if (record.zi_cmd != ZINJECT_UNINITIALIZED && error != 0) { + (void) fprintf(stderr, "error type -e incompatible " + "with delay injection\n"); + libzfs_fini(g_zfs); + return (1); + } + if (error == EACCES) { if (type != TYPE_DATA) { (void) fprintf(stderr, "decryption errors " @@ -1425,8 +1482,12 @@ main(int argc, char **argv) * not found. */ error = ECKSUM; - } else { + } else if (record.zi_cmd == ZINJECT_UNINITIALIZED) { record.zi_cmd = ZINJECT_DATA_FAULT; + if (!error) + error = EIO; + } else { + record.zi_iotype = io_type; } if (translate_record(type, argv[0], range, level, &record, pool, @@ -1434,8 +1495,6 @@ main(int argc, char **argv) libzfs_fini(g_zfs); return (1); } - if (!error) - error = EIO; } /* diff --git a/sys/contrib/openzfs/cmd/zpool/zpool_iter.c b/sys/contrib/openzfs/cmd/zpool/zpool_iter.c index 2eec9a95e24c..fef602736705 100644 --- a/sys/contrib/openzfs/cmd/zpool/zpool_iter.c +++ b/sys/contrib/openzfs/cmd/zpool/zpool_iter.c @@ -26,6 +26,7 @@ /* * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>. + * Copyright (c) 2025, Klara, Inc. */ #include <libintl.h> @@ -52,7 +53,7 @@ typedef struct zpool_node { zpool_handle_t *zn_handle; uu_avl_node_t zn_avlnode; - int zn_mark; + hrtime_t zn_last_refresh; } zpool_node_t; struct zpool_list { @@ -62,6 +63,7 @@ struct zpool_list { uu_avl_pool_t *zl_pool; zprop_list_t **zl_proplist; zfs_type_t zl_type; + hrtime_t zl_last_refresh; }; static int @@ -81,26 +83,30 @@ zpool_compare(const void *larg, const void *rarg, void *unused) * of known pools. */ static int -add_pool(zpool_handle_t *zhp, void *data) +add_pool(zpool_handle_t *zhp, zpool_list_t *zlp) { - zpool_list_t *zlp = data; - zpool_node_t *node = safe_malloc(sizeof (zpool_node_t)); + zpool_node_t *node, *new = safe_malloc(sizeof (zpool_node_t)); uu_avl_index_t idx; - node->zn_handle = zhp; - uu_avl_node_init(node, &node->zn_avlnode, zlp->zl_pool); - if (uu_avl_find(zlp->zl_avl, node, NULL, &idx) == NULL) { + new->zn_handle = zhp; + uu_avl_node_init(new, &new->zn_avlnode, zlp->zl_pool); + + node = uu_avl_find(zlp->zl_avl, new, NULL, &idx); + if (node == NULL) { if (zlp->zl_proplist && zpool_expand_proplist(zhp, zlp->zl_proplist, zlp->zl_type, zlp->zl_literal) != 0) { zpool_close(zhp); - free(node); + free(new); return (-1); } - uu_avl_insert(zlp->zl_avl, node, idx); + new->zn_last_refresh = zlp->zl_last_refresh; + uu_avl_insert(zlp->zl_avl, new, idx); } else { + zpool_refresh_stats_from_handle(node->zn_handle, zhp); + node->zn_last_refresh = zlp->zl_last_refresh; zpool_close(zhp); - free(node); + free(new); return (-1); } @@ -108,6 +114,18 @@ add_pool(zpool_handle_t *zhp, void *data) } /* + * add_pool(), but always returns 0. This allows zpool_iter() to continue + * even if a pool exists in the tree, or we fail to get the properties for + * a new one. + */ +static int +add_pool_cb(zpool_handle_t *zhp, void *data) +{ + (void) add_pool(zhp, data); + return (0); +} + +/* * Create a list of pools based on the given arguments. If we're given no * arguments, then iterate over all pools in the system and add them to the AVL * tree. Otherwise, add only those pool explicitly specified on the command @@ -135,9 +153,10 @@ pool_list_get(int argc, char **argv, zprop_list_t **proplist, zfs_type_t type, zlp->zl_type = type; zlp->zl_literal = literal; + zlp->zl_last_refresh = gethrtime(); if (argc == 0) { - (void) zpool_iter(g_zfs, add_pool, zlp); + (void) zpool_iter(g_zfs, add_pool_cb, zlp); zlp->zl_findall = B_TRUE; } else { int i; @@ -159,15 +178,61 @@ pool_list_get(int argc, char **argv, zprop_list_t **proplist, zfs_type_t type, } /* - * Search for any new pools, adding them to the list. We only add pools when no - * options were given on the command line. Otherwise, we keep the list fixed as - * those that were explicitly specified. + * Refresh the state of all pools on the list. Additionally, if no options were + * given on the command line, add any new pools and remove any that are no + * longer available. */ -void -pool_list_update(zpool_list_t *zlp) +int +pool_list_refresh(zpool_list_t *zlp) { - if (zlp->zl_findall) - (void) zpool_iter(g_zfs, add_pool, zlp); + zlp->zl_last_refresh = gethrtime(); + + if (!zlp->zl_findall) { + /* + * This list is a fixed list of pools, so we must not add + * or remove any. Just walk over them and refresh their + * state. + */ + int navail = 0; + for (zpool_node_t *node = uu_avl_first(zlp->zl_avl); + node != NULL; node = uu_avl_next(zlp->zl_avl, node)) { + boolean_t missing; + zpool_refresh_stats(node->zn_handle, &missing); + navail += !missing; + node->zn_last_refresh = zlp->zl_last_refresh; + } + return (navail); + } + + /* Search for any new pools and add them to the list. */ + (void) zpool_iter(g_zfs, add_pool_cb, zlp); + + /* Walk the list of existing pools, and update or remove them. */ + zpool_node_t *node, *next; + for (node = uu_avl_first(zlp->zl_avl); node != NULL; node = next) { + next = uu_avl_next(zlp->zl_avl, node); + + /* + * Skip any that were refreshed and are online; they were added + * by zpool_iter() and are already up to date. + */ + if (node->zn_last_refresh == zlp->zl_last_refresh && + zpool_get_state(node->zn_handle) != POOL_STATE_UNAVAIL) + continue; + + /* Refresh and remove if necessary. */ + boolean_t missing; + zpool_refresh_stats(node->zn_handle, &missing); + if (missing) { + uu_avl_remove(zlp->zl_avl, node); + zpool_close(node->zn_handle); + free(node); + } else { + node->zn_last_refresh = zlp->zl_last_refresh; + } + } + + return (uu_avl_numnodes(zlp->zl_avl)); } /* @@ -191,23 +256,6 @@ pool_list_iter(zpool_list_t *zlp, int unavail, zpool_iter_f func, } /* - * Remove the given pool from the list. When running iostat, we want to remove - * those pools that no longer exist. - */ -void -pool_list_remove(zpool_list_t *zlp, zpool_handle_t *zhp) -{ - zpool_node_t search, *node; - - search.zn_handle = zhp; - if ((node = uu_avl_find(zlp->zl_avl, &search, NULL, NULL)) != NULL) { - uu_avl_remove(zlp->zl_avl, node); - zpool_close(node->zn_handle); - free(node); - } -} - -/* * Free all the handles associated with this list. */ void diff --git a/sys/contrib/openzfs/cmd/zpool/zpool_main.c b/sys/contrib/openzfs/cmd/zpool/zpool_main.c index 2c46ad0df895..1feec55c0e8b 100644 --- a/sys/contrib/openzfs/cmd/zpool/zpool_main.c +++ b/sys/contrib/openzfs/cmd/zpool/zpool_main.c @@ -33,7 +33,7 @@ * Copyright (c) 2017, Intel Corporation. * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com> * Copyright (c) 2021, Colm Buckley <colm@tuatha.org> - * Copyright (c) 2021, 2023, Klara Inc. + * Copyright (c) 2021, 2023, 2025, Klara, Inc. * Copyright (c) 2021, 2025 Hewlett Packard Enterprise Development LP. */ @@ -5761,24 +5761,6 @@ children: return (ret); } -static int -refresh_iostat(zpool_handle_t *zhp, void *data) -{ - iostat_cbdata_t *cb = data; - boolean_t missing; - - /* - * If the pool has disappeared, remove it from the list and continue. - */ - if (zpool_refresh_stats(zhp, &missing) != 0) - return (-1); - - if (missing) - pool_list_remove(cb->cb_list, zhp); - - return (0); -} - /* * Callback to print out the iostats for the given pool. */ @@ -6359,15 +6341,14 @@ get_namewidth_iostat(zpool_handle_t *zhp, void *data) * This command can be tricky because we want to be able to deal with pool * creation/destruction as well as vdev configuration changes. The bulk of this * processing is handled by the pool_list_* routines in zpool_iter.c. We rely - * on pool_list_update() to detect the addition of new pools. Configuration - * changes are all handled within libzfs. + * on pool_list_refresh() to detect the addition and removal of pools. + * Configuration changes are all handled within libzfs. */ int zpool_do_iostat(int argc, char **argv) { int c; int ret; - int npools; float interval = 0; unsigned long count = 0; zpool_list_t *list; @@ -6618,10 +6599,24 @@ zpool_do_iostat(int argc, char **argv) return (1); } + int last_npools = 0; for (;;) { - if ((npools = pool_list_count(list)) == 0) + /* + * Refresh all pools in list, adding or removing pools as + * necessary. + */ + int npools = pool_list_refresh(list); + if (npools == 0) { (void) fprintf(stderr, gettext("no pools available\n")); - else { + } else { + /* + * If the list of pools has changed since last time + * around, reset the iteration count to force the + * header to be redisplayed. + */ + if (last_npools != npools) + cb.cb_iteration = 0; + /* * If this is the first iteration and -y was supplied * we skip any printing. @@ -6630,15 +6625,6 @@ zpool_do_iostat(int argc, char **argv) cb.cb_iteration == 0); /* - * Refresh all statistics. This is done as an - * explicit step before calculating the maximum name - * width, so that any * configuration changes are - * properly accounted for. - */ - (void) pool_list_iter(list, B_FALSE, refresh_iostat, - &cb); - - /* * Iterate over all pools to determine the maximum width * for the pool / device name column across all pools. */ @@ -6691,6 +6677,7 @@ zpool_do_iostat(int argc, char **argv) if (skip) { (void) fflush(stdout); (void) fsleep(interval); + last_npools = npools; continue; } @@ -6728,6 +6715,8 @@ zpool_do_iostat(int argc, char **argv) (void) fflush(stdout); (void) fsleep(interval); + + last_npools = npools; } pool_list_free(list); diff --git a/sys/contrib/openzfs/cmd/zpool/zpool_util.h b/sys/contrib/openzfs/cmd/zpool/zpool_util.h index 5ab7cb9750f1..3af23c52bd45 100644 --- a/sys/contrib/openzfs/cmd/zpool/zpool_util.h +++ b/sys/contrib/openzfs/cmd/zpool/zpool_util.h @@ -76,11 +76,10 @@ typedef struct zpool_list zpool_list_t; zpool_list_t *pool_list_get(int, char **, zprop_list_t **, zfs_type_t, boolean_t, int *); -void pool_list_update(zpool_list_t *); +int pool_list_refresh(zpool_list_t *); int pool_list_iter(zpool_list_t *, int unavail, zpool_iter_f, void *); void pool_list_free(zpool_list_t *); int pool_list_count(zpool_list_t *); -void pool_list_remove(zpool_list_t *, zpool_handle_t *); extern libzfs_handle_t *g_zfs; diff --git a/sys/contrib/openzfs/cmd/zpool/zpool_vdev.c b/sys/contrib/openzfs/cmd/zpool/zpool_vdev.c index 684b46a2d673..088c0108e911 100644 --- a/sys/contrib/openzfs/cmd/zpool/zpool_vdev.c +++ b/sys/contrib/openzfs/cmd/zpool/zpool_vdev.c @@ -609,22 +609,28 @@ get_replication(nvlist_t *nvroot, boolean_t fatal) ZPOOL_CONFIG_PATH, &path) == 0); /* + * Skip active spares they should never cause + * the pool to be evaluated as inconsistent. + */ + if (is_spare(NULL, path)) + continue; + + /* * If we have a raidz/mirror that combines disks - * with files, report it as an error. + * with files, only report it as an error when + * fatal is set to ensure all the replication + * checks aren't skipped in check_replication(). */ - if (!dontreport && type != NULL && + if (fatal && !dontreport && type != NULL && strcmp(type, childtype) != 0) { if (ret != NULL) free(ret); ret = NULL; - if (fatal) - vdev_error(gettext( - "mismatched replication " - "level: %s contains both " - "files and devices\n"), - rep.zprl_type); - else - return (NULL); + vdev_error(gettext( + "mismatched replication " + "level: %s contains both " + "files and devices\n"), + rep.zprl_type); dontreport = B_TRUE; } diff --git a/sys/contrib/openzfs/contrib/intel_qat/readme.md b/sys/contrib/openzfs/contrib/intel_qat/readme.md index 7e45d395bb80..04c299b6404c 100644 --- a/sys/contrib/openzfs/contrib/intel_qat/readme.md +++ b/sys/contrib/openzfs/contrib/intel_qat/readme.md @@ -8,7 +8,7 @@ This contrib contains community compatibility patches to get Intel QAT working o These patches are based on the following Intel QAT version: [1.7.l.4.10.0-00014](https://01.org/sites/default/files/downloads/qat1.7.l.4.10.0-00014.tar.gz) -When using QAT with above kernels versions, the following patches needs to be applied using: +When using QAT with the above kernel versions, the following patches need to be applied using: patch -p1 < _$PATCH_ _Where $PATCH refers to the path of the patch in question_ diff --git a/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/test/test_libzfs_core.py b/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/test/test_libzfs_core.py index 971aa1d0d493..bad1af2d1671 100644 --- a/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/test/test_libzfs_core.py +++ b/sys/contrib/openzfs/contrib/pyzfs/libzfs_core/test/test_libzfs_core.py @@ -4223,7 +4223,7 @@ class _TempPool(object): self.getRoot().reset() return - # On the Buildbot builders this may fail with "pool is busy" + # On the CI builders this may fail with "pool is busy" # Retry 5 times before raising an error retry = 0 while True: diff --git a/sys/contrib/openzfs/etc/init.d/README.md b/sys/contrib/openzfs/etc/init.d/README.md index da780fdc1222..3852dd9a6b2e 100644 --- a/sys/contrib/openzfs/etc/init.d/README.md +++ b/sys/contrib/openzfs/etc/init.d/README.md @@ -1,5 +1,5 @@ DESCRIPTION - These script were written with the primary intention of being portable and + These scripts were written with the primary intention of being portable and usable on as many systems as possible. This is, in practice, usually not possible. But the intention is there. diff --git a/sys/contrib/openzfs/include/libzfs.h b/sys/contrib/openzfs/include/libzfs.h index 3fcdc176a621..14930fb90622 100644 --- a/sys/contrib/openzfs/include/libzfs.h +++ b/sys/contrib/openzfs/include/libzfs.h @@ -479,6 +479,8 @@ _LIBZFS_H zpool_status_t zpool_import_status(nvlist_t *, const char **, _LIBZFS_H nvlist_t *zpool_get_config(zpool_handle_t *, nvlist_t **); _LIBZFS_H nvlist_t *zpool_get_features(zpool_handle_t *); _LIBZFS_H int zpool_refresh_stats(zpool_handle_t *, boolean_t *); +_LIBZFS_H void zpool_refresh_stats_from_handle(zpool_handle_t *, + zpool_handle_t *); _LIBZFS_H int zpool_get_errlog(zpool_handle_t *, nvlist_t **); _LIBZFS_H void zpool_add_propname(zpool_handle_t *, const char *); diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h index 076dab8ba6dc..214f3ea0e787 100644 --- a/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h +++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h @@ -542,24 +542,6 @@ blk_generic_alloc_queue(make_request_fn make_request, int node_id) } #endif /* !HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */ -/* - * All the io_*() helper functions below can operate on a bio, or a rq, but - * not both. The older submit_bio() codepath will pass a bio, and the - * newer blk-mq codepath will pass a rq. - */ -static inline int -io_data_dir(struct bio *bio, struct request *rq) -{ - if (rq != NULL) { - if (op_is_write(req_op(rq))) { - return (WRITE); - } else { - return (READ); - } - } - return (bio_data_dir(bio)); -} - static inline int io_is_flush(struct bio *bio, struct request *rq) { diff --git a/sys/contrib/openzfs/include/sys/fs/zfs.h b/sys/contrib/openzfs/include/sys/fs/zfs.h index 49ab9d3db795..662fd81c5ee1 100644 --- a/sys/contrib/openzfs/include/sys/fs/zfs.h +++ b/sys/contrib/openzfs/include/sys/fs/zfs.h @@ -748,6 +748,8 @@ typedef struct zpool_load_policy { #define ZPOOL_CONFIG_METASLAB_SHIFT "metaslab_shift" #define ZPOOL_CONFIG_ASHIFT "ashift" #define ZPOOL_CONFIG_ASIZE "asize" +#define ZPOOL_CONFIG_MIN_ALLOC "min_alloc" +#define ZPOOL_CONFIG_MAX_ALLOC "max_alloc" #define ZPOOL_CONFIG_DTL "DTL" #define ZPOOL_CONFIG_SCAN_STATS "scan_stats" /* not stored on disk */ #define ZPOOL_CONFIG_REMOVAL_STATS "removal_stats" /* not stored on disk */ diff --git a/sys/contrib/openzfs/include/sys/range_tree.h b/sys/contrib/openzfs/include/sys/range_tree.h index 0f6884682459..0f6def36f9f6 100644 --- a/sys/contrib/openzfs/include/sys/range_tree.h +++ b/sys/contrib/openzfs/include/sys/range_tree.h @@ -238,8 +238,7 @@ zfs_rs_set_end_raw(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t end) } static inline void -zfs_zfs_rs_set_fill_raw(zfs_range_seg_t *rs, zfs_range_tree_t *rt, - uint64_t fill) +zfs_rs_set_fill_raw(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t fill) { ASSERT3U(rt->rt_type, <=, ZFS_RANGE_SEG_NUM_TYPES); switch (rt->rt_type) { @@ -277,7 +276,7 @@ static inline void zfs_rs_set_fill(zfs_range_seg_t *rs, zfs_range_tree_t *rt, uint64_t fill) { ASSERT(IS_P2ALIGNED(fill, 1ULL << rt->rt_shift)); - zfs_zfs_rs_set_fill_raw(rs, rt, fill >> rt->rt_shift); + zfs_rs_set_fill_raw(rs, rt, fill >> rt->rt_shift); } typedef void zfs_range_tree_func_t(void *arg, uint64_t start, uint64_t size); diff --git a/sys/contrib/openzfs/include/sys/spa.h b/sys/contrib/openzfs/include/sys/spa.h index 66db16b33c51..f172f2af6f07 100644 --- a/sys/contrib/openzfs/include/sys/spa.h +++ b/sys/contrib/openzfs/include/sys/spa.h @@ -1030,7 +1030,7 @@ extern void spa_import_progress_set_notes_nolog(spa_t *spa, extern int spa_config_tryenter(spa_t *spa, int locks, const void *tag, krw_t rw); extern void spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw); -extern void spa_config_enter_mmp(spa_t *spa, int locks, const void *tag, +extern void spa_config_enter_priority(spa_t *spa, int locks, const void *tag, krw_t rw); extern void spa_config_exit(spa_t *spa, int locks, const void *tag); extern int spa_config_held(spa_t *spa, int locks, krw_t rw); @@ -1084,6 +1084,7 @@ extern pool_state_t spa_state(spa_t *spa); extern spa_load_state_t spa_load_state(spa_t *spa); extern uint64_t spa_freeze_txg(spa_t *spa); extern uint64_t spa_get_worst_case_asize(spa_t *spa, uint64_t lsize); +extern void spa_get_min_alloc_range(spa_t *spa, uint64_t *min, uint64_t *max); extern uint64_t spa_get_dspace(spa_t *spa); extern uint64_t spa_get_checkpoint_space(spa_t *spa); extern uint64_t spa_get_slop_space(spa_t *spa); diff --git a/sys/contrib/openzfs/include/sys/spa_impl.h b/sys/contrib/openzfs/include/sys/spa_impl.h index 07a959db3447..62b062984d36 100644 --- a/sys/contrib/openzfs/include/sys/spa_impl.h +++ b/sys/contrib/openzfs/include/sys/spa_impl.h @@ -265,6 +265,7 @@ struct spa { uint64_t spa_min_ashift; /* of vdevs in normal class */ uint64_t spa_max_ashift; /* of vdevs in normal class */ uint64_t spa_min_alloc; /* of vdevs in normal class */ + uint64_t spa_max_alloc; /* of vdevs in normal class */ uint64_t spa_gcd_alloc; /* of vdevs in normal class */ uint64_t spa_config_guid; /* config pool guid */ uint64_t spa_load_guid; /* spa_load initialized guid */ diff --git a/sys/contrib/openzfs/include/sys/zfs_ioctl.h b/sys/contrib/openzfs/include/sys/zfs_ioctl.h index 8174242abdac..cfe11f43bb8e 100644 --- a/sys/contrib/openzfs/include/sys/zfs_ioctl.h +++ b/sys/contrib/openzfs/include/sys/zfs_ioctl.h @@ -455,6 +455,7 @@ typedef enum zinject_type { ZINJECT_DECRYPT_FAULT, ZINJECT_DELAY_IMPORT, ZINJECT_DELAY_EXPORT, + ZINJECT_DELAY_READY, } zinject_type_t; typedef enum zinject_iotype { diff --git a/sys/contrib/openzfs/include/sys/zio.h b/sys/contrib/openzfs/include/sys/zio.h index a8acb83b4c2f..acb0a03a36b2 100644 --- a/sys/contrib/openzfs/include/sys/zio.h +++ b/sys/contrib/openzfs/include/sys/zio.h @@ -718,6 +718,7 @@ extern void zio_handle_ignored_writes(zio_t *zio); extern hrtime_t zio_handle_io_delay(zio_t *zio); extern void zio_handle_import_delay(spa_t *spa, hrtime_t elapsed); extern void zio_handle_export_delay(spa_t *spa, hrtime_t elapsed); +extern hrtime_t zio_handle_ready_delay(zio_t *zio); /* * Checksum ereport functions diff --git a/sys/contrib/openzfs/lib/libuutil/libuutil.abi b/sys/contrib/openzfs/lib/libuutil/libuutil.abi index 6c736c61e4a5..2a740afa07ca 100644 --- a/sys/contrib/openzfs/lib/libuutil/libuutil.abi +++ b/sys/contrib/openzfs/lib/libuutil/libuutil.abi @@ -616,6 +616,7 @@ <array-type-def dimensions='1' type-id='de572c22' size-in-bits='1472' id='6d3c2f42'> <subrange length='23' type-id='7359adad' id='fdd0f594'/> </array-type-def> + <type-decl name='long long int' size-in-bits='64' id='1eb56b1e'/> <array-type-def dimensions='1' type-id='3a47d82b' size-in-bits='256' id='a133ec23'> <subrange length='4' type-id='7359adad' id='16fe7105'/> </array-type-def> @@ -1020,13 +1021,6 @@ <array-type-def dimensions='1' type-id='03085adc' size-in-bits='192' id='083f8d58'> <subrange length='3' type-id='7359adad' id='56f209d2'/> </array-type-def> - <array-type-def dimensions='1' type-id='d315442e' size-in-bits='16' id='811205dc'> - <subrange length='1' type-id='7359adad' id='52f813b4'/> - </array-type-def> - <array-type-def dimensions='1' type-id='d3130597' size-in-bits='768' id='f63f23b9'> - <subrange length='12' type-id='7359adad' id='84827bdc'/> - </array-type-def> - <type-decl name='long long int' size-in-bits='64' id='1eb56b1e'/> <class-decl name='mnttab' size-in-bits='256' is-struct='yes' visibility='default' id='1b055409'> <data-member access='public' layout-offset-in-bits='0'> <var-decl name='mnt_special' type-id='26a90f95' visibility='default'/> @@ -1061,93 +1055,6 @@ <var-decl name='mnt_minor' type-id='3502e3ff' visibility='default'/> </data-member> </class-decl> - <typedef-decl name='__u16' type-id='8efea9e5' id='d315442e'/> - <typedef-decl name='__s32' type-id='95e97e5e' id='3158a266'/> - <typedef-decl name='__u32' type-id='f0981eeb' id='3f1a6b60'/> - <typedef-decl name='__s64' type-id='1eb56b1e' id='49659421'/> - <typedef-decl name='__u64' type-id='3a47d82b' id='d3130597'/> - <class-decl name='statx_timestamp' size-in-bits='128' is-struct='yes' visibility='default' id='94101016'> - <data-member access='public' layout-offset-in-bits='0'> - <var-decl name='tv_sec' type-id='49659421' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='64'> - <var-decl name='tv_nsec' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='96'> - <var-decl name='__reserved' type-id='3158a266' visibility='default'/> - </data-member> - </class-decl> - <class-decl name='statx' size-in-bits='2048' is-struct='yes' visibility='default' id='720b04c5'> - <data-member access='public' layout-offset-in-bits='0'> - <var-decl name='stx_mask' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='32'> - <var-decl name='stx_blksize' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='64'> - <var-decl name='stx_attributes' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='128'> - <var-decl name='stx_nlink' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='160'> - <var-decl name='stx_uid' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='192'> - <var-decl name='stx_gid' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='224'> - <var-decl name='stx_mode' type-id='d315442e' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='240'> - <var-decl name='__spare0' type-id='811205dc' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='256'> - <var-decl name='stx_ino' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='320'> - <var-decl name='stx_size' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='384'> - <var-decl name='stx_blocks' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='448'> - <var-decl name='stx_attributes_mask' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='512'> - <var-decl name='stx_atime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='640'> - <var-decl name='stx_btime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='768'> - <var-decl name='stx_ctime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='896'> - <var-decl name='stx_mtime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1024'> - <var-decl name='stx_rdev_major' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1056'> - <var-decl name='stx_rdev_minor' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1088'> - <var-decl name='stx_dev_major' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1120'> - <var-decl name='stx_dev_minor' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1152'> - <var-decl name='stx_mnt_id' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1216'> - <var-decl name='__spare2' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1280'> - <var-decl name='__spare3' type-id='f63f23b9' visibility='default'/> - </data-member> - </class-decl> <class-decl name='mntent' size-in-bits='320' is-struct='yes' visibility='default' id='56fe4a37'> <data-member access='public' layout-offset-in-bits='0'> <var-decl name='mnt_fsname' type-id='26a90f95' visibility='default'/> @@ -1237,8 +1144,6 @@ <pointer-type-def type-id='1b055409' size-in-bits='64' id='9d424d31'/> <pointer-type-def type-id='0bbec9cd' size-in-bits='64' id='62f7a03d'/> <qualified-type-def type-id='62f7a03d' restrict='yes' id='f1cadedf'/> - <pointer-type-def type-id='720b04c5' size-in-bits='64' id='936b8e35'/> - <qualified-type-def type-id='936b8e35' restrict='yes' id='31d265b7'/> <function-decl name='getmntent_r' visibility='default' binding='global' size-in-bits='64'> <parameter type-id='e75a27e9'/> <parameter type-id='3cad23cd'/> @@ -1254,14 +1159,6 @@ <parameter type-id='95e97e5e'/> <return type-id='26a90f95'/> </function-decl> - <function-decl name='statx' visibility='default' binding='global' size-in-bits='64'> - <parameter type-id='95e97e5e'/> - <parameter type-id='9d26089a'/> - <parameter type-id='95e97e5e'/> - <parameter type-id='f0981eeb'/> - <parameter type-id='31d265b7'/> - <return type-id='95e97e5e'/> - </function-decl> <function-decl name='__fprintf_chk' visibility='default' binding='global' size-in-bits='64'> <parameter type-id='e75a27e9'/> <parameter type-id='95e97e5e'/> diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs.abi b/sys/contrib/openzfs/lib/libzfs/libzfs.abi index 184ea4a55b43..f988d27a286a 100644 --- a/sys/contrib/openzfs/lib/libzfs/libzfs.abi +++ b/sys/contrib/openzfs/lib/libzfs/libzfs.abi @@ -571,6 +571,7 @@ <elf-symbol name='zpool_props_refresh' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> <elf-symbol name='zpool_read_label' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> <elf-symbol name='zpool_refresh_stats' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> + <elf-symbol name='zpool_refresh_stats_from_handle' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> <elf-symbol name='zpool_reguid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> <elf-symbol name='zpool_reopen_one' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> <elf-symbol name='zpool_scan' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> @@ -641,7 +642,7 @@ <elf-symbol name='sa_protocol_names' size='16' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> <elf-symbol name='spa_feature_table' size='2632' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> <elf-symbol name='zfeature_checks_disable' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> - <elf-symbol name='zfs_deleg_perm_tab' size='528' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> + <elf-symbol name='zfs_deleg_perm_tab' size='544' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> <elf-symbol name='zfs_history_event_names' size='328' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> <elf-symbol name='zfs_max_dataset_nesting' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> <elf-symbol name='zfs_userquota_prop_prefixes' size='96' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/> @@ -1458,103 +1459,8 @@ </function-decl> </abi-instr> <abi-instr address-size='64' path='lib/libspl/os/linux/getmntany.c' language='LANG_C99'> - <array-type-def dimensions='1' type-id='d315442e' size-in-bits='16' id='811205dc'> - <subrange length='1' type-id='7359adad' id='52f813b4'/> - </array-type-def> - <array-type-def dimensions='1' type-id='d3130597' size-in-bits='768' id='f63f23b9'> - <subrange length='12' type-id='7359adad' id='84827bdc'/> - </array-type-def> - <typedef-decl name='__u16' type-id='8efea9e5' id='d315442e'/> - <typedef-decl name='__s32' type-id='95e97e5e' id='3158a266'/> - <typedef-decl name='__u32' type-id='f0981eeb' id='3f1a6b60'/> - <typedef-decl name='__s64' type-id='1eb56b1e' id='49659421'/> - <typedef-decl name='__u64' type-id='3a47d82b' id='d3130597'/> - <class-decl name='statx_timestamp' size-in-bits='128' is-struct='yes' visibility='default' id='94101016'> - <data-member access='public' layout-offset-in-bits='0'> - <var-decl name='tv_sec' type-id='49659421' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='64'> - <var-decl name='tv_nsec' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='96'> - <var-decl name='__reserved' type-id='3158a266' visibility='default'/> - </data-member> - </class-decl> - <class-decl name='statx' size-in-bits='2048' is-struct='yes' visibility='default' id='720b04c5'> - <data-member access='public' layout-offset-in-bits='0'> - <var-decl name='stx_mask' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='32'> - <var-decl name='stx_blksize' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='64'> - <var-decl name='stx_attributes' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='128'> - <var-decl name='stx_nlink' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='160'> - <var-decl name='stx_uid' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='192'> - <var-decl name='stx_gid' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='224'> - <var-decl name='stx_mode' type-id='d315442e' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='240'> - <var-decl name='__spare0' type-id='811205dc' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='256'> - <var-decl name='stx_ino' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='320'> - <var-decl name='stx_size' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='384'> - <var-decl name='stx_blocks' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='448'> - <var-decl name='stx_attributes_mask' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='512'> - <var-decl name='stx_atime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='640'> - <var-decl name='stx_btime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='768'> - <var-decl name='stx_ctime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='896'> - <var-decl name='stx_mtime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1024'> - <var-decl name='stx_rdev_major' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1056'> - <var-decl name='stx_rdev_minor' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1088'> - <var-decl name='stx_dev_major' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1120'> - <var-decl name='stx_dev_minor' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1152'> - <var-decl name='stx_mnt_id' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1216'> - <var-decl name='__spare2' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1280'> - <var-decl name='__spare3' type-id='f63f23b9' visibility='default'/> - </data-member> - </class-decl> <pointer-type-def type-id='56fe4a37' size-in-bits='64' id='b6b61d2f'/> <qualified-type-def type-id='b6b61d2f' restrict='yes' id='3cad23cd'/> - <pointer-type-def type-id='720b04c5' size-in-bits='64' id='936b8e35'/> - <qualified-type-def type-id='936b8e35' restrict='yes' id='31d265b7'/> <function-decl name='getmntent_r' visibility='default' binding='global' size-in-bits='64'> <parameter type-id='e75a27e9'/> <parameter type-id='3cad23cd'/> @@ -1566,14 +1472,6 @@ <parameter type-id='822cd80b'/> <return type-id='95e97e5e'/> </function-decl> - <function-decl name='statx' visibility='default' binding='global' size-in-bits='64'> - <parameter type-id='95e97e5e'/> - <parameter type-id='9d26089a'/> - <parameter type-id='95e97e5e'/> - <parameter type-id='f0981eeb'/> - <parameter type-id='31d265b7'/> - <return type-id='95e97e5e'/> - </function-decl> </abi-instr> <abi-instr address-size='64' path='lib/libspl/timestamp.c' language='LANG_C99'> <typedef-decl name='nl_item' type-id='95e97e5e' id='03b79a94'/> @@ -3194,6 +3092,10 @@ <parameter type-id='dace003f'/> <return type-id='80f4b756'/> </function-decl> + <function-decl name='fnvlist_dup' visibility='default' binding='global' size-in-bits='64'> + <parameter type-id='22cce67b'/> + <return type-id='5ce45b60'/> + </function-decl> <function-decl name='fnvpair_value_nvlist' visibility='default' binding='global' size-in-bits='64'> <parameter type-id='3fa542f0'/> <return type-id='5ce45b60'/> @@ -3238,6 +3140,11 @@ <parameter type-id='37e3bd22' name='missing'/> <return type-id='95e97e5e'/> </function-decl> + <function-decl name='zpool_refresh_stats_from_handle' mangled-name='zpool_refresh_stats_from_handle' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_refresh_stats_from_handle'> + <parameter type-id='4c81de99' name='dzhp'/> + <parameter type-id='4c81de99' name='szhp'/> + <return type-id='48b5725f'/> + </function-decl> <function-decl name='zpool_skip_pool' mangled-name='zpool_skip_pool' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_skip_pool'> <parameter type-id='80f4b756' name='poolname'/> <return type-id='c19b74c3'/> @@ -9398,10 +9305,6 @@ <parameter type-id='5ce45b60'/> <return type-id='48b5725f'/> </function-decl> - <function-decl name='fnvlist_dup' visibility='default' binding='global' size-in-bits='64'> - <parameter type-id='22cce67b'/> - <return type-id='5ce45b60'/> - </function-decl> <function-decl name='spl_pagesize' mangled-name='spl_pagesize' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='spl_pagesize'> <return type-id='b59d7dce'/> </function-decl> @@ -9774,8 +9677,8 @@ </function-decl> </abi-instr> <abi-instr address-size='64' path='module/zcommon/zfs_deleg.c' language='LANG_C99'> - <array-type-def dimensions='1' type-id='fa1870fd' size-in-bits='4224' id='55e705e7'> - <subrange length='33' type-id='7359adad' id='6a5934df'/> + <array-type-def dimensions='1' type-id='fa1870fd' size-in-bits='4352' id='55f84f08'> + <subrange length='34' type-id='7359adad' id='6a6a7e00'/> </array-type-def> <array-type-def dimensions='1' type-id='fa1870fd' size-in-bits='infinite' id='7c00e69d'> <subrange length='infinite' id='031f2035'/> @@ -9805,30 +9708,31 @@ <enumerator name='ZFS_DELEG_NOTE_PROMOTE' value='5'/> <enumerator name='ZFS_DELEG_NOTE_RENAME' value='6'/> <enumerator name='ZFS_DELEG_NOTE_SEND' value='7'/> - <enumerator name='ZFS_DELEG_NOTE_RECEIVE' value='8'/> - <enumerator name='ZFS_DELEG_NOTE_ALLOW' value='9'/> - <enumerator name='ZFS_DELEG_NOTE_USERPROP' value='10'/> - <enumerator name='ZFS_DELEG_NOTE_MOUNT' value='11'/> - <enumerator name='ZFS_DELEG_NOTE_SHARE' value='12'/> - <enumerator name='ZFS_DELEG_NOTE_USERQUOTA' value='13'/> - <enumerator name='ZFS_DELEG_NOTE_GROUPQUOTA' value='14'/> - <enumerator name='ZFS_DELEG_NOTE_USERUSED' value='15'/> - <enumerator name='ZFS_DELEG_NOTE_GROUPUSED' value='16'/> - <enumerator name='ZFS_DELEG_NOTE_USEROBJQUOTA' value='17'/> - <enumerator name='ZFS_DELEG_NOTE_GROUPOBJQUOTA' value='18'/> - <enumerator name='ZFS_DELEG_NOTE_USEROBJUSED' value='19'/> - <enumerator name='ZFS_DELEG_NOTE_GROUPOBJUSED' value='20'/> - <enumerator name='ZFS_DELEG_NOTE_HOLD' value='21'/> - <enumerator name='ZFS_DELEG_NOTE_RELEASE' value='22'/> - <enumerator name='ZFS_DELEG_NOTE_DIFF' value='23'/> - <enumerator name='ZFS_DELEG_NOTE_BOOKMARK' value='24'/> - <enumerator name='ZFS_DELEG_NOTE_LOAD_KEY' value='25'/> - <enumerator name='ZFS_DELEG_NOTE_CHANGE_KEY' value='26'/> - <enumerator name='ZFS_DELEG_NOTE_PROJECTUSED' value='27'/> - <enumerator name='ZFS_DELEG_NOTE_PROJECTQUOTA' value='28'/> - <enumerator name='ZFS_DELEG_NOTE_PROJECTOBJUSED' value='29'/> - <enumerator name='ZFS_DELEG_NOTE_PROJECTOBJQUOTA' value='30'/> - <enumerator name='ZFS_DELEG_NOTE_NONE' value='31'/> + <enumerator name='ZFS_DELEG_NOTE_SEND_RAW' value='8'/> + <enumerator name='ZFS_DELEG_NOTE_RECEIVE' value='9'/> + <enumerator name='ZFS_DELEG_NOTE_ALLOW' value='10'/> + <enumerator name='ZFS_DELEG_NOTE_USERPROP' value='11'/> + <enumerator name='ZFS_DELEG_NOTE_MOUNT' value='12'/> + <enumerator name='ZFS_DELEG_NOTE_SHARE' value='13'/> + <enumerator name='ZFS_DELEG_NOTE_USERQUOTA' value='14'/> + <enumerator name='ZFS_DELEG_NOTE_GROUPQUOTA' value='15'/> + <enumerator name='ZFS_DELEG_NOTE_USERUSED' value='16'/> + <enumerator name='ZFS_DELEG_NOTE_GROUPUSED' value='17'/> + <enumerator name='ZFS_DELEG_NOTE_USEROBJQUOTA' value='18'/> + <enumerator name='ZFS_DELEG_NOTE_GROUPOBJQUOTA' value='19'/> + <enumerator name='ZFS_DELEG_NOTE_USEROBJUSED' value='20'/> + <enumerator name='ZFS_DELEG_NOTE_GROUPOBJUSED' value='21'/> + <enumerator name='ZFS_DELEG_NOTE_HOLD' value='22'/> + <enumerator name='ZFS_DELEG_NOTE_RELEASE' value='23'/> + <enumerator name='ZFS_DELEG_NOTE_DIFF' value='24'/> + <enumerator name='ZFS_DELEG_NOTE_BOOKMARK' value='25'/> + <enumerator name='ZFS_DELEG_NOTE_LOAD_KEY' value='26'/> + <enumerator name='ZFS_DELEG_NOTE_CHANGE_KEY' value='27'/> + <enumerator name='ZFS_DELEG_NOTE_PROJECTUSED' value='28'/> + <enumerator name='ZFS_DELEG_NOTE_PROJECTQUOTA' value='29'/> + <enumerator name='ZFS_DELEG_NOTE_PROJECTOBJUSED' value='30'/> + <enumerator name='ZFS_DELEG_NOTE_PROJECTOBJQUOTA' value='31'/> + <enumerator name='ZFS_DELEG_NOTE_NONE' value='32'/> </enum-decl> <typedef-decl name='zfs_deleg_note_t' type-id='729d4547' id='4613c173'/> <class-decl name='zfs_deleg_perm_tab' size-in-bits='128' is-struct='yes' visibility='default' id='5aa05c1f'> diff --git a/sys/contrib/openzfs/lib/libzfs/libzfs_config.c b/sys/contrib/openzfs/lib/libzfs/libzfs_config.c index 0d2102191389..9d704e4303ff 100644 --- a/sys/contrib/openzfs/lib/libzfs/libzfs_config.c +++ b/sys/contrib/openzfs/lib/libzfs/libzfs_config.c @@ -308,6 +308,23 @@ zpool_refresh_stats(zpool_handle_t *zhp, boolean_t *missing) } /* + * Copies the pool config and state from szhp to dzhp. szhp and dzhp must + * represent the same pool. Used by pool_list_refresh() to avoid another + * round-trip into the kernel to get stats already collected earlier in the + * function. + */ +void +zpool_refresh_stats_from_handle(zpool_handle_t *dzhp, zpool_handle_t *szhp) +{ + VERIFY0(strcmp(dzhp->zpool_name, szhp->zpool_name)); + nvlist_free(dzhp->zpool_old_config); + dzhp->zpool_old_config = dzhp->zpool_config; + dzhp->zpool_config = fnvlist_dup(szhp->zpool_config); + dzhp->zpool_config_size = szhp->zpool_config_size; + dzhp->zpool_state = szhp->zpool_state; +} + +/* * The following environment variables are undocumented * and should be used for testing purposes only: * diff --git a/sys/contrib/openzfs/lib/libzfs_core/libzfs_core.abi b/sys/contrib/openzfs/lib/libzfs_core/libzfs_core.abi index 7464b3adb254..263cad045f7a 100644 --- a/sys/contrib/openzfs/lib/libzfs_core/libzfs_core.abi +++ b/sys/contrib/openzfs/lib/libzfs_core/libzfs_core.abi @@ -617,6 +617,7 @@ <array-type-def dimensions='1' type-id='de572c22' size-in-bits='1472' id='6d3c2f42'> <subrange length='23' type-id='7359adad' id='fdd0f594'/> </array-type-def> + <type-decl name='long long int' size-in-bits='64' id='1eb56b1e'/> <array-type-def dimensions='1' type-id='3a47d82b' size-in-bits='256' id='a133ec23'> <subrange length='4' type-id='7359adad' id='16fe7105'/> </array-type-def> @@ -988,13 +989,6 @@ </function-decl> </abi-instr> <abi-instr address-size='64' path='lib/libspl/os/linux/getmntany.c' language='LANG_C99'> - <array-type-def dimensions='1' type-id='d315442e' size-in-bits='16' id='811205dc'> - <subrange length='1' type-id='7359adad' id='52f813b4'/> - </array-type-def> - <array-type-def dimensions='1' type-id='d3130597' size-in-bits='768' id='f63f23b9'> - <subrange length='12' type-id='7359adad' id='84827bdc'/> - </array-type-def> - <type-decl name='long long int' size-in-bits='64' id='1eb56b1e'/> <class-decl name='mnttab' size-in-bits='256' is-struct='yes' visibility='default' id='1b055409'> <data-member access='public' layout-offset-in-bits='0'> <var-decl name='mnt_special' type-id='26a90f95' visibility='default'/> @@ -1029,93 +1023,6 @@ <var-decl name='mnt_minor' type-id='3502e3ff' visibility='default'/> </data-member> </class-decl> - <typedef-decl name='__u16' type-id='8efea9e5' id='d315442e'/> - <typedef-decl name='__s32' type-id='95e97e5e' id='3158a266'/> - <typedef-decl name='__u32' type-id='f0981eeb' id='3f1a6b60'/> - <typedef-decl name='__s64' type-id='1eb56b1e' id='49659421'/> - <typedef-decl name='__u64' type-id='3a47d82b' id='d3130597'/> - <class-decl name='statx_timestamp' size-in-bits='128' is-struct='yes' visibility='default' id='94101016'> - <data-member access='public' layout-offset-in-bits='0'> - <var-decl name='tv_sec' type-id='49659421' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='64'> - <var-decl name='tv_nsec' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='96'> - <var-decl name='__reserved' type-id='3158a266' visibility='default'/> - </data-member> - </class-decl> - <class-decl name='statx' size-in-bits='2048' is-struct='yes' visibility='default' id='720b04c5'> - <data-member access='public' layout-offset-in-bits='0'> - <var-decl name='stx_mask' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='32'> - <var-decl name='stx_blksize' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='64'> - <var-decl name='stx_attributes' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='128'> - <var-decl name='stx_nlink' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='160'> - <var-decl name='stx_uid' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='192'> - <var-decl name='stx_gid' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='224'> - <var-decl name='stx_mode' type-id='d315442e' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='240'> - <var-decl name='__spare0' type-id='811205dc' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='256'> - <var-decl name='stx_ino' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='320'> - <var-decl name='stx_size' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='384'> - <var-decl name='stx_blocks' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='448'> - <var-decl name='stx_attributes_mask' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='512'> - <var-decl name='stx_atime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='640'> - <var-decl name='stx_btime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='768'> - <var-decl name='stx_ctime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='896'> - <var-decl name='stx_mtime' type-id='94101016' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1024'> - <var-decl name='stx_rdev_major' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1056'> - <var-decl name='stx_rdev_minor' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1088'> - <var-decl name='stx_dev_major' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1120'> - <var-decl name='stx_dev_minor' type-id='3f1a6b60' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1152'> - <var-decl name='stx_mnt_id' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1216'> - <var-decl name='__spare2' type-id='d3130597' visibility='default'/> - </data-member> - <data-member access='public' layout-offset-in-bits='1280'> - <var-decl name='__spare3' type-id='f63f23b9' visibility='default'/> - </data-member> - </class-decl> <class-decl name='mntent' size-in-bits='320' is-struct='yes' visibility='default' id='56fe4a37'> <data-member access='public' layout-offset-in-bits='0'> <var-decl name='mnt_fsname' type-id='26a90f95' visibility='default'/> @@ -1191,8 +1098,6 @@ <pointer-type-def type-id='1b055409' size-in-bits='64' id='9d424d31'/> <pointer-type-def type-id='0bbec9cd' size-in-bits='64' id='62f7a03d'/> <qualified-type-def type-id='62f7a03d' restrict='yes' id='f1cadedf'/> - <pointer-type-def type-id='720b04c5' size-in-bits='64' id='936b8e35'/> - <qualified-type-def type-id='936b8e35' restrict='yes' id='31d265b7'/> <function-decl name='getmntent_r' visibility='default' binding='global' size-in-bits='64'> <parameter type-id='e75a27e9'/> <parameter type-id='3cad23cd'/> @@ -1208,14 +1113,6 @@ <parameter type-id='95e97e5e'/> <return type-id='26a90f95'/> </function-decl> - <function-decl name='statx' visibility='default' binding='global' size-in-bits='64'> - <parameter type-id='95e97e5e'/> - <parameter type-id='9d26089a'/> - <parameter type-id='95e97e5e'/> - <parameter type-id='f0981eeb'/> - <parameter type-id='31d265b7'/> - <return type-id='95e97e5e'/> - </function-decl> <function-decl name='stat64' visibility='default' binding='global' size-in-bits='64'> <parameter type-id='9d26089a'/> <parameter type-id='f1cadedf'/> diff --git a/sys/contrib/openzfs/man/man8/zinject.8 b/sys/contrib/openzfs/man/man8/zinject.8 index 1d9e43aed5ec..704f6a7accd8 100644 --- a/sys/contrib/openzfs/man/man8/zinject.8 +++ b/sys/contrib/openzfs/man/man8/zinject.8 @@ -138,6 +138,20 @@ This injector is automatically cleared after the import is finished. . .It Xo .Nm zinject +.Fl E Ar delay +.Op Fl a +.Op Fl m +.Op Fl f Ar freq +.Op Fl l Ar level +.Op Fl r Ar range +.Op Fl T Ar iotype +.Op Fl t Ar type Ns | Ns Fl b Ar bookmark +.Xc +Inject pipeline ready stage delays for the given object or bookmark. +The delay is specified in milliseconds. +. +.It Xo +.Nm zinject .Fl I .Op Fl s Ar seconds Ns | Ns Fl g Ar txgs .Ar pool diff --git a/sys/contrib/openzfs/man/man8/zpool-upgrade.8 b/sys/contrib/openzfs/man/man8/zpool-upgrade.8 index cf69060da5ce..adae47f82eb1 100644 --- a/sys/contrib/openzfs/man/man8/zpool-upgrade.8 +++ b/sys/contrib/openzfs/man/man8/zpool-upgrade.8 @@ -65,10 +65,10 @@ property). .Cm upgrade .Fl v .Xc -Displays legacy ZFS versions supported by the this version of ZFS. +Displays legacy ZFS versions supported by this version of ZFS. See .Xr zpool-features 7 -for a description of feature flags features supported by this version of ZFS. +for a description of features supported by this version of ZFS. .It Xo .Nm zpool .Cm upgrade diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c index bac166fcd89e..967a018640e1 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c @@ -484,7 +484,28 @@ zvol_request_impl(zvol_state_t *zv, struct bio *bio, struct request *rq, fstrans_cookie_t cookie = spl_fstrans_mark(); uint64_t offset = io_offset(bio, rq); uint64_t size = io_size(bio, rq); - int rw = io_data_dir(bio, rq); + int rw; + + if (rq != NULL) { + /* + * Flush & trim requests go down the zvol_write codepath. Or + * more specifically: + * + * If request is a write, or if it's op_is_sync() and not a + * read, or if it's a flush, or if it's a discard, then send the + * request down the write path. + */ + if (op_is_write(rq->cmd_flags) || + (op_is_sync(rq->cmd_flags) && req_op(rq) != REQ_OP_READ) || + req_op(rq) == REQ_OP_FLUSH || + op_is_discard(rq->cmd_flags)) { + rw = WRITE; + } else { + rw = READ; + } + } else { + rw = bio_data_dir(bio); + } if (unlikely(zv->zv_flags & ZVOL_REMOVING)) { zvol_end_io(bio, rq, SET_ERROR(ENXIO)); diff --git a/sys/contrib/openzfs/module/zcommon/zfs_prop.c b/sys/contrib/openzfs/module/zcommon/zfs_prop.c index 864e3898b365..9190ae0362ea 100644 --- a/sys/contrib/openzfs/module/zcommon/zfs_prop.c +++ b/sys/contrib/openzfs/module/zcommon/zfs_prop.c @@ -364,8 +364,8 @@ zfs_prop_init(void) static const zprop_index_t xattr_table[] = { { "off", ZFS_XATTR_OFF }, - { "on", ZFS_XATTR_SA }, { "sa", ZFS_XATTR_SA }, + { "on", ZFS_XATTR_SA }, { "dir", ZFS_XATTR_DIR }, { NULL } }; diff --git a/sys/contrib/openzfs/module/zfs/dnode.c b/sys/contrib/openzfs/module/zfs/dnode.c index 6c150d31c669..e88d394b5229 100644 --- a/sys/contrib/openzfs/module/zfs/dnode.c +++ b/sys/contrib/openzfs/module/zfs/dnode.c @@ -2656,6 +2656,32 @@ dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset, } /* + * Adjust *offset to the next (or previous) block byte offset at lvl. + * Returns FALSE if *offset would overflow or underflow. + */ +static boolean_t +dnode_next_block(dnode_t *dn, int flags, uint64_t *offset, int lvl) +{ + int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; + int span = lvl * epbs + dn->dn_datablkshift; + uint64_t blkid, maxblkid; + + if (span >= 8 * sizeof (uint64_t)) + return (B_FALSE); + + blkid = *offset >> span; + maxblkid = 1ULL << (8 * sizeof (*offset) - span); + if (!(flags & DNODE_FIND_BACKWARDS) && blkid + 1 < maxblkid) + *offset = (blkid + 1) << span; + else if ((flags & DNODE_FIND_BACKWARDS) && blkid > 0) + *offset = (blkid << span) - 1; + else + return (B_FALSE); + + return (B_TRUE); +} + +/* * Find the next hole, data, or sparse region at or after *offset. * The value 'blkfill' tells us how many items we expect to find * in an L0 data block; this value is 1 for normal objects, @@ -2682,7 +2708,7 @@ int dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset, int minlvl, uint64_t blkfill, uint64_t txg) { - uint64_t initial_offset = *offset; + uint64_t matched = *offset; int lvl, maxlvl; int error = 0; @@ -2706,16 +2732,36 @@ dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset, maxlvl = dn->dn_phys->dn_nlevels; - for (lvl = minlvl; lvl <= maxlvl; lvl++) { + for (lvl = minlvl; lvl <= maxlvl; ) { error = dnode_next_offset_level(dn, flags, offset, lvl, blkfill, txg); - if (error != ESRCH) + if (error == 0 && lvl > minlvl) { + --lvl; + matched = *offset; + } else if (error == ESRCH && lvl < maxlvl && + dnode_next_block(dn, flags, &matched, lvl)) { + /* + * Continue search at next/prev offset in lvl+1 block. + * + * Usually we only search upwards at the start of the + * search as higher level blocks point at a matching + * minlvl block in most cases, but we backtrack if not. + * + * This can happen for txg > 0 searches if the block + * contains only BPs/dnodes freed at that txg. It also + * happens if we are still syncing out the tree, and + * some BP's at higher levels are not updated yet. + * + * We must adjust offset to avoid coming back to the + * same offset and getting stuck looping forever. This + * also deals with the case where offset is already at + * the beginning or end of the object. + */ + ++lvl; + *offset = matched; + } else { break; - } - - while (error == 0 && --lvl >= minlvl) { - error = dnode_next_offset_level(dn, - flags, offset, lvl, blkfill, txg); + } } /* @@ -2727,9 +2773,6 @@ dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset, error = 0; } - if (error == 0 && (flags & DNODE_FIND_BACKWARDS ? - initial_offset < *offset : initial_offset > *offset)) - error = SET_ERROR(ESRCH); out: if (!(flags & DNODE_FIND_HAVELOCK)) rw_exit(&dn->dn_struct_rwlock); diff --git a/sys/contrib/openzfs/module/zfs/mmp.c b/sys/contrib/openzfs/module/zfs/mmp.c index 7db72b9b04b0..fd46127b6068 100644 --- a/sys/contrib/openzfs/module/zfs/mmp.c +++ b/sys/contrib/openzfs/module/zfs/mmp.c @@ -446,7 +446,7 @@ mmp_write_uberblock(spa_t *spa) uint64_t offset; hrtime_t lock_acquire_time = gethrtime(); - spa_config_enter_mmp(spa, SCL_STATE, mmp_tag, RW_READER); + spa_config_enter_priority(spa, SCL_STATE, mmp_tag, RW_READER); lock_acquire_time = gethrtime() - lock_acquire_time; if (lock_acquire_time > (MSEC2NSEC(MMP_MIN_INTERVAL) / 10)) zfs_dbgmsg("MMP SCL_STATE acquisition pool '%s' took %llu ns " diff --git a/sys/contrib/openzfs/module/zfs/range_tree.c b/sys/contrib/openzfs/module/zfs/range_tree.c index ea2d2c7227c8..d73195f1a21f 100644 --- a/sys/contrib/openzfs/module/zfs/range_tree.c +++ b/sys/contrib/openzfs/module/zfs/range_tree.c @@ -585,7 +585,7 @@ zfs_range_tree_remove_impl(zfs_range_tree_t *rt, uint64_t start, uint64_t size, * the size, since we do not support removing partial segments * of range trees with gaps. */ - zfs_zfs_rs_set_fill_raw(rs, rt, zfs_rs_get_end_raw(rs, rt) - + zfs_rs_set_fill_raw(rs, rt, zfs_rs_get_end_raw(rs, rt) - zfs_rs_get_start_raw(rs, rt)); zfs_range_tree_stat_incr(rt, &rs_tmp); diff --git a/sys/contrib/openzfs/module/zfs/spa_config.c b/sys/contrib/openzfs/module/zfs/spa_config.c index cf28955b0c50..f615591e826b 100644 --- a/sys/contrib/openzfs/module/zfs/spa_config.c +++ b/sys/contrib/openzfs/module/zfs/spa_config.c @@ -372,6 +372,8 @@ spa_config_generate(spa_t *spa, vdev_t *vd, uint64_t txg, int getstats) fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, txg); fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, spa_guid(spa)); fnvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA, spa->spa_errata); + fnvlist_add_uint64(config, ZPOOL_CONFIG_MIN_ALLOC, spa->spa_min_alloc); + fnvlist_add_uint64(config, ZPOOL_CONFIG_MAX_ALLOC, spa->spa_max_alloc); if (spa->spa_comment != NULL) fnvlist_add_string(config, ZPOOL_CONFIG_COMMENT, spa->spa_comment); diff --git a/sys/contrib/openzfs/module/zfs/spa_misc.c b/sys/contrib/openzfs/module/zfs/spa_misc.c index 6f7c060f97f8..0bead6d49666 100644 --- a/sys/contrib/openzfs/module/zfs/spa_misc.c +++ b/sys/contrib/openzfs/module/zfs/spa_misc.c @@ -510,7 +510,7 @@ spa_config_tryenter(spa_t *spa, int locks, const void *tag, krw_t rw) static void spa_config_enter_impl(spa_t *spa, int locks, const void *tag, krw_t rw, - int mmp_flag) + int priority_flag) { (void) tag; int wlocks_held = 0; @@ -526,7 +526,7 @@ spa_config_enter_impl(spa_t *spa, int locks, const void *tag, krw_t rw, mutex_enter(&scl->scl_lock); if (rw == RW_READER) { while (scl->scl_writer || - (!mmp_flag && scl->scl_write_wanted)) { + (!priority_flag && scl->scl_write_wanted)) { cv_wait(&scl->scl_cv, &scl->scl_lock); } } else { @@ -551,7 +551,7 @@ spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) } /* - * The spa_config_enter_mmp() allows the mmp thread to cut in front of + * The spa_config_enter_priority() allows the mmp thread to cut in front of * outstanding write lock requests. This is needed since the mmp updates are * time sensitive and failure to service them promptly will result in a * suspended pool. This pool suspension has been seen in practice when there is @@ -560,7 +560,7 @@ spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) */ void -spa_config_enter_mmp(spa_t *spa, int locks, const void *tag, krw_t rw) +spa_config_enter_priority(spa_t *spa, int locks, const void *tag, krw_t rw) { spa_config_enter_impl(spa, locks, tag, rw, 1); } @@ -806,6 +806,7 @@ spa_add(const char *name, nvlist_t *config, const char *altroot) spa->spa_min_ashift = INT_MAX; spa->spa_max_ashift = 0; spa->spa_min_alloc = INT_MAX; + spa->spa_max_alloc = 0; spa->spa_gcd_alloc = INT_MAX; /* Reset cached value */ @@ -1865,6 +1866,19 @@ spa_get_worst_case_asize(spa_t *spa, uint64_t lsize) } /* + * Return the range of minimum allocation sizes for the normal allocation + * class. This can be used by external consumers of the DMU to estimate + * potential wasted capacity when setting the recordsize for an object. + * This is mainly for dRAID pools which always pad to a full stripe width. + */ +void +spa_get_min_alloc_range(spa_t *spa, uint64_t *min_alloc, uint64_t *max_alloc) +{ + *min_alloc = spa->spa_min_alloc; + *max_alloc = spa->spa_max_alloc; +} + +/* * Return the amount of slop space in bytes. It is typically 1/32 of the pool * (3.2%), minus the embedded log space. On very small pools, it may be * slightly larger than this. On very large pools, it will be capped to @@ -3085,6 +3099,7 @@ EXPORT_SYMBOL(spa_version); EXPORT_SYMBOL(spa_state); EXPORT_SYMBOL(spa_load_state); EXPORT_SYMBOL(spa_freeze_txg); +EXPORT_SYMBOL(spa_get_min_alloc_range); /* for Lustre */ EXPORT_SYMBOL(spa_get_dspace); EXPORT_SYMBOL(spa_update_dspace); EXPORT_SYMBOL(spa_deflate); diff --git a/sys/contrib/openzfs/module/zfs/vdev.c b/sys/contrib/openzfs/module/zfs/vdev.c index fc6d445f9785..654e034de9e1 100644 --- a/sys/contrib/openzfs/module/zfs/vdev.c +++ b/sys/contrib/openzfs/module/zfs/vdev.c @@ -1497,12 +1497,14 @@ vdev_spa_set_alloc(spa_t *spa, uint64_t min_alloc) { if (min_alloc < spa->spa_min_alloc) spa->spa_min_alloc = min_alloc; - if (spa->spa_gcd_alloc == INT_MAX) { + + if (min_alloc > spa->spa_max_alloc) + spa->spa_max_alloc = min_alloc; + + if (spa->spa_gcd_alloc == INT_MAX) spa->spa_gcd_alloc = min_alloc; - } else { - spa->spa_gcd_alloc = vdev_gcd(min_alloc, - spa->spa_gcd_alloc); - } + else + spa->spa_gcd_alloc = vdev_gcd(min_alloc, spa->spa_gcd_alloc); } void @@ -1560,8 +1562,7 @@ vdev_metaslab_group_create(vdev_t *vd) if (vd->vdev_ashift < spa->spa_min_ashift) spa->spa_min_ashift = vd->vdev_ashift; - uint64_t min_alloc = vdev_get_min_alloc(vd); - vdev_spa_set_alloc(spa, min_alloc); + vdev_spa_set_alloc(spa, vdev_get_min_alloc(vd)); } } } diff --git a/sys/contrib/openzfs/module/zfs/vdev_label.c b/sys/contrib/openzfs/module/zfs/vdev_label.c index c44f654b0261..0d4fdaa77ba0 100644 --- a/sys/contrib/openzfs/module/zfs/vdev_label.c +++ b/sys/contrib/openzfs/module/zfs/vdev_label.c @@ -511,6 +511,8 @@ vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats, fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASHIFT, vd->vdev_ashift); fnvlist_add_uint64(nv, ZPOOL_CONFIG_ASIZE, vd->vdev_asize); + fnvlist_add_uint64(nv, ZPOOL_CONFIG_MIN_ALLOC, + vdev_get_min_alloc(vd)); fnvlist_add_uint64(nv, ZPOOL_CONFIG_IS_LOG, vd->vdev_islog); if (vd->vdev_noalloc) { fnvlist_add_uint64(nv, ZPOOL_CONFIG_NONALLOCATING, diff --git a/sys/contrib/openzfs/module/zfs/zio.c b/sys/contrib/openzfs/module/zfs/zio.c index 4cf8912d4269..aeea58bedfe4 100644 --- a/sys/contrib/openzfs/module/zfs/zio.c +++ b/sys/contrib/openzfs/module/zfs/zio.c @@ -4574,8 +4574,29 @@ zio_vdev_io_start(zio_t *zio) ASSERT0(zio->io_child_error[ZIO_CHILD_VDEV]); if (vd == NULL) { - if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) - spa_config_enter(spa, SCL_ZIO, zio, RW_READER); + if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) { + /* + * A deadlock workaround. The ddt_prune_unique_entries() + * -> prune_candidates_sync() code path takes the + * SCL_ZIO reader lock and may request it again here. + * If there is another thread who wants the SCL_ZIO + * writer lock, then scl_write_wanted will be set. + * Thus, the spa_config_enter_priority() is used to + * ignore pending writer requests. + * + * The locking should be revised to remove the need + * for this workaround. If that's not workable then + * it should only be applied to the zios involved in + * the pruning process. This impacts the read/write + * I/O balance while pruning. + */ + if (spa->spa_active_ddt_prune) + spa_config_enter_priority(spa, SCL_ZIO, zio, + RW_READER); + else + spa_config_enter(spa, SCL_ZIO, zio, + RW_READER); + } /* * The mirror_ops handle multiple DVAs in a single BP. @@ -5305,6 +5326,16 @@ zio_ready(zio_t *zio) return (NULL); } + if (zio_injection_enabled) { + hrtime_t target = zio_handle_ready_delay(zio); + if (target != 0 && zio->io_target_timestamp == 0) { + zio->io_stage >>= 1; + zio->io_target_timestamp = target; + zio_delay_interrupt(zio); + return (NULL); + } + } + if (zio->io_ready) { ASSERT(IO_IS_ALLOCATING(zio)); ASSERT(BP_GET_BIRTH(bp) == zio->io_txg || diff --git a/sys/contrib/openzfs/module/zfs/zio_inject.c b/sys/contrib/openzfs/module/zfs/zio_inject.c index 981a1be4847c..287577018ed1 100644 --- a/sys/contrib/openzfs/module/zfs/zio_inject.c +++ b/sys/contrib/openzfs/module/zfs/zio_inject.c @@ -827,6 +827,44 @@ zio_handle_export_delay(spa_t *spa, hrtime_t elapsed) zio_handle_pool_delay(spa, elapsed, ZINJECT_DELAY_EXPORT); } +/* + * For testing, inject a delay before ready state. + */ +hrtime_t +zio_handle_ready_delay(zio_t *zio) +{ + inject_handler_t *handler; + hrtime_t now = gethrtime(); + hrtime_t target = 0; + + /* + * Ignore I/O not associated with any logical data. + */ + if (zio->io_logical == NULL) + return (0); + + rw_enter(&inject_lock, RW_READER); + + for (handler = list_head(&inject_handlers); handler != NULL; + handler = list_next(&inject_handlers, handler)) { + if (zio->io_spa != handler->zi_spa || + handler->zi_record.zi_cmd != ZINJECT_DELAY_READY) + continue; + + /* If this handler matches, inject the delay */ + if (zio_match_iotype(zio, handler->zi_record.zi_iotype) && + zio_match_handler(&zio->io_logical->io_bookmark, + zio->io_bp ? BP_GET_TYPE(zio->io_bp) : DMU_OT_NONE, + zio_match_dva(zio), &handler->zi_record, zio->io_error)) { + target = now + (hrtime_t)handler->zi_record.zi_timer; + break; + } + } + + rw_exit(&inject_lock); + return (target); +} + static int zio_calculate_range(const char *pool, zinject_record_t *record) { diff --git a/sys/contrib/openzfs/scripts/zfs-tests.sh b/sys/contrib/openzfs/scripts/zfs-tests.sh index 04f3b6f32cb8..5a0a1a609448 100755 --- a/sys/contrib/openzfs/scripts/zfs-tests.sh +++ b/sys/contrib/openzfs/scripts/zfs-tests.sh @@ -38,6 +38,7 @@ DEBUG="" CLEANUP="yes" CLEANUPALL="no" KMSG="" +TIMEOUT_DEBUG="" LOOPBACK="yes" STACK_TRACER="no" FILESIZE="4G" @@ -364,6 +365,7 @@ OPTIONS: -k Disable cleanup after test failure -K Log test names to /dev/kmsg -f Use files only, disables block device tests + -O Dump debugging info to /dev/kmsg on test timeout -S Enable stack tracer (negative performance impact) -c Only create and populate constrained path -R Automatically rerun failing tests @@ -402,7 +404,7 @@ $0 -x EOF } -while getopts 'hvqxkKfScRmn:d:Ds:r:?t:T:u:I:' OPTION; do +while getopts 'hvqxkKfScRmOn:d:Ds:r:?t:T:u:I:' OPTION; do case $OPTION in h) usage @@ -445,6 +447,9 @@ while getopts 'hvqxkKfScRmn:d:Ds:r:?t:T:u:I:' OPTION; do export NFS=1 . "$nfsfile" ;; + O) + TIMEOUT_DEBUG="yes" + ;; d) FILEDIR="$OPTARG" ;; @@ -773,6 +778,7 @@ msg "${TEST_RUNNER}" \ "${DEBUG:+-D}" \ "${KMEMLEAK:+-m}" \ "${KMSG:+-K}" \ + "${TIMEOUT_DEBUG:+-O}" \ "-c \"${RUNFILES}\"" \ "-T \"${TAGS}\"" \ "-i \"${STF_SUITE}\"" \ @@ -783,6 +789,7 @@ msg "${TEST_RUNNER}" \ ${DEBUG:+-D} \ ${KMEMLEAK:+-m} \ ${KMSG:+-K} \ + ${TIMEOUT_DEBUG:+-O} \ -c "${RUNFILES}" \ -T "${TAGS}" \ -i "${STF_SUITE}" \ diff --git a/sys/contrib/openzfs/tests/runfiles/common.run b/sys/contrib/openzfs/tests/runfiles/common.run index 2b002830c82f..9f531411fbe1 100644 --- a/sys/contrib/openzfs/tests/runfiles/common.run +++ b/sys/contrib/openzfs/tests/runfiles/common.run @@ -168,10 +168,10 @@ tags = ['functional', 'cli_root', 'zinject'] tests = ['zdb_002_pos', 'zdb_003_pos', 'zdb_004_pos', 'zdb_005_pos', 'zdb_006_pos', 'zdb_args_neg', 'zdb_args_pos', 'zdb_block_size_histogram', 'zdb_checksum', 'zdb_decompress', - 'zdb_display_block', 'zdb_encrypted', 'zdb_label_checksum', - 'zdb_object_range_neg', 'zdb_object_range_pos', 'zdb_objset_id', - 'zdb_decompress_zstd', 'zdb_recover', 'zdb_recover_2', 'zdb_backup', - 'zdb_tunables'] + 'zdb_display_block', 'zdb_encrypted', 'zdb_encrypted_raw', + 'zdb_label_checksum', 'zdb_object_range_neg', 'zdb_object_range_pos', + 'zdb_objset_id', 'zdb_decompress_zstd', 'zdb_recover', 'zdb_recover_2', + 'zdb_backup', 'zdb_tunables'] pre = post = tags = ['functional', 'cli_root', 'zdb'] @@ -395,8 +395,9 @@ tags = ['functional', 'cli_root', 'zpool'] [tests/functional/cli_root/zpool_add] tests = ['zpool_add_001_pos', 'zpool_add_002_pos', 'zpool_add_003_pos', 'zpool_add_004_pos', 'zpool_add_006_pos', 'zpool_add_007_neg', - 'zpool_add_008_neg', 'zpool_add_009_neg', 'zpool_add_010_pos', - 'add-o_ashift', 'add_prop_ashift', 'zpool_add_dryrun_output'] + 'zpool_add_008_neg', 'zpool_add_009_neg', 'zpool_add_warn_create', + 'zpool_add_warn_degraded', 'zpool_add_warn_removal', 'add-o_ashift', + 'add_prop_ashift', 'zpool_add_dryrun_output'] tags = ['functional', 'cli_root', 'zpool_add'] [tests/functional/cli_root/zpool_attach] @@ -490,6 +491,10 @@ tests = ['zpool_import_001_pos', 'zpool_import_002_pos', tags = ['functional', 'cli_root', 'zpool_import'] timeout = 1200 +[tests/functional/cli_root/zpool_iostat] +tests = ['zpool_iostat_interval_all', 'zpool_iostat_interval_some'] +tags = ['functional', 'cli_root', 'zpool_iostat'] + [tests/functional/cli_root/zpool_labelclear] tests = ['zpool_labelclear_active', 'zpool_labelclear_exported', 'zpool_labelclear_removed', 'zpool_labelclear_valid'] @@ -1085,7 +1090,8 @@ tags = ['functional', 'write_dirs'] [tests/functional/xattr] tests = ['xattr_001_pos', 'xattr_002_neg', 'xattr_003_neg', 'xattr_004_pos', 'xattr_005_pos', 'xattr_006_pos', 'xattr_007_neg', - 'xattr_011_pos', 'xattr_012_pos', 'xattr_013_pos', 'xattr_compat'] + 'xattr_011_pos', 'xattr_012_pos', 'xattr_013_pos', 'xattr_014_pos', + 'xattr_compat'] tags = ['functional', 'xattr'] [tests/functional/zvol/zvol_ENOSPC] diff --git a/sys/contrib/openzfs/tests/runfiles/sanity.run b/sys/contrib/openzfs/tests/runfiles/sanity.run index b56ffc3a4a2d..249b415029c4 100644 --- a/sys/contrib/openzfs/tests/runfiles/sanity.run +++ b/sys/contrib/openzfs/tests/runfiles/sanity.run @@ -622,7 +622,7 @@ tags = ['functional', 'vdev_zaps'] [tests/functional/xattr] tests = ['xattr_001_pos', 'xattr_002_neg', 'xattr_003_neg', 'xattr_004_pos', 'xattr_005_pos', 'xattr_006_pos', 'xattr_007_neg', - 'xattr_011_pos', 'xattr_013_pos', 'xattr_compat'] + 'xattr_011_pos', 'xattr_013_pos', 'xattr_014_pos', 'xattr_compat'] tags = ['functional', 'xattr'] [tests/functional/zvol/zvol_ENOSPC] diff --git a/sys/contrib/openzfs/tests/test-runner/bin/test-runner.py.in b/sys/contrib/openzfs/tests/test-runner/bin/test-runner.py.in index 2158208be6e5..d2c1185e4a94 100755 --- a/sys/contrib/openzfs/tests/test-runner/bin/test-runner.py.in +++ b/sys/contrib/openzfs/tests/test-runner/bin/test-runner.py.in @@ -34,6 +34,7 @@ from select import select from subprocess import PIPE from subprocess import Popen from subprocess import check_output +from subprocess import run from threading import Timer from time import time, CLOCK_MONOTONIC from os.path import exists @@ -187,6 +188,63 @@ User: %s ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) def kill_cmd(self, proc, options, kmemleak, keyboard_interrupt=False): + + """ + We're about to kill a command due to a timeout. + If we're running with the -O option, then dump debug info about the + process with the highest CPU usage to /dev/kmsg (Linux only). This can + help debug the timeout. + + Debug info includes: + - 30 lines from 'top' + - /proc/<PID>/stack output of process with highest CPU usage + - Last lines strace-ing process with highest CPU usage + """ + if exists("/dev/kmsg"): + c = """ +TOP_OUT="$(COLUMNS=160 top -b -n 1 | head -n 30)" +read -r PID CMD <<< $(echo "$TOP_OUT" | /usr/bin/awk \ +"/COMMAND/{ + print_next=1 + next +} +{ + if (print_next == 1) { + print \\$1\\" \\"\\$12 + exit + } +}") +echo "##### ZTS timeout debug #####" +echo "----- top -----" +echo "$TOP_OUT" +echo "----- /proc/$PID/stack ($CMD)) -----" +cat /proc/$PID/stack +echo "----- strace ($CMD) -----" +TMPFILE="$(mktemp --suffix=ZTS)" +/usr/bin/strace -k --stack-traces -p $PID &> "$TMPFILE" & +sleep 0.1 +killall strace +tail -n 30 $TMPFILE +rm "$TMPFILE" +echo "##### /proc/sysrq-trigger stack #####" +""" + c = "sudo bash -c '" + c + "'" + data = run(c, capture_output=True, shell=True, text=True) + out = data.stdout + try: + kp = Popen([SUDO, "sh", "-c", + "echo '" + out + "' > /dev/kmsg"]) + kp.wait() + + """ + Trigger kernel stack traces + """ + kp = Popen([SUDO, "sh", "-c", + "echo l > /proc/sysrq-trigger"]) + kp.wait() + except Exception: + pass + """ Kill a running command due to timeout, or ^C from the keyboard. If sudo is required, this user was verified previously. @@ -1129,6 +1187,9 @@ def parse_args(): parser.add_option('-o', action='callback', callback=options_cb, default=BASEDIR, dest='outputdir', type='string', metavar='outputdir', help='Specify an output directory.') + parser.add_option('-O', action='store_true', default=False, + dest='timeout_debug', + help='Dump debugging info to /dev/kmsg on test timeout') parser.add_option('-i', action='callback', callback=options_cb, default=TESTDIR, dest='testdir', type='string', metavar='testdir', help='Specify a test directory.') diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am index 1517f90e99a5..678c01b58f94 100644 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am @@ -197,6 +197,7 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \ functional/cli_root/zpool_import/blockfiles/unclean_export.dat.bz2 \ functional/cli_root/zpool_import/zpool_import.cfg \ functional/cli_root/zpool_import/zpool_import.kshlib \ + functional/cli_root/zpool_iostat/zpool_iostat.kshlib \ functional/cli_root/zpool_initialize/zpool_initialize.kshlib \ functional/cli_root/zpool_labelclear/labelclear.cfg \ functional/cli_root/zpool_remove/zpool_remove.cfg \ @@ -640,6 +641,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zdb/zdb_decompress_zstd.ksh \ functional/cli_root/zdb/zdb_display_block.ksh \ functional/cli_root/zdb/zdb_encrypted.ksh \ + functional/cli_root/zdb/zdb_encrypted_raw.ksh \ functional/cli_root/zdb/zdb_label_checksum.ksh \ functional/cli_root/zdb/zdb_object_range_neg.ksh \ functional/cli_root/zdb/zdb_object_range_pos.ksh \ @@ -1027,7 +1029,9 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zpool_add/zpool_add_007_neg.ksh \ functional/cli_root/zpool_add/zpool_add_008_neg.ksh \ functional/cli_root/zpool_add/zpool_add_009_neg.ksh \ - functional/cli_root/zpool_add/zpool_add_010_pos.ksh \ + functional/cli_root/zpool_add/zpool_add_warn_create.ksh \ + functional/cli_root/zpool_add/zpool_add_warn_degraded.ksh \ + functional/cli_root/zpool_add/zpool_add_warn_removal.ksh \ functional/cli_root/zpool_add/zpool_add_dryrun_output.ksh \ functional/cli_root/zpool_attach/attach-o_ashift.ksh \ functional/cli_root/zpool_attach/cleanup.ksh \ @@ -1178,6 +1182,10 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zpool_import/zpool_import_parallel_admin.ksh \ functional/cli_root/zpool_import/zpool_import_parallel_neg.ksh \ functional/cli_root/zpool_import/zpool_import_parallel_pos.ksh \ + functional/cli_root/zpool_iostat/setup.ksh \ + functional/cli_root/zpool_iostat/cleanup.ksh \ + functional/cli_root/zpool_iostat/zpool_iostat_interval_all.ksh \ + functional/cli_root/zpool_iostat/zpool_iostat_interval_some.ksh \ functional/cli_root/zpool_initialize/cleanup.ksh \ functional/cli_root/zpool_initialize/zpool_initialize_attach_detach_add_remove.ksh \ functional/cli_root/zpool_initialize/zpool_initialize_fault_export_import_online.ksh \ @@ -2226,6 +2234,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/xattr/xattr_011_pos.ksh \ functional/xattr/xattr_012_pos.ksh \ functional/xattr/xattr_013_pos.ksh \ + functional/xattr/xattr_014_pos.ksh \ functional/xattr/xattr_compat.ksh \ functional/zap_shrink/cleanup.ksh \ functional/zap_shrink/zap_shrink_001_pos.ksh \ diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted_raw.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted_raw.ksh new file mode 100755 index 000000000000..85d267d5402f --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted_raw.ksh @@ -0,0 +1,75 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2023, Klara Inc. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib + +# +# DESCRIPTION: +# 'zdb -K ...' should enable reading from a raw-encrypted dataset +# +# STRATEGY: +# 1. Create an encrypted dataset +# 2. Write some data to a file +# 3. Run zdb -dddd on the file, confirm it can't be read +# 4. Run zdb -K ... -ddddd on the file, confirm it can be read +# + +verify_runnable "both" + +dataset="$TESTPOOL/$TESTFS2" +file="$TESTDIR2/somefile" +keyfile="$TEST_BASE_DIR/keyfile" + +function cleanup +{ + datasetexists "$dataset" && destroy_dataset "$dataset" -f + rm -f "$keyfile" + default_cleanup_noexit +} + +log_onexit cleanup + +log_must default_setup_noexit $DISKS + +log_assert "'zdb -K' should enable reading from a raw-encrypted dataset" + +# The key must be 32 bytes long. +echo -n "$RAWKEY" > "$keyfile" + +log_must zfs create -o mountpoint="$TESTDIR2" \ + -o encryption=on -o keyformat=raw -o keylocation="file://$keyfile" \ + "$dataset" + +echo 'my great encrypted text' > "$file" + +typeset -i obj=$(ls -i "$file" | cut -d' ' -f1) +typeset -i size=$(wc -c < "$file") + +log_note "test file $file is objid $obj, size $size" + +sync_pool "$TESTPOOL" true + +log_must eval "zdb -dddd $dataset $obj | grep -q 'object encrypted'" + +log_must eval "zdb -K $keyfile -dddd $dataset $obj | grep -q 'size\s$size$'" + +log_pass "'zdb -K' enables reading from a raw-encrypted dataset" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib index 091d65bb4f33..74780bb02141 100644 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add.kshlib @@ -27,6 +27,7 @@ # # Copyright (c) 2012, 2016 by Delphix. All rights reserved. +# Copyright 2025 by Lawrence Livermore National Security, LLC. # . $STF_SUITE/include/libtest.shlib @@ -89,3 +90,44 @@ function save_dump_dev fi echo $dumpdev } + +function zpool_create_add_setup +{ + typeset -i i=0 + + while ((i < 10)); do + log_must truncate -s $MINVDEVSIZE $TEST_BASE_DIR/vdev$i + + eval vdev$i=$TEST_BASE_DIR/vdev$i + ((i += 1)) + done + + if is_linux; then + vdev_lo="$(losetup -f "$vdev4" --show)" + elif is_freebsd; then + vdev_lo=/dev/"$(mdconfig -a -t vnode -f "$vdev4")" + else + vdev_lo="$(lofiadm -a "$vdev4")" + fi +} + +function zpool_create_add_cleanup +{ + datasetexists $TESTPOOL1 && destroy_pool $TESTPOOL1 + + if [[ -e $vdev_lo ]]; then + if is_linux; then + log_must losetup -d "$vdev_lo" + elif is_freebsd; then + log_must mdconfig -d -u "$vdev_lo" + else + log_must lofiadm -d "$vdev_lo" + fi + fi + + typeset -i i=0 + while ((i < 10)); do + rm -f $TEST_BASE_DIR/vdev$i + ((i += 1)) + done +} diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_010_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_create.ksh index df085a2ec746..661e55998d8d 100755 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_010_pos.ksh +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_create.ksh @@ -23,67 +23,51 @@ # # Copyright 2009 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. -# - -# -# Copyright (c) 2012, 2016 by Delphix. All rights reserved. +# Copyright 2012, 2016 by Delphix. All rights reserved. +# Copyright 2025 by Lawrence Livermore National Security, LLC. # . $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/cli_root/zpool_create/zpool_create.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib # # DESCRIPTION: -# Verify zpool add succeed when adding vdevs with matching redundancy. +# Verify zpool add succeeds when adding vdevs with matching redundancy +# and warns with differing redundancy for a healthy pool. # # STRATEGY: # 1. Create several files == $MINVDEVSIZE. # 2. Verify 'zpool add' succeeds with matching redundancy. # 3. Verify 'zpool add' warns with differing redundancy. -# 4. Verify 'zpool add' warns with differing redundancy after removal. # verify_runnable "global" -function cleanup -{ - datasetexists $TESTPOOL1 && destroy_pool $TESTPOOL1 - - typeset -i i=0 - while ((i < 10)); do - rm -f $TEST_BASE_DIR/vdev$i - ((i += 1)) - done -} - +log_assert "Verify 'zpool add' warns for differing redundancy." +log_onexit zpool_create_add_cleanup -log_assert "Verify 'zpool add' succeed with keywords combination." -log_onexit cleanup +zpool_create_add_setup -# 1. Create several files == $MINVDEVSIZE. typeset -i i=0 -while ((i < 10)); do - log_must truncate -s $MINVDEVSIZE $TEST_BASE_DIR/vdev$i - - eval vdev$i=$TEST_BASE_DIR/vdev$i - ((i += 1)) -done +typeset -i j=0 set -A redundancy0_create_args \ "$vdev0" set -A redundancy1_create_args \ "mirror $vdev0 $vdev1" \ - "raidz1 $vdev0 $vdev1" + "raidz1 $vdev0 $vdev1" \ + "draid1:1s $vdev0 $vdev1 $vdev9" set -A redundancy2_create_args \ "mirror $vdev0 $vdev1 $vdev2" \ - "raidz2 $vdev0 $vdev1 $vdev2" + "raidz2 $vdev0 $vdev1 $vdev2" \ + "draid2:1s $vdev0 $vdev1 $vdev2 $vdev9" set -A redundancy3_create_args \ "mirror $vdev0 $vdev1 $vdev2 $vdev3" \ - "raidz3 $vdev0 $vdev1 $vdev2 $vdev3" + "raidz3 $vdev0 $vdev1 $vdev2 $vdev3" \ + "draid3:1s $vdev0 $vdev1 $vdev2 $vdev3 $vdev9" set -A redundancy0_add_args \ "$vdev5" \ @@ -93,21 +77,19 @@ set -A redundancy1_add_args \ "mirror $vdev5 $vdev6" \ "raidz1 $vdev5 $vdev6" \ "raidz1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \ - "mirror $vdev5 $vdev6 raidz1 $vdev7 $vdev8" + "mirror $vdev5 $vdev6 raidz1 $vdev7 $vdev8" \ + "draid1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \ + "mirror $vdev5 $vdev6 draid1 $vdev7 $vdev8" set -A redundancy2_add_args \ "mirror $vdev5 $vdev6 $vdev7" \ - "raidz2 $vdev5 $vdev6 $vdev7" + "raidz2 $vdev5 $vdev6 $vdev7" \ + "draid2 $vdev5 $vdev6 $vdev7" set -A redundancy3_add_args \ "mirror $vdev5 $vdev6 $vdev7 $vdev8" \ - "raidz3 $vdev5 $vdev6 $vdev7 $vdev8" - -set -A log_args "log" "$vdev4" -set -A cache_args "cache" "$vdev4" -set -A spare_args "spare" "$vdev4" - -typeset -i j=0 + "raidz3 $vdev5 $vdev6 $vdev7 $vdev8" \ + "draid3 $vdev5 $vdev6 $vdev7 $vdev8" function zpool_create_add { @@ -148,30 +130,6 @@ function zpool_create_forced_add done } -function zpool_create_rm_add -{ - typeset -n create_args=$1 - typeset -n add_args=$2 - typeset -n rm_args=$3 - - i=0 - while ((i < ${#create_args[@]})); do - j=0 - while ((j < ${#add_args[@]})); do - log_must zpool create $TESTPOOL1 ${create_args[$i]} - log_must zpool add $TESTPOOL1 ${rm_args[0]} ${rm_args[1]} - log_must zpool add $TESTPOOL1 ${add_args[$j]} - log_must zpool remove $TESTPOOL1 ${rm_args[1]} - log_mustnot zpool add $TESTPOOL1 ${rm_args[1]} - log_must zpool add $TESTPOOL1 ${rm_args[0]} ${rm_args[1]} - log_must zpool destroy -f $TESTPOOL1 - - ((j += 1)) - done - ((i += 1)) - done -} - # 2. Verify 'zpool add' succeeds with matching redundancy. zpool_create_add redundancy0_create_args redundancy0_add_args zpool_create_add redundancy1_create_args redundancy1_add_args @@ -195,17 +153,4 @@ zpool_create_forced_add redundancy3_create_args redundancy0_add_args zpool_create_forced_add redundancy3_create_args redundancy1_add_args zpool_create_forced_add redundancy3_create_args redundancy2_add_args -# 4. Verify 'zpool add' warns with differing redundancy after removal. -zpool_create_rm_add redundancy1_create_args redundancy1_add_args log_args -zpool_create_rm_add redundancy2_create_args redundancy2_add_args log_args -zpool_create_rm_add redundancy3_create_args redundancy3_add_args log_args - -zpool_create_rm_add redundancy1_create_args redundancy1_add_args cache_args -zpool_create_rm_add redundancy2_create_args redundancy2_add_args cache_args -zpool_create_rm_add redundancy3_create_args redundancy3_add_args cache_args - -zpool_create_rm_add redundancy1_create_args redundancy1_add_args spare_args -zpool_create_rm_add redundancy2_create_args redundancy2_add_args spare_args -zpool_create_rm_add redundancy3_create_args redundancy3_add_args spare_args - -log_pass "'zpool add' succeed with keywords combination." +log_pass "Verify 'zpool add' warns for differing redundancy." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_degraded.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_degraded.ksh new file mode 100755 index 000000000000..313eb3666f27 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_degraded.ksh @@ -0,0 +1,204 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Copyright 2012, 2016 by Delphix. All rights reserved. +# Copyright 2025 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib + +# +# DESCRIPTION: +# Verify zpool add succeeds when adding vdevs with matching redundancy +# and warns with differing redundancy for a degraded pool. +# +# STRATEGY: +# 1. Create several files == $MINVDEVSIZE. +# 2. Verify 'zpool add' succeeds with matching redundancy +# 3. Verify 'zpool add' warns with differing redundancy when +# a. Degraded pool with replaced mismatch vdev (file vs disk) +# b. Degraded pool dRAID distributed spare active +# c. Degraded pool hot spare active +# + +verify_runnable "global" + +log_assert "Verify 'zpool add' warns for differing redundancy." +log_onexit zpool_create_add_cleanup + +zpool_create_add_setup + +set -A redundancy1_create_args \ + "mirror $vdev0 $vdev1" \ + "raidz1 $vdev0 $vdev1" \ + "draid1:1s $vdev0 $vdev1 $vdev9" + +set -A redundancy2_create_args \ + "mirror $vdev0 $vdev1 $vdev2" \ + "raidz2 $vdev0 $vdev1 $vdev2" \ + "draid2:1s $vdev0 $vdev1 $vdev2 $vdev9" + +set -A redundancy3_create_args \ + "mirror $vdev0 $vdev1 $vdev2 $vdev3" \ + "raidz3 $vdev0 $vdev1 $vdev2 $vdev3" \ + "draid3:1s $vdev0 $vdev1 $vdev2 $vdev3 $vdev9" + +set -A redundancy1_add_args \ + "mirror $vdev5 $vdev6" \ + "raidz1 $vdev5 $vdev6" \ + "raidz1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \ + "mirror $vdev5 $vdev6 raidz1 $vdev7 $vdev8" \ + "draid1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \ + "mirror $vdev5 $vdev6 draid1 $vdev7 $vdev8" + +set -A redundancy2_add_args \ + "mirror $vdev5 $vdev6 $vdev7" \ + "raidz2 $vdev5 $vdev6 $vdev7" \ + "draid2 $vdev5 $vdev6 $vdev7" + +set -A redundancy3_add_args \ + "mirror $vdev5 $vdev6 $vdev7 $vdev8" \ + "raidz3 $vdev5 $vdev6 $vdev7 $vdev8" \ + "draid3 $vdev5 $vdev6 $vdev7 $vdev8" + +set -A redundancy1_create_draid_args \ + "draid1:1s $vdev0 $vdev1 $vdev2" + +set -A redundancy2_create_draid_args \ + "draid2:1s $vdev0 $vdev1 $vdev2 $vdev3" + +set -A redundancy3_create_draid_args \ + "draid3:1s $vdev0 $vdev1 $vdev2 $vdev3 $vdev9" + +set -A redundancy1_create_spare_args \ + "mirror $vdev0 $vdev1 spare $vdev_lo" \ + "raidz1 $vdev0 $vdev1 spare $vdev_lo" \ + "draid1 $vdev0 $vdev1 spare $vdev_lo" + +set -A redundancy2_create_spare_args \ + "mirror $vdev0 $vdev1 $vdev2 spare $vdev_lo" \ + "raidz2 $vdev0 $vdev1 $vdev2 spare $vdev_lo" \ + "draid2 $vdev0 $vdev1 $vdev2 spare $vdev_lo" + +set -A redundancy3_create_spare_args \ + "mirror $vdev0 $vdev1 $vdev2 $vdev3 spare $vdev_lo" \ + "raidz3 $vdev0 $vdev1 $vdev2 $vdev3 spare $vdev_lo" \ + "draid3 $vdev0 $vdev1 $vdev2 $vdev3 spare $vdev_lo" + +set -A replace_args "$vdev1" "$vdev_lo" +set -A draid1_args "$vdev1" "draid1-0-0" +set -A draid2_args "$vdev1" "draid2-0-0" +set -A draid3_args "$vdev1" "draid3-0-0" + +typeset -i i=0 +typeset -i j=0 + +function zpool_create_degraded_add +{ + typeset -n create_args=$1 + typeset -n add_args=$2 + typeset -n rm_args=$3 + + i=0 + while ((i < ${#create_args[@]})); do + j=0 + while ((j < ${#add_args[@]})); do + log_must zpool create $TESTPOOL1 ${create_args[$i]} + log_must zpool offline -f $TESTPOOL1 ${rm_args[0]} + log_must zpool replace -w $TESTPOOL1 ${rm_args[0]} ${rm_args[1]} + log_must zpool add $TESTPOOL1 ${add_args[$j]} + log_must zpool destroy -f $TESTPOOL1 + log_must zpool labelclear -f ${rm_args[0]} + + ((j += 1)) + done + ((i += 1)) + done +} + +function zpool_create_forced_degraded_add +{ + typeset -n create_args=$1 + typeset -n add_args=$2 + typeset -n rm_args=$3 + + i=0 + while ((i < ${#create_args[@]})); do + j=0 + while ((j < ${#add_args[@]})); do + log_must zpool create $TESTPOOL1 ${create_args[$i]} + log_must zpool offline -f $TESTPOOL1 ${rm_args[0]} + log_must zpool replace -w $TESTPOOL1 ${rm_args[0]} ${rm_args[1]} + log_mustnot zpool add $TESTPOOL1 ${add_args[$j]} + log_must zpool add --allow-replication-mismatch $TESTPOOL1 ${add_args[$j]} + log_must zpool destroy -f $TESTPOOL1 + log_must zpool labelclear -f ${rm_args[0]} + + ((j += 1)) + done + ((i += 1)) + done +} + +# 2. Verify 'zpool add' succeeds with matching redundancy and a degraded pool. +zpool_create_degraded_add redundancy1_create_args redundancy1_add_args replace_args +zpool_create_degraded_add redundancy2_create_args redundancy2_add_args replace_args +zpool_create_degraded_add redundancy3_create_args redundancy3_add_args replace_args + +# 3. Verify 'zpool add' warns with differing redundancy and a degraded pool. +# +# a. Degraded pool with replaced mismatch vdev (file vs disk) +zpool_create_forced_degraded_add redundancy1_create_args redundancy2_add_args replace_args +zpool_create_forced_degraded_add redundancy1_create_args redundancy3_add_args replace_args + +zpool_create_forced_degraded_add redundancy2_create_args redundancy1_add_args replace_args +zpool_create_forced_degraded_add redundancy2_create_args redundancy3_add_args replace_args + +zpool_create_forced_degraded_add redundancy3_create_args redundancy1_add_args replace_args +zpool_create_forced_degraded_add redundancy3_create_args redundancy2_add_args replace_args + +# b. Degraded pool dRAID distributed spare active + +zpool_create_forced_degraded_add redundancy1_create_draid_args redundancy2_add_args draid1_args +zpool_create_forced_degraded_add redundancy1_create_draid_args redundancy3_add_args draid1_args + +zpool_create_forced_degraded_add redundancy2_create_draid_args redundancy1_add_args draid2_args +zpool_create_forced_degraded_add redundancy2_create_draid_args redundancy3_add_args draid2_args + +zpool_create_forced_degraded_add redundancy3_create_draid_args redundancy1_add_args draid3_args +zpool_create_forced_degraded_add redundancy3_create_draid_args redundancy2_add_args draid3_args + +# c. Degraded pool hot spare active +zpool_create_forced_degraded_add redundancy1_create_spare_args redundancy2_add_args replace_args +zpool_create_forced_degraded_add redundancy1_create_spare_args redundancy3_add_args replace_args + +zpool_create_forced_degraded_add redundancy2_create_spare_args redundancy1_add_args replace_args +zpool_create_forced_degraded_add redundancy2_create_spare_args redundancy3_add_args replace_args + +zpool_create_forced_degraded_add redundancy3_create_spare_args redundancy1_add_args replace_args +zpool_create_forced_degraded_add redundancy3_create_spare_args redundancy2_add_args replace_args + +log_pass "Verify 'zpool add' warns for differing redundancy." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_removal.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_removal.ksh new file mode 100755 index 000000000000..782858e301ac --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_add/zpool_add_warn_removal.ksh @@ -0,0 +1,126 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Copyright 2012, 2016 by Delphix. All rights reserved. +# Copyright 2025 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_add/zpool_add.kshlib + +# +# DESCRIPTION: +# Verify zpool add succeeds when adding vdevs with matching redundancy +# and warns with differing redundancy after removal. +# +# STRATEGY: +# 1. Create several files == $MINVDEVSIZE. +# 2. Verify 'zpool add' warns with differing redundancy after removal. +# + +verify_runnable "global" + +log_assert "Verify 'zpool add' warns for differing redundancy." +log_onexit zpool_create_add_cleanup + +zpool_create_add_setup + +typeset -i i=0 +typeset -i j=0 + +set -A redundancy1_create_args \ + "mirror $vdev0 $vdev1" \ + "raidz1 $vdev0 $vdev1" \ + "draid1:1s $vdev0 $vdev1 $vdev9" + +set -A redundancy2_create_args \ + "mirror $vdev0 $vdev1 $vdev2" \ + "raidz2 $vdev0 $vdev1 $vdev2" \ + "draid2:1s $vdev0 $vdev1 $vdev2 $vdev9" + +set -A redundancy3_create_args \ + "mirror $vdev0 $vdev1 $vdev2 $vdev3" \ + "raidz3 $vdev0 $vdev1 $vdev2 $vdev3" \ + "draid3:1s $vdev0 $vdev1 $vdev2 $vdev3 $vdev9" + +set -A redundancy1_add_args \ + "mirror $vdev5 $vdev6" \ + "raidz1 $vdev5 $vdev6" \ + "raidz1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \ + "mirror $vdev5 $vdev6 raidz1 $vdev7 $vdev8" \ + "draid1 $vdev5 $vdev6 mirror $vdev7 $vdev8" \ + "mirror $vdev5 $vdev6 draid1 $vdev7 $vdev8" + +set -A redundancy2_add_args \ + "mirror $vdev5 $vdev6 $vdev7" \ + "raidz2 $vdev5 $vdev6 $vdev7" \ + "draid2 $vdev5 $vdev6 $vdev7" + +set -A redundancy3_add_args \ + "mirror $vdev5 $vdev6 $vdev7 $vdev8" \ + "raidz3 $vdev5 $vdev6 $vdev7 $vdev8" \ + "draid3 $vdev5 $vdev6 $vdev7 $vdev8" + +set -A log_args "log" "$vdev_lo" +set -A cache_args "cache" "$vdev_lo" +set -A spare_args "spare" "$vdev_lo" + + +function zpool_create_rm_add +{ + typeset -n create_args=$1 + typeset -n add_args=$2 + typeset -n rm_args=$3 + + i=0 + while ((i < ${#create_args[@]})); do + j=0 + while ((j < ${#add_args[@]})); do + log_must zpool create $TESTPOOL1 ${create_args[$i]} + log_must zpool add $TESTPOOL1 ${rm_args[0]} ${rm_args[1]} + log_must zpool add $TESTPOOL1 ${add_args[$j]} + log_must zpool remove $TESTPOOL1 ${rm_args[1]} + log_mustnot zpool add $TESTPOOL1 ${rm_args[1]} + log_must zpool add $TESTPOOL1 ${rm_args[0]} ${rm_args[1]} + log_must zpool destroy -f $TESTPOOL1 + + ((j += 1)) + done + ((i += 1)) + done +} + +# 2. Verify 'zpool add' warns with differing redundancy after removal. +zpool_create_rm_add redundancy1_create_args redundancy1_add_args log_args +zpool_create_rm_add redundancy2_create_args redundancy2_add_args log_args +zpool_create_rm_add redundancy3_create_args redundancy3_add_args log_args + +zpool_create_rm_add redundancy1_create_args redundancy1_add_args cache_args +zpool_create_rm_add redundancy2_create_args redundancy2_add_args cache_args +zpool_create_rm_add redundancy3_create_args redundancy3_add_args cache_args + +zpool_create_rm_add redundancy1_create_args redundancy1_add_args spare_args +zpool_create_rm_add redundancy2_create_args redundancy2_add_args spare_args +zpool_create_rm_add redundancy3_create_args redundancy3_add_args spare_args diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/cleanup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/cleanup.ksh new file mode 100755 index 000000000000..099b5426031d --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/cleanup.ksh @@ -0,0 +1,30 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025, Klara, Inc. +# +# +. $STF_SUITE/include/libtest.shlib + +log_pass diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/setup.ksh new file mode 100755 index 000000000000..3529a0ccc015 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/setup.ksh @@ -0,0 +1,32 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025, Klara, Inc. +# +# +. $STF_SUITE/include/libtest.shlib + +verify_runnable "global" + +log_pass diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib new file mode 100644 index 000000000000..ea4b0bd2756d --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib @@ -0,0 +1,235 @@ +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025, Klara, Inc. +# + +# Since we want to make sure that iostat responds correctly as pools appear and +# disappear, we run it in the background and capture its output to a file. +# Once we're done, we parse the output and ensure it matches what we'd expect +# from the operations we performed. +# +# Because iostat is producing output every interval, it may produce the "same" +# output for each step of the change; in fact, we want that to make sure we +# don't miss anything. So, we describe what we expect as a series of "chunks". +# Each chunk is a particular kind of output, which may repeat. Current known +# chunk types are: +# +# NOPOOL: the text "no pools available" +# HEADER: three lines, starting with "capacity", "pool" and "----" respectively. +# (the rough shape of the normal iostat header). +# POOL1: a line starting with "pool1" (stats line for a pool of that name) +# POOL2: a line starting with "pool2" +# POOLBOTH: three lines, starting with "pool1", "pool2" (either order) and +# "-----" respectively. (the pool stat output for multiple pools) +# +# (the parser may produce other chunks in a failed parse to assist with +# debugging, but they should never be part of the "wanted" output See the +# parser commentary below). +# +# To help recognise the start of a new interval output, we run iostat with the +# -T u option, which will output a numeric timestamp before each header or +# second-or-later pool stat after the header. +# +# To keep the test run shorter, we use a subsecond interval, but to make sure +# nothing is missed, we sleep for three intervals after each change. + +typeset _iostat_out=$(mktemp) +typeset _iostat_pid="" + +function cleanup_iostat { + if [[ -n $_iostat_pid ]] ; then + kill -KILL $_iostat_pid || true + fi + rm -f $_iostat_out +} + +function start_iostat { + zpool iostat -T u $@ 0.1 > $_iostat_out 2>&1 & + _iostat_pid=$! +} + +function stop_iostat { + kill -TERM $_iostat_pid + wait $_iostat_pid + _iostat_pid="" +} + +function delay_iostat { + sleep 0.3 +} + +typeset -a _iostat_expect +function expect_iostat { + typeset chunk=$1 + _iostat_expect+=($chunk) +} + +# Parse the output The `state` var is used to track state across +# multiple lines. The `last` var and the `_got_iostat` function are used +# to record the completed chunks, and to collapse repetitions. +typeset -a _iostat_got +typeset _iostat_last="" +typeset _iostat_state="" + +function _got_iostat { + typeset chunk=$1 + if [[ -n $chunk && $_iostat_last != $chunk ]] ; then + _iostat_last=$chunk + _iostat_got+=($chunk) + fi + _iostat_state="" +} + +function verify_iostat { + + cat $_iostat_out | while read line ; do + + # The "no pools available" text has no timestamp or other + # header, and should never appear in the middle of multiline + # chunk, so we can close any in-flight state. + if [[ $line = "no pools available" ]] ; then + _got_iostat $_iostat_state + _got_iostat "NOPOOL" + continue + fi + + # A run of digits alone on the line is a timestamp (the `-T u` + # switch to `iostat`). It closes any in-flight state as a + # complete chunk, and indicates the start of a new chunk. + if [[ -z ${line/#+([0-9])/} ]] ; then + _got_iostat $_iostat_state + _iostat_state="TIMESTAMP" + continue + fi + + # For this test, the first word of each line should be unique, + # so we extract it and use it for simplicity. + typeset first=${line%% *} + + # Header is emitted whenever the pool list changes. It has + # three lines: + # + # capacity operations bandwidth + # pool alloc free read write read write + # ---------- ----- ----- ----- ----- ----- ----- + # + # Each line moves the state; when we get to a run of dashes, we + # commit. Note that we check for one-or-more dashes, because + # the width can vary depending on the length of pool name. + # + if [[ $_iostat_state = "TIMESTAMP" && + $first = "capacity" ]] ; then + _iostat_state="INHEADER1" + continue + fi + if [[ $_iostat_state = "INHEADER1" && + $first = "pool" ]] ; then + _iostat_state="INHEADER2" + continue + fi + if [[ $_iostat_state = "INHEADER2" && + -z ${first/#+(-)/} ]] ; then + # Headers never repeat, so if the last committed chunk + # was a header, we commit this one as EXTRAHEADER so we + # can see it in the error output. + if [[ $_iostat_last = "HEADER" ]] ; then + _got_iostat "EXTRAHEADER" + elif [[ $_iostat_last != "EXTRAHEADER" ]] ; then + _got_iostat "HEADER" + fi + _iostat_state="HEADER" + continue + fi + + # A pool stat line looks like: + # + # pool1 147K 240M 0 0 0 0 + # + # If there are multiple pools, iostat follows them with a + # separator of dashed lines: + # + # pool1 147K 240M 0 0 0 0 + # pool2 147K 240M 0 0 0 0 + # ---------- ----- ----- ----- ----- ----- ----- + # + # Stats rows always start after a timestamp or a header. If the + # header was emitted, we won't see a timestamp here (it goes + # before the header). + # + # Because our test exercises both pools on their own and + # together, we allow pools in either order. In practice they + # are sorted, but that's a side-effect of the implementation + # (see zpool_compare()), so we're not going to rely on it here. + if [[ $first = "pool1" ]] || [[ $first = "pool2" ]] ; then + + # First line, track which one we saw. If it's a + # standalone line, it will be committed by the next + # NOPOOL or TIMESTAMP above (or the `_got_iostat` after + # the loop if this is the last line). + if [[ $_iostat_state == "TIMESTAMP" || + $_iostat_state == "HEADER" ]] ; then + if [[ $first = "pool1" ]] ; then + _iostat_state="POOL1" + elif [[ $first = "pool2" ]] ; then + _iostat_state="POOL2" + fi + continue + fi + + # If this is the second pool, we're in a multi-pool + # block, and need to look for the separator to close it + # out. + if [[ $_iostat_state = "POOL1" && $first = "pool2" ]] || + [[ $_iostat_state = "POOL2" && $first = "pool1" ]] ; + then + _iostat_state="INPOOLBOTH" + continue + fi + fi + + # Separator after the stats block. + if [[ $_iostat_state = "INPOOLBOTH" && + -z ${first/#+(-)/} ]] ; then + _got_iostat "POOLBOTH" + continue + fi + + # Anything else will fall through to here. We commit any + # in-flight state, then "UNKNOWN", all to help with debugging.. + if [[ $_iostat_state != "UNKNOWN" ]] ; then + _got_iostat $_iostat_state + _got_iostat "UNKNOWN" + fi + done + + # Close out any remaining state. + _got_iostat $_iostat_state + + # Compare what we wanted with what we got, and pass/fail the test! + if [[ "${_iostat_expect[*]}" != "${_iostat_got[*]}" ]] ; then + log_note "expected: ${_iostat_expect[*]}" + log_note " got: ${_iostat_got[*]}" + log_fail "zpool iostat did not produce expected output" + fi +} diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_all.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_all.ksh new file mode 100755 index 000000000000..8e040058ec3e --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_all.ksh @@ -0,0 +1,90 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025, Klara, Inc. +# + +# `zpool iostat <N>` should keep running and update the pools it displays as +# pools are created/destroyed/imported/export. + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib + +typeset vdev1=$(mktemp) +typeset vdev2=$(mktemp) + +function cleanup { + cleanup_iostat + + poolexists pool1 && destroy_pool pool1 + poolexists pool2 && destroy_pool pool2 + rm -f $vdev1 $vdev2 +} + +log_must mkfile $MINVDEVSIZE $vdev1 $vdev2 + +expect_iostat "NOPOOL" + +start_iostat + +delay_iostat + +expect_iostat "HEADER" +expect_iostat "POOL1" +log_must zpool create pool1 $vdev1 +delay_iostat + +expect_iostat "HEADER" +expect_iostat "POOLBOTH" +log_must zpool create pool2 $vdev2 +delay_iostat + +expect_iostat "NOPOOL" +log_must zpool export -a +delay_iostat + +expect_iostat "HEADER" +expect_iostat "POOL2" +log_must zpool import -d $vdev2 pool2 +delay_iostat + +expect_iostat "HEADER" +expect_iostat "POOLBOTH" +log_must zpool import -d $vdev1 pool1 +delay_iostat + +expect_iostat "HEADER" +expect_iostat "POOL2" +log_must zpool destroy pool1 +delay_iostat + +expect_iostat "NOPOOL" +log_must zpool destroy pool2 +delay_iostat + +stop_iostat + +verify_iostat + +log_pass "zpool iostat in interval mode follows pool updates" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_some.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_some.ksh new file mode 100755 index 000000000000..ab1f258aa1cd --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_iostat/zpool_iostat_interval_some.ksh @@ -0,0 +1,80 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025, Klara, Inc. +# + +# `zpool iostat <pools> <N>` should keep running and only show the listed pools. + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_iostat/zpool_iostat.kshlib + +typeset vdev1=$(mktemp) +typeset vdev2=$(mktemp) + +function cleanup { + cleanup_iostat + + poolexists pool1 && destroy_pool pool1 + poolexists pool2 && destroy_pool pool2 + rm -f $vdev1 $vdev2 +} + +log_must mkfile $MINVDEVSIZE $vdev1 $vdev2 + +log_must zpool create pool1 $vdev1 +delay_iostat + +expect_iostat "HEADER" +expect_iostat "POOL1" +start_iostat pool1 +delay_iostat + +log_must zpool create pool2 $vdev2 +delay_iostat + +expect_iostat "NOPOOL" +log_must zpool export -a +delay_iostat + +log_must zpool import -d $vdev2 pool2 +delay_iostat + +expect_iostat "HEADER" +expect_iostat "POOL1" +log_must zpool import -d $vdev1 pool1 +delay_iostat + +expect_iostat "NOPOOL" +log_must zpool destroy pool1 +delay_iostat + +log_must zpool destroy pool2 +delay_iostat + +stop_iostat + +verify_iostat + +log_pass "zpool iostat in interval mode with pools follows listed pool updates" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib index 0a402e71ee68..345239b88680 100644 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib @@ -1234,10 +1234,10 @@ function verify_fs_aedsx typeset oldval set -A modes "on" "off" oldval=$(get_prop $perm $fs) - if [[ $oldval == "on" ]]; then - n=1 - elif [[ $oldval == "off" ]]; then + if [[ $oldval == "off" ]]; then n=0 + else + n=1 fi log_note "$user zfs set $perm=${modes[$n]} $fs" user_run $user zfs set $perm=${modes[$n]} $fs diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/setup.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/setup.ksh index 26153aafbc02..0e79e9b8b70c 100755 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/setup.ksh +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/setup.ksh @@ -39,6 +39,6 @@ verify_runnable "global" # create a pool without any features -log_must mkfile 128m $TMPDEV +log_must truncate -s $MINVDEVSIZE $TMPDEV log_pass diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/upgrade_readonly_pool.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/upgrade_readonly_pool.ksh index d6bd69b7e134..e81d07794689 100755 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/upgrade_readonly_pool.ksh +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/upgrade/upgrade_readonly_pool.ksh @@ -35,17 +35,19 @@ verify_runnable "global" -TESTFILE="$TESTDIR/file.bin" - log_assert "User accounting upgrade should not be executed on readonly pool" log_onexit cleanup_upgrade # 1. Create a pool with the feature@userobj_accounting disabled to simulate # a legacy pool from a previous ZFS version. -log_must zpool create -d -m $TESTDIR $TESTPOOL $TMPDEV +log_must zpool create -d $TESTPOOL $TMPDEV +log_must zfs create $TESTPOOL/$TESTFS + +MNTPNT=$(get_prop mountpoint $TESTPOOL/$TESTFS) +TESTFILE="$MNTPNT/file.bin" # 2. Create a file on the "legecy" dataset -log_must touch $TESTDIR/file.bin +log_must touch $TESTFILE # 3. Enable feature@userobj_accounting on the pool and verify it is only # "enabled" and not "active": upgrading starts when the filesystem is mounted @@ -54,12 +56,12 @@ log_must test "enabled" == "$(get_pool_prop 'feature@userobj_accounting' $TESTPO # 4. Export the pool and re-import is readonly, without mounting any filesystem log_must zpool export $TESTPOOL -log_must zpool import -o readonly=on -N -d "$(dirname $TMPDEV)" $TESTPOOL +log_must zpool import -o readonly=on -N -d $TEST_BASE_DIR $TESTPOOL # 5. Try to mount the root dataset manually without the "ro" option, then verify # filesystem status and the pool feature status (not "active") to ensure the # pool "readonly" status is enforced. -log_must mount -t zfs -o zfsutil $TESTPOOL $TESTDIR +log_must zfs mount -R $TESTPOOL log_must stat "$TESTFILE" log_mustnot touch "$TESTFILE" log_must test "enabled" == "$(get_pool_prop 'feature@userobj_accounting' $TESTPOOL)" diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/xattr/xattr_014_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/xattr/xattr_014_pos.ksh new file mode 100755 index 000000000000..d4c9a0a41816 --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/xattr/xattr_014_pos.ksh @@ -0,0 +1,53 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025 by Klara, Inc. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/xattr/xattr_common.kshlib + +# +# DESCRIPTION: +# The default xattr should be shown as 'sa', not 'on', for clarity. +# +# STRATEGY: +# 1. Create a filesystem. +# 2. Verify that the xattra is shown as 'sa'. +# 3. Manually set the value to 'dir', 'sa', 'on', and 'off'. +# 4. Verify that it is shown as 'dir', 'sa', 'sa', and 'off. +# + +log_assert "The default and specific xattr values are displayed correctly." + +set -A args "dir" "sa" "on" "off" +set -A display "dir" "sa" "sa" "off" + +log_must eval "[[ 'sa' == '$(zfs get -Hpo value xattr $TESTPOOL)' ]]" + +for i in `seq 0 3`; do + log_must zfs set xattr="${args[$i]}" $TESTPOOL + log_must eval "[[ '${display[$i]}' == '$(zfs get -Hpo value xattr $TESTPOOL)' ]]" +done +log_pass "The default and specific xattr values are displayed correctly." diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_fua.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_fua.ksh index 571a698eb63a..502ebada22dc 100755 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_fua.ksh +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_fua.ksh @@ -50,17 +50,53 @@ fi typeset datafile1="$(mktemp -t zvol_misc_fua1.XXXXXX)" typeset datafile2="$(mktemp -t zvol_misc_fua2.XXXXXX)" +typeset datafile3="$(mktemp -t zvol_misc_fua3_log.XXXXXX)" typeset zvolpath=${ZVOL_DEVDIR}/$TESTPOOL/$TESTVOL +typeset DISK1=${DISKS%% *} function cleanup { - rm "$datafile1" "$datafile2" + log_must zpool remove $TESTPOOL $datafile3 + rm "$datafile1" "$datafile2" "$datafile2" +} + +# Prints the total number of sync writes for a vdev +# $1: vdev +function get_sync +{ + zpool iostat -p -H -v -r $TESTPOOL $1 | \ + awk '/[0-9]+$/{s+=$4+$5} END{print s}' } function do_test { # Wait for udev to create symlinks to our zvol block_device_wait $zvolpath + # Write using sync (creates FLUSH calls after writes, but not FUA) + old_vdev_writes=$(get_sync $DISK1) + old_log_writes=$(get_sync $datafile3) + + log_must fio --name=write_iops --size=5M \ + --ioengine=libaio --verify=0 --bs=4K \ + --iodepth=1 --rw=randwrite --group_reporting=1 \ + --filename=$zvolpath --sync=1 + + vdev_writes=$(( $(get_sync $DISK1) - $old_vdev_writes)) + log_writes=$(( $(get_sync $datafile3) - $old_log_writes)) + + # When we're doing sync writes, we should see many more writes go to + # the log vs the first vdev. Experiments show anywhere from a 160-320x + # ratio of writes to the log vs the first vdev (due to some straggler + # writes to the first vdev). + # + # Check that we have a large ratio (100x) of sync writes going to the + # log device + ratio=$(($log_writes / $vdev_writes)) + log_note "Got $log_writes log writes, $vdev_writes vdev writes." + if [ $ratio -lt 100 ] ; then + log_fail "Expected > 100x more log writes than vdev writes. " + fi + # Create a data file log_must dd if=/dev/urandom of="$datafile1" bs=1M count=5 @@ -81,6 +117,8 @@ log_assert "Verify that a ZFS volume can do Force Unit Access (FUA)" log_onexit cleanup log_must zfs set compression=off $TESTPOOL/$TESTVOL +log_must truncate -s 100M $datafile3 +log_must zpool add $TESTPOOL log $datafile3 log_note "Testing without blk-mq" diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index 8b5908f5219a..b7316ea5f387 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -869,7 +869,7 @@ _thread_lock(struct thread *td) WITNESS_LOCK(&m->lock_object, LOP_EXCLUSIVE, file, line); return; } - _mtx_release_lock_quick(m); + atomic_store_rel_ptr(&m->mtx_lock, MTX_UNOWNED); slowpath_unlocked: spinlock_exit(); slowpath_noirq: @@ -959,7 +959,7 @@ retry: } if (m == td->td_lock) break; - _mtx_release_lock_quick(m); + atomic_store_rel_ptr(&m->mtx_lock, MTX_UNOWNED); } LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, line); @@ -1071,7 +1071,7 @@ __mtx_unlock_sleep(volatile uintptr_t *c, uintptr_t v) * can be removed from the hash list if it is empty. */ turnstile_chain_lock(&m->lock_object); - _mtx_release_lock_quick(m); + atomic_store_rel_ptr(&m->mtx_lock, MTX_UNOWNED); ts = turnstile_lookup(&m->lock_object); MPASS(ts != NULL); if (LOCK_LOG_TEST(&m->lock_object, opts)) diff --git a/sys/kern/sys_pipe.c b/sys/kern/sys_pipe.c index 30527fdd4fd0..57ebe8dc85f0 100644 --- a/sys/kern/sys_pipe.c +++ b/sys/kern/sys_pipe.c @@ -567,7 +567,7 @@ pipespace_new(struct pipe *cpipe, int size) static int curfail = 0; static struct timeval lastfail; - KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked")); + PIPE_LOCK_ASSERT(cpipe, MA_NOTOWNED); KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW), ("pipespace: resize of direct writes not allowed")); retry: @@ -1679,8 +1679,7 @@ static void pipe_free_kmem(struct pipe *cpipe) { - KASSERT(!mtx_owned(PIPE_MTX(cpipe)), - ("pipe_free_kmem: pipe mutex locked")); + PIPE_LOCK_ASSERT(cpipe, MA_NOTOWNED); if (cpipe->pipe_buffer.buffer != NULL) { atomic_subtract_long(&amountpipekva, cpipe->pipe_buffer.size); diff --git a/sys/modules/zfs/zfs_config.h b/sys/modules/zfs/zfs_config.h index c595030ed4a0..db1b6f33a8ef 100644 --- a/sys/modules/zfs/zfs_config.h +++ b/sys/modules/zfs/zfs_config.h @@ -843,7 +843,7 @@ /* #undef ZFS_DEVICE_MINOR */ /* Define the project alias string. */ -#define ZFS_META_ALIAS "zfs-2.4.99-72-FreeBSD_gb2196fbed" +#define ZFS_META_ALIAS "zfs-2.4.99-95-FreeBSD_g5605a6d79" /* Define the project author. */ #define ZFS_META_AUTHOR "OpenZFS" @@ -852,7 +852,7 @@ /* #undef ZFS_META_DATA */ /* Define the maximum compatible kernel version. */ -#define ZFS_META_KVER_MAX "6.16" +#define ZFS_META_KVER_MAX "6.17" /* Define the minimum compatible kernel version. */ #define ZFS_META_KVER_MIN "4.18" @@ -873,7 +873,7 @@ #define ZFS_META_NAME "zfs" /* Define the project release. */ -#define ZFS_META_RELEASE "72-FreeBSD_gb2196fbed" +#define ZFS_META_RELEASE "95-FreeBSD_g5605a6d79" /* Define the project version. */ #define ZFS_META_VERSION "2.4.99" diff --git a/sys/modules/zfs/zfs_gitrev.h b/sys/modules/zfs/zfs_gitrev.h index 9eae1e8573c0..8a1802f5480b 100644 --- a/sys/modules/zfs/zfs_gitrev.h +++ b/sys/modules/zfs/zfs_gitrev.h @@ -1 +1 @@ -#define ZFS_META_GITREV "zfs-2.4.99-72-gb2196fbed" +#define ZFS_META_GITREV "zfs-2.4.99-95-g5605a6d79" diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index b534a74626bc..83300d4eb593 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -221,10 +221,6 @@ void _thread_lock(struct thread *); #define _mtx_release_lock(mp, tid) \ atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), MTX_UNOWNED) -/* Release mtx_lock quickly, assuming we own it. */ -#define _mtx_release_lock_quick(mp) \ - atomic_store_rel_ptr(&(mp)->mtx_lock, MTX_UNOWNED) - #define _mtx_release_lock_fetch(mp, vp) \ atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, (vp), MTX_UNOWNED) @@ -332,7 +328,7 @@ void _thread_lock(struct thread *); (mp)->mtx_recurse--; \ else { \ LOCKSTAT_PROFILE_RELEASE_SPIN_LOCK(spin__release, mp); \ - _mtx_release_lock_quick((mp)); \ + atomic_store_rel_ptr(&(mp)->mtx_lock, MTX_UNOWNED); \ } \ spinlock_exit(); \ }) |