diff options
Diffstat (limited to 'sys')
286 files changed, 16823 insertions, 2325 deletions
diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC index 385eb9667652..81427b5b18b6 100644 --- a/sys/amd64/conf/GENERIC +++ b/sys/amd64/conf/GENERIC @@ -184,6 +184,9 @@ device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s device nvme # base NVMe driver device nvd # expose NVMe namespaces as disks, depends on nvme +# Universal Flash Storage Host Controller Interface support +device ufshci # UFS host controller + # Intel Volume Management Device (VMD) support device vmd @@ -383,6 +386,7 @@ options HID_DEBUG # enable debug msgs device hid # Generic HID support device hidbus # Generic HID Bus options IICHID_SAMPLING # Workaround missing GPIO INTR support +options U2F_MAKE_UHID_ALIAS # install /dev/uhid alias for /dev/u2f/ # EFI devices device efidev # EFI pseudo-device diff --git a/sys/amd64/vmm/io/vioapic.c b/sys/amd64/vmm/io/vioapic.c index 8869dc1383e6..7df6193d6dc0 100644 --- a/sys/amd64/vmm/io/vioapic.c +++ b/sys/amd64/vmm/io/vioapic.c @@ -130,6 +130,15 @@ vioapic_send_intr(struct vioapic *vioapic, int pin) vector = low & IOART_INTVEC; dest = high >> APIC_ID_SHIFT; + /* + * Ideally we'd just call lapic_intr_msi() here with the + * constructed MSI instead of interpreting it for ourselves. + * But until/unless we support emulated IOMMUs with interrupt + * remapping, interpretation is simple. We just need to mask + * in the Extended Destination ID bits for the 15-bit + * enlightenment (http://david.woodhou.se/ExtDestId.pdf) + */ + dest |= ((high & APIC_EXT_ID_MASK) >> APIC_EXT_ID_SHIFT) << 8; vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector); } diff --git a/sys/amd64/vmm/vmm_lapic.c b/sys/amd64/vmm/vmm_lapic.c index fd511733492e..0cae01f172ec 100644 --- a/sys/amd64/vmm/vmm_lapic.c +++ b/sys/amd64/vmm/vmm_lapic.c @@ -115,6 +115,11 @@ lapic_intr_msi(struct vm *vm, uint64_t addr, uint64_t msg) * physical otherwise. */ dest = (addr >> 12) & 0xff; + /* + * Extended Destination ID support uses bits 5-11 of the address: + * http://david.woodhou.se/ExtDestId.pdf + */ + dest |= ((addr >> 5) & 0x7f) << 8; phys = ((addr & (MSI_X86_ADDR_RH | MSI_X86_ADDR_LOG)) != (MSI_X86_ADDR_RH | MSI_X86_ADDR_LOG)); delmode = msg & APIC_DELMODE_MASK; diff --git a/sys/amd64/vmm/x86.c b/sys/amd64/vmm/x86.c index 366f1da9f850..2e2224595ab4 100644 --- a/sys/amd64/vmm/x86.c +++ b/sys/amd64/vmm/x86.c @@ -48,7 +48,12 @@ SYSCTL_DECL(_hw_vmm); static SYSCTL_NODE(_hw_vmm, OID_AUTO, topology, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, NULL); -#define CPUID_VM_HIGH 0x40000000 +#define CPUID_VM_SIGNATURE 0x40000000 +#define CPUID_BHYVE_FEATURES 0x40000001 +#define CPUID_VM_HIGH CPUID_BHYVE_FEATURES + +/* Features advertised in CPUID_BHYVE_FEATURES %eax */ +#define CPUID_BHYVE_FEAT_EXT_DEST_ID (1UL << 0) /* MSI Extended Dest ID */ static const char bhyve_id[12] = "bhyve bhyve "; @@ -100,7 +105,7 @@ x86_emulate_cpuid(struct vcpu *vcpu, uint64_t *rax, uint64_t *rbx, if (cpu_exthigh != 0 && func >= 0x80000000) { if (func > cpu_exthigh) func = cpu_exthigh; - } else if (func >= 0x40000000) { + } else if (func >= CPUID_VM_SIGNATURE) { if (func > CPUID_VM_HIGH) func = CPUID_VM_HIGH; } else if (func > cpu_high) { @@ -601,13 +606,20 @@ x86_emulate_cpuid(struct vcpu *vcpu, uint64_t *rax, uint64_t *rbx, regs[3] = 0; break; - case 0x40000000: + case CPUID_VM_SIGNATURE: regs[0] = CPUID_VM_HIGH; bcopy(bhyve_id, ®s[1], 4); bcopy(bhyve_id + 4, ®s[2], 4); bcopy(bhyve_id + 8, ®s[3], 4); break; + case CPUID_BHYVE_FEATURES: + regs[0] = CPUID_BHYVE_FEAT_EXT_DEST_ID; + regs[1] = 0; + regs[2] = 0; + regs[3] = 0; + break; + default: default_leaf: /* diff --git a/sys/arm/arm/pmu_fdt.c b/sys/arm/arm/pmu_fdt.c index 3e733f3e1b18..dd6087652e38 100644 --- a/sys/arm/arm/pmu_fdt.c +++ b/sys/arm/arm/pmu_fdt.c @@ -152,7 +152,7 @@ pmu_parse_intr(device_t dev, struct pmu_softc *sc) if (intr_is_per_cpu(sc->irq[0].res)) { if (has_affinity) { device_printf(dev, - "Per CPU interupt have declared affinity\n"); + "Per CPU interrupt have declared affinity\n"); err = ENXIO; goto done; } @@ -179,7 +179,7 @@ pmu_parse_intr(device_t dev, struct pmu_softc *sc) if (intr_is_per_cpu(sc->irq[i].res)) { - device_printf(dev, "Unexpected per CPU interupt\n"); + device_printf(dev, "Unexpected per CPU interrupt\n"); err = ENXIO; goto done; } diff --git a/sys/arm/conf/GENERIC b/sys/arm/conf/GENERIC index 7394f3842d43..26b0c7bf0294 100644 --- a/sys/arm/conf/GENERIC +++ b/sys/arm/conf/GENERIC @@ -261,6 +261,7 @@ device aw_thermal # Allwinner Thermal Sensor Controller # HID support device hid # Generic HID support device hidbus # Generic HID Bus +options U2F_MAKE_UHID_ALIAS # install /dev/uhid alias for /dev/u2f/ # Flattened Device Tree options FDT # Configure using FDT/DTB data diff --git a/sys/arm64/arm64/elf32_machdep.c b/sys/arm64/arm64/elf32_machdep.c index 7cd5327b9f1b..5c81c6cdce3d 100644 --- a/sys/arm64/arm64/elf32_machdep.c +++ b/sys/arm64/arm64/elf32_machdep.c @@ -195,7 +195,7 @@ freebsd32_fetch_syscall_args(struct thread *td) register_t *ap; struct syscall_args *sa; int error, i, nap, narg; - unsigned int args[4]; + unsigned int args[6]; nap = 4; p = td->td_proc; diff --git a/sys/arm64/conf/std.dev b/sys/arm64/conf/std.dev index c5c364ffda04..719f272426dd 100644 --- a/sys/arm64/conf/std.dev +++ b/sys/arm64/conf/std.dev @@ -115,6 +115,7 @@ device mmcsd # mmc/sd flash cards options HID_DEBUG # enable debug msgs device hid # Generic HID support device hidbus # Generic HID Bus +options U2F_MAKE_UHID_ALIAS # install /dev/uhid alias for /dev/u2f/ # Firmware device mmio_sram # Generic on-chip SRAM diff --git a/sys/cddl/boot/zfs/zfsimpl.h b/sys/cddl/boot/zfs/zfsimpl.h index 83d964360343..c9de1fe4c391 100644 --- a/sys/cddl/boot/zfs/zfsimpl.h +++ b/sys/cddl/boot/zfs/zfsimpl.h @@ -536,6 +536,12 @@ typedef struct zio_gbh { offsetof(vdev_label_t, vl_uberblock[(n) << VDEV_UBERBLOCK_SHIFT(vd)]) #define VDEV_UBERBLOCK_SIZE(vd) (1ULL << VDEV_UBERBLOCK_SHIFT(vd)) +#define ASHIFT_UBERBLOCK_SHIFT(ashift) \ + MIN(MAX(ashift, UBERBLOCK_SHIFT), \ + MAX_UBERBLOCK_SHIFT) +#define ASHIFT_UBERBLOCK_SIZE(ashift) \ + (1ULL << ASHIFT_UBERBLOCK_SHIFT(ashift)) + typedef struct vdev_phys { char vp_nvlist[VDEV_PHYS_SIZE - sizeof (zio_eck_t)]; zio_eck_t vp_zbt; @@ -2015,11 +2021,11 @@ typedef struct vdev_indirect_config { typedef struct vdev { STAILQ_ENTRY(vdev) v_childlink; /* link in parent's child list */ - STAILQ_ENTRY(vdev) v_alllink; /* link in global vdev list */ vdev_list_t v_children; /* children of this vdev */ const char *v_name; /* vdev name */ uint64_t v_guid; /* vdev guid */ - uint64_t v_txg; /* most recent transaction */ + uint64_t v_label; /* label instantiated from (top vdev) */ + uint64_t v_txg; /* most recent transaction (top vdev) */ uint64_t v_id; /* index in parent */ uint64_t v_psize; /* physical device capacity */ int v_ashift; /* offset to block shift */ diff --git a/sys/compat/freebsd32/freebsd32_syscall.h b/sys/compat/freebsd32/freebsd32_syscall.h index 971510ebb6b6..90cd21a80923 100644 --- a/sys/compat/freebsd32/freebsd32_syscall.h +++ b/sys/compat/freebsd32/freebsd32_syscall.h @@ -83,8 +83,8 @@ /* 76 is obsolete vhangup */ /* 77 is obsolete vlimit */ #define FREEBSD32_SYS_mincore 78 -#define FREEBSD32_SYS_getgroups 79 -#define FREEBSD32_SYS_setgroups 80 +#define FREEBSD32_SYS_freebsd14_getgroups 79 +#define FREEBSD32_SYS_freebsd14_setgroups 80 #define FREEBSD32_SYS_getpgrp 81 #define FREEBSD32_SYS_setpgid 82 #define FREEBSD32_SYS_freebsd32_setitimer 83 @@ -513,4 +513,6 @@ #define FREEBSD32_SYS_exterrctl 592 #define FREEBSD32_SYS_inotify_add_watch_at 593 #define FREEBSD32_SYS_inotify_rm_watch 594 -#define FREEBSD32_SYS_MAXSYSCALL 595 +#define FREEBSD32_SYS_getgroups 595 +#define FREEBSD32_SYS_setgroups 596 +#define FREEBSD32_SYS_MAXSYSCALL 597 diff --git a/sys/compat/freebsd32/freebsd32_syscalls.c b/sys/compat/freebsd32/freebsd32_syscalls.c index 79c414b675b1..f0f8d26554b5 100644 --- a/sys/compat/freebsd32/freebsd32_syscalls.c +++ b/sys/compat/freebsd32/freebsd32_syscalls.c @@ -84,8 +84,8 @@ const char *freebsd32_syscallnames[] = { "obs_vhangup", /* 76 = obsolete vhangup */ "obs_vlimit", /* 77 = obsolete vlimit */ "mincore", /* 78 = mincore */ - "getgroups", /* 79 = getgroups */ - "setgroups", /* 80 = setgroups */ + "compat14.getgroups", /* 79 = freebsd14 getgroups */ + "compat14.setgroups", /* 80 = freebsd14 setgroups */ "getpgrp", /* 81 = getpgrp */ "setpgid", /* 82 = setpgid */ "freebsd32_setitimer", /* 83 = freebsd32_setitimer */ @@ -600,4 +600,6 @@ const char *freebsd32_syscallnames[] = { "exterrctl", /* 592 = exterrctl */ "inotify_add_watch_at", /* 593 = inotify_add_watch_at */ "inotify_rm_watch", /* 594 = inotify_rm_watch */ + "getgroups", /* 595 = getgroups */ + "setgroups", /* 596 = setgroups */ }; diff --git a/sys/compat/freebsd32/freebsd32_sysent.c b/sys/compat/freebsd32/freebsd32_sysent.c index 1a4b0d87722c..12f1a346c3e9 100644 --- a/sys/compat/freebsd32/freebsd32_sysent.c +++ b/sys/compat/freebsd32/freebsd32_sysent.c @@ -146,8 +146,8 @@ struct sysent freebsd32_sysent[] = { { .sy_narg = 0, .sy_call = (sy_call_t *)nosys, .sy_auevent = AUE_NULL, .sy_flags = 0, .sy_thrcnt = SY_THR_ABSENT }, /* 76 = obsolete vhangup */ { .sy_narg = 0, .sy_call = (sy_call_t *)nosys, .sy_auevent = AUE_NULL, .sy_flags = 0, .sy_thrcnt = SY_THR_ABSENT }, /* 77 = obsolete vlimit */ { .sy_narg = AS(mincore_args), .sy_call = (sy_call_t *)sys_mincore, .sy_auevent = AUE_MINCORE, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 78 = mincore */ - { .sy_narg = AS(getgroups_args), .sy_call = (sy_call_t *)sys_getgroups, .sy_auevent = AUE_GETGROUPS, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 79 = getgroups */ - { .sy_narg = AS(setgroups_args), .sy_call = (sy_call_t *)sys_setgroups, .sy_auevent = AUE_SETGROUPS, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 80 = setgroups */ + { compat14(AS(freebsd14_getgroups_args),getgroups), .sy_auevent = AUE_GETGROUPS, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 79 = freebsd14 getgroups */ + { compat14(AS(freebsd14_setgroups_args),setgroups), .sy_auevent = AUE_SETGROUPS, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 80 = freebsd14 setgroups */ { .sy_narg = 0, .sy_call = (sy_call_t *)sys_getpgrp, .sy_auevent = AUE_GETPGRP, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 81 = getpgrp */ { .sy_narg = AS(setpgid_args), .sy_call = (sy_call_t *)sys_setpgid, .sy_auevent = AUE_SETPGRP, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 82 = setpgid */ { .sy_narg = AS(freebsd32_setitimer_args), .sy_call = (sy_call_t *)freebsd32_setitimer, .sy_auevent = AUE_SETITIMER, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 83 = freebsd32_setitimer */ @@ -662,4 +662,6 @@ struct sysent freebsd32_sysent[] = { { .sy_narg = AS(exterrctl_args), .sy_call = (sy_call_t *)sys_exterrctl, .sy_auevent = AUE_NULL, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 592 = exterrctl */ { .sy_narg = AS(inotify_add_watch_at_args), .sy_call = (sy_call_t *)sys_inotify_add_watch_at, .sy_auevent = AUE_INOTIFY, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 593 = inotify_add_watch_at */ { .sy_narg = AS(inotify_rm_watch_args), .sy_call = (sy_call_t *)sys_inotify_rm_watch, .sy_auevent = AUE_INOTIFY, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 594 = inotify_rm_watch */ + { .sy_narg = AS(getgroups_args), .sy_call = (sy_call_t *)sys_getgroups, .sy_auevent = AUE_GETGROUPS, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 595 = getgroups */ + { .sy_narg = AS(setgroups_args), .sy_call = (sy_call_t *)sys_setgroups, .sy_auevent = AUE_SETGROUPS, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 596 = setgroups */ }; diff --git a/sys/compat/freebsd32/freebsd32_systrace_args.c b/sys/compat/freebsd32/freebsd32_systrace_args.c index f9dc514bee7d..e471c5148021 100644 --- a/sys/compat/freebsd32/freebsd32_systrace_args.c +++ b/sys/compat/freebsd32/freebsd32_systrace_args.c @@ -457,22 +457,6 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args) *n_args = 3; break; } - /* getgroups */ - case 79: { - struct getgroups_args *p = params; - iarg[a++] = p->gidsetsize; /* int */ - uarg[a++] = (intptr_t)p->gidset; /* gid_t * */ - *n_args = 2; - break; - } - /* setgroups */ - case 80: { - struct setgroups_args *p = params; - iarg[a++] = p->gidsetsize; /* int */ - uarg[a++] = (intptr_t)p->gidset; /* const gid_t * */ - *n_args = 2; - break; - } /* getpgrp */ case 81: { *n_args = 0; @@ -3413,6 +3397,22 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args) *n_args = 2; break; } + /* getgroups */ + case 595: { + struct getgroups_args *p = params; + iarg[a++] = p->gidsetsize; /* int */ + uarg[a++] = (intptr_t)p->gidset; /* gid_t * */ + *n_args = 2; + break; + } + /* setgroups */ + case 596: { + struct setgroups_args *p = params; + iarg[a++] = p->gidsetsize; /* int */ + uarg[a++] = (intptr_t)p->gidset; /* const gid_t * */ + *n_args = 2; + break; + } default: *n_args = 0; break; @@ -4112,32 +4112,6 @@ systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) break; }; break; - /* getgroups */ - case 79: - switch (ndx) { - case 0: - p = "int"; - break; - case 1: - p = "userland gid_t *"; - break; - default: - break; - }; - break; - /* setgroups */ - case 80: - switch (ndx) { - case 0: - p = "int"; - break; - case 1: - p = "userland const gid_t *"; - break; - default: - break; - }; - break; /* getpgrp */ case 81: break; @@ -9222,6 +9196,32 @@ systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) break; }; break; + /* getgroups */ + case 595: + switch (ndx) { + case 0: + p = "int"; + break; + case 1: + p = "userland gid_t *"; + break; + default: + break; + }; + break; + /* setgroups */ + case 596: + switch (ndx) { + case 0: + p = "int"; + break; + case 1: + p = "userland const gid_t *"; + break; + default: + break; + }; + break; default: break; }; @@ -9488,16 +9488,6 @@ systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) if (ndx == 0 || ndx == 1) p = "int"; break; - /* getgroups */ - case 79: - if (ndx == 0 || ndx == 1) - p = "int"; - break; - /* setgroups */ - case 80: - if (ndx == 0 || ndx == 1) - p = "int"; - break; /* getpgrp */ case 81: /* setpgid */ @@ -11130,6 +11120,16 @@ systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) if (ndx == 0 || ndx == 1) p = "int"; break; + /* getgroups */ + case 595: + if (ndx == 0 || ndx == 1) + p = "int"; + break; + /* setgroups */ + case 596: + if (ndx == 0 || ndx == 1) + p = "int"; + break; default: break; }; diff --git a/sys/compat/linux/linux_netlink.c b/sys/compat/linux/linux_netlink.c index f51838ee00d7..927f3689e2b6 100644 --- a/sys/compat/linux/linux_netlink.c +++ b/sys/compat/linux/linux_netlink.c @@ -242,24 +242,9 @@ nlmsg_copy_nla(const struct nlattr *nla_orig, struct nl_writer *nw) } /* - * Translate a FreeBSD interface name to a Linux interface name. - */ -static bool -nlmsg_translate_ifname_nla(struct nlattr *nla, struct nl_writer *nw) -{ - char ifname[LINUX_IFNAMSIZ]; - - if (ifname_bsd_to_linux_name((char *)(nla + 1), ifname, - sizeof(ifname)) <= 0) - return (false); - return (nlattr_add_string(nw, IFLA_IFNAME, ifname)); -} - -#define LINUX_NLA_UNHANDLED -1 -/* * Translate a FreeBSD attribute to a Linux attribute. - * Returns LINUX_NLA_UNHANDLED when the attribute is not processed - * and the caller must take care of it, otherwise the result is returned. + * Returns false when the attribute is not processed and the caller must take + * care of it. */ static int nlmsg_translate_all_nla(struct nlmsghdr *hdr, struct nlattr *nla, @@ -271,22 +256,27 @@ nlmsg_translate_all_nla(struct nlmsghdr *hdr, struct nlattr *nla, case NL_RTM_DELLINK: case NL_RTM_GETLINK: switch (nla->nla_type) { - case IFLA_IFNAME: - return (nlmsg_translate_ifname_nla(nla, nw)); + case IFLA_IFNAME: { + char ifname[LINUX_IFNAMSIZ]; + + if (ifname_bsd_to_linux_name((char *)(nla + 1), ifname, + sizeof(ifname)) > 0) + return (true); + break; + } default: break; } default: break; } - return (LINUX_NLA_UNHANDLED); + return (false); } static bool nlmsg_copy_all_nla(struct nlmsghdr *hdr, int raw_hdrlen, struct nl_writer *nw) { struct nlattr *nla; - int ret; int hdrlen = NETLINK_ALIGN(raw_hdrlen); int attrs_len = hdr->nlmsg_len - sizeof(struct nlmsghdr) - hdrlen; @@ -297,15 +287,12 @@ nlmsg_copy_all_nla(struct nlmsghdr *hdr, int raw_hdrlen, struct nl_writer *nw) if (nla->nla_len < sizeof(struct nlattr)) { return (false); } - ret = nlmsg_translate_all_nla(hdr, nla, nw); - if (ret == LINUX_NLA_UNHANDLED) - ret = nlmsg_copy_nla(nla, nw); - if (!ret) + if (!nlmsg_translate_all_nla(hdr, nla, nw) && + !nlmsg_copy_nla(nla, nw)) return (false); } return (true); } -#undef LINUX_NLA_UNHANDLED static unsigned int rtnl_if_flags_to_linux(unsigned int if_flags) @@ -563,22 +550,15 @@ nlmsg_to_linux(struct nlmsghdr *hdr, struct nlpcb *nlp, struct nl_writer *nw) } } -static bool -nlmsgs_to_linux(struct nl_writer *nw, struct nlpcb *nlp) +static struct nl_buf * +nlmsgs_to_linux(struct nl_buf *orig, struct nlpcb *nlp) { - struct nl_buf *nb, *orig; - u_int offset, msglen, orig_messages; + struct nl_writer nw; + u_int offset, msglen; - RT_LOG(LOG_DEBUG3, "%p: in %u bytes %u messages", __func__, - nw->buf->datalen, nw->num_messages); - - orig = nw->buf; - nb = nl_buf_alloc(orig->datalen + SCRATCH_BUFFER_SIZE, M_NOWAIT); - if (__predict_false(nb == NULL)) - return (false); - nw->buf = nb; - orig_messages = nw->num_messages; - nw->num_messages = 0; + if (__predict_false(!nl_writer_unicast(&nw, + orig->datalen + SCRATCH_BUFFER_SIZE, nlp, false))) + return (NULL); /* Assume correct headers. Buffer IS mutable */ for (offset = 0; @@ -587,22 +567,18 @@ nlmsgs_to_linux(struct nl_writer *nw, struct nlpcb *nlp) struct nlmsghdr *hdr = (struct nlmsghdr *)&orig->data[offset]; msglen = NLMSG_ALIGN(hdr->nlmsg_len); - if (!nlmsg_to_linux(hdr, nlp, nw)) { + if (!nlmsg_to_linux(hdr, nlp, &nw)) { RT_LOG(LOG_DEBUG, "failed to process msg type %d", hdr->nlmsg_type); - nl_buf_free(nb); - nw->buf = orig; - nw->num_messages = orig_messages; - return (false); + nl_buf_free(nw.buf); + return (NULL); } } - MPASS(nw->num_messages == orig_messages); - MPASS(nw->buf == nb); - nl_buf_free(orig); - RT_LOG(LOG_DEBUG3, "%p: out %u bytes", __func__, offset); + RT_LOG(LOG_DEBUG3, "%p: in %u bytes %u messages", __func__, + nw.buf->datalen, nw.num_messages); - return (true); + return (nw.buf); } static struct linux_netlink_provider linux_netlink_v1 = { diff --git a/sys/compat/linuxkpi/common/include/linux/ieee80211.h b/sys/compat/linuxkpi/common/include/linux/ieee80211.h index 3644ef80861b..5851ac08f083 100644 --- a/sys/compat/linuxkpi/common/include/linux/ieee80211.h +++ b/sys/compat/linuxkpi/common/include/linux/ieee80211.h @@ -35,6 +35,7 @@ #include <asm/unaligned.h> #include <linux/kernel.h> #include <linux/bitops.h> +#include <linux/bitfield.h> #include <linux/if_ether.h> /* linux_80211.c */ @@ -121,7 +122,20 @@ enum ieee80211_rate_control_changed_flags { /* 802.11-2016, 9.4.2.158.3 Supported VHT-MCS and NSS Set field. */ #define IEEE80211_VHT_EXT_NSS_BW_CAPABLE (1 << 13) /* part of tx_highest */ -#define IEEE80211_VHT_MAX_AMPDU_1024K 7 /* 9.4.2.56.3 A-MPDU Parameters field, Table 9-163 */ +/* + * 802.11-2020, 9.4.2.157.2 VHT Capabilities Information field, + * Table 9-271-Subfields of the VHT Capabilities Information field (continued). + */ +enum ieee80211_vht_max_ampdu_len_exp { + IEEE80211_VHT_MAX_AMPDU_8K = 0, + IEEE80211_VHT_MAX_AMPDU_16K = 1, + IEEE80211_VHT_MAX_AMPDU_32K = 2, + IEEE80211_VHT_MAX_AMPDU_64K = 3, + IEEE80211_VHT_MAX_AMPDU_128K = 4, + IEEE80211_VHT_MAX_AMPDU_256K = 5, + IEEE80211_VHT_MAX_AMPDU_512K = 6, + IEEE80211_VHT_MAX_AMPDU_1024K = 7, +}; #define IEEE80211_WEP_IV_LEN 3 /* net80211: IEEE80211_WEP_IVLEN */ #define IEEE80211_WEP_ICV_LEN 4 diff --git a/sys/compat/linuxkpi/common/include/net/cfg80211.h b/sys/compat/linuxkpi/common/include/net/cfg80211.h index 18b34f0e90ec..044f348ef08b 100644 --- a/sys/compat/linuxkpi/common/include/net/cfg80211.h +++ b/sys/compat/linuxkpi/common/include/net/cfg80211.h @@ -36,6 +36,7 @@ #include <linux/mutex.h> #include <linux/if_ether.h> #include <linux/ethtool.h> +#include <linux/debugfs.h> #include <linux/device.h> #include <linux/netdevice.h> #include <linux/random.h> @@ -722,8 +723,10 @@ struct linuxkpi_ieee80211_regdomain { #define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU 0x04 #define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY 0x08 #define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_32US 0x10 +#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US 0x10 #define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY 0x20 #define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_64US 0x40 +#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US 0x40 #define VENDOR_CMD_RAW_DATA (void *)(uintptr_t)(-ENOENT) @@ -2065,6 +2068,18 @@ nl80211_chan_width_to_mhz(enum nl80211_chan_width width) } static __inline ssize_t +wiphy_locked_debugfs_read(struct wiphy *wiphy, struct file *file, + char *buf, size_t bufsize, const char __user *userbuf, size_t count, + loff_t *ppos, + ssize_t (*handler)(struct wiphy *, struct file *, char *, size_t, void *), + void *data) +{ + TODO(); + return (-ENXIO); +} + + +static __inline ssize_t wiphy_locked_debugfs_write(struct wiphy *wiphy, struct file *file, char *buf, size_t bufsize, const char __user *userbuf, size_t count, ssize_t (*handler)(struct wiphy *, struct file *, char *, size_t, void *), diff --git a/sys/compat/linuxkpi/common/include/net/mac80211.h b/sys/compat/linuxkpi/common/include/net/mac80211.h index af3199c38939..2ed595095f9e 100644 --- a/sys/compat/linuxkpi/common/include/net/mac80211.h +++ b/sys/compat/linuxkpi/common/include/net/mac80211.h @@ -1184,7 +1184,7 @@ struct wireless_dev *linuxkpi_ieee80211_vif_to_wdev(struct ieee80211_vif *); void linuxkpi_ieee80211_connection_loss(struct ieee80211_vif *); void linuxkpi_ieee80211_beacon_loss(struct ieee80211_vif *); struct sk_buff *linuxkpi_ieee80211_probereq_get(struct ieee80211_hw *, - uint8_t *, uint8_t *, size_t, size_t); + const uint8_t *, const uint8_t *, size_t, size_t); void linuxkpi_ieee80211_tx_status(struct ieee80211_hw *, struct sk_buff *); void linuxkpi_ieee80211_tx_status_ext(struct ieee80211_hw *, struct ieee80211_tx_status *); @@ -2161,8 +2161,8 @@ ieee80211_nullfunc_get(struct ieee80211_hw *hw, struct ieee80211_vif *vif, } static __inline struct sk_buff * -ieee80211_probereq_get(struct ieee80211_hw *hw, uint8_t *addr, - uint8_t *ssid, size_t ssid_len, size_t tailroom) +ieee80211_probereq_get(struct ieee80211_hw *hw, const uint8_t *addr, + const uint8_t *ssid, size_t ssid_len, size_t tailroom) { return (linuxkpi_ieee80211_probereq_get(hw, addr, ssid, ssid_len, diff --git a/sys/compat/linuxkpi/common/src/linux_80211.c b/sys/compat/linuxkpi/common/src/linux_80211.c index 1d00e8da8f9a..a7d6003843ba 100644 --- a/sys/compat/linuxkpi/common/src/linux_80211.c +++ b/sys/compat/linuxkpi/common/src/linux_80211.c @@ -7668,8 +7668,8 @@ linuxkpi_ieee80211_queue_work(struct ieee80211_hw *hw, } struct sk_buff * -linuxkpi_ieee80211_probereq_get(struct ieee80211_hw *hw, uint8_t *addr, - uint8_t *ssid, size_t ssid_len, size_t tailroom) +linuxkpi_ieee80211_probereq_get(struct ieee80211_hw *hw, const uint8_t *addr, + const uint8_t *ssid, size_t ssid_len, size_t tailroom) { struct sk_buff *skb; struct ieee80211_frame *wh; diff --git a/sys/conf/NOTES b/sys/conf/NOTES index 2458756ae350..92e98aa57ebf 100644 --- a/sys/conf/NOTES +++ b/sys/conf/NOTES @@ -2436,7 +2436,7 @@ options HID_DEBUG # enable debug msgs device hidbus # HID bus device hidmap # HID to evdev mapping device hidraw # Raw access driver -options HIDRAW_MAKE_UHID_ALIAS # install /dev/uhid alias +options HIDRAW_MAKE_UHID_ALIAS # install /dev/uhid alias for /dev/hidraw device hconf # Multitouch configuration TLC device hcons # Consumer controls device hgame # Generic game controllers @@ -2446,6 +2446,8 @@ device hmt # HID multitouch (MS-compatible) device hpen # Generic pen driver device hsctrl # System controls device ps4dshock # Sony PS4 DualShock 4 gamepad driver +device u2f # FIDO/U2F authenticator +options U2F_MAKE_UHID_ALIAS # install /dev/uhid alias for /dev/u2f/ device xb360gp # XBox 360 gamepad driver ##################################################################### diff --git a/sys/conf/files b/sys/conf/files index be65ed20d6aa..d89813c70355 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -1750,6 +1750,7 @@ dev/hid/hpen.c optional hpen dev/hid/hsctrl.c optional hsctrl dev/hid/ietp.c optional ietp dev/hid/ps4dshock.c optional ps4dshock +dev/hid/u2f.c optional u2f dev/hid/xb360gp.c optional xb360gp dev/hifn/hifn7751.c optional hifn dev/hptiop/hptiop.c optional hptiop scbus @@ -2280,6 +2281,8 @@ dev/ixgbe/ixgbe_x540.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_x550.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" +dev/ixgbe/ixgbe_e610.c optional ix inet | ixv inet \ + compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb.c optional ix inet | ixv inet \ compile-with "${NORMAL_C} -I$S/dev/ixgbe" dev/ixgbe/ixgbe_dcb_82598.c optional ix inet | ixv inet \ @@ -3448,7 +3451,6 @@ dev/virtio/mmio/virtio_mmio.c optional virtio_mmio dev/virtio/mmio/virtio_mmio_acpi.c optional virtio_mmio acpi dev/virtio/mmio/virtio_mmio_cmdline.c optional virtio_mmio dev/virtio/mmio/virtio_mmio_fdt.c optional virtio_mmio fdt -dev/virtio/mmio/virtio_mmio_if.m optional virtio_mmio dev/virtio/network/if_vtnet.c optional vtnet dev/virtio/balloon/virtio_balloon.c optional virtio_balloon dev/virtio/block/virtio_blk.c optional virtio_blk diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index 80548320c3fc..c12ab9db030a 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -13,14 +13,14 @@ include "conf/files.x86" # elf-vdso.so.o standard \ dependency "$S/amd64/amd64/sigtramp.S assym.inc $S/conf/vdso_amd64.ldscript $S/tools/amd64_vdso.sh" \ - compile-with "env AWK='${AWK}' NM='${NM}' LD='${LD}' CC='${CC}' DEBUG='${DEBUG}' OBJCOPY='${OBJCOPY}' ELFDUMP='${ELFDUMP}' S='${S}' sh $S/tools/amd64_vdso.sh" \ + compile-with "env AWK='${AWK}' NM='${NM}' LD='${LD}' CC='${CC}' OBJCOPY='${OBJCOPY}' ELFDUMP='${ELFDUMP}' S='${S}' sh $S/tools/amd64_vdso.sh" \ no-ctfconvert \ no-implicit-rule before-depend \ clean "elf-vdso.so.o elf-vdso.so.1 vdso_offsets.h sigtramp.pico" # elf-vdso32.so.o optional compat_freebsd32 \ dependency "$S/amd64/ia32/ia32_sigtramp.S ia32_assym.h $S/conf/vdso_amd64_ia32.ldscript $S/tools/amd64_ia32_vdso.sh" \ - compile-with "env AWK='${AWK}' NM='${NM}' LD='${LD}' CC='${CC}' DEBUG='${DEBUG}' OBJCOPY='${OBJCOPY}' ELFDUMP='${ELFDUMP}' S='${S}' sh $S/tools/amd64_ia32_vdso.sh" \ + compile-with "env AWK='${AWK}' NM='${NM}' LD='${LD}' CC='${CC}' OBJCOPY='${OBJCOPY}' ELFDUMP='${ELFDUMP}' S='${S}' sh $S/tools/amd64_ia32_vdso.sh" \ no-ctfconvert \ no-implicit-rule before-depend \ clean "elf-vdso32.so.o elf-vdso32.so.1 vdso_ia32_offsets.h ia32_sigtramp.pico" @@ -419,6 +419,9 @@ contrib/openzfs/module/icp/asm-x86_64/blake3/blake3_avx512.S optional zfs com contrib/openzfs/module/icp/asm-x86_64/blake3/blake3_sse2.S optional zfs compile-with "${ZFS_S}" contrib/openzfs/module/icp/asm-x86_64/blake3/blake3_sse41.S optional zfs compile-with "${ZFS_S}" +# zfs AVX2 implementation of aes-gcm from BoringSSL +contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S optional zfs compile-with "${ZFS_S}" + # zfs sha2 hash support zfs-sha256-x86_64.o optional zfs \ dependency "$S/contrib/openzfs/module/icp/asm-x86_64/sha2/sha256-x86_64.S" \ diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 641001efab5e..45966fea8041 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -178,6 +178,8 @@ crypto/des/des_enc.c optional netsmb crypto/openssl/ossl_aarch64.c optional ossl crypto/openssl/aarch64/chacha-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}" +crypto/openssl/aarch64/chacha-armv8-sve.S optional ossl \ + compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/poly1305-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/sha1-armv8.S optional ossl \ diff --git a/sys/conf/kern.post.mk b/sys/conf/kern.post.mk index 0e4ffd92724e..bb3c7af82a4d 100644 --- a/sys/conf/kern.post.mk +++ b/sys/conf/kern.post.mk @@ -372,6 +372,19 @@ _ILINKS+= x86 _ILINKS+= i386 .endif +.if ${MK_REPRODUCIBLE_BUILD} != "no" +PREFIX_SYSDIR=/usr/src/sys +PREFIX_OBJDIR=/usr/obj/usr/src/${MACHINE}.${MACHINE_CPUARCH}/sys/${KERN_IDENT} +CFLAGS+= -ffile-prefix-map=${SYSDIR}=${PREFIX_SYSDIR} +CFLAGS+= -ffile-prefix-map=${.OBJDIR}=${PREFIX_OBJDIR} +.if defined(SYSROOT) +CFLAGS+= -ffile-prefix-map=${SYSROOT}=/sysroot +.endif +.else +PREFIX_SYSDIR=${SYSDIR} +PREFIX_OBJDIR=${.OBJDIR} +.endif + # Ensure that the link exists without depending on it when it exists. # Ensure that debug info references the path in the source tree. .for _link in ${_ILINKS} @@ -379,9 +392,9 @@ _ILINKS+= i386 ${SRCS} ${DEPENDOBJS}: ${_link} .endif .if ${_link} == "machine" -CFLAGS+= -fdebug-prefix-map=./machine=${SYSDIR}/${MACHINE}/include +CFLAGS+= -fdebug-prefix-map=./machine=${PREFIX_SYSDIR}/${MACHINE}/include .else -CFLAGS+= -fdebug-prefix-map=./${_link}=${SYSDIR}/${_link}/include +CFLAGS+= -fdebug-prefix-map=./${_link}=${PREFIX_SYSDIR}/${_link}/include .endif .endfor @@ -454,7 +467,7 @@ config.o env.o hints.o vers.o vnode_if.o: NEWVERS_ENV+= MAKE="${MAKE}" .if ${MK_REPRODUCIBLE_BUILD} != "no" -NEWVERS_ARGS+= -R +NEWVERS_ARGS+= -R -d ${PREFIX_OBJDIR} .endif vers.c: .NOMETA_CMP $S/conf/newvers.sh $S/sys/param.h ${SYSTEM_DEP:Nvers.*} ${NEWVERS_ENV} sh $S/conf/newvers.sh ${NEWVERS_ARGS} ${KERN_IDENT} diff --git a/sys/conf/kern.pre.mk b/sys/conf/kern.pre.mk index 78178065e15b..1fcfd6467e7f 100644 --- a/sys/conf/kern.pre.mk +++ b/sys/conf/kern.pre.mk @@ -214,7 +214,8 @@ ZFS_CFLAGS+= -I$S/contrib/openzfs/module/icp/include \ .if ${MACHINE_ARCH} == "amd64" ZFS_CFLAGS+= -D__x86_64 -DHAVE_SSE2 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2 \ - -DHAVE_AVX -DHAVE_AVX2 -DHAVE_AVX512F -DHAVE_AVX512VL -DHAVE_AVX512BW + -DHAVE_AVX -DHAVE_AVX2 -DHAVE_AVX512F -DHAVE_AVX512VL -DHAVE_AVX512BW \ + -DHAVE_VAES -DHAVE_VPCLMULQDQ .endif .if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "powerpc" || \ diff --git a/sys/conf/kmod.mk b/sys/conf/kmod.mk index 645c04cdd135..0fd2d4050cf1 100644 --- a/sys/conf/kmod.mk +++ b/sys/conf/kmod.mk @@ -303,6 +303,25 @@ all: ${PROG} beforedepend: ${_ILINKS} beforebuild: ${_ILINKS} +.if ${MK_REPRODUCIBLE_BUILD} != "no" +PREFIX_SYSDIR=/usr/src/sys +CFLAGS+= -ffile-prefix-map=${SYSDIR}=${PREFIX_SYSDIR} +.if defined(KERNBUILDDIR) +PREFIX_KERNBUILDDIR=/usr/obj/usr/src/${MACHINE}.${MACHINE_CPUARCH}/sys/${KERNBUILDDIR:T} +PREFIX_OBJDIR=${PREFIX_KERNBUILDDIR}/modules/usr/src/sys/modules/${.OBJDIR:T} +CFLAGS+= -ffile-prefix-map=${KERNBUILDDIR}=${PREFIX_KERNBUILDDIR} +.else +PREFIX_OBJDIR=/usr/obj/usr/src/${MACHINE}.${MACHINE_CPUARCH}/sys/modules/${.OBJDIR:T} +.endif +CFLAGS+= -ffile-prefix-map=${.OBJDIR}=${PREFIX_OBJDIR} +.if defined(SYSROOT) +CFLAGS+= -ffile-prefix-map=${SYSROOT}=/sysroot +.endif +.else +PREFIX_SYSDIR=${SYSDIR} +PREFIX_OBJDIR=${.OBJDIR} +.endif + # Ensure that the links exist without depending on it when it exists which # causes all the modules to be rebuilt when the directory pointed to changes. # Ensure that debug info references the path in the source tree. @@ -311,9 +330,9 @@ beforebuild: ${_ILINKS} OBJS_DEPEND_GUESS+= ${_link} .endif .if ${_link} == "machine" -CFLAGS+= -fdebug-prefix-map=./machine=${SYSDIR}/${MACHINE}/include +CFLAGS+= -fdebug-prefix-map=./machine=${PREFIX_SYSDIR}/${MACHINE}/include .else -CFLAGS+= -fdebug-prefix-map=./${_link}=${SYSDIR}/${_link}/include +CFLAGS+= -fdebug-prefix-map=./${_link}=${PREFIX_SYSDIR}/${_link}/include .endif .endfor diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 66926805052c..8b60da95741e 100644 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -110,14 +110,18 @@ COPYRIGHT="$COPYRIGHT # We expand include_metadata later since we may set it to the # future value of modified. +builddir=$(pwd) include_metadata=yes modified=no -while getopts crRvV: opt; do +while getopts cd:rRvV: opt; do case "$opt" in c) echo "$COPYRIGHT" exit 0 ;; + d) + builddir=$OPTARG + ;; r) include_metadata=no ;; @@ -187,7 +191,7 @@ fi touch version v=$(cat version) u=${USER:-root} -d=$(pwd) +d=$builddir h=${HOSTNAME:-$(hostname)} if [ -n "$SOURCE_DATE_EPOCH" ]; then if ! t=$(date -ur $SOURCE_DATE_EPOCH 2>/dev/null); then diff --git a/sys/conf/options b/sys/conf/options index a637b0b74a77..4009ba2b4843 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -235,11 +235,6 @@ TCP_REQUEST_TRK opt_global.h TCP_ACCOUNTING opt_global.h TCP_BBR opt_inet.h TCP_RACK opt_inet.h -# -# TCP SaD Detection is an experimental Sack attack Detection (SaD) -# algorithm that uses "normal" behaviour with SACK's to detect -# a possible attack. It is strictly experimental at this point. -# TURNSTILE_PROFILING UMTX_PROFILING UMTX_CHAINS opt_global.h @@ -1009,6 +1004,7 @@ IICHID_DEBUG opt_hid.h IICHID_SAMPLING opt_hid.h HKBD_DFLT_KEYMAP opt_hkbd.h HIDRAW_MAKE_UHID_ALIAS opt_hid.h +U2F_MAKE_UHID_ALIAS opt_hid.h # kenv options # The early kernel environment (loader environment, config(8)-provided static) diff --git a/sys/contrib/dev/acpica/changes.txt b/sys/contrib/dev/acpica/changes.txt index 435540b254f1..4e3cf4f2f41c 100644 --- a/sys/contrib/dev/acpica/changes.txt +++ b/sys/contrib/dev/acpica/changes.txt @@ -1,11 +1,29 @@ ---------------------------------------- +7 August 2025. Summary of changes for version 20250807: + +Major changes: + +Added option to skip the global lock for SMM - Huacai Chen + +Fixed non-NUL terminated string implementations - Ahmed Salem + +Fixed CCEL and CDAT templates - Ahmed Salem + +Fixed a major Linux kernel bug (UAF) that was triggered by unequal number of method parameters (definition) vs arguments (invocation) in different places - Peter Williams, Hans de Goede, Rafael Wysocki + +Define distinct D3 states (D3Hot and D3Cold) that help clarify the device behavior support - Aymeric Wibo + +A few cleanups, improvements to existing table supports, small fixes, spelling corrections etc. + + +---------------------------------------- 4 April 2025. Summary of changes for version 20250404: Major changes: Update all the copyright continuation year to 2025 in the license header of all files -Add complete support for 3 new ACPI tables ? MRRM,ERDT and RIMT (Tony Luck & V L Sunil) +Add complete support for 3 new ACPI tables - MRRM,ERDT and RIMT (Tony Luck & V L Sunil) Add a license file to the project which is a great improvement (Dionna Glaze) @@ -21,11 +39,11 @@ Major changes: Fix 2 critical CVE addressing memory leaks - Seunghun Han -EINJ V2 updates ? Zaid Alali (Ampere Computing) +EINJ V2 updates - Zaid Alali (Ampere Computing) -CDAT updates ? Ira Weiny (Intel Corporation) +CDAT updates - Ira Weiny (Intel Corporation) -Fix mutex handling, don?t release ones that were never acquired ? Daniil Tatianin +Fix mutex handling, do not release ones that were never acquired - Daniil Tatianin Experiment with new tag name format Ryyyy_mm_dd to solve chronological sorting problems @@ -39,7 +57,7 @@ Fix the acpixf.h file which caused issues for the last release (before this) 202 Fix the pointer offset for the SLIC table -Verify the local environment and GitHub commits are all in sync which was a problem with the second from last release (before this)20240322 (aka 20240323 – date issue) +Verify the local environment and GitHub commits are all in sync which was a problem with the second from last release (before this)20240322 (aka 20240323 - date issue) diff --git a/sys/contrib/dev/acpica/common/adisasm.c b/sys/contrib/dev/acpica/common/adisasm.c index 96cd6c7f5d3c..83125098cbd1 100644 --- a/sys/contrib/dev/acpica/common/adisasm.c +++ b/sys/contrib/dev/acpica/common/adisasm.c @@ -481,12 +481,12 @@ AdDisassembleOneTable ( "FieldName : FieldValue (in hex)\n */\n\n"); AcpiDmDumpDataTable (Table); - fprintf (stderr, "Acpi Data Table [%4.4s] decoded\n", + fprintf (stdout, "Acpi Data Table [%4.4s] decoded\n", AcpiGbl_CDAT ? (char *) AcpiGbl_CDAT : Table->Signature); if (File) { - fprintf (stderr, "Formatted output: %s - %u bytes\n", + fprintf (stdout, "Formatted output: %s - %u bytes\n", DisasmFilename, CmGetFileSize (File)); } @@ -584,16 +584,16 @@ AdDisassembleOneTable ( AcpiDmDumpDataTable (Table); - fprintf (stderr, "Disassembly completed\n"); + fprintf (stdout, "Disassembly completed\n"); if (File) { - fprintf (stderr, "ASL Output: %s - %u bytes\n", + fprintf (stdout, "ASL Output: %s - %u bytes\n", DisasmFilename, CmGetFileSize (File)); } if (AslGbl_MapfileFlag) { - fprintf (stderr, "%14s %s - %u bytes\n", + fprintf (stdout, "%14s %s - %u bytes\n", AslGbl_FileDescs[ASL_FILE_MAP_OUTPUT].ShortDescription, AslGbl_Files[ASL_FILE_MAP_OUTPUT].Filename, FlGetFileSize (ASL_FILE_MAP_OUTPUT)); @@ -630,7 +630,7 @@ AdReparseOneTable ( ACPI_COMMENT_ADDR_NODE *AddrListHead; - fprintf (stderr, + fprintf (stdout, "\nFound %u external control methods, " "reparsing with new information\n", AcpiDmGetUnresolvedExternalMethodCount ()); diff --git a/sys/contrib/dev/acpica/common/ahtable.c b/sys/contrib/dev/acpica/common/ahtable.c index 898b2d09f609..587bf61016f0 100644 --- a/sys/contrib/dev/acpica/common/ahtable.c +++ b/sys/contrib/dev/acpica/common/ahtable.c @@ -265,6 +265,7 @@ const AH_TABLE AcpiGbl_SupportedTables[] = {ACPI_SIG_SSDT, "Secondary System Description Table (AML table)"}, {ACPI_SIG_STAO, "Status Override Table"}, {ACPI_SIG_SVKL, "Storage Volume Key Location Table"}, + {ACPI_SIG_SWFT, "SoundWire File Table"}, {ACPI_SIG_TCPA, "Trusted Computing Platform Alliance Table"}, {ACPI_SIG_TDEL, "TD-Event Log Table"}, {ACPI_SIG_TPM2, "Trusted Platform Module hardware interface Table"}, diff --git a/sys/contrib/dev/acpica/common/dmtable.c b/sys/contrib/dev/acpica/common/dmtable.c index fcff97a304ae..702f4f7965e4 100644 --- a/sys/contrib/dev/acpica/common/dmtable.c +++ b/sys/contrib/dev/acpica/common/dmtable.c @@ -721,6 +721,7 @@ const ACPI_DMTABLE_DATA AcpiDmTableData[] = {ACPI_SIG_SRAT, NULL, AcpiDmDumpSrat, DtCompileSrat, TemplateSrat}, {ACPI_SIG_STAO, NULL, AcpiDmDumpStao, DtCompileStao, TemplateStao}, {ACPI_SIG_SVKL, AcpiDmTableInfoSvkl, AcpiDmDumpSvkl, DtCompileSvkl, TemplateSvkl}, + {ACPI_SIG_SWFT, NULL, NULL, NULL, NULL}, {ACPI_SIG_TCPA, NULL, AcpiDmDumpTcpa, DtCompileTcpa, TemplateTcpa}, {ACPI_SIG_TDEL, AcpiDmTableInfoTdel, NULL, NULL, TemplateTdel}, {ACPI_SIG_TPM2, AcpiDmTableInfoTpm2, AcpiDmDumpTpm2, DtCompileTpm2, TemplateTpm2}, diff --git a/sys/contrib/dev/acpica/common/dmtbdump2.c b/sys/contrib/dev/acpica/common/dmtbdump2.c index 822920d2ea94..d29a60be0f67 100644 --- a/sys/contrib/dev/acpica/common/dmtbdump2.c +++ b/sys/contrib/dev/acpica/common/dmtbdump2.c @@ -2637,7 +2637,7 @@ AcpiDmDumpRhct ( RhctIsaString, RhctIsaString->IsaLength, AcpiDmTableInfoRhctIsa1); if (Subtable->Length > IsaPadOffset) { - Status = AcpiDmDumpTable (Table->Length, Offset + SubtableOffset, + Status = AcpiDmDumpTable (Table->Length, Offset + IsaPadOffset, ACPI_ADD_PTR (UINT8, Subtable, IsaPadOffset), (Subtable->Length - IsaPadOffset), AcpiDmTableInfoRhctIsaPad); } diff --git a/sys/contrib/dev/acpica/common/dmtbinfo2.c b/sys/contrib/dev/acpica/common/dmtbinfo2.c index 9ecf877fcfb0..b7c6d3b8d536 100644 --- a/sys/contrib/dev/acpica/common/dmtbinfo2.c +++ b/sys/contrib/dev/acpica/common/dmtbinfo2.c @@ -2180,7 +2180,7 @@ ACPI_DMTABLE_INFO AcpiDmTableInfoRhct[] = ACPI_DMTABLE_INFO AcpiDmTableInfoRhctNodeHdr[] = { {ACPI_DMT_RHCT, ACPI_RHCTH_OFFSET (Type), "Subtable Type", 0}, - {ACPI_DMT_UINT16, ACPI_RHCTH_OFFSET (Length), "Length", 0}, + {ACPI_DMT_UINT16, ACPI_RHCTH_OFFSET (Length), "Length", DT_LENGTH}, {ACPI_DMT_UINT16, ACPI_RHCTH_OFFSET (Revision), "Revision", 0}, ACPI_DMT_TERMINATOR }; diff --git a/sys/contrib/dev/acpica/common/dmtbinfo3.c b/sys/contrib/dev/acpica/common/dmtbinfo3.c index 75b580e0d890..0935fc86aff9 100644 --- a/sys/contrib/dev/acpica/common/dmtbinfo3.c +++ b/sys/contrib/dev/acpica/common/dmtbinfo3.c @@ -200,7 +200,7 @@ ACPI_DMTABLE_INFO AcpiDmTableInfoCcel[] = { {ACPI_DMT_UINT8, ACPI_CCEL_OFFSET (CCType), "CC Type", 0}, {ACPI_DMT_UINT8, ACPI_CCEL_OFFSET (CCSubType), "CC Sub Type", 0}, - {ACPI_DMT_UINT32, ACPI_CCEL_OFFSET (Reserved), "Reserved", 0}, + {ACPI_DMT_UINT16, ACPI_CCEL_OFFSET (Reserved), "Reserved", 0}, {ACPI_DMT_UINT64, ACPI_CCEL_OFFSET (LogAreaMinimumLength), "Log Area Minimum Length", 0}, {ACPI_DMT_UINT64, ACPI_CCEL_OFFSET (LogAreaStartAddress), "Log Area Start Address", 0}, ACPI_DMT_TERMINATOR diff --git a/sys/contrib/dev/acpica/compiler/aslanalyze.c b/sys/contrib/dev/acpica/compiler/aslanalyze.c index 17e2674817a9..625611a630de 100644 --- a/sys/contrib/dev/acpica/compiler/aslanalyze.c +++ b/sys/contrib/dev/acpica/compiler/aslanalyze.c @@ -572,10 +572,22 @@ ApCheckForGpeNameConflict ( ACPI_PARSE_OBJECT *NextOp; UINT32 GpeNumber; char Name[ACPI_NAMESEG_SIZE + 1]; - char Target[ACPI_NAMESEG_SIZE]; + char Target[ACPI_NAMESEG_SIZE] ACPI_NONSTRING; - /* Need a null-terminated string version of NameSeg */ + /** + * Need a null-terminated string version of NameSeg + * + * NOTE: during a review on Name[ACPI_NAMESEG_SIZE + 1] having an extra + * byte[1], compiler testing exhibited a difference in behavior between + * GCC and Clang[2] (at least; MSVC may also exhibit the same) in + * how optimization is done. The extra byte is needed to ensure + * the signature does not get mangled, subsequently avoiding + * GpeNumber being a completely different return value from strtoul. + * + * [1] https://github.com/acpica/acpica/pull/1019#discussion_r2058687704 + * [2] https://github.com/acpica/acpica/pull/1019#discussion_r2061953039 + */ ACPI_MOVE_32_TO_32 (Name, Op->Asl.NameSeg); Name[ACPI_NAMESEG_SIZE] = 0; diff --git a/sys/contrib/dev/acpica/compiler/aslrestype2s.c b/sys/contrib/dev/acpica/compiler/aslrestype2s.c index 096862290384..f47402d4e025 100644 --- a/sys/contrib/dev/acpica/compiler/aslrestype2s.c +++ b/sys/contrib/dev/acpica/compiler/aslrestype2s.c @@ -1469,7 +1469,7 @@ RsDoCsi2SerialBusDescriptor ( case 2: /* Local Port Instance [Integer] (_PRT) */ - RsSetFlagBits16 ((UINT16 *) &Descriptor->Csi2SerialBus.TypeSpecificFlags, InitializerOp, 0, 0); + RsSetFlagBits16 ((UINT16 *) &Descriptor->Csi2SerialBus.TypeSpecificFlags, InitializerOp, 2, 0); RsCreateMultiBitField (InitializerOp, ACPI_RESTAG_LOCALPORT, CurrentByteOffset + ASL_RESDESC_OFFSET (Csi2SerialBus.TypeSpecificFlags), 2, 6); break; diff --git a/sys/contrib/dev/acpica/compiler/dttable2.c b/sys/contrib/dev/acpica/compiler/dttable2.c index 6203a382ad62..754880346299 100644 --- a/sys/contrib/dev/acpica/compiler/dttable2.c +++ b/sys/contrib/dev/acpica/compiler/dttable2.c @@ -1929,24 +1929,30 @@ DtCompileRhct ( { ACPI_STATUS Status; ACPI_RHCT_NODE_HEADER *RhctHeader; - ACPI_RHCT_HART_INFO *RhctHartInfo = NULL; + ACPI_RHCT_HART_INFO *RhctHartInfo; DT_SUBTABLE *Subtable; DT_SUBTABLE *ParentTable; ACPI_DMTABLE_INFO *InfoTable; DT_FIELD **PFieldList = (DT_FIELD **) List; DT_FIELD *SubtableStart; + ACPI_TABLE_RHCT *Table; + BOOLEAN FirstNode = TRUE; /* Compile the main table */ + ParentTable = DtPeekSubtable (); Status = DtCompileTable (PFieldList, AcpiDmTableInfoRhct, &Subtable); if (ACPI_FAILURE (Status)) { return (Status); } + DtInsertSubtable (ParentTable, Subtable); + Table = ACPI_CAST_PTR (ACPI_TABLE_RHCT, ParentTable->Buffer); + Table->NodeCount = 0; + Table->NodeOffset = sizeof (ACPI_TABLE_RHCT); - ParentTable = DtPeekSubtable (); while (*PFieldList) { SubtableStart = *PFieldList; @@ -1961,7 +1967,10 @@ DtCompileRhct ( } DtInsertSubtable (ParentTable, Subtable); RhctHeader = ACPI_CAST_PTR (ACPI_RHCT_NODE_HEADER, Subtable->Buffer); - RhctHeader->Length = (UINT16)(Subtable->Length); + + DtPushSubtable (Subtable); + ParentTable = DtPeekSubtable (); + Table->NodeCount++; switch (RhctHeader->Type) { @@ -1999,37 +2008,54 @@ DtCompileRhct ( return (Status); } DtInsertSubtable (ParentTable, Subtable); - RhctHeader->Length += (UINT16)(Subtable->Length); + if (FirstNode) + { + Table->NodeOffset = ACPI_PTR_DIFF(ParentTable->Buffer, Table); + FirstNode = FALSE; + } /* Compile RHCT subtable additionals */ switch (RhctHeader->Type) { - case ACPI_RHCT_NODE_TYPE_HART_INFO: + case ACPI_RHCT_NODE_TYPE_ISA_STRING: - RhctHartInfo = ACPI_SUB_PTR (ACPI_RHCT_HART_INFO, - Subtable->Buffer, sizeof (ACPI_RHCT_NODE_HEADER)); - if (RhctHartInfo) + /* + * Padding - Variable-length data + * Optionally allows the padding of the ISA string to be used + * for filling this field. + */ + Status = DtCompileTable (PFieldList, AcpiDmTableInfoRhctIsaPad, + &Subtable); + if (ACPI_FAILURE (Status)) + { + return (Status); + } + if (Subtable) { + DtInsertSubtable (ParentTable, Subtable); + } + break; - RhctHartInfo->NumOffsets = 0; - while (*PFieldList) - { - Status = DtCompileTable (PFieldList, - AcpiDmTableInfoRhctHartInfo2, &Subtable); - if (ACPI_FAILURE (Status)) - { - return (Status); - } - if (!Subtable) - { - break; - } + case ACPI_RHCT_NODE_TYPE_HART_INFO: - DtInsertSubtable (ParentTable, Subtable); - RhctHeader->Length += (UINT16)(Subtable->Length); - RhctHartInfo->NumOffsets++; + RhctHartInfo = ACPI_CAST_PTR (ACPI_RHCT_HART_INFO, + Subtable->Buffer); + RhctHartInfo->NumOffsets = 0; + while (*PFieldList) + { + Status = DtCompileTable (PFieldList, + AcpiDmTableInfoRhctHartInfo2, &Subtable); + if (ACPI_FAILURE (Status)) + { + return (Status); } + if (!Subtable) + { + break; + } + DtInsertSubtable (ParentTable, Subtable); + RhctHartInfo->NumOffsets++; } break; @@ -2037,6 +2063,9 @@ DtCompileRhct ( break; } + + DtPopSubtable (); + ParentTable = DtPeekSubtable (); } return (AE_OK); diff --git a/sys/contrib/dev/acpica/compiler/dttemplate.c b/sys/contrib/dev/acpica/compiler/dttemplate.c index 67b13bb82d1b..d7140712d4e6 100644 --- a/sys/contrib/dev/acpica/compiler/dttemplate.c +++ b/sys/contrib/dev/acpica/compiler/dttemplate.c @@ -255,7 +255,7 @@ DtCreateTemplates ( if (AcpiGbl_Optind < 3) { - fprintf (stderr, "Creating default template: [DSDT]\n"); + fprintf (stdout, "Creating default template: [DSDT]\n"); Status = DtCreateOneTemplateFile (ACPI_SIG_DSDT, 0); goto Exit; } @@ -411,7 +411,7 @@ DtCreateAllTemplates ( ACPI_STATUS Status; - fprintf (stderr, "Creating all supported Template files\n"); + fprintf (stdout, "Creating all supported Template files\n"); /* Walk entire ACPI table data structure */ @@ -421,8 +421,13 @@ DtCreateAllTemplates ( if (TableData->Template) { - Status = DtCreateOneTemplate (TableData->Signature, - 0, TableData); + if (ACPI_COMPARE_NAMESEG (TableData->Signature, ACPI_SIG_CDAT)) + /* Special handling of CDAT */ + Status = DtCreateOneTemplate (TableData->Signature, + 0, NULL); + else + Status = DtCreateOneTemplate (TableData->Signature, + 0, TableData); if (ACPI_FAILURE (Status)) { return (Status); @@ -563,7 +568,7 @@ DtCreateOneTemplate ( } else { - /* Special ACPI tables - DSDT, SSDT, OSDT, FACS, RSDP */ + /* Special ACPI tables - DSDT, SSDT, OSDT, FACS, RSDP, CDAT */ AcpiOsPrintf (" (AML byte code table)\n"); AcpiOsPrintf (" */\n"); @@ -621,6 +626,11 @@ DtCreateOneTemplate ( AcpiDmDumpDataTable (ACPI_CAST_PTR (ACPI_TABLE_HEADER, TemplateRsdp)); } + else if (ACPI_COMPARE_NAMESEG (Signature, ACPI_SIG_CDAT)) + { + AcpiDmDumpCdat (ACPI_CAST_PTR (ACPI_TABLE_HEADER, + TemplateCdat)); + } else { fprintf (stderr, @@ -632,14 +642,14 @@ DtCreateOneTemplate ( if (TableCount == 0) { - fprintf (stderr, + fprintf (stdout, "Created ACPI table template for [%4.4s], " "written to \"%s\"\n", Signature, DisasmFilename); } else { - fprintf (stderr, + fprintf (stdout, "Created ACPI table templates for [%4.4s] " "and %u [SSDT] in same file, written to \"%s\"\n", Signature, TableCount, DisasmFilename); diff --git a/sys/contrib/dev/acpica/compiler/dttemplate.h b/sys/contrib/dev/acpica/compiler/dttemplate.h index 0fdd90f73a23..51a34be5c36b 100644 --- a/sys/contrib/dev/acpica/compiler/dttemplate.h +++ b/sys/contrib/dev/acpica/compiler/dttemplate.h @@ -389,7 +389,7 @@ const unsigned char TemplateBoot[] = const unsigned char TemplateCcel[] = { 0x43,0x43,0x45,0x4C,0x38,0x00,0x00,0x00, /* 00000000 "CCEL8..." */ - 0x04,0x1C,0x49,0x4E,0x54,0x45,0x4C,0x20, /* 00000008 "..INTEL " */ + 0x04,0x2E,0x49,0x4E,0x54,0x45,0x4C,0x20, /* 00000008 "..INTEL " */ 0x54,0x65,0x6D,0x70,0x6C,0x61,0x74,0x65, /* 00000010 "Template" */ 0x00,0x00,0x00,0x00,0x49,0x4E,0x54,0x4C, /* 00000018 "....INTL" */ 0x30,0x09,0x21,0x20,0x00,0x00,0x00,0x00, /* 00000020 "0.! ...." */ @@ -1951,25 +1951,25 @@ const unsigned char TemplateRgrt[] = const unsigned char TemplateRhct[] = { - 0x52,0x48,0x43,0x54,0x96,0x00,0x00,0x00, /* 00000000 "RHCT|..." */ - 0x01,0x24,0x4F,0x45,0x4D,0x43,0x41,0x00, /* 00000008 "..OEMCA." */ + 0x52,0x48,0x43,0x54,0x96,0x00,0x00,0x00, /* 00000000 "RHCT...." */ + 0x01,0x6D,0x4F,0x45,0x4D,0x43,0x41,0x00, /* 00000008 ".mOEMCA." */ 0x54,0x45,0x4D,0x50,0x4C,0x41,0x54,0x45, /* 00000010 "TEMPLATE" */ 0x01,0x00,0x00,0x00,0x49,0x4E,0x54,0x4C, /* 00000018 "....INTL" */ - 0x28,0x09,0x22,0x20,0x00,0x00,0x00,0x00, /* 00000020 "... ...." */ + 0x04,0x04,0x25,0x20,0x00,0x00,0x00,0x00, /* 00000020 "..% ...." */ 0x80,0x96,0x98,0x00,0x00,0x00,0x00,0x00, /* 00000028 "........" */ - 0x02,0x00,0x00,0x00,0x38,0x00,0x00,0x00, /* 00000030 "....8..." */ - 0x00,0x00,0x34,0x00,0x01,0x00,0x2B,0x00, /* 00000038 "..4...*." */ + 0x04,0x00,0x00,0x00,0x38,0x00,0x00,0x00, /* 00000030 "....8..." */ + 0x00,0x00,0x34,0x00,0x01,0x00,0x2B,0x00, /* 00000038 "..4...+." */ 0x72,0x76,0x36,0x34,0x69,0x6D,0x61,0x66, /* 00000040 "rv64imaf" */ 0x64,0x63,0x68,0x5F,0x7A,0x69,0x63,0x73, /* 00000048 "dch_zics" */ 0x72,0x5F,0x7A,0x69,0x66,0x65,0x6E,0x63, /* 00000050 "r_zifenc" */ 0x65,0x69,0x5F,0x7A,0x62,0x61,0x5F,0x7A, /* 00000058 "ei_zba_z" */ 0x62,0x62,0x5F,0x7A,0x62,0x63,0x5F,0x7A, /* 00000060 "bb_zbc_z" */ - 0x62,0x73,0x00,0x00,0xFF,0xFF,0x18,0x00, /* 00000068 "bs......" */ - 0x01,0x00,0x03,0x00,0x00,0x00,0x00,0x00, /* 00000070 "........" */ - 0x38,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, /* 00000078 "........" */ - 0x8E,0x00,0x00,0x00,0x01,0x00,0x0A,0x00, /* 00000080 "........" */ - 0x01,0x00,0x00,0x06,0x06,0x06,0x02,0x00, /* 00000088 "........" */ - 0x08,0x00,0x01,0x00,0x00,0x02 /* 00000090 "........" */ + 0x62,0x73,0x00,0x00,0x01,0x00,0x0A,0x00, /* 00000068 "bs......" */ + 0x01,0x00,0x00,0x06,0x06,0x06,0x02,0x00, /* 00000070 "........" */ + 0x08,0x00,0x01,0x00,0x00,0x02,0xFF,0xFF, /* 00000078 "........" */ + 0x18,0x00,0x01,0x00,0x03,0x00,0x00,0x00, /* 00000080 "........" */ + 0x00,0x00,0x3B,0x00,0x00,0x00,0x6C,0x00, /* 00000088 "..;...l." */ + 0x00,0x00,0x76,0x00,0x00,0x00 /* 00000090 "..v..." */ }; const unsigned char TemplateRimt[] = diff --git a/sys/contrib/dev/acpica/compiler/dtutils.c b/sys/contrib/dev/acpica/compiler/dtutils.c index f2463f74b8fc..18ea18cefdd6 100644 --- a/sys/contrib/dev/acpica/compiler/dtutils.c +++ b/sys/contrib/dev/acpica/compiler/dtutils.c @@ -623,6 +623,7 @@ DtGetFieldLength ( case ACPI_DMT_NFIT: case ACPI_DMT_PCI_PATH: case ACPI_DMT_PHAT: + case ACPI_DMT_RHCT: ByteLength = 2; break; diff --git a/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c b/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c index dd8cf4889885..551cf8178d94 100644 --- a/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c +++ b/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c @@ -778,7 +778,7 @@ AcpiDmCsi2SerialBusDescriptor ( AcpiOsPrintf (" 0x%2.2X, 0x%2.2X,\n", Resource->Csi2SerialBus.TypeSpecificFlags & 0x03, - Resource->Csi2SerialBus.TypeSpecificFlags & 0xFC); + (Resource->Csi2SerialBus.TypeSpecificFlags & 0xFC) >> 2); /* ResourceSource is a required field */ diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c b/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c index 8b6efc070b1b..becdb95f8b83 100644 --- a/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c +++ b/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c @@ -646,8 +646,6 @@ AcpiDsCallControlMethod ( ACPI_WALK_STATE *NextWalkState = NULL; ACPI_OPERAND_OBJECT *ObjDesc; ACPI_EVALUATE_INFO *Info; - UINT32 i; - ACPI_FUNCTION_TRACE_PTR (DsCallControlMethod, ThisWalkState); @@ -670,6 +668,23 @@ AcpiDsCallControlMethod ( return_ACPI_STATUS (AE_NULL_OBJECT); } + if (ThisWalkState->NumOperands < ObjDesc->Method.ParamCount) + { + ACPI_ERROR ((AE_INFO, "Missing argument(s) for method [%4.4s]", + AcpiUtGetNodeName (MethodNode))); + + return_ACPI_STATUS (AE_AML_TOO_FEW_ARGUMENTS); + } + + else if (ThisWalkState->NumOperands > ObjDesc->Method.ParamCount) + { + ACPI_ERROR ((AE_INFO, "Too many arguments for method [%4.4s]", + AcpiUtGetNodeName (MethodNode))); + + return_ACPI_STATUS (AE_AML_TOO_MANY_ARGUMENTS); + } + + /* Init for new method, possibly wait on method mutex */ Status = AcpiDsBeginMethodExecution ( @@ -726,15 +741,7 @@ AcpiDsCallControlMethod ( * Delete the operands on the previous walkstate operand stack * (they were copied to new objects) */ - for (i = 0; i < ObjDesc->Method.ParamCount; i++) - { - AcpiUtRemoveReference (ThisWalkState->Operands [i]); - ThisWalkState->Operands [i] = NULL; - } - - /* Clear the operand stack */ - - ThisWalkState->NumOperands = 0; + AcpiDsClearOperands (ThisWalkState); ACPI_DEBUG_PRINT ((ACPI_DB_DISPATCH, "**** Begin nested execution of [%4.4s] **** WalkState=%p\n", diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c b/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c index 42e1aa505d02..2c45e8c91f57 100644 --- a/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c +++ b/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c @@ -357,6 +357,7 @@ AcpiDsMethodDataInitArgs ( Index++; } + AcpiExTraceArgs(Params, Index); ACPI_DEBUG_PRINT ((ACPI_DB_EXEC, "%u args passed to method\n", Index)); return_ACPI_STATUS (AE_OK); diff --git a/sys/contrib/dev/acpica/components/events/evglock.c b/sys/contrib/dev/acpica/components/events/evglock.c index 872e7b499a8f..395ca14fb315 100644 --- a/sys/contrib/dev/acpica/components/events/evglock.c +++ b/sys/contrib/dev/acpica/components/events/evglock.c @@ -195,6 +195,11 @@ AcpiEvInitGlobalLockHandler ( return_ACPI_STATUS (AE_OK); } + if (!AcpiGbl_UseGlobalLock) + { + return_ACPI_STATUS (AE_OK); + } + /* Attempt installation of the global lock handler */ Status = AcpiInstallFixedEventHandler (ACPI_EVENT_GLOBAL, diff --git a/sys/contrib/dev/acpica/components/executer/extrace.c b/sys/contrib/dev/acpica/components/executer/extrace.c index 0eceb0ffccb1..d54d4908ca65 100644 --- a/sys/contrib/dev/acpica/components/executer/extrace.c +++ b/sys/contrib/dev/acpica/components/executer/extrace.c @@ -269,6 +269,68 @@ AcpiExGetTraceEventName ( #endif +/******************************************************************************* + * + * FUNCTION: AcpiExTraceArgs + * + * PARAMETERS: Params - AML method arguments + * Count - numer of method arguments + * + * RETURN: None + * + * DESCRIPTION: Trace any arguments + * + ******************************************************************************/ + +void +AcpiExTraceArgs(ACPI_OPERAND_OBJECT **Params, UINT32 Count) +{ + UINT32 i; + + ACPI_FUNCTION_NAME(ExTraceArgs); + + for (i = 0; i < Count; i++) + { + ACPI_OPERAND_OBJECT *obj_desc = Params[i]; + + if (!i) + { + ACPI_DEBUG_PRINT((ACPI_DB_TRACE_POINT, " ")); + } + + switch (obj_desc->Common.Type) + { + case ACPI_TYPE_INTEGER: + ACPI_DEBUG_PRINT_RAW((ACPI_DB_TRACE_POINT, "%lx", obj_desc->Integer.Value)); + break; + + case ACPI_TYPE_STRING: + if (!obj_desc->String.Length) + { + ACPI_DEBUG_PRINT_RAW((ACPI_DB_TRACE_POINT, "NULL")); + break; + } + if (ACPI_IS_DEBUG_ENABLED(ACPI_LV_TRACE_POINT, _COMPONENT)) + { + AcpiUtPrintString(obj_desc->String.Pointer, ACPI_UINT8_MAX); + } + break; + + default: + ACPI_DEBUG_PRINT_RAW((ACPI_DB_TRACE_POINT, "Unknown")); + break; + } + + if ((i + 1) == Count) + { + ACPI_DEBUG_PRINT_RAW((ACPI_DB_TRACE_POINT, "\n")); + } + else + { + ACPI_DEBUG_PRINT_RAW((ACPI_DB_TRACE_POINT, ", ")); + } + } +} /******************************************************************************* * @@ -299,9 +361,9 @@ AcpiExTracePoint ( if (Pathname) { ACPI_DEBUG_PRINT ((ACPI_DB_TRACE_POINT, - "%s %s [0x%p:%s] execution.\n", + "%s %s [%s] execution.\n", AcpiExGetTraceEventName (Type), Begin ? "Begin" : "End", - Aml, Pathname)); + Pathname)); } else { diff --git a/sys/contrib/dev/acpica/components/parser/psopinfo.c b/sys/contrib/dev/acpica/components/parser/psopinfo.c index 21c2b831ef24..1db32f4e8246 100644 --- a/sys/contrib/dev/acpica/components/parser/psopinfo.c +++ b/sys/contrib/dev/acpica/components/parser/psopinfo.c @@ -180,8 +180,8 @@ const ACPI_OPCODE_INFO * AcpiPsGetOpcodeInfo ( UINT16 Opcode) { -#ifdef ACPI_DEBUG_OUTPUT - const char *OpcodeName = "Unknown AML opcode"; +#if defined ACPI_ASL_COMPILER && defined ACPI_DEBUG_OUTPUT + const char *OpcodeName = "Unknown AML opcode"; #endif ACPI_FUNCTION_NAME (PsGetOpcodeInfo); @@ -207,7 +207,7 @@ AcpiPsGetOpcodeInfo ( #if defined ACPI_ASL_COMPILER && defined ACPI_DEBUG_OUTPUT #include <contrib/dev/acpica/compiler/asldefine.h> - + switch (Opcode) { case AML_RAW_DATA_BYTE: @@ -249,12 +249,12 @@ AcpiPsGetOpcodeInfo ( default: break; } -#endif /* Unknown AML opcode */ ACPI_DEBUG_PRINT ((ACPI_DB_EXEC, "%s [%4.4X]\n", OpcodeName, Opcode)); +#endif return (&AcpiGbl_AmlOpInfo [_UNK]); } diff --git a/sys/contrib/dev/acpica/components/tables/tbprint.c b/sys/contrib/dev/acpica/components/tables/tbprint.c index 7211673c42a2..8b812a890a07 100644 --- a/sys/contrib/dev/acpica/components/tables/tbprint.c +++ b/sys/contrib/dev/acpica/components/tables/tbprint.c @@ -279,6 +279,14 @@ AcpiTbPrintTableHeader ( ACPI_CAST_PTR (ACPI_TABLE_RSDP, Header)->Revision, LocalHeader.OemId)); } + else if (AcpiGbl_CDAT && !AcpiUtValidNameseg (Header->Signature)) + { + /* CDAT does not use the common ACPI table header */ + + ACPI_INFO (("%-4.4s 0x%8.8X%8.8X %06X", + ACPI_SIG_CDAT, ACPI_FORMAT_UINT64 (Address), + ACPI_CAST_PTR (ACPI_TABLE_CDAT, Header)->Length)); + } else { /* Standard ACPI table with full common header */ diff --git a/sys/contrib/dev/acpica/components/utilities/utnonansi.c b/sys/contrib/dev/acpica/components/utilities/utnonansi.c index bfbe1194ceae..f8b3a29e3283 100644 --- a/sys/contrib/dev/acpica/components/utilities/utnonansi.c +++ b/sys/contrib/dev/acpica/components/utilities/utnonansi.c @@ -353,7 +353,7 @@ AcpiUtSafeStrncpy ( { /* Always terminate destination string */ - memcpy (Dest, Source, DestSize); + strncpy (Dest, Source, DestSize); Dest[DestSize - 1] = 0; } diff --git a/sys/contrib/dev/acpica/include/acdebug.h b/sys/contrib/dev/acpica/include/acdebug.h index e335752148b9..63f39051a8ac 100644 --- a/sys/contrib/dev/acpica/include/acdebug.h +++ b/sys/contrib/dev/acpica/include/acdebug.h @@ -187,7 +187,7 @@ typedef struct acpi_db_execute_walk { UINT32 Count; UINT32 MaxCount; - char NameSeg[ACPI_NAMESEG_SIZE + 1] ACPI_NONSTRING; + char NameSeg[ACPI_NAMESEG_SIZE + 1]; } ACPI_DB_EXECUTE_WALK; diff --git a/sys/contrib/dev/acpica/include/acexcep.h b/sys/contrib/dev/acpica/include/acexcep.h index 57f98ab4540f..7216e0d49148 100644 --- a/sys/contrib/dev/acpica/include/acexcep.h +++ b/sys/contrib/dev/acpica/include/acexcep.h @@ -322,8 +322,11 @@ typedef struct acpi_exception_info #define AE_AML_TARGET_TYPE EXCEP_AML (0x0023) #define AE_AML_PROTOCOL EXCEP_AML (0x0024) #define AE_AML_BUFFER_LENGTH EXCEP_AML (0x0025) +#define AE_AML_TOO_FEW_ARGUMENTS EXCEP_AML (0x0026) +#define AE_AML_TOO_MANY_ARGUMENTS EXCEP_AML (0x0027) -#define AE_CODE_AML_MAX 0x0025 + +#define AE_CODE_AML_MAX 0x0027 /* @@ -456,7 +459,9 @@ static const ACPI_EXCEPTION_INFO AcpiGbl_ExceptionNames_Aml[] = EXCEP_TXT ("AE_AML_UNINITIALIZED_NODE", "A namespace node is uninitialized or unresolved"), EXCEP_TXT ("AE_AML_TARGET_TYPE", "A target operand of an incorrect type was encountered"), EXCEP_TXT ("AE_AML_PROTOCOL", "Violation of a fixed ACPI protocol"), - EXCEP_TXT ("AE_AML_BUFFER_LENGTH", "The length of the buffer is invalid/incorrect") + EXCEP_TXT ("AE_AML_BUFFER_LENGTH", "The length of the buffer is invalid/incorrect"), + EXCEP_TXT ("AE_AML_TOO_FEW_ARGUMENTS", "There are fewer than expected method arguments"), + EXCEP_TXT ("AE_AML_TOO_MANY_ARGUMENTS", "There are too many arguments for this method") }; static const ACPI_EXCEPTION_INFO AcpiGbl_ExceptionNames_Ctrl[] = diff --git a/sys/contrib/dev/acpica/include/acinterp.h b/sys/contrib/dev/acpica/include/acinterp.h index 74166384f172..b7f9e8f615e4 100644 --- a/sys/contrib/dev/acpica/include/acinterp.h +++ b/sys/contrib/dev/acpica/include/acinterp.h @@ -280,6 +280,10 @@ AcpiExTracePoint ( UINT8 *Aml, char *Pathname); +void +AcpiExTraceArgs( + ACPI_OPERAND_OBJECT **Params, + UINT32 Count); /* * exfield - ACPI AML (p-code) execution - field manipulation diff --git a/sys/contrib/dev/acpica/include/acpixf.h b/sys/contrib/dev/acpica/include/acpixf.h index 193b0e6a70dc..b5961e21bb9b 100644 --- a/sys/contrib/dev/acpica/include/acpixf.h +++ b/sys/contrib/dev/acpica/include/acpixf.h @@ -154,7 +154,7 @@ /* Current ACPICA subsystem version in YYYYMMDD format */ -#define ACPI_CA_VERSION 0x20250404 +#define ACPI_CA_VERSION 0x20250807 #include <contrib/dev/acpica/include/acconfig.h> #include <contrib/dev/acpica/include/actypes.h> @@ -358,6 +358,12 @@ ACPI_INIT_GLOBAL (UINT8, AcpiGbl_OsiData, 0); ACPI_INIT_GLOBAL (BOOLEAN, AcpiGbl_ReducedHardware, FALSE); /* + * ACPI Global Lock is mainly used for systems with SMM, so no-SMM systems + * (such as LoongArch) may not have and not use Global Lock. + */ +ACPI_INIT_GLOBAL (BOOLEAN, AcpiGbl_UseGlobalLock, TRUE); + +/* * Maximum timeout for While() loop iterations before forced method abort. * This mechanism is intended to prevent infinite loops during interpreter * execution within a host kernel. diff --git a/sys/contrib/dev/acpica/include/actbl.h b/sys/contrib/dev/acpica/include/actbl.h index eafd5d8a0f8b..ae52bd452c90 100644 --- a/sys/contrib/dev/acpica/include/actbl.h +++ b/sys/contrib/dev/acpica/include/actbl.h @@ -220,7 +220,7 @@ typedef struct acpi_table_header char OemId[ACPI_OEM_ID_SIZE] ACPI_NONSTRING; /* ASCII OEM identification */ char OemTableId[ACPI_OEM_TABLE_ID_SIZE] ACPI_NONSTRING; /* ASCII OEM table identification */ UINT32 OemRevision; /* OEM revision number */ - char AslCompilerId[ACPI_NAMESEG_SIZE]; /* ASCII ASL compiler vendor ID */ + char AslCompilerId[ACPI_NAMESEG_SIZE] ACPI_NONSTRING; /* ASCII ASL compiler vendor ID */ UINT32 AslCompilerRevision; /* ASL compiler version */ } ACPI_TABLE_HEADER; diff --git a/sys/contrib/dev/acpica/include/actbl1.h b/sys/contrib/dev/acpica/include/actbl1.h index 876b721068c6..ec04f0a0ab9f 100644 --- a/sys/contrib/dev/acpica/include/actbl1.h +++ b/sys/contrib/dev/acpica/include/actbl1.h @@ -262,7 +262,7 @@ typedef struct acpi_whea_header /* Larger subtable header (when Length can exceed 255) */ -typedef struct acpi_subtable_header_16 +typedef struct acpi_subtbl_hdr_16 { UINT16 Type; UINT16 Length; diff --git a/sys/contrib/dev/acpica/include/actbl2.h b/sys/contrib/dev/acpica/include/actbl2.h index 4899929b2d45..a74b6d555a3a 100644 --- a/sys/contrib/dev/acpica/include/actbl2.h +++ b/sys/contrib/dev/acpica/include/actbl2.h @@ -201,6 +201,7 @@ #define ACPI_SIG_SDEI "SDEI" /* Software Delegated Exception Interface Table */ #define ACPI_SIG_SDEV "SDEV" /* Secure Devices table */ #define ACPI_SIG_SVKL "SVKL" /* Storage Volume Key Location Table */ +#define ACPI_SIG_SWFT "SWFT" /* SoundWire File Table */ #define ACPI_SIG_TDEL "TDEL" /* TD Event Log Table */ @@ -4094,6 +4095,30 @@ enum acpi_svkl_format ACPI_SVKL_FORMAT_RESERVED = 1 /* 1 and greater are reserved */ }; +/******************************************************************************* + * + * SWFT - SoundWire File Table + * as described in Discovery and Configuration (DisCo) Specification + * for SoundWire® + * Version 1 + * + ******************************************************************************/ + +typedef struct acpi_table_swft +{ + ACPI_TABLE_HEADER Header; /* Common ACPI table header */ + +} ACPI_TABLE_SWFT; + +typedef struct acpi_swft_file +{ + UINT16 VendorID; + UINT32 FileID; + UINT16 FileVersion; + UINT16 FileLength; + UINT8 FileData[]; + +} ACPI_SWFT_FILE; /******************************************************************************* * diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh index 885a64037f89..70a2364f1fc6 100755 --- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh +++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh @@ -109,7 +109,7 @@ case "$OS" in KSRC="$FREEBSD_SNAP/../amd64/$FreeBSD/src.txz" ;; freebsd15-0c) - FreeBSD="15.0-CURRENT" + FreeBSD="15.0-PRERELEASE" OSNAME="FreeBSD $FreeBSD" OSv="freebsd14.0" URLxz="$FREEBSD_SNAP/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI-ufs.raw.xz" diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh index 17e976ebcc39..2807d9e77127 100755 --- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh +++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh @@ -5,12 +5,13 @@ # # Usage: # -# qemu-4-build-vm.sh OS [--enable-debug][--dkms][--poweroff] -# [--release][--repo][--tarball] +# qemu-4-build-vm.sh OS [--enable-debug][--dkms][--patch-level NUM] +# [--poweroff][--release][--repo][--tarball] # # OS: OS name like 'fedora41' # --enable-debug: Build RPMs with '--enable-debug' (for testing) # --dkms: Build DKMS RPMs as well +# --patch-level NUM: Use a custom patch level number for packages. # --poweroff: Power-off the VM after building # --release Build zfs-release*.rpm as well # --repo After building everything, copy RPMs into /tmp/repo @@ -21,6 +22,7 @@ ENABLE_DEBUG="" DKMS="" +PATCH_LEVEL="" POWEROFF="" RELEASE="" REPO="" @@ -35,6 +37,11 @@ while [[ $# -gt 0 ]]; do DKMS=1 shift ;; + --patch-level) + PATCH_LEVEL=$2 + shift + shift + ;; --poweroff) POWEROFF=1 shift @@ -215,6 +222,10 @@ function rpm_build_and_install() { run ./autogen.sh echo "##[endgroup]" + if [ -n "$PATCH_LEVEL" ] ; then + sed -i -E 's/(Release:\s+)1/\1'$PATCH_LEVEL'/g' META + fi + echo "##[group]Configure" run ./configure --enable-debuginfo $extra echo "##[endgroup]" @@ -328,7 +339,13 @@ fi # almalinux9.5 # fedora42 source /etc/os-release -sudo hostname "$ID$VERSION_ID" + if which hostnamectl &> /dev/null ; then + # Fedora 42+ use hostnamectl + sudo hostnamectl set-hostname "$ID$VERSION_ID" + sudo hostnamectl set-hostname --pretty "$ID$VERSION_ID" +else + sudo hostname "$ID$VERSION_ID" +fi # save some sysinfo uname -a > /var/tmp/uname.txt diff --git a/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml b/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml index 5b5afe746859..d8a95954fe1a 100644 --- a/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml +++ b/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml @@ -32,6 +32,11 @@ on: options: - "Build RPMs" - "Test repo" + patch_level: + type: string + required: false + default: "" + description: "(optional) patch level number" repo_url: type: string required: false @@ -78,7 +83,13 @@ jobs: mkdir -p /tmp/repo ssh zfs@vm0 '$HOME/zfs/.github/workflows/scripts/qemu-test-repo-vm.sh' ${{ github.event.inputs.repo_url }} else - .github/workflows/scripts/qemu-4-build.sh --repo --release --dkms --tarball ${{ matrix.os }} + EXTRA="" + if [ -n "${{ github.event.inputs.patch_level }}" ] ; then + EXTRA="--patch-level ${{ github.event.inputs.patch_level }}" + fi + + .github/workflows/scripts/qemu-4-build.sh $EXTRA \ + --repo --release --dkms --tarball ${{ matrix.os }} fi - name: Prepare artifacts diff --git a/sys/contrib/openzfs/.mailmap b/sys/contrib/openzfs/.mailmap index b6d942c000b8..e6f09c6c9d43 100644 --- a/sys/contrib/openzfs/.mailmap +++ b/sys/contrib/openzfs/.mailmap @@ -23,6 +23,7 @@ # These maps are making names consistent where they have varied but the email # address has never changed. In most cases, the full name is in the # Signed-off-by of a commit with a matching author. +Achill Gilgenast <achill@achill.org> Ahelenia ZiemiaÅ„ska <nabijaczleweli@gmail.com> Ahelenia ZiemiaÅ„ska <nabijaczleweli@nabijaczleweli.xyz> Alex John <alex@stty.io> @@ -37,6 +38,7 @@ Crag Wang <crag0715@gmail.com> Damian Szuberski <szuberskidamian@gmail.com> Daniel Kolesa <daniel@octaforge.org> Debabrata Banerjee <dbavatar@gmail.com> +Diwakar Kristappagari <diwakar-k@hpe.com> Finix Yan <yanchongwen@hotmail.com> Gaurav Kumar <gauravk.18@gmail.com> Gionatan Danti <g.danti@assyoma.it> @@ -145,6 +147,7 @@ Gaurav Kumar <gauravk.18@gmail.com> <gaurkuma@users.noreply.github.com> George Gaydarov <git@gg7.io> <gg7@users.noreply.github.com> Georgy Yakovlev <gyakovlev@gentoo.org> <168902+gyakovlev@users.noreply.github.com> Gerardwx <gerardw@alum.mit.edu> <Gerardwx@users.noreply.github.com> +Germano Massullo <germano.massullo@gmail.com> <Germano0@users.noreply.github.com> Gian-Carlo DeFazio <defazio1@llnl.gov> <defaziogiancarlo@users.noreply.github.com> Giuseppe Di Natale <dinatale2@llnl.gov> <dinatale2@users.noreply.github.com> Hajo Möller <dasjoe@gmail.com> <dasjoe@users.noreply.github.com> @@ -164,6 +167,7 @@ John Ramsden <johnramsden@riseup.net> <johnramsden@users.noreply.github.com> Jonathon Fernyhough <jonathon@m2x.dev> <559369+jonathonf@users.noreply.github.com> Jose Luis Duran <jlduran@gmail.com> <jlduran@users.noreply.github.com> Justin Hibbits <chmeeedalf@gmail.com> <chmeeedalf@users.noreply.github.com> +Kaitlin Hoang <kthoang@amazon.com> <khoang98@users.noreply.github.com> Kevin Greene <kevin.greene@delphix.com> <104801862+kxgreene@users.noreply.github.com> Kevin Jin <lostking2008@hotmail.com> <33590050+jxdking@users.noreply.github.com> Kevin P. Fleming <kevin@km6g.us> <kpfleming@users.noreply.github.com> diff --git a/sys/contrib/openzfs/AUTHORS b/sys/contrib/openzfs/AUTHORS index a9d249a66f1e..6c34c07f39ef 100644 --- a/sys/contrib/openzfs/AUTHORS +++ b/sys/contrib/openzfs/AUTHORS @@ -10,6 +10,7 @@ PAST MAINTAINERS: CONTRIBUTORS: Aaron Fineman <abyxcos@gmail.com> + Achill Gilgenast <achill@achill.org> Adam D. Moss <c@yotes.com> Adam Leventhal <ahl@delphix.com> Adam Stevko <adam.stevko@gmail.com> @@ -59,6 +60,7 @@ CONTRIBUTORS: Andreas Buschmann <andreas.buschmann@tech.net.de> Andreas Dilger <adilger@intel.com> Andreas Vögele <andreas@andreasvoegele.com> + Andres <a-d-j-i@users.noreply.github.com> Andrew Barnes <barnes333@gmail.com> Andrew Hamilton <ahamilto@tjhsst.edu> Andrew Innes <andrew.c12@gmail.com> @@ -72,6 +74,7 @@ CONTRIBUTORS: Andrey Prokopenko <job@terem.fr> Andrey Vesnovaty <andrey.vesnovaty@gmail.com> Andriy Gapon <avg@freebsd.org> + Andriy Tkachuk <andriy.tkachuk@seagate.com> Andy Bakun <github@thwartedefforts.org> Andy Fiddaman <omnios@citrus-it.co.uk> Aniruddha Shankar <k@191a.net> @@ -120,6 +123,7 @@ CONTRIBUTORS: Caleb James DeLisle <calebdelisle@lavabit.com> Cameron Harr <harr1@llnl.gov> Cao Xuewen <cao.xuewen@zte.com.cn> + Carl George <carlwgeorge@gmail.com> Carlo Landmeter <clandmeter@gmail.com> Carlos Alberto Lopez Perez <clopez@igalia.com> Cedric Maunoury <cedric.maunoury@gmail.com> @@ -200,6 +204,7 @@ CONTRIBUTORS: Dimitri John Ledkov <xnox@ubuntu.com> Dimitry Andric <dimitry@andric.com> Dirkjan Bussink <d.bussink@gmail.com> + Diwakar Kristappagari <diwakar-k@hpe.com> Dmitry Khasanov <pik4ez@gmail.com> Dominic Pearson <dsp@technoanimal.net> Dominik Hassler <hadfl@omniosce.org> @@ -250,6 +255,7 @@ CONTRIBUTORS: George Wilson <gwilson@delphix.com> Georgy Yakovlev <ya@sysdump.net> Gerardwx <gerardw@alum.mit.edu> + Germano Massullo <germano.massullo@gmail.com> Gian-Carlo DeFazio <defazio1@llnl.gov> Gionatan Danti <g.danti@assyoma.it> Giuseppe Di Natale <guss80@gmail.com> @@ -287,6 +293,7 @@ CONTRIBUTORS: Igor K <igor@dilos.org> Igor Kozhukhov <ikozhukhov@gmail.com> Igor Lvovsky <ilvovsky@gmail.com> + Igor Ostapenko <pm@igoro.pro> ilbsmart <wgqimut@gmail.com> Ilkka Sovanto <github@ilkka.kapsi.fi> illiliti <illiliti@protonmail.com> @@ -326,6 +333,7 @@ CONTRIBUTORS: Jinshan Xiong <jinshan.xiong@intel.com> Jitendra Patidar <jitendra.patidar@nutanix.com> JK Dingwall <james@dingwall.me.uk> + Joel Low <joel@joelsplace.sg> Joe Stein <joe.stein@delphix.com> John-Mark Gurney <jmg@funkthat.com> John Albietz <inthecloud247@gmail.com> @@ -374,6 +382,7 @@ CONTRIBUTORS: Kevin Jin <lostking2008@hotmail.com> Kevin P. Fleming <kevin@km6g.us> Kevin Tanguy <kevin.tanguy@ovh.net> + khoang98 <khoang98@users.noreply.github.com> KireinaHoro <i@jsteward.moe> Kjeld Schouten-Lebbing <kjeld@schouten-lebbing.nl> Kleber TarcÃsio <klebertarcisio@yahoo.com.br> @@ -447,6 +456,7 @@ CONTRIBUTORS: Max Zettlmeißl <max@zettlmeissl.de> Md Islam <mdnahian@outlook.com> megari <megari@iki.fi> + Meriel Luna Mittelbach <lunarlambda@gmail.com> Michael D Labriola <michael.d.labriola@gmail.com> Michael Franzl <michael@franzl.name> Michael Gebetsroither <michael@mgeb.org> @@ -494,6 +504,7 @@ CONTRIBUTORS: Orivej Desh <orivej@gmx.fr> Pablo Correa Gómez <ablocorrea@hotmail.com> Palash Gandhi <pbg4930@rit.edu> + Patrick Fasano <patrick@patrickfasano.com> Patrick Mooney <pmooney@pfmooney.com> Patrik Greco <sikevux@sikevux.se> Paul B. Henson <henson@acm.org> @@ -535,6 +546,7 @@ CONTRIBUTORS: Remy Blank <remy.blank@pobox.com> renelson <bnelson@nelsonbe.com> Reno Reckling <e-github@wthack.de> + René Wirnata <rene.wirnata@pandascience.net> Ricardo M. Correia <ricardo.correia@oracle.com> Riccardo Schirone <rschirone91@gmail.com> Richard Allen <belperite@gmail.com> @@ -640,6 +652,7 @@ CONTRIBUTORS: tleydxdy <shironeko.github@tesaguri.club> Tobin Harding <me@tobin.cc> Todd Seidelmann <seidelma@users.noreply.github.com> + Todd Zullinger <tmz@pobox.com> Tom Caputi <tcaputi@datto.com> Tom Matthews <tom@axiom-partners.com> Tomohiro Kusumi <kusumi.tomohiro@gmail.com> diff --git a/sys/contrib/openzfs/META b/sys/contrib/openzfs/META index 47f0795bfa11..1a9c671feac6 100644 --- a/sys/contrib/openzfs/META +++ b/sys/contrib/openzfs/META @@ -6,5 +6,5 @@ Release: 1 Release-Tags: relext License: CDDL Author: OpenZFS -Linux-Maximum: 6.15 +Linux-Maximum: 6.16 Linux-Minimum: 4.18 diff --git a/sys/contrib/openzfs/cmd/zdb/zdb.c b/sys/contrib/openzfs/cmd/zdb/zdb.c index 66d5fbd6adbe..a5f23be2aaaf 100644 --- a/sys/contrib/openzfs/cmd/zdb/zdb.c +++ b/sys/contrib/openzfs/cmd/zdb/zdb.c @@ -127,6 +127,7 @@ static zfs_range_tree_t *mos_refd_objs; static spa_t *spa; static objset_t *os; static boolean_t kernel_init_done; +static boolean_t corruption_found = B_FALSE; static void snprintf_blkptr_compact(char *, size_t, const blkptr_t *, boolean_t); @@ -250,6 +251,7 @@ sublivelist_verify_func(void *args, dsl_deadlist_entry_t *dle) &e->svbr_blk, B_TRUE); (void) printf("\tERROR: %d unmatched FREE(s): %s\n", e->svbr_refcnt, blkbuf); + corruption_found = B_TRUE; } zfs_btree_destroy(&sv->sv_pair); @@ -405,6 +407,7 @@ verify_livelist_allocs(metaslab_verify_t *mv, uint64_t txg, (u_longlong_t)DVA_GET_ASIZE(&found->svb_dva), (u_longlong_t)found->svb_allocated_txg, (u_longlong_t)txg); + corruption_found = B_TRUE; } } } @@ -426,6 +429,7 @@ metaslab_spacemap_validation_cb(space_map_entry_t *sme, void *arg) (u_longlong_t)txg, (u_longlong_t)offset, (u_longlong_t)size, (u_longlong_t)mv->mv_vdid, (u_longlong_t)mv->mv_msid); + corruption_found = B_TRUE; } else { zfs_range_tree_add(mv->mv_allocated, offset, size); @@ -439,6 +443,7 @@ metaslab_spacemap_validation_cb(space_map_entry_t *sme, void *arg) (u_longlong_t)txg, (u_longlong_t)offset, (u_longlong_t)size, (u_longlong_t)mv->mv_vdid, (u_longlong_t)mv->mv_msid); + corruption_found = B_TRUE; } else { zfs_range_tree_remove(mv->mv_allocated, offset, size); @@ -526,6 +531,7 @@ mv_populate_livelist_allocs(metaslab_verify_t *mv, sublivelist_verify_t *sv) (u_longlong_t)DVA_GET_VDEV(&svb->svb_dva), (u_longlong_t)DVA_GET_OFFSET(&svb->svb_dva), (u_longlong_t)DVA_GET_ASIZE(&svb->svb_dva)); + corruption_found = B_TRUE; continue; } @@ -542,6 +548,7 @@ mv_populate_livelist_allocs(metaslab_verify_t *mv, sublivelist_verify_t *sv) (u_longlong_t)DVA_GET_VDEV(&svb->svb_dva), (u_longlong_t)DVA_GET_OFFSET(&svb->svb_dva), (u_longlong_t)DVA_GET_ASIZE(&svb->svb_dva)); + corruption_found = B_TRUE; continue; } @@ -655,6 +662,7 @@ livelist_metaslab_validate(spa_t *spa) } (void) printf("ERROR: Found livelist blocks marked as allocated " "for indirect vdevs:\n"); + corruption_found = B_TRUE; zfs_btree_index_t *where = NULL; sublivelist_verify_block_t *svb; @@ -827,7 +835,7 @@ usage(void) (void) fprintf(stderr, "Specify an option more than once (e.g. -bb) " "to make only that option verbose\n"); (void) fprintf(stderr, "Default is to dump everything non-verbosely\n"); - zdb_exit(1); + zdb_exit(2); } static void @@ -2583,19 +2591,17 @@ snprintf_blkptr_compact(char *blkbuf, size_t buflen, const blkptr_t *bp, } } -static void +static u_longlong_t print_indirect(spa_t *spa, blkptr_t *bp, const zbookmark_phys_t *zb, const dnode_phys_t *dnp) { char blkbuf[BP_SPRINTF_LEN]; + u_longlong_t offset; int l; - if (!BP_IS_EMBEDDED(bp)) { - ASSERT3U(BP_GET_TYPE(bp), ==, dnp->dn_type); - ASSERT3U(BP_GET_LEVEL(bp), ==, zb->zb_level); - } + offset = (u_longlong_t)blkid2offset(dnp, bp, zb); - (void) printf("%16llx ", (u_longlong_t)blkid2offset(dnp, bp, zb)); + (void) printf("%16llx ", offset); ASSERT(zb->zb_level >= 0); @@ -2610,19 +2616,38 @@ print_indirect(spa_t *spa, blkptr_t *bp, const zbookmark_phys_t *zb, snprintf_blkptr_compact(blkbuf, sizeof (blkbuf), bp, B_FALSE); if (dump_opt['Z'] && BP_GET_COMPRESS(bp) == ZIO_COMPRESS_ZSTD) snprintf_zstd_header(spa, blkbuf, sizeof (blkbuf), bp); - (void) printf("%s\n", blkbuf); + (void) printf("%s", blkbuf); + + if (!BP_IS_EMBEDDED(bp)) { + if (BP_GET_TYPE(bp) != dnp->dn_type) { + (void) printf(" (ERROR: Block pointer type " + "(%llu) does not match dnode type (%hhu))", + BP_GET_TYPE(bp), dnp->dn_type); + corruption_found = B_TRUE; + } + if (BP_GET_LEVEL(bp) != zb->zb_level) { + (void) printf(" (ERROR: Block pointer level " + "(%llu) does not match bookmark level (%lld))", + BP_GET_LEVEL(bp), (u_longlong_t)zb->zb_level); + corruption_found = B_TRUE; + } + } + (void) printf("\n"); + + return (offset); } static int visit_indirect(spa_t *spa, const dnode_phys_t *dnp, blkptr_t *bp, const zbookmark_phys_t *zb) { + u_longlong_t offset; int err = 0; if (BP_GET_BIRTH(bp) == 0) return (0); - print_indirect(spa, bp, zb, dnp); + offset = print_indirect(spa, bp, zb, dnp); if (BP_GET_LEVEL(bp) > 0 && !BP_IS_HOLE(bp)) { arc_flags_t flags = ARC_FLAG_WAIT; @@ -2652,8 +2677,15 @@ visit_indirect(spa_t *spa, const dnode_phys_t *dnp, break; fill += BP_GET_FILL(cbp); } - if (!err) - ASSERT3U(fill, ==, BP_GET_FILL(bp)); + if (!err) { + if (fill != BP_GET_FILL(bp)) { + (void) printf("%16llx: Block pointer " + "fill (%llu) does not match calculated " + "value (%llu)\n", offset, BP_GET_FILL(bp), + (u_longlong_t)fill); + corruption_found = B_TRUE; + } + } arc_buf_destroy(buf, &buf); } @@ -2909,6 +2941,7 @@ dump_full_bpobj(bpobj_t *bpo, const char *name, int indent) (void) printf("ERROR %u while trying to open " "subobj id %llu\n", error, (u_longlong_t)subobj); + corruption_found = B_TRUE; continue; } dump_full_bpobj(&subbpo, "subobj", indent + 1); @@ -3088,6 +3121,7 @@ bpobj_count_refd(bpobj_t *bpo) (void) printf("ERROR %u while trying to open " "subobj id %llu\n", error, (u_longlong_t)subobj); + corruption_found = B_TRUE; continue; } bpobj_count_refd(&subbpo); @@ -9634,7 +9668,7 @@ main(int argc, char **argv) } else if (objset_str && !zdb_numeric(objset_str + 1) && dump_opt['N']) { printf("Supply a numeric objset ID with -N\n"); - error = 1; + error = 2; goto fini; } } else { @@ -9936,5 +9970,8 @@ fini: if (kernel_init_done) kernel_fini(); + if (corruption_found && error == 0) + error = 3; + return (error); } diff --git a/sys/contrib/openzfs/config/kernel-mkdir.m4 b/sys/contrib/openzfs/config/kernel-mkdir.m4 index c1aebc387abe..78b32447c593 100644 --- a/sys/contrib/openzfs/config/kernel-mkdir.m4 +++ b/sys/contrib/openzfs/config/kernel-mkdir.m4 @@ -84,6 +84,8 @@ AC_DEFUN([ZFS_AC_KERNEL_MKDIR], [ AC_DEFINE(HAVE_IOPS_MKDIR_DENTRY, 1, [iops->mkdir() returns struct dentry*]) ],[ + AC_MSG_RESULT(no) + dnl # dnl # 6.3 API change dnl # mkdir() takes struct mnt_idmap * as the first arg diff --git a/sys/contrib/openzfs/config/toolchain-simd.m4 b/sys/contrib/openzfs/config/toolchain-simd.m4 index 344807fc830c..f18c91007cde 100644 --- a/sys/contrib/openzfs/config/toolchain-simd.m4 +++ b/sys/contrib/openzfs/config/toolchain-simd.m4 @@ -24,6 +24,8 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [ ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE + ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VAES + ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VPCLMULQDQ ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES @@ -447,6 +449,48 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE], [ ]) dnl # +dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VAES +dnl # +AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VAES], [ + AC_MSG_CHECKING([whether host toolchain supports VAES]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([ + [ + int main() + { + __asm__ __volatile__("vaesenc %ymm0, %ymm1, %ymm0"); + return (0); + } + ]])], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_VAES], 1, [Define if host toolchain supports VAES]) + ], [ + AC_MSG_RESULT([no]) + ]) +]) + +dnl # +dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VPCLMULQDQ +dnl # +AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VPCLMULQDQ], [ + AC_MSG_CHECKING([whether host toolchain supports VPCLMULQDQ]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([ + [ + int main() + { + __asm__ __volatile__("vpclmulqdq %0, %%ymm4, %%ymm3, %%ymm5" :: "i"(0)); + return (0); + } + ]])], [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_VPCLMULQDQ], 1, [Define if host toolchain supports VPCLMULQDQ]) + ], [ + AC_MSG_RESULT([no]) + ]) +]) + +dnl # dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE dnl # AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE], [ diff --git a/sys/contrib/openzfs/contrib/debian/control b/sys/contrib/openzfs/contrib/debian/control index 96a2bdd88665..c5358dedc0fd 100644 --- a/sys/contrib/openzfs/contrib/debian/control +++ b/sys/contrib/openzfs/contrib/debian/control @@ -100,8 +100,8 @@ Depends: ${misc:Depends}, ${shlibs:Depends} # The libcurl4 is loaded through dlopen("libcurl.so.4"). # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=988521 Recommends: libcurl4 -Breaks: libzfs2, libzfs4, libzfs4linux, libzfs6linux -Replaces: libzfs2, libzfs4, libzfs4linux, libzfs6linux +Breaks: libzfs2, libzfs4, libzfs4linux, libzfs6linux, openzfs-libzfs4 +Replaces: libzfs2, libzfs4, libzfs4linux, libzfs6linux, openzfs-libzfs4 Conflicts: libzfs6linux Description: OpenZFS filesystem library for Linux - general support OpenZFS is a storage platform that encompasses the functionality of @@ -128,8 +128,8 @@ Package: openzfs-libzpool6 Section: contrib/libs Architecture: linux-any Depends: ${misc:Depends}, ${shlibs:Depends} -Breaks: libzpool2, libzpool5, libzpool5linux, libzpool6linux -Replaces: libzpool2, libzpool5, libzpool5linux, libzpool6linux +Breaks: libzpool2, libzpool5, libzpool6linux +Replaces: libzpool2, libzpool5, libzpool6linux Conflicts: libzpool6linux Description: OpenZFS pool library for Linux OpenZFS is a storage platform that encompasses the functionality of diff --git a/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/LICENSE b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/LICENSE new file mode 100644 index 000000000000..04c03a37e0cb --- /dev/null +++ b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/LICENSE @@ -0,0 +1,253 @@ +BoringSSL is a fork of OpenSSL. As such, large parts of it fall under OpenSSL +licensing. Files that are completely new have a Google copyright and an ISC +license. This license is reproduced at the bottom of this file. + +Contributors to BoringSSL are required to follow the CLA rules for Chromium: +https://cla.developers.google.com/clas + +Files in third_party/ have their own licenses, as described therein. The MIT +license, for third_party/fiat, which, unlike other third_party directories, is +compiled into non-test libraries, is included below. + +The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the +OpenSSL License and the original SSLeay license apply to the toolkit. See below +for the actual license texts. Actually both licenses are BSD-style Open Source +licenses. In case of any license issues related to OpenSSL please contact +openssl-core@openssl.org. + +The following are Google-internal bug numbers where explicit permission from +some authors is recorded for use of their work. (This is purely for our own +record keeping.) + 27287199 + 27287880 + 27287883 + 263291445 + + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + + +ISC license used for completely new code in BoringSSL: + +/* Copyright 2015 The BoringSSL Authors + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ + + +The code in third_party/fiat carries the MIT license: + +Copyright (c) 2015-2016 the fiat-crypto authors (see +https://github.com/mit-plv/fiat-crypto/blob/master/AUTHORS). + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Licenses for support code +------------------------- + +Parts of the TLS test suite are under the Go license. This code is not included +in BoringSSL (i.e. libcrypto and libssl) when compiled, however, so +distributing code linked against BoringSSL does not trigger this license: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +BoringSSL uses the Chromium test infrastructure to run a continuous build, +trybots etc. The scripts which manage this, and the script for generating build +metadata, are under the Chromium license. Distributing code linked against +BoringSSL does not trigger this license. + +Copyright 2015 The Chromium Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/README b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/README new file mode 100644 index 000000000000..aa6fb6d477fa --- /dev/null +++ b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/README @@ -0,0 +1,11 @@ +This directory contains the original BoringSSL [1] GCM x86-64 assembly +files [2]. + +The assembler files where then further modified to fit the ICP conventions. + +The main purpose to include these files (and the original ones) here, is to +serve as a reference if upstream changes need to be applied to the files +included and modified in the ICP. + +[1] https://github.com/google/boringssl +[2] https://github.com/google/boringssl/blob/d5440dd2c2c500ac2d3bba4afec47a054b4d99ae/gen/bcm/aes-gcm-avx2-x86_64-linux.S diff --git a/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/aes-gcm-avx2-x86_64-linux.S b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/aes-gcm-avx2-x86_64-linux.S new file mode 100644 index 000000000000..e7327c9de872 --- /dev/null +++ b/sys/contrib/openzfs/contrib/icp/gcm-simd/boringssl/aes-gcm-avx2-x86_64-linux.S @@ -0,0 +1,1328 @@ +// SPDX-License-Identifier: Apache-2.0 +// This file is generated from a similarly-named Perl script in the BoringSSL +// source tree. Do not edit by hand. + +#include <openssl/asm_base.h> + +#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__) +.section .rodata +.align 16 + + +.Lbswap_mask: +.quad 0x08090a0b0c0d0e0f, 0x0001020304050607 + + + + + + + + +.Lgfpoly: +.quad 1, 0xc200000000000000 + + +.Lgfpoly_and_internal_carrybit: +.quad 1, 0xc200000000000001 + +.align 32 + +.Lctr_pattern: +.quad 0, 0 +.quad 1, 0 +.Linc_2blocks: +.quad 2, 0 +.quad 2, 0 + +.text +.globl gcm_init_vpclmulqdq_avx2 +.hidden gcm_init_vpclmulqdq_avx2 +.type gcm_init_vpclmulqdq_avx2,@function +.align 32 +gcm_init_vpclmulqdq_avx2: +.cfi_startproc + +_CET_ENDBR + + + + + + vpshufd $0x4e,(%rsi),%xmm3 + + + + + + vpshufd $0xd3,%xmm3,%xmm0 + vpsrad $31,%xmm0,%xmm0 + vpaddq %xmm3,%xmm3,%xmm3 + vpand .Lgfpoly_and_internal_carrybit(%rip),%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + + vbroadcasti128 .Lgfpoly(%rip),%ymm6 + + + vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 + vpclmulqdq $0x01,%xmm3,%xmm3,%xmm1 + vpclmulqdq $0x10,%xmm3,%xmm3,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vpclmulqdq $0x01,%xmm0,%xmm6,%xmm2 + vpshufd $0x4e,%xmm0,%xmm0 + vpxor %xmm0,%xmm1,%xmm1 + vpxor %xmm2,%xmm1,%xmm1 + vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5 + vpclmulqdq $0x01,%xmm1,%xmm6,%xmm0 + vpshufd $0x4e,%xmm1,%xmm1 + vpxor %xmm1,%xmm5,%xmm5 + vpxor %xmm0,%xmm5,%xmm5 + + + + vinserti128 $1,%xmm3,%ymm5,%ymm3 + vinserti128 $1,%xmm5,%ymm5,%ymm5 + + + vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 + vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 + vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 + vpshufd $0x4e,%ymm0,%ymm0 + vpxor %ymm0,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 + vpshufd $0x4e,%ymm1,%ymm1 + vpxor %ymm1,%ymm4,%ymm4 + vpxor %ymm0,%ymm4,%ymm4 + + + + vmovdqu %ymm3,96(%rdi) + vmovdqu %ymm4,64(%rdi) + + + + vpunpcklqdq %ymm3,%ymm4,%ymm0 + vpunpckhqdq %ymm3,%ymm4,%ymm1 + vpxor %ymm1,%ymm0,%ymm0 + vmovdqu %ymm0,128+32(%rdi) + + + vpclmulqdq $0x00,%ymm5,%ymm4,%ymm0 + vpclmulqdq $0x01,%ymm5,%ymm4,%ymm1 + vpclmulqdq $0x10,%ymm5,%ymm4,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 + vpshufd $0x4e,%ymm0,%ymm0 + vpxor %ymm0,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x11,%ymm5,%ymm4,%ymm3 + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 + vpshufd $0x4e,%ymm1,%ymm1 + vpxor %ymm1,%ymm3,%ymm3 + vpxor %ymm0,%ymm3,%ymm3 + + vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 + vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 + vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 + vpshufd $0x4e,%ymm0,%ymm0 + vpxor %ymm0,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 + vpshufd $0x4e,%ymm1,%ymm1 + vpxor %ymm1,%ymm4,%ymm4 + vpxor %ymm0,%ymm4,%ymm4 + + vmovdqu %ymm3,32(%rdi) + vmovdqu %ymm4,0(%rdi) + + + + vpunpcklqdq %ymm3,%ymm4,%ymm0 + vpunpckhqdq %ymm3,%ymm4,%ymm1 + vpxor %ymm1,%ymm0,%ymm0 + vmovdqu %ymm0,128(%rdi) + + vzeroupper + ret + +.cfi_endproc +.size gcm_init_vpclmulqdq_avx2, . - gcm_init_vpclmulqdq_avx2 +.globl gcm_gmult_vpclmulqdq_avx2 +.hidden gcm_gmult_vpclmulqdq_avx2 +.type gcm_gmult_vpclmulqdq_avx2,@function +.align 32 +gcm_gmult_vpclmulqdq_avx2: +.cfi_startproc + +_CET_ENDBR + + + + vmovdqu (%rdi),%xmm0 + vmovdqu .Lbswap_mask(%rip),%xmm1 + vmovdqu 128-16(%rsi),%xmm2 + vmovdqu .Lgfpoly(%rip),%xmm3 + vpshufb %xmm1,%xmm0,%xmm0 + + vpclmulqdq $0x00,%xmm2,%xmm0,%xmm4 + vpclmulqdq $0x01,%xmm2,%xmm0,%xmm5 + vpclmulqdq $0x10,%xmm2,%xmm0,%xmm6 + vpxor %xmm6,%xmm5,%xmm5 + vpclmulqdq $0x01,%xmm4,%xmm3,%xmm6 + vpshufd $0x4e,%xmm4,%xmm4 + vpxor %xmm4,%xmm5,%xmm5 + vpxor %xmm6,%xmm5,%xmm5 + vpclmulqdq $0x11,%xmm2,%xmm0,%xmm0 + vpclmulqdq $0x01,%xmm5,%xmm3,%xmm4 + vpshufd $0x4e,%xmm5,%xmm5 + vpxor %xmm5,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 + + + vpshufb %xmm1,%xmm0,%xmm0 + vmovdqu %xmm0,(%rdi) + ret + +.cfi_endproc +.size gcm_gmult_vpclmulqdq_avx2, . - gcm_gmult_vpclmulqdq_avx2 +.globl gcm_ghash_vpclmulqdq_avx2 +.hidden gcm_ghash_vpclmulqdq_avx2 +.type gcm_ghash_vpclmulqdq_avx2,@function +.align 32 +gcm_ghash_vpclmulqdq_avx2: +.cfi_startproc + +_CET_ENDBR + + + + + + + vmovdqu .Lbswap_mask(%rip),%xmm6 + vmovdqu .Lgfpoly(%rip),%xmm7 + + + vmovdqu (%rdi),%xmm5 + vpshufb %xmm6,%xmm5,%xmm5 + + + cmpq $32,%rcx + jb .Lghash_lastblock + + + + vinserti128 $1,%xmm6,%ymm6,%ymm6 + vinserti128 $1,%xmm7,%ymm7,%ymm7 + + cmpq $127,%rcx + jbe .Lghash_loop_1x + + + vmovdqu 128(%rsi),%ymm8 + vmovdqu 128+32(%rsi),%ymm9 +.Lghash_loop_4x: + + vmovdqu 0(%rdx),%ymm1 + vpshufb %ymm6,%ymm1,%ymm1 + vmovdqu 0(%rsi),%ymm2 + vpxor %ymm5,%ymm1,%ymm1 + vpclmulqdq $0x00,%ymm2,%ymm1,%ymm3 + vpclmulqdq $0x11,%ymm2,%ymm1,%ymm5 + vpunpckhqdq %ymm1,%ymm1,%ymm0 + vpxor %ymm1,%ymm0,%ymm0 + vpclmulqdq $0x00,%ymm8,%ymm0,%ymm4 + + vmovdqu 32(%rdx),%ymm1 + vpshufb %ymm6,%ymm1,%ymm1 + vmovdqu 32(%rsi),%ymm2 + vpclmulqdq $0x00,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm3,%ymm3 + vpclmulqdq $0x11,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm5,%ymm5 + vpunpckhqdq %ymm1,%ymm1,%ymm0 + vpxor %ymm1,%ymm0,%ymm0 + vpclmulqdq $0x10,%ymm8,%ymm0,%ymm0 + vpxor %ymm0,%ymm4,%ymm4 + + vmovdqu 64(%rdx),%ymm1 + vpshufb %ymm6,%ymm1,%ymm1 + vmovdqu 64(%rsi),%ymm2 + vpclmulqdq $0x00,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm3,%ymm3 + vpclmulqdq $0x11,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm5,%ymm5 + vpunpckhqdq %ymm1,%ymm1,%ymm0 + vpxor %ymm1,%ymm0,%ymm0 + vpclmulqdq $0x00,%ymm9,%ymm0,%ymm0 + vpxor %ymm0,%ymm4,%ymm4 + + + vmovdqu 96(%rdx),%ymm1 + vpshufb %ymm6,%ymm1,%ymm1 + vmovdqu 96(%rsi),%ymm2 + vpclmulqdq $0x00,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm3,%ymm3 + vpclmulqdq $0x11,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm5,%ymm5 + vpunpckhqdq %ymm1,%ymm1,%ymm0 + vpxor %ymm1,%ymm0,%ymm0 + vpclmulqdq $0x10,%ymm9,%ymm0,%ymm0 + vpxor %ymm0,%ymm4,%ymm4 + + vpxor %ymm3,%ymm4,%ymm4 + vpxor %ymm5,%ymm4,%ymm4 + + + vbroadcasti128 .Lgfpoly(%rip),%ymm2 + vpclmulqdq $0x01,%ymm3,%ymm2,%ymm0 + vpshufd $0x4e,%ymm3,%ymm3 + vpxor %ymm3,%ymm4,%ymm4 + vpxor %ymm0,%ymm4,%ymm4 + + vpclmulqdq $0x01,%ymm4,%ymm2,%ymm0 + vpshufd $0x4e,%ymm4,%ymm4 + vpxor %ymm4,%ymm5,%ymm5 + vpxor %ymm0,%ymm5,%ymm5 + vextracti128 $1,%ymm5,%xmm0 + vpxor %xmm0,%xmm5,%xmm5 + + subq $-128,%rdx + addq $-128,%rcx + cmpq $127,%rcx + ja .Lghash_loop_4x + + + cmpq $32,%rcx + jb .Lghash_loop_1x_done +.Lghash_loop_1x: + vmovdqu (%rdx),%ymm0 + vpshufb %ymm6,%ymm0,%ymm0 + vpxor %ymm0,%ymm5,%ymm5 + vmovdqu 128-32(%rsi),%ymm0 + vpclmulqdq $0x00,%ymm0,%ymm5,%ymm1 + vpclmulqdq $0x01,%ymm0,%ymm5,%ymm2 + vpclmulqdq $0x10,%ymm0,%ymm5,%ymm3 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x01,%ymm1,%ymm7,%ymm3 + vpshufd $0x4e,%ymm1,%ymm1 + vpxor %ymm1,%ymm2,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x11,%ymm0,%ymm5,%ymm5 + vpclmulqdq $0x01,%ymm2,%ymm7,%ymm1 + vpshufd $0x4e,%ymm2,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpxor %ymm1,%ymm5,%ymm5 + + vextracti128 $1,%ymm5,%xmm0 + vpxor %xmm0,%xmm5,%xmm5 + addq $32,%rdx + subq $32,%rcx + cmpq $32,%rcx + jae .Lghash_loop_1x +.Lghash_loop_1x_done: + + +.Lghash_lastblock: + testq %rcx,%rcx + jz .Lghash_done + vmovdqu (%rdx),%xmm0 + vpshufb %xmm6,%xmm0,%xmm0 + vpxor %xmm0,%xmm5,%xmm5 + vmovdqu 128-16(%rsi),%xmm0 + vpclmulqdq $0x00,%xmm0,%xmm5,%xmm1 + vpclmulqdq $0x01,%xmm0,%xmm5,%xmm2 + vpclmulqdq $0x10,%xmm0,%xmm5,%xmm3 + vpxor %xmm3,%xmm2,%xmm2 + vpclmulqdq $0x01,%xmm1,%xmm7,%xmm3 + vpshufd $0x4e,%xmm1,%xmm1 + vpxor %xmm1,%xmm2,%xmm2 + vpxor %xmm3,%xmm2,%xmm2 + vpclmulqdq $0x11,%xmm0,%xmm5,%xmm5 + vpclmulqdq $0x01,%xmm2,%xmm7,%xmm1 + vpshufd $0x4e,%xmm2,%xmm2 + vpxor %xmm2,%xmm5,%xmm5 + vpxor %xmm1,%xmm5,%xmm5 + + +.Lghash_done: + + vpshufb %xmm6,%xmm5,%xmm5 + vmovdqu %xmm5,(%rdi) + + vzeroupper + ret + +.cfi_endproc +.size gcm_ghash_vpclmulqdq_avx2, . - gcm_ghash_vpclmulqdq_avx2 +.globl aes_gcm_enc_update_vaes_avx2 +.hidden aes_gcm_enc_update_vaes_avx2 +.type aes_gcm_enc_update_vaes_avx2,@function +.align 32 +aes_gcm_enc_update_vaes_avx2: +.cfi_startproc + +_CET_ENDBR + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-16 + + movq 16(%rsp),%r12 +#ifdef BORINGSSL_DISPATCH_TEST +.extern BORINGSSL_function_hit +.hidden BORINGSSL_function_hit + movb $1,BORINGSSL_function_hit+8(%rip) +#endif + vbroadcasti128 .Lbswap_mask(%rip),%ymm0 + + + + vmovdqu (%r12),%xmm1 + vpshufb %xmm0,%xmm1,%xmm1 + vbroadcasti128 (%r8),%ymm11 + vpshufb %ymm0,%ymm11,%ymm11 + + + + movl 240(%rcx),%r10d + leal -20(,%r10,4),%r10d + + + + + leaq 96(%rcx,%r10,4),%r11 + vbroadcasti128 (%rcx),%ymm9 + vbroadcasti128 (%r11),%ymm10 + + + vpaddd .Lctr_pattern(%rip),%ymm11,%ymm11 + + + + cmpq $127,%rdx + jbe .Lcrypt_loop_4x_done__func1 + + vmovdqu 128(%r9),%ymm7 + vmovdqu 128+32(%r9),%ymm8 + + + + vmovdqu .Linc_2blocks(%rip),%ymm2 + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm14 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm15 + vpaddd %ymm2,%ymm11,%ymm11 + + + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + vpxor %ymm9,%ymm14,%ymm14 + vpxor %ymm9,%ymm15,%ymm15 + + leaq 16(%rcx),%rax +.Lvaesenc_loop_first_4_vecs__func1: + vbroadcasti128 (%rax),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + addq $16,%rax + cmpq %rax,%r11 + jne .Lvaesenc_loop_first_4_vecs__func1 + vpxor 0(%rdi),%ymm10,%ymm2 + vpxor 32(%rdi),%ymm10,%ymm3 + vpxor 64(%rdi),%ymm10,%ymm5 + vpxor 96(%rdi),%ymm10,%ymm6 + vaesenclast %ymm2,%ymm12,%ymm12 + vaesenclast %ymm3,%ymm13,%ymm13 + vaesenclast %ymm5,%ymm14,%ymm14 + vaesenclast %ymm6,%ymm15,%ymm15 + vmovdqu %ymm12,0(%rsi) + vmovdqu %ymm13,32(%rsi) + vmovdqu %ymm14,64(%rsi) + vmovdqu %ymm15,96(%rsi) + + subq $-128,%rdi + addq $-128,%rdx + cmpq $127,%rdx + jbe .Lghash_last_ciphertext_4x__func1 +.align 16 +.Lcrypt_loop_4x__func1: + + + + + vmovdqu .Linc_2blocks(%rip),%ymm2 + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm14 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm15 + vpaddd %ymm2,%ymm11,%ymm11 + + + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + vpxor %ymm9,%ymm14,%ymm14 + vpxor %ymm9,%ymm15,%ymm15 + + cmpl $24,%r10d + jl .Laes128__func1 + je .Laes192__func1 + + vbroadcasti128 -208(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vbroadcasti128 -192(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + +.Laes192__func1: + vbroadcasti128 -176(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vbroadcasti128 -160(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + +.Laes128__func1: + prefetcht0 512(%rdi) + prefetcht0 512+64(%rdi) + + vmovdqu 0(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 0(%r9),%ymm4 + vpxor %ymm1,%ymm3,%ymm3 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm7,%ymm2,%ymm6 + + vbroadcasti128 -144(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vbroadcasti128 -128(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vmovdqu 32(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 32(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm7,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -112(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vmovdqu 64(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 64(%r9),%ymm4 + + vbroadcasti128 -96(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + + vbroadcasti128 -80(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + + vmovdqu 96(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + + vbroadcasti128 -64(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vmovdqu 96(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -48(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm1,%ymm6,%ymm6 + + + vbroadcasti128 .Lgfpoly(%rip),%ymm4 + vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 + vpshufd $0x4e,%ymm5,%ymm5 + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -32(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 + vpshufd $0x4e,%ymm6,%ymm6 + vpxor %ymm6,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + + vbroadcasti128 -16(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vextracti128 $1,%ymm1,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + + + subq $-128,%rsi + vpxor 0(%rdi),%ymm10,%ymm2 + vpxor 32(%rdi),%ymm10,%ymm3 + vpxor 64(%rdi),%ymm10,%ymm5 + vpxor 96(%rdi),%ymm10,%ymm6 + vaesenclast %ymm2,%ymm12,%ymm12 + vaesenclast %ymm3,%ymm13,%ymm13 + vaesenclast %ymm5,%ymm14,%ymm14 + vaesenclast %ymm6,%ymm15,%ymm15 + vmovdqu %ymm12,0(%rsi) + vmovdqu %ymm13,32(%rsi) + vmovdqu %ymm14,64(%rsi) + vmovdqu %ymm15,96(%rsi) + + subq $-128,%rdi + + addq $-128,%rdx + cmpq $127,%rdx + ja .Lcrypt_loop_4x__func1 +.Lghash_last_ciphertext_4x__func1: + + vmovdqu 0(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 0(%r9),%ymm4 + vpxor %ymm1,%ymm3,%ymm3 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm7,%ymm2,%ymm6 + + vmovdqu 32(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 32(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm7,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vmovdqu 64(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 64(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + + vmovdqu 96(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 96(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm1,%ymm6,%ymm6 + + + vbroadcasti128 .Lgfpoly(%rip),%ymm4 + vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 + vpshufd $0x4e,%ymm5,%ymm5 + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm2,%ymm6,%ymm6 + + vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 + vpshufd $0x4e,%ymm6,%ymm6 + vpxor %ymm6,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + vextracti128 $1,%ymm1,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + + subq $-128,%rsi +.Lcrypt_loop_4x_done__func1: + + testq %rdx,%rdx + jz .Ldone__func1 + + + + + + leaq 128(%r9),%r8 + subq %rdx,%r8 + + + vpxor %xmm5,%xmm5,%xmm5 + vpxor %xmm6,%xmm6,%xmm6 + vpxor %xmm7,%xmm7,%xmm7 + + cmpq $64,%rdx + jb .Llessthan64bytes__func1 + + + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + leaq 16(%rcx),%rax +.Lvaesenc_loop_tail_1__func1: + vbroadcasti128 (%rax),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + addq $16,%rax + cmpq %rax,%r11 + jne .Lvaesenc_loop_tail_1__func1 + vaesenclast %ymm10,%ymm12,%ymm12 + vaesenclast %ymm10,%ymm13,%ymm13 + + + vmovdqu 0(%rdi),%ymm2 + vmovdqu 32(%rdi),%ymm3 + vpxor %ymm2,%ymm12,%ymm12 + vpxor %ymm3,%ymm13,%ymm13 + vmovdqu %ymm12,0(%rsi) + vmovdqu %ymm13,32(%rsi) + + + vpshufb %ymm0,%ymm12,%ymm12 + vpshufb %ymm0,%ymm13,%ymm13 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + vmovdqu 32(%r8),%ymm3 + vpclmulqdq $0x00,%ymm2,%ymm12,%ymm5 + vpclmulqdq $0x01,%ymm2,%ymm12,%ymm6 + vpclmulqdq $0x10,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm2,%ymm12,%ymm7 + vpclmulqdq $0x00,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm7,%ymm7 + + addq $64,%r8 + addq $64,%rdi + addq $64,%rsi + subq $64,%rdx + jz .Lreduce__func1 + + vpxor %xmm1,%xmm1,%xmm1 + + +.Llessthan64bytes__func1: + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + leaq 16(%rcx),%rax +.Lvaesenc_loop_tail_2__func1: + vbroadcasti128 (%rax),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + addq $16,%rax + cmpq %rax,%r11 + jne .Lvaesenc_loop_tail_2__func1 + vaesenclast %ymm10,%ymm12,%ymm12 + vaesenclast %ymm10,%ymm13,%ymm13 + + + + + cmpq $32,%rdx + jb .Lxor_one_block__func1 + je .Lxor_two_blocks__func1 + +.Lxor_three_blocks__func1: + vmovdqu 0(%rdi),%ymm2 + vmovdqu 32(%rdi),%xmm3 + vpxor %ymm2,%ymm12,%ymm12 + vpxor %xmm3,%xmm13,%xmm13 + vmovdqu %ymm12,0(%rsi) + vmovdqu %xmm13,32(%rsi) + + vpshufb %ymm0,%ymm12,%ymm12 + vpshufb %xmm0,%xmm13,%xmm13 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + vmovdqu 32(%r8),%xmm3 + vpclmulqdq $0x00,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm7,%ymm7 + jmp .Lghash_mul_one_vec_unreduced__func1 + +.Lxor_two_blocks__func1: + vmovdqu (%rdi),%ymm2 + vpxor %ymm2,%ymm12,%ymm12 + vmovdqu %ymm12,(%rsi) + vpshufb %ymm0,%ymm12,%ymm12 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + jmp .Lghash_mul_one_vec_unreduced__func1 + +.Lxor_one_block__func1: + vmovdqu (%rdi),%xmm2 + vpxor %xmm2,%xmm12,%xmm12 + vmovdqu %xmm12,(%rsi) + vpshufb %xmm0,%xmm12,%xmm12 + vpxor %xmm1,%xmm12,%xmm12 + vmovdqu (%r8),%xmm2 + +.Lghash_mul_one_vec_unreduced__func1: + vpclmulqdq $0x00,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm7,%ymm7 + +.Lreduce__func1: + + vbroadcasti128 .Lgfpoly(%rip),%ymm2 + vpclmulqdq $0x01,%ymm5,%ymm2,%ymm3 + vpshufd $0x4e,%ymm5,%ymm5 + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm3,%ymm6,%ymm6 + vpclmulqdq $0x01,%ymm6,%ymm2,%ymm3 + vpshufd $0x4e,%ymm6,%ymm6 + vpxor %ymm6,%ymm7,%ymm7 + vpxor %ymm3,%ymm7,%ymm7 + vextracti128 $1,%ymm7,%xmm1 + vpxor %xmm7,%xmm1,%xmm1 + +.Ldone__func1: + + vpshufb %xmm0,%xmm1,%xmm1 + vmovdqu %xmm1,(%r12) + + vzeroupper + popq %r12 +.cfi_adjust_cfa_offset -8 +.cfi_restore %r12 + ret + +.cfi_endproc +.size aes_gcm_enc_update_vaes_avx2, . - aes_gcm_enc_update_vaes_avx2 +.globl aes_gcm_dec_update_vaes_avx2 +.hidden aes_gcm_dec_update_vaes_avx2 +.type aes_gcm_dec_update_vaes_avx2,@function +.align 32 +aes_gcm_dec_update_vaes_avx2: +.cfi_startproc + +_CET_ENDBR + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-16 + + movq 16(%rsp),%r12 + vbroadcasti128 .Lbswap_mask(%rip),%ymm0 + + + + vmovdqu (%r12),%xmm1 + vpshufb %xmm0,%xmm1,%xmm1 + vbroadcasti128 (%r8),%ymm11 + vpshufb %ymm0,%ymm11,%ymm11 + + + + movl 240(%rcx),%r10d + leal -20(,%r10,4),%r10d + + + + + leaq 96(%rcx,%r10,4),%r11 + vbroadcasti128 (%rcx),%ymm9 + vbroadcasti128 (%r11),%ymm10 + + + vpaddd .Lctr_pattern(%rip),%ymm11,%ymm11 + + + + cmpq $127,%rdx + jbe .Lcrypt_loop_4x_done__func2 + + vmovdqu 128(%r9),%ymm7 + vmovdqu 128+32(%r9),%ymm8 +.align 16 +.Lcrypt_loop_4x__func2: + + + + + vmovdqu .Linc_2blocks(%rip),%ymm2 + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm14 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm15 + vpaddd %ymm2,%ymm11,%ymm11 + + + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + vpxor %ymm9,%ymm14,%ymm14 + vpxor %ymm9,%ymm15,%ymm15 + + cmpl $24,%r10d + jl .Laes128__func2 + je .Laes192__func2 + + vbroadcasti128 -208(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vbroadcasti128 -192(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + +.Laes192__func2: + vbroadcasti128 -176(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vbroadcasti128 -160(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + +.Laes128__func2: + prefetcht0 512(%rdi) + prefetcht0 512+64(%rdi) + + vmovdqu 0(%rdi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 0(%r9),%ymm4 + vpxor %ymm1,%ymm3,%ymm3 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm7,%ymm2,%ymm6 + + vbroadcasti128 -144(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vbroadcasti128 -128(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vmovdqu 32(%rdi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 32(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm7,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -112(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vmovdqu 64(%rdi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 64(%r9),%ymm4 + + vbroadcasti128 -96(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + + vbroadcasti128 -80(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + + vmovdqu 96(%rdi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + + vbroadcasti128 -64(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vmovdqu 96(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -48(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm1,%ymm6,%ymm6 + + + vbroadcasti128 .Lgfpoly(%rip),%ymm4 + vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 + vpshufd $0x4e,%ymm5,%ymm5 + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -32(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 + vpshufd $0x4e,%ymm6,%ymm6 + vpxor %ymm6,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + + vbroadcasti128 -16(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vextracti128 $1,%ymm1,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + + + + vpxor 0(%rdi),%ymm10,%ymm2 + vpxor 32(%rdi),%ymm10,%ymm3 + vpxor 64(%rdi),%ymm10,%ymm5 + vpxor 96(%rdi),%ymm10,%ymm6 + vaesenclast %ymm2,%ymm12,%ymm12 + vaesenclast %ymm3,%ymm13,%ymm13 + vaesenclast %ymm5,%ymm14,%ymm14 + vaesenclast %ymm6,%ymm15,%ymm15 + vmovdqu %ymm12,0(%rsi) + vmovdqu %ymm13,32(%rsi) + vmovdqu %ymm14,64(%rsi) + vmovdqu %ymm15,96(%rsi) + + subq $-128,%rdi + subq $-128,%rsi + addq $-128,%rdx + cmpq $127,%rdx + ja .Lcrypt_loop_4x__func2 +.Lcrypt_loop_4x_done__func2: + + testq %rdx,%rdx + jz .Ldone__func2 + + + + + + leaq 128(%r9),%r8 + subq %rdx,%r8 + + + vpxor %xmm5,%xmm5,%xmm5 + vpxor %xmm6,%xmm6,%xmm6 + vpxor %xmm7,%xmm7,%xmm7 + + cmpq $64,%rdx + jb .Llessthan64bytes__func2 + + + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + leaq 16(%rcx),%rax +.Lvaesenc_loop_tail_1__func2: + vbroadcasti128 (%rax),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + addq $16,%rax + cmpq %rax,%r11 + jne .Lvaesenc_loop_tail_1__func2 + vaesenclast %ymm10,%ymm12,%ymm12 + vaesenclast %ymm10,%ymm13,%ymm13 + + + vmovdqu 0(%rdi),%ymm2 + vmovdqu 32(%rdi),%ymm3 + vpxor %ymm2,%ymm12,%ymm12 + vpxor %ymm3,%ymm13,%ymm13 + vmovdqu %ymm12,0(%rsi) + vmovdqu %ymm13,32(%rsi) + + + vpshufb %ymm0,%ymm2,%ymm12 + vpshufb %ymm0,%ymm3,%ymm13 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + vmovdqu 32(%r8),%ymm3 + vpclmulqdq $0x00,%ymm2,%ymm12,%ymm5 + vpclmulqdq $0x01,%ymm2,%ymm12,%ymm6 + vpclmulqdq $0x10,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm2,%ymm12,%ymm7 + vpclmulqdq $0x00,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm7,%ymm7 + + addq $64,%r8 + addq $64,%rdi + addq $64,%rsi + subq $64,%rdx + jz .Lreduce__func2 + + vpxor %xmm1,%xmm1,%xmm1 + + +.Llessthan64bytes__func2: + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + leaq 16(%rcx),%rax +.Lvaesenc_loop_tail_2__func2: + vbroadcasti128 (%rax),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + addq $16,%rax + cmpq %rax,%r11 + jne .Lvaesenc_loop_tail_2__func2 + vaesenclast %ymm10,%ymm12,%ymm12 + vaesenclast %ymm10,%ymm13,%ymm13 + + + + + cmpq $32,%rdx + jb .Lxor_one_block__func2 + je .Lxor_two_blocks__func2 + +.Lxor_three_blocks__func2: + vmovdqu 0(%rdi),%ymm2 + vmovdqu 32(%rdi),%xmm3 + vpxor %ymm2,%ymm12,%ymm12 + vpxor %xmm3,%xmm13,%xmm13 + vmovdqu %ymm12,0(%rsi) + vmovdqu %xmm13,32(%rsi) + + vpshufb %ymm0,%ymm2,%ymm12 + vpshufb %xmm0,%xmm3,%xmm13 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + vmovdqu 32(%r8),%xmm3 + vpclmulqdq $0x00,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm7,%ymm7 + jmp .Lghash_mul_one_vec_unreduced__func2 + +.Lxor_two_blocks__func2: + vmovdqu (%rdi),%ymm2 + vpxor %ymm2,%ymm12,%ymm12 + vmovdqu %ymm12,(%rsi) + vpshufb %ymm0,%ymm2,%ymm12 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + jmp .Lghash_mul_one_vec_unreduced__func2 + +.Lxor_one_block__func2: + vmovdqu (%rdi),%xmm2 + vpxor %xmm2,%xmm12,%xmm12 + vmovdqu %xmm12,(%rsi) + vpshufb %xmm0,%xmm2,%xmm12 + vpxor %xmm1,%xmm12,%xmm12 + vmovdqu (%r8),%xmm2 + +.Lghash_mul_one_vec_unreduced__func2: + vpclmulqdq $0x00,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm7,%ymm7 + +.Lreduce__func2: + + vbroadcasti128 .Lgfpoly(%rip),%ymm2 + vpclmulqdq $0x01,%ymm5,%ymm2,%ymm3 + vpshufd $0x4e,%ymm5,%ymm5 + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm3,%ymm6,%ymm6 + vpclmulqdq $0x01,%ymm6,%ymm2,%ymm3 + vpshufd $0x4e,%ymm6,%ymm6 + vpxor %ymm6,%ymm7,%ymm7 + vpxor %ymm3,%ymm7,%ymm7 + vextracti128 $1,%ymm7,%xmm1 + vpxor %xmm7,%xmm1,%xmm1 + +.Ldone__func2: + + vpshufb %xmm0,%xmm1,%xmm1 + vmovdqu %xmm1,(%r12) + + vzeroupper + popq %r12 +.cfi_adjust_cfa_offset -8 +.cfi_restore %r12 + ret + +.cfi_endproc +.size aes_gcm_dec_update_vaes_avx2, . - aes_gcm_dec_update_vaes_avx2 +#endif diff --git a/sys/contrib/openzfs/contrib/initramfs/scripts/zfs b/sys/contrib/openzfs/contrib/initramfs/scripts/zfs index c569b2528368..67707e9d80f4 100644 --- a/sys/contrib/openzfs/contrib/initramfs/scripts/zfs +++ b/sys/contrib/openzfs/contrib/initramfs/scripts/zfs @@ -979,7 +979,8 @@ mountroot() touch /run/zfs_unlock_complete if [ -e /run/zfs_unlock_complete_notify ]; then - read -r < /run/zfs_unlock_complete_notify + # shellcheck disable=SC2034 + read -r zfs_unlock_complete_notify < /run/zfs_unlock_complete_notify fi # ------------ diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_x86.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_x86.h index e8004e18c4a4..326f471d7c9b 100644 --- a/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_x86.h +++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_x86.h @@ -598,6 +598,32 @@ zfs_movbe_available(void) } /* + * Check if VAES instruction set is available + */ +static inline boolean_t +zfs_vaes_available(void) +{ +#if defined(X86_FEATURE_VAES) + return (!!boot_cpu_has(X86_FEATURE_VAES)); +#else + return (B_FALSE); +#endif +} + +/* + * Check if VPCLMULQDQ instruction set is available + */ +static inline boolean_t +zfs_vpclmulqdq_available(void) +{ +#if defined(X86_FEATURE_VPCLMULQDQ) + return (!!boot_cpu_has(X86_FEATURE_VPCLMULQDQ)); +#else + return (B_FALSE); +#endif +} + +/* * Check if SHA_NI instruction set is available */ static inline boolean_t diff --git a/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h b/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h index 955462c85d10..e34ea46b3fe8 100644 --- a/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h +++ b/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h @@ -139,18 +139,18 @@ #define ZCW_TP_STRUCT_ENTRY \ __field(lwb_t *, zcw_lwb) \ __field(boolean_t, zcw_done) \ - __field(int, zcw_zio_error) \ + __field(int, zcw_error) \ #define ZCW_TP_FAST_ASSIGN \ __entry->zcw_lwb = zcw->zcw_lwb; \ __entry->zcw_done = zcw->zcw_done; \ - __entry->zcw_zio_error = zcw->zcw_zio_error; + __entry->zcw_error = zcw->zcw_error; #define ZCW_TP_PRINTK_FMT \ "zcw { lwb %p done %u error %u }" #define ZCW_TP_PRINTK_ARGS \ - __entry->zcw_lwb, __entry->zcw_done, __entry->zcw_zio_error + __entry->zcw_lwb, __entry->zcw_done, __entry->zcw_error /* * Generic support for two argument tracepoints of the form: diff --git a/sys/contrib/openzfs/include/sys/spa.h b/sys/contrib/openzfs/include/sys/spa.h index db6de332ae67..66db16b33c51 100644 --- a/sys/contrib/openzfs/include/sys/spa.h +++ b/sys/contrib/openzfs/include/sys/spa.h @@ -880,7 +880,6 @@ extern kcondvar_t spa_namespace_cv; #define SPA_CONFIG_UPDATE_VDEVS 1 extern void spa_write_cachefile(spa_t *, boolean_t, boolean_t, boolean_t); -extern void spa_config_load(void); extern int spa_all_configs(uint64_t *generation, nvlist_t **pools); extern void spa_config_set(spa_t *spa, nvlist_t *config); extern nvlist_t *spa_config_generate(spa_t *spa, vdev_t *vd, uint64_t txg, @@ -1244,7 +1243,6 @@ extern void vdev_mirror_stat_fini(void); /* Initialization and termination */ extern void spa_init(spa_mode_t mode); extern void spa_fini(void); -extern void spa_boot_init(void *); /* properties */ extern int spa_prop_set(spa_t *spa, nvlist_t *nvp); diff --git a/sys/contrib/openzfs/include/sys/zil_impl.h b/sys/contrib/openzfs/include/sys/zil_impl.h index 44b776e16b52..ea1364a7e35a 100644 --- a/sys/contrib/openzfs/include/sys/zil_impl.h +++ b/sys/contrib/openzfs/include/sys/zil_impl.h @@ -41,8 +41,8 @@ extern "C" { * * An lwb will start out in the "new" state, and transition to the "opened" * state via a call to zil_lwb_write_open() on first itx assignment. When - * transitioning from "new" to "opened" the zilog's "zl_issuer_lock" must be - * held. + * transitioning from "new" to "opened" the zilog's "zl_issuer_lock" and + * LWB's "lwb_lock" must be held. * * After the lwb is "opened", it can be assigned number of itxs and transition * into the "closed" state via zil_lwb_write_close() when full or on timeout. @@ -100,16 +100,22 @@ typedef enum { * holding the "zl_issuer_lock". After the lwb is issued, the zilog's * "zl_lock" is used to protect the lwb against concurrent access. */ +typedef enum { + LWB_FLAG_SLIM = (1<<0), /* log block has slim format */ + LWB_FLAG_SLOG = (1<<1), /* lwb_blk is on SLOG device */ + LWB_FLAG_CRASHED = (1<<2), /* lwb is on the crash list */ +} lwb_flag_t; + typedef struct lwb { zilog_t *lwb_zilog; /* back pointer to log struct */ blkptr_t lwb_blk; /* on disk address of this log blk */ - boolean_t lwb_slim; /* log block has slim format */ - boolean_t lwb_slog; /* lwb_blk is on SLOG device */ + lwb_flag_t lwb_flags; /* extra info about this lwb */ int lwb_error; /* log block allocation error */ int lwb_nmax; /* max bytes in the buffer */ int lwb_nused; /* # used bytes in buffer */ int lwb_nfilled; /* # filled bytes in buffer */ int lwb_sz; /* size of block and buffer */ + int lwb_min_sz; /* min size for range allocation */ lwb_state_t lwb_state; /* the state of this lwb */ char *lwb_buf; /* log write buffer */ zio_t *lwb_child_zio; /* parent zio for children */ @@ -124,7 +130,7 @@ typedef struct lwb { list_t lwb_itxs; /* list of itx's */ list_t lwb_waiters; /* list of zil_commit_waiter's */ avl_tree_t lwb_vdev_tree; /* vdevs to flush after lwb write */ - kmutex_t lwb_vdev_lock; /* protects lwb_vdev_tree */ + kmutex_t lwb_lock; /* protects lwb_vdev_tree and size */ } lwb_t; /* @@ -149,7 +155,7 @@ typedef struct zil_commit_waiter { list_node_t zcw_node; /* linkage in lwb_t:lwb_waiter list */ lwb_t *zcw_lwb; /* back pointer to lwb when linked */ boolean_t zcw_done; /* B_TRUE when "done", else B_FALSE */ - int zcw_zio_error; /* contains the zio io_error value */ + int zcw_error; /* result to return from zil_commit() */ } zil_commit_waiter_t; /* diff --git a/sys/contrib/openzfs/include/sys/zio.h b/sys/contrib/openzfs/include/sys/zio.h index 4f46eab3db89..353805fcb969 100644 --- a/sys/contrib/openzfs/include/sys/zio.h +++ b/sys/contrib/openzfs/include/sys/zio.h @@ -360,26 +360,26 @@ struct zbookmark_err_phys { (zb)->zb_blkid == ZB_ROOT_BLKID) typedef struct zio_prop { - enum zio_checksum zp_checksum; - enum zio_compress zp_compress; + enum zio_checksum zp_checksum:8; + enum zio_compress zp_compress:8; uint8_t zp_complevel; uint8_t zp_level; uint8_t zp_copies; uint8_t zp_gang_copies; - dmu_object_type_t zp_type; - boolean_t zp_dedup; - boolean_t zp_dedup_verify; - boolean_t zp_nopwrite; - boolean_t zp_brtwrite; - boolean_t zp_encrypt; - boolean_t zp_byteorder; - boolean_t zp_direct_write; - boolean_t zp_rewrite; + dmu_object_type_t zp_type:8; + dmu_object_type_t zp_storage_type:8; + boolean_t zp_dedup:1; + boolean_t zp_dedup_verify:1; + boolean_t zp_nopwrite:1; + boolean_t zp_brtwrite:1; + boolean_t zp_encrypt:1; + boolean_t zp_byteorder:1; + boolean_t zp_direct_write:1; + boolean_t zp_rewrite:1; + uint32_t zp_zpl_smallblk; uint8_t zp_salt[ZIO_DATA_SALT_LEN]; uint8_t zp_iv[ZIO_DATA_IV_LEN]; uint8_t zp_mac[ZIO_DATA_MAC_LEN]; - uint32_t zp_zpl_smallblk; - dmu_object_type_t zp_storage_type; } zio_prop_t; typedef struct zio_cksum_report zio_cksum_report_t; @@ -622,7 +622,8 @@ extern zio_t *zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, zio_flag_t flags); extern int zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, - blkptr_t *new_bp, uint64_t size, boolean_t *slog); + blkptr_t *new_bp, uint64_t min_size, uint64_t max_size, boolean_t *slog, + boolean_t allow_larger); extern void zio_flush(zio_t *zio, vdev_t *vd); extern void zio_shrink(zio_t *zio, uint64_t size); diff --git a/sys/contrib/openzfs/include/sys/zvol_impl.h b/sys/contrib/openzfs/include/sys/zvol_impl.h index f3dd9f26f23c..5422e66832c0 100644 --- a/sys/contrib/openzfs/include/sys/zvol_impl.h +++ b/sys/contrib/openzfs/include/sys/zvol_impl.h @@ -20,7 +20,7 @@ * CDDL HEADER END */ /* - * Copyright (c) 2024, Klara, Inc. + * Copyright (c) 2024, 2025, Klara, Inc. */ #ifndef _SYS_ZVOL_IMPL_H @@ -56,6 +56,7 @@ typedef struct zvol_state { atomic_t zv_suspend_ref; /* refcount for suspend */ krwlock_t zv_suspend_lock; /* suspend lock */ kcondvar_t zv_removing_cv; /* ready to remove minor */ + list_node_t zv_remove_node; /* node on removal list */ struct zvol_state_os *zv_zso; /* private platform state */ boolean_t zv_threading; /* volthreading property */ } zvol_state_t; @@ -135,7 +136,7 @@ int zvol_os_rename_minor(zvol_state_t *zv, const char *newname); int zvol_os_create_minor(const char *name); int zvol_os_update_volsize(zvol_state_t *zv, uint64_t volsize); boolean_t zvol_os_is_zvol(const char *path); -void zvol_os_clear_private(zvol_state_t *zv); +void zvol_os_remove_minor(zvol_state_t *zv); void zvol_os_set_disk_ro(zvol_state_t *zv, int flags); void zvol_os_set_capacity(zvol_state_t *zv, uint64_t capacity); diff --git a/sys/contrib/openzfs/lib/libicp/Makefile.am b/sys/contrib/openzfs/lib/libicp/Makefile.am index ce24d13a760f..23adba10bc44 100644 --- a/sys/contrib/openzfs/lib/libicp/Makefile.am +++ b/sys/contrib/openzfs/lib/libicp/Makefile.am @@ -69,6 +69,7 @@ nodist_libicp_la_SOURCES += \ module/icp/asm-x86_64/aes/aes_aesni.S \ module/icp/asm-x86_64/modes/gcm_pclmulqdq.S \ module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S \ + module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S \ module/icp/asm-x86_64/modes/ghash-x86_64.S \ module/icp/asm-x86_64/sha2/sha256-x86_64.S \ module/icp/asm-x86_64/sha2/sha512-x86_64.S \ diff --git a/sys/contrib/openzfs/lib/libspl/include/sys/simd.h b/sys/contrib/openzfs/lib/libspl/include/sys/simd.h index 1ef24f5a7d39..4772a5416b2e 100644 --- a/sys/contrib/openzfs/lib/libspl/include/sys/simd.h +++ b/sys/contrib/openzfs/lib/libspl/include/sys/simd.h @@ -102,7 +102,9 @@ typedef enum cpuid_inst_sets { AES, PCLMULQDQ, MOVBE, - SHA_NI + SHA_NI, + VAES, + VPCLMULQDQ } cpuid_inst_sets_t; /* @@ -127,6 +129,8 @@ typedef struct cpuid_feature_desc { #define _AES_BIT (1U << 25) #define _PCLMULQDQ_BIT (1U << 1) #define _MOVBE_BIT (1U << 22) +#define _VAES_BIT (1U << 9) +#define _VPCLMULQDQ_BIT (1U << 10) #define _SHA_NI_BIT (1U << 29) /* @@ -157,6 +161,8 @@ static const cpuid_feature_desc_t cpuid_features[] = { [PCLMULQDQ] = {1U, 0U, _PCLMULQDQ_BIT, ECX }, [MOVBE] = {1U, 0U, _MOVBE_BIT, ECX }, [SHA_NI] = {7U, 0U, _SHA_NI_BIT, EBX }, + [VAES] = {7U, 0U, _VAES_BIT, ECX }, + [VPCLMULQDQ] = {7U, 0U, _VPCLMULQDQ_BIT, ECX }, }; /* @@ -231,6 +237,8 @@ CPUID_FEATURE_CHECK(aes, AES); CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ); CPUID_FEATURE_CHECK(movbe, MOVBE); CPUID_FEATURE_CHECK(shani, SHA_NI); +CPUID_FEATURE_CHECK(vaes, VAES); +CPUID_FEATURE_CHECK(vpclmulqdq, VPCLMULQDQ); /* * Detect register set support @@ -382,6 +390,24 @@ zfs_shani_available(void) } /* + * Check if VAES instruction is available + */ +static inline boolean_t +zfs_vaes_available(void) +{ + return (__cpuid_has_vaes()); +} + +/* + * Check if VPCLMULQDQ instruction is available + */ +static inline boolean_t +zfs_vpclmulqdq_available(void) +{ + return (__cpuid_has_vpclmulqdq()); +} + +/* * AVX-512 family of instruction sets: * * AVX512F Foundation diff --git a/sys/contrib/openzfs/lib/libzpool/kernel.c b/sys/contrib/openzfs/lib/libzpool/kernel.c index e63153a03370..fea2f81458f9 100644 --- a/sys/contrib/openzfs/lib/libzpool/kernel.c +++ b/sys/contrib/openzfs/lib/libzpool/kernel.c @@ -38,6 +38,7 @@ #include <sys/processor.h> #include <sys/rrwlock.h> #include <sys/spa.h> +#include <sys/spa_impl.h> #include <sys/stat.h> #include <sys/systeminfo.h> #include <sys/time.h> @@ -811,6 +812,79 @@ umem_out_of_memory(void) return (0); } +static void +spa_config_load(void) +{ + void *buf = NULL; + nvlist_t *nvlist, *child; + nvpair_t *nvpair; + char *pathname; + zfs_file_t *fp; + zfs_file_attr_t zfa; + uint64_t fsize; + int err; + + /* + * Open the configuration file. + */ + pathname = kmem_alloc(MAXPATHLEN, KM_SLEEP); + + (void) snprintf(pathname, MAXPATHLEN, "%s", spa_config_path); + + err = zfs_file_open(pathname, O_RDONLY, 0, &fp); + if (err) + err = zfs_file_open(ZPOOL_CACHE_BOOT, O_RDONLY, 0, &fp); + + kmem_free(pathname, MAXPATHLEN); + + if (err) + return; + + if (zfs_file_getattr(fp, &zfa)) + goto out; + + fsize = zfa.zfa_size; + buf = kmem_alloc(fsize, KM_SLEEP); + + /* + * Read the nvlist from the file. + */ + if (zfs_file_read(fp, buf, fsize, NULL) < 0) + goto out; + + /* + * Unpack the nvlist. + */ + if (nvlist_unpack(buf, fsize, &nvlist, KM_SLEEP) != 0) + goto out; + + /* + * Iterate over all elements in the nvlist, creating a new spa_t for + * each one with the specified configuration. + */ + mutex_enter(&spa_namespace_lock); + nvpair = NULL; + while ((nvpair = nvlist_next_nvpair(nvlist, nvpair)) != NULL) { + if (nvpair_type(nvpair) != DATA_TYPE_NVLIST) + continue; + + child = fnvpair_value_nvlist(nvpair); + + if (spa_lookup(nvpair_name(nvpair)) != NULL) + continue; + (void) spa_add(nvpair_name(nvpair), child, NULL); + } + mutex_exit(&spa_namespace_lock); + + nvlist_free(nvlist); + +out: + if (buf != NULL) + kmem_free(buf, fsize); + + zfs_file_close(fp); +} + void kernel_init(int mode) { @@ -835,6 +909,7 @@ kernel_init(int mode) zstd_init(); spa_init((spa_mode_t)mode); + spa_config_load(); fletcher_4_init(); diff --git a/sys/contrib/openzfs/lib/libzutil/zutil_import.c b/sys/contrib/openzfs/lib/libzutil/zutil_import.c index a4a6e76a1d09..08367f4c064d 100644 --- a/sys/contrib/openzfs/lib/libzutil/zutil_import.c +++ b/sys/contrib/openzfs/lib/libzutil/zutil_import.c @@ -1903,30 +1903,43 @@ zpool_find_config(libpc_handle_t *hdl, const char *target, nvlist_t **configp, *sepp = '\0'; pools = zpool_search_import(hdl, args); + if (pools == NULL) { + zutil_error_aux(hdl, dgettext(TEXT_DOMAIN, "no pools found")); + (void) zutil_error_fmt(hdl, LPC_UNKNOWN, dgettext(TEXT_DOMAIN, + "failed to find config for pool '%s'"), targetdup); + free(targetdup); + return (ENOENT); + } - if (pools != NULL) { - nvpair_t *elem = NULL; - while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) { - VERIFY0(nvpair_value_nvlist(elem, &config)); - if (pool_match(config, targetdup)) { - count++; - if (match != NULL) { - /* multiple matches found */ - continue; - } else { - match = fnvlist_dup(config); - } + nvpair_t *elem = NULL; + while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) { + VERIFY0(nvpair_value_nvlist(elem, &config)); + if (pool_match(config, targetdup)) { + count++; + if (match != NULL) { + /* multiple matches found */ + continue; + } else { + match = fnvlist_dup(config); } } - fnvlist_free(pools); } + fnvlist_free(pools); if (count == 0) { + zutil_error_aux(hdl, dgettext(TEXT_DOMAIN, + "no matching pools")); + (void) zutil_error_fmt(hdl, LPC_UNKNOWN, dgettext(TEXT_DOMAIN, + "failed to find config for pool '%s'"), targetdup); free(targetdup); return (ENOENT); } if (count > 1) { + zutil_error_aux(hdl, dgettext(TEXT_DOMAIN, + "more than one matching pool")); + (void) zutil_error_fmt(hdl, LPC_UNKNOWN, dgettext(TEXT_DOMAIN, + "failed to find config for pool '%s'"), targetdup); free(targetdup); fnvlist_free(match); return (EINVAL); diff --git a/sys/contrib/openzfs/man/man4/zfs.4 b/sys/contrib/openzfs/man/man4/zfs.4 index 4a5f9fd93f4f..5c7958667f92 100644 --- a/sys/contrib/openzfs/man/man4/zfs.4 +++ b/sys/contrib/openzfs/man/man4/zfs.4 @@ -941,10 +941,6 @@ The target number of bytes the ARC should leave as free memory on the system. If zero, equivalent to the bigger of .Sy 512 KiB No and Sy all_system_memory/64 . . -.It Sy zfs_autoimport_disable Ns = Ns Sy 1 Ns | Ns 0 Pq int -Disable pool import at module load by ignoring the cache file -.Pq Sy spa_config_path . -. .It Sy zfs_checksum_events_per_second Ns = Ns Sy 20 Ns /s Pq uint Rate limit checksum events to this many per second. Note that this should not be set below the ZED thresholds diff --git a/sys/contrib/openzfs/man/man8/zdb.8 b/sys/contrib/openzfs/man/man8/zdb.8 index 3984aaac5866..0a5b6af73fdb 100644 --- a/sys/contrib/openzfs/man/man8/zdb.8 +++ b/sys/contrib/openzfs/man/man8/zdb.8 @@ -15,7 +15,7 @@ .\" Copyright (c) 2017 Lawrence Livermore National Security, LLC. .\" Copyright (c) 2017 Intel Corporation. .\" -.Dd October 27, 2024 +.Dd April 23, 2025 .Dt ZDB 8 .Os . @@ -531,6 +531,18 @@ option, with more occurrences enabling more verbosity. If no options are specified, all information about the named pool will be displayed at default verbosity. . +.Sh EXIT STATUS +The +.Nm +utility exits +.Sy 0 +on success, +.Sy 1 +if a fatal error occurs, +.Sy 2 +if invalid command line options were specified, or +.Sy 3 +if on-disk corruption was detected, but was not fatal. .Sh EXAMPLES .Ss Example 1 : No Display the configuration of imported pool Ar rpool .Bd -literal diff --git a/sys/contrib/openzfs/man/man8/zfs-send.8 b/sys/contrib/openzfs/man/man8/zfs-send.8 index c920a5a48798..f7c6b840303c 100644 --- a/sys/contrib/openzfs/man/man8/zfs-send.8 +++ b/sys/contrib/openzfs/man/man8/zfs-send.8 @@ -173,8 +173,10 @@ The receiving system must have the feature enabled. If the .Sy lz4_compress -feature is active on the sending system, then the receiving system must have -that feature enabled as well. +or +.Sy zstd_compress +features are active on the sending system, then the receiving system must have +the corresponding features enabled as well. Datasets that are sent with this flag may not be received as an encrypted dataset, since encrypted datasets cannot use the .Sy embedded_data @@ -201,8 +203,10 @@ property for details .Pc . If the .Sy lz4_compress -feature is active on the sending system, then the receiving system must have -that feature enabled as well. +or +.Sy zstd_compress +features are active on the sending system, then the receiving system must have +the corresponding features enabled as well. If the .Sy large_blocks feature is enabled on the sending system but the @@ -357,8 +361,10 @@ property for details .Pc . If the .Sy lz4_compress -feature is active on the sending system, then the receiving system must have -that feature enabled as well. +or +.Sy zstd_compress +features are active on the sending system, then the receiving system must have +the corresponding features enabled as well. If the .Sy large_blocks feature is enabled on the sending system but the @@ -400,8 +406,10 @@ The receiving system must have the feature enabled. If the .Sy lz4_compress -feature is active on the sending system, then the receiving system must have -that feature enabled as well. +or +.Sy zstd_compress +features are active on the sending system, then the receiving system must have +the corresponding features enabled as well. Datasets that are sent with this flag may not be received as an encrypted dataset, since encrypted datasets cannot use the diff --git a/sys/contrib/openzfs/module/Kbuild.in b/sys/contrib/openzfs/module/Kbuild.in index 3d6f288fa5da..362d2295e091 100644 --- a/sys/contrib/openzfs/module/Kbuild.in +++ b/sys/contrib/openzfs/module/Kbuild.in @@ -135,6 +135,7 @@ ICP_OBJS_X86_64 := \ asm-x86_64/sha2/sha256-x86_64.o \ asm-x86_64/sha2/sha512-x86_64.o \ asm-x86_64/modes/aesni-gcm-x86_64.o \ + asm-x86_64/modes/aesni-gcm-avx2-vaes.o \ asm-x86_64/modes/gcm_pclmulqdq.o \ asm-x86_64/modes/ghash-x86_64.o diff --git a/sys/contrib/openzfs/module/icp/algs/modes/gcm.c b/sys/contrib/openzfs/module/icp/algs/modes/gcm.c index c2a982b5a376..3cfa5b8165ce 100644 --- a/sys/contrib/openzfs/module/icp/algs/modes/gcm.c +++ b/sys/contrib/openzfs/module/icp/algs/modes/gcm.c @@ -46,6 +46,9 @@ #define IMPL_CYCLE (UINT32_MAX-1) #ifdef CAN_USE_GCM_ASM #define IMPL_AVX (UINT32_MAX-2) +#if CAN_USE_GCM_ASM >= 2 +#define IMPL_AVX2 (UINT32_MAX-3) +#endif #endif #define GCM_IMPL_READ(i) (*(volatile uint32_t *) &(i)) static uint32_t icp_gcm_impl = IMPL_FASTEST; @@ -56,17 +59,16 @@ static uint32_t user_sel_impl = IMPL_FASTEST; boolean_t gcm_avx_can_use_movbe = B_FALSE; /* * Whether to use the optimized openssl gcm and ghash implementations. - * Set to true if module parameter icp_gcm_impl == "avx". */ -static boolean_t gcm_use_avx = B_FALSE; -#define GCM_IMPL_USE_AVX (*(volatile boolean_t *)&gcm_use_avx) +static gcm_impl gcm_impl_used = GCM_IMPL_GENERIC; +#define GCM_IMPL_USED (*(volatile gcm_impl *)&gcm_impl_used) extern boolean_t ASMABI atomic_toggle_boolean_nv(volatile boolean_t *); static inline boolean_t gcm_avx_will_work(void); -static inline void gcm_set_avx(boolean_t); -static inline boolean_t gcm_toggle_avx(void); -static inline size_t gcm_simd_get_htab_size(boolean_t); +static inline boolean_t gcm_avx2_will_work(void); +static inline void gcm_use_impl(gcm_impl impl); +static inline gcm_impl gcm_toggle_impl(void); static int gcm_mode_encrypt_contiguous_blocks_avx(gcm_ctx_t *, char *, size_t, crypto_data_t *, size_t); @@ -89,7 +91,7 @@ gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length, void (*xor_block)(uint8_t *, uint8_t *)) { #ifdef CAN_USE_GCM_ASM - if (ctx->gcm_use_avx == B_TRUE) + if (ctx->impl != GCM_IMPL_GENERIC) return (gcm_mode_encrypt_contiguous_blocks_avx( ctx, data, length, out, block_size)); #endif @@ -208,7 +210,7 @@ gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size, { (void) copy_block; #ifdef CAN_USE_GCM_ASM - if (ctx->gcm_use_avx == B_TRUE) + if (ctx->impl != GCM_IMPL_GENERIC) return (gcm_encrypt_final_avx(ctx, out, block_size)); #endif @@ -374,7 +376,7 @@ gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size, void (*xor_block)(uint8_t *, uint8_t *)) { #ifdef CAN_USE_GCM_ASM - if (ctx->gcm_use_avx == B_TRUE) + if (ctx->impl != GCM_IMPL_GENERIC) return (gcm_decrypt_final_avx(ctx, out, block_size)); #endif @@ -631,23 +633,23 @@ gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, ((aes_key_t *)gcm_ctx->gcm_keysched)->ops->needs_byteswap; if (GCM_IMPL_READ(icp_gcm_impl) != IMPL_CYCLE) { - gcm_ctx->gcm_use_avx = GCM_IMPL_USE_AVX; + gcm_ctx->impl = GCM_IMPL_USED; } else { /* - * Handle the "cycle" implementation by creating avx and - * non-avx contexts alternately. + * Handle the "cycle" implementation by creating different + * contexts, one per implementation. */ - gcm_ctx->gcm_use_avx = gcm_toggle_avx(); + gcm_ctx->impl = gcm_toggle_impl(); - /* The avx impl. doesn't handle byte swapped key schedules. */ - if (gcm_ctx->gcm_use_avx == B_TRUE && needs_bswap == B_TRUE) { - gcm_ctx->gcm_use_avx = B_FALSE; + /* The AVX impl. doesn't handle byte swapped key schedules. */ + if (needs_bswap == B_TRUE) { + gcm_ctx->impl = GCM_IMPL_GENERIC; } /* - * If this is a GCM context, use the MOVBE and the BSWAP + * If this is an AVX context, use the MOVBE and the BSWAP * variants alternately. */ - if (gcm_ctx->gcm_use_avx == B_TRUE && + if (gcm_ctx->impl == GCM_IMPL_AVX && zfs_movbe_available() == B_TRUE) { (void) atomic_toggle_boolean_nv( (volatile boolean_t *)&gcm_avx_can_use_movbe); @@ -658,12 +660,13 @@ gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, * still they could be created by the aes generic implementation. * Make sure not to use them since we'll corrupt data if we do. */ - if (gcm_ctx->gcm_use_avx == B_TRUE && needs_bswap == B_TRUE) { - gcm_ctx->gcm_use_avx = B_FALSE; + if (gcm_ctx->impl != GCM_IMPL_GENERIC && needs_bswap == B_TRUE) { + gcm_ctx->impl = GCM_IMPL_GENERIC; cmn_err_once(CE_WARN, "ICP: Can't use the aes generic or cycle implementations " - "in combination with the gcm avx implementation!"); + "in combination with the gcm avx or avx2-vaes " + "implementation!"); cmn_err_once(CE_WARN, "ICP: Falling back to a compatible implementation, " "aes-gcm performance will likely be degraded."); @@ -672,36 +675,20 @@ gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, "restore performance."); } - /* Allocate Htab memory as needed. */ - if (gcm_ctx->gcm_use_avx == B_TRUE) { - size_t htab_len = gcm_simd_get_htab_size(gcm_ctx->gcm_use_avx); - - if (htab_len == 0) { - return (CRYPTO_MECHANISM_PARAM_INVALID); - } - gcm_ctx->gcm_htab_len = htab_len; - gcm_ctx->gcm_Htable = - kmem_alloc(htab_len, KM_SLEEP); - - if (gcm_ctx->gcm_Htable == NULL) { - return (CRYPTO_HOST_MEMORY); - } + /* + * AVX implementations use Htable with sizes depending on + * implementation. + */ + if (gcm_ctx->impl != GCM_IMPL_GENERIC) { + rv = gcm_init_avx(gcm_ctx, iv, iv_len, aad, aad_len, + block_size); } - /* Avx and non avx context initialization differs from here on. */ - if (gcm_ctx->gcm_use_avx == B_FALSE) { + else #endif /* ifdef CAN_USE_GCM_ASM */ - if (gcm_init(gcm_ctx, iv, iv_len, aad, aad_len, block_size, - encrypt_block, copy_block, xor_block) != CRYPTO_SUCCESS) { - rv = CRYPTO_MECHANISM_PARAM_INVALID; - } -#ifdef CAN_USE_GCM_ASM - } else { - if (gcm_init_avx(gcm_ctx, iv, iv_len, aad, aad_len, - block_size) != CRYPTO_SUCCESS) { - rv = CRYPTO_MECHANISM_PARAM_INVALID; - } + if (gcm_init(gcm_ctx, iv, iv_len, aad, aad_len, block_size, + encrypt_block, copy_block, xor_block) != CRYPTO_SUCCESS) { + rv = CRYPTO_MECHANISM_PARAM_INVALID; } -#endif /* ifdef CAN_USE_GCM_ASM */ return (rv); } @@ -767,6 +754,9 @@ gcm_impl_get_ops(void) break; #ifdef CAN_USE_GCM_ASM case IMPL_AVX: +#if CAN_USE_GCM_ASM >= 2 + case IMPL_AVX2: +#endif /* * Make sure that we return a valid implementation while * switching to the avx implementation since there still @@ -828,6 +818,13 @@ gcm_impl_init(void) * Use the avx implementation if it's available and the implementation * hasn't changed from its default value of fastest on module load. */ +#if CAN_USE_GCM_ASM >= 2 + if (gcm_avx2_will_work()) { + if (GCM_IMPL_READ(user_sel_impl) == IMPL_FASTEST) { + gcm_use_impl(GCM_IMPL_AVX2); + } + } else +#endif if (gcm_avx_will_work()) { #ifdef HAVE_MOVBE if (zfs_movbe_available() == B_TRUE) { @@ -835,7 +832,7 @@ gcm_impl_init(void) } #endif if (GCM_IMPL_READ(user_sel_impl) == IMPL_FASTEST) { - gcm_set_avx(B_TRUE); + gcm_use_impl(GCM_IMPL_AVX); } } #endif @@ -852,6 +849,7 @@ static const struct { { "fastest", IMPL_FASTEST }, #ifdef CAN_USE_GCM_ASM { "avx", IMPL_AVX }, + { "avx2-vaes", IMPL_AVX2 }, #endif }; @@ -887,7 +885,13 @@ gcm_impl_set(const char *val) /* Check mandatory options */ for (i = 0; i < ARRAY_SIZE(gcm_impl_opts); i++) { #ifdef CAN_USE_GCM_ASM +#if CAN_USE_GCM_ASM >= 2 /* Ignore avx implementation if it won't work. */ + if (gcm_impl_opts[i].sel == IMPL_AVX2 && + !gcm_avx2_will_work()) { + continue; + } +#endif if (gcm_impl_opts[i].sel == IMPL_AVX && !gcm_avx_will_work()) { continue; } @@ -915,11 +919,17 @@ gcm_impl_set(const char *val) * Use the avx implementation if available and the requested one is * avx or fastest. */ +#if CAN_USE_GCM_ASM >= 2 + if (gcm_avx2_will_work() == B_TRUE && + (impl == IMPL_AVX2 || impl == IMPL_FASTEST)) { + gcm_use_impl(GCM_IMPL_AVX2); + } else +#endif if (gcm_avx_will_work() == B_TRUE && (impl == IMPL_AVX || impl == IMPL_FASTEST)) { - gcm_set_avx(B_TRUE); + gcm_use_impl(GCM_IMPL_AVX); } else { - gcm_set_avx(B_FALSE); + gcm_use_impl(GCM_IMPL_GENERIC); } #endif @@ -952,6 +962,12 @@ icp_gcm_impl_get(char *buffer, zfs_kernel_param_t *kp) for (i = 0; i < ARRAY_SIZE(gcm_impl_opts); i++) { #ifdef CAN_USE_GCM_ASM /* Ignore avx implementation if it won't work. */ +#if CAN_USE_GCM_ASM >= 2 + if (gcm_impl_opts[i].sel == IMPL_AVX2 && + !gcm_avx2_will_work()) { + continue; + } +#endif if (gcm_impl_opts[i].sel == IMPL_AVX && !gcm_avx_will_work()) { continue; } @@ -993,9 +1009,6 @@ MODULE_PARM_DESC(icp_gcm_impl, "Select gcm implementation."); /* Clear the FPU registers since they hold sensitive internal state. */ #define clear_fpu_regs() clear_fpu_regs_avx() -#define GHASH_AVX(ctx, in, len) \ - gcm_ghash_avx((ctx)->gcm_ghash, (const uint64_t *)(ctx)->gcm_Htable, \ - in, len) #define gcm_incr_counter_block(ctx) gcm_incr_counter_block_by(ctx, 1) @@ -1010,20 +1023,77 @@ MODULE_PARM_DESC(icp_gcm_impl, "Select gcm implementation."); static uint32_t gcm_avx_chunk_size = ((32 * 1024) / GCM_AVX_MIN_DECRYPT_BYTES) * GCM_AVX_MIN_DECRYPT_BYTES; +/* + * GCM definitions: uint128_t is copied from include/crypto/modes.h + * Avoiding u128 because it is already defined in kernel sources. + */ +typedef struct { + uint64_t hi, lo; +} uint128_t; + extern void ASMABI clear_fpu_regs_avx(void); extern void ASMABI gcm_xor_avx(const uint8_t *src, uint8_t *dst); extern void ASMABI aes_encrypt_intel(const uint32_t rk[], int nr, const uint32_t pt[4], uint32_t ct[4]); extern void ASMABI gcm_init_htab_avx(uint64_t *Htable, const uint64_t H[2]); +#if CAN_USE_GCM_ASM >= 2 +extern void ASMABI gcm_init_vpclmulqdq_avx2(uint128_t Htable[16], + const uint64_t H[2]); +#endif extern void ASMABI gcm_ghash_avx(uint64_t ghash[2], const uint64_t *Htable, const uint8_t *in, size_t len); +#if CAN_USE_GCM_ASM >= 2 +extern void ASMABI gcm_ghash_vpclmulqdq_avx2(uint64_t ghash[2], + const uint64_t *Htable, const uint8_t *in, size_t len); +#endif +static inline void GHASH_AVX(gcm_ctx_t *ctx, const uint8_t *in, size_t len) +{ + switch (ctx->impl) { +#if CAN_USE_GCM_ASM >= 2 + case GCM_IMPL_AVX2: + gcm_ghash_vpclmulqdq_avx2(ctx->gcm_ghash, + (const uint64_t *)ctx->gcm_Htable, in, len); + break; +#endif + + case GCM_IMPL_AVX: + gcm_ghash_avx(ctx->gcm_ghash, + (const uint64_t *)ctx->gcm_Htable, in, len); + break; + + default: + VERIFY(B_FALSE); + } +} +typedef size_t ASMABI aesni_gcm_encrypt_impl(const uint8_t *, uint8_t *, + size_t, const void *, uint64_t *, const uint64_t *Htable, uint64_t *); extern size_t ASMABI aesni_gcm_encrypt(const uint8_t *, uint8_t *, size_t, const void *, uint64_t *, uint64_t *); +#if CAN_USE_GCM_ASM >= 2 +extern void ASMABI aes_gcm_enc_update_vaes_avx2(const uint8_t *in, + uint8_t *out, size_t len, const void *key, const uint8_t ivec[16], + const uint128_t Htable[16], uint8_t Xi[16]); +#endif +typedef size_t ASMABI aesni_gcm_decrypt_impl(const uint8_t *, uint8_t *, + size_t, const void *, uint64_t *, const uint64_t *Htable, uint64_t *); extern size_t ASMABI aesni_gcm_decrypt(const uint8_t *, uint8_t *, size_t, const void *, uint64_t *, uint64_t *); +#if CAN_USE_GCM_ASM >= 2 +extern void ASMABI aes_gcm_dec_update_vaes_avx2(const uint8_t *in, + uint8_t *out, size_t len, const void *key, const uint8_t ivec[16], + const uint128_t Htable[16], uint8_t Xi[16]); +#endif + +static inline boolean_t +gcm_avx2_will_work(void) +{ + return (kfpu_allowed() && + zfs_avx2_available() && zfs_vaes_available() && + zfs_vpclmulqdq_available()); +} static inline boolean_t gcm_avx_will_work(void) @@ -1035,33 +1105,67 @@ gcm_avx_will_work(void) } static inline void -gcm_set_avx(boolean_t val) +gcm_use_impl(gcm_impl impl) { - if (gcm_avx_will_work() == B_TRUE) { - atomic_swap_32(&gcm_use_avx, val); + switch (impl) { +#if CAN_USE_GCM_ASM >= 2 + case GCM_IMPL_AVX2: + if (gcm_avx2_will_work() == B_TRUE) { + atomic_swap_32(&gcm_impl_used, impl); + return; + } + + zfs_fallthrough; +#endif + + case GCM_IMPL_AVX: + if (gcm_avx_will_work() == B_TRUE) { + atomic_swap_32(&gcm_impl_used, impl); + return; + } + + zfs_fallthrough; + + default: + atomic_swap_32(&gcm_impl_used, GCM_IMPL_GENERIC); } } static inline boolean_t -gcm_toggle_avx(void) +gcm_impl_will_work(gcm_impl impl) { - if (gcm_avx_will_work() == B_TRUE) { - return (atomic_toggle_boolean_nv(&GCM_IMPL_USE_AVX)); - } else { - return (B_FALSE); + switch (impl) { +#if CAN_USE_GCM_ASM >= 2 + case GCM_IMPL_AVX2: + return (gcm_avx2_will_work()); +#endif + + case GCM_IMPL_AVX: + return (gcm_avx_will_work()); + + default: + return (B_TRUE); } } -static inline size_t -gcm_simd_get_htab_size(boolean_t simd_mode) +static inline gcm_impl +gcm_toggle_impl(void) { - switch (simd_mode) { - case B_TRUE: - return (2 * 6 * 2 * sizeof (uint64_t)); + gcm_impl current_impl, new_impl; + do { /* handle races */ + current_impl = atomic_load_32(&gcm_impl_used); + new_impl = current_impl; + while (B_TRUE) { /* handle incompatble implementations */ + new_impl = (new_impl + 1) % GCM_IMPL_MAX; + if (gcm_impl_will_work(new_impl)) { + break; + } + } - default: - return (0); - } + } while (atomic_cas_32(&gcm_impl_used, current_impl, new_impl) != + current_impl); + + return (new_impl); } @@ -1077,6 +1181,50 @@ gcm_incr_counter_block_by(gcm_ctx_t *ctx, int n) ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; } +static size_t aesni_gcm_encrypt_avx(const uint8_t *in, uint8_t *out, + size_t len, const void *key, uint64_t *iv, const uint64_t *Htable, + uint64_t *Xip) +{ + (void) Htable; + return (aesni_gcm_encrypt(in, out, len, key, iv, Xip)); +} + +#if CAN_USE_GCM_ASM >= 2 +// kSizeTWithoutLower4Bits is a mask that can be used to zero the lower four +// bits of a |size_t|. +// This is from boringssl/crypto/fipsmodule/aes/gcm.cc.inc +static const size_t kSizeTWithoutLower4Bits = (size_t)-16; + +/* The following CRYPTO methods are from boringssl/crypto/internal.h */ +static inline uint32_t CRYPTO_bswap4(uint32_t x) { + return (__builtin_bswap32(x)); +} + +static inline uint32_t CRYPTO_load_u32_be(const void *in) { + uint32_t v; + memcpy(&v, in, sizeof (v)); + return (CRYPTO_bswap4(v)); +} + +static inline void CRYPTO_store_u32_be(void *out, uint32_t v) { + v = CRYPTO_bswap4(v); + memcpy(out, &v, sizeof (v)); +} + +static size_t aesni_gcm_encrypt_avx2(const uint8_t *in, uint8_t *out, + size_t len, const void *key, uint64_t *iv, const uint64_t *Htable, + uint64_t *Xip) +{ + uint8_t *ivec = (uint8_t *)iv; + len &= kSizeTWithoutLower4Bits; + aes_gcm_enc_update_vaes_avx2(in, out, len, key, ivec, + (const uint128_t *)Htable, (uint8_t *)Xip); + CRYPTO_store_u32_be(&ivec[12], + CRYPTO_load_u32_be(&ivec[12]) + len / 16); + return (len); +} +#endif /* if CAN_USE_GCM_ASM >= 2 */ + /* * Encrypt multiple blocks of data in GCM mode. * This is done in gcm_avx_chunk_size chunks, utilizing AVX assembler routines @@ -1091,8 +1239,15 @@ gcm_mode_encrypt_contiguous_blocks_avx(gcm_ctx_t *ctx, char *data, size_t done = 0; uint8_t *datap = (uint8_t *)data; size_t chunk_size = (size_t)GCM_CHUNK_SIZE_READ; + aesni_gcm_encrypt_impl *encrypt_blocks = +#if CAN_USE_GCM_ASM >= 2 + ctx->impl == GCM_IMPL_AVX2 ? + aesni_gcm_encrypt_avx2 : +#endif + aesni_gcm_encrypt_avx; const aes_key_t *key = ((aes_key_t *)ctx->gcm_keysched); uint64_t *ghash = ctx->gcm_ghash; + uint64_t *htable = ctx->gcm_Htable; uint64_t *cb = ctx->gcm_cb; uint8_t *ct_buf = NULL; uint8_t *tmp = (uint8_t *)ctx->gcm_tmp; @@ -1156,8 +1311,8 @@ gcm_mode_encrypt_contiguous_blocks_avx(gcm_ctx_t *ctx, char *data, /* Do the bulk encryption in chunk_size blocks. */ for (; bleft >= chunk_size; bleft -= chunk_size) { kfpu_begin(); - done = aesni_gcm_encrypt( - datap, ct_buf, chunk_size, key, cb, ghash); + done = encrypt_blocks( + datap, ct_buf, chunk_size, key, cb, htable, ghash); clear_fpu_regs(); kfpu_end(); @@ -1180,7 +1335,8 @@ gcm_mode_encrypt_contiguous_blocks_avx(gcm_ctx_t *ctx, char *data, /* Bulk encrypt the remaining data. */ kfpu_begin(); if (bleft >= GCM_AVX_MIN_ENCRYPT_BYTES) { - done = aesni_gcm_encrypt(datap, ct_buf, bleft, key, cb, ghash); + done = encrypt_blocks(datap, ct_buf, bleft, key, cb, htable, + ghash); if (done == 0) { rv = CRYPTO_FAILED; goto out; @@ -1293,6 +1449,29 @@ gcm_encrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size) return (CRYPTO_SUCCESS); } +static size_t aesni_gcm_decrypt_avx(const uint8_t *in, uint8_t *out, + size_t len, const void *key, uint64_t *iv, const uint64_t *Htable, + uint64_t *Xip) +{ + (void) Htable; + return (aesni_gcm_decrypt(in, out, len, key, iv, Xip)); +} + +#if CAN_USE_GCM_ASM >= 2 +static size_t aesni_gcm_decrypt_avx2(const uint8_t *in, uint8_t *out, + size_t len, const void *key, uint64_t *iv, const uint64_t *Htable, + uint64_t *Xip) +{ + uint8_t *ivec = (uint8_t *)iv; + len &= kSizeTWithoutLower4Bits; + aes_gcm_dec_update_vaes_avx2(in, out, len, key, ivec, + (const uint128_t *)Htable, (uint8_t *)Xip); + CRYPTO_store_u32_be(&ivec[12], + CRYPTO_load_u32_be(&ivec[12]) + len / 16); + return (len); +} +#endif /* if CAN_USE_GCM_ASM >= 2 */ + /* * Finalize decryption: We just have accumulated crypto text, so now we * decrypt it here inplace. @@ -1306,10 +1485,17 @@ gcm_decrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size) B_FALSE); size_t chunk_size = (size_t)GCM_CHUNK_SIZE_READ; + aesni_gcm_decrypt_impl *decrypt_blocks = +#if CAN_USE_GCM_ASM >= 2 + ctx->impl == GCM_IMPL_AVX2 ? + aesni_gcm_decrypt_avx2 : +#endif + aesni_gcm_decrypt_avx; size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len; uint8_t *datap = ctx->gcm_pt_buf; const aes_key_t *key = ((aes_key_t *)ctx->gcm_keysched); uint32_t *cb = (uint32_t *)ctx->gcm_cb; + uint64_t *htable = ctx->gcm_Htable; uint64_t *ghash = ctx->gcm_ghash; uint32_t *tmp = (uint32_t *)ctx->gcm_tmp; int rv = CRYPTO_SUCCESS; @@ -1322,8 +1508,8 @@ gcm_decrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size) */ for (bleft = pt_len; bleft >= chunk_size; bleft -= chunk_size) { kfpu_begin(); - done = aesni_gcm_decrypt(datap, datap, chunk_size, - (const void *)key, ctx->gcm_cb, ghash); + done = decrypt_blocks(datap, datap, chunk_size, + (const void *)key, ctx->gcm_cb, htable, ghash); clear_fpu_regs(); kfpu_end(); if (done != chunk_size) { @@ -1334,8 +1520,8 @@ gcm_decrypt_final_avx(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size) /* Decrypt remainder, which is less than chunk size, in one go. */ kfpu_begin(); if (bleft >= GCM_AVX_MIN_DECRYPT_BYTES) { - done = aesni_gcm_decrypt(datap, datap, bleft, - (const void *)key, ctx->gcm_cb, ghash); + done = decrypt_blocks(datap, datap, bleft, + (const void *)key, ctx->gcm_cb, htable, ghash); if (done == 0) { clear_fpu_regs(); kfpu_end(); @@ -1424,13 +1610,42 @@ gcm_init_avx(gcm_ctx_t *ctx, const uint8_t *iv, size_t iv_len, ASSERT3S(((aes_key_t *)ctx->gcm_keysched)->ops->needs_byteswap, ==, B_FALSE); + size_t htab_len = 0; +#if CAN_USE_GCM_ASM >= 2 + if (ctx->impl == GCM_IMPL_AVX2) { + /* + * BoringSSL's API specifies uint128_t[16] for htab; but only + * uint128_t[12] are used. + * See https://github.com/google/boringssl/blob/ + * 813840dd094f9e9c1b00a7368aa25e656554221f1/crypto/fipsmodule/ + * modes/asm/aes-gcm-avx2-x86_64.pl#L198-L200 + */ + htab_len = (2 * 8 * sizeof (uint128_t)); + } else +#endif /* CAN_USE_GCM_ASM >= 2 */ + { + htab_len = (2 * 6 * sizeof (uint128_t)); + } + + ctx->gcm_Htable = kmem_alloc(htab_len, KM_SLEEP); + if (ctx->gcm_Htable == NULL) { + return (CRYPTO_HOST_MEMORY); + } + /* Init H (encrypt zero block) and create the initial counter block. */ memset(H, 0, sizeof (ctx->gcm_H)); kfpu_begin(); aes_encrypt_intel(keysched, aes_rounds, (const uint32_t *)H, (uint32_t *)H); - gcm_init_htab_avx(ctx->gcm_Htable, H); +#if CAN_USE_GCM_ASM >= 2 + if (ctx->impl == GCM_IMPL_AVX2) { + gcm_init_vpclmulqdq_avx2((uint128_t *)ctx->gcm_Htable, H); + } else +#endif /* if CAN_USE_GCM_ASM >= 2 */ + { + gcm_init_htab_avx(ctx->gcm_Htable, H); + } if (iv_len == 12) { memcpy(cb, iv, 12); diff --git a/sys/contrib/openzfs/module/icp/algs/modes/modes.c b/sys/contrib/openzfs/module/icp/algs/modes/modes.c index 343591cd9691..ef3c1806e4b6 100644 --- a/sys/contrib/openzfs/module/icp/algs/modes/modes.c +++ b/sys/contrib/openzfs/module/icp/algs/modes/modes.c @@ -171,7 +171,7 @@ gcm_clear_ctx(gcm_ctx_t *ctx) explicit_memset(ctx->gcm_remainder, 0, sizeof (ctx->gcm_remainder)); explicit_memset(ctx->gcm_H, 0, sizeof (ctx->gcm_H)); #if defined(CAN_USE_GCM_ASM) - if (ctx->gcm_use_avx == B_TRUE) { + if (ctx->impl != GCM_IMPL_GENERIC) { ASSERT3P(ctx->gcm_Htable, !=, NULL); explicit_memset(ctx->gcm_Htable, 0, ctx->gcm_htab_len); kmem_free(ctx->gcm_Htable, ctx->gcm_htab_len); diff --git a/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl new file mode 100644 index 000000000000..04c03a37e0cb --- /dev/null +++ b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl @@ -0,0 +1,253 @@ +BoringSSL is a fork of OpenSSL. As such, large parts of it fall under OpenSSL +licensing. Files that are completely new have a Google copyright and an ISC +license. This license is reproduced at the bottom of this file. + +Contributors to BoringSSL are required to follow the CLA rules for Chromium: +https://cla.developers.google.com/clas + +Files in third_party/ have their own licenses, as described therein. The MIT +license, for third_party/fiat, which, unlike other third_party directories, is +compiled into non-test libraries, is included below. + +The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the +OpenSSL License and the original SSLeay license apply to the toolkit. See below +for the actual license texts. Actually both licenses are BSD-style Open Source +licenses. In case of any license issues related to OpenSSL please contact +openssl-core@openssl.org. + +The following are Google-internal bug numbers where explicit permission from +some authors is recorded for use of their work. (This is purely for our own +record keeping.) + 27287199 + 27287880 + 27287883 + 263291445 + + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + + +ISC license used for completely new code in BoringSSL: + +/* Copyright 2015 The BoringSSL Authors + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ + + +The code in third_party/fiat carries the MIT license: + +Copyright (c) 2015-2016 the fiat-crypto authors (see +https://github.com/mit-plv/fiat-crypto/blob/master/AUTHORS). + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Licenses for support code +------------------------- + +Parts of the TLS test suite are under the Go license. This code is not included +in BoringSSL (i.e. libcrypto and libssl) when compiled, however, so +distributing code linked against BoringSSL does not trigger this license: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +BoringSSL uses the Chromium test infrastructure to run a continuous build, +trybots etc. The scripts which manage this, and the script for generating build +metadata, are under the Chromium license. Distributing code linked against +BoringSSL does not trigger this license. + +Copyright 2015 The Chromium Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl.descrip b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl.descrip new file mode 100644 index 000000000000..f63a67a4d2ae --- /dev/null +++ b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.boringssl.descrip @@ -0,0 +1 @@ +PORTIONS OF AES GCM and GHASH FUNCTIONALITY diff --git a/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S new file mode 100644 index 000000000000..3d1b045127e2 --- /dev/null +++ b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S @@ -0,0 +1,1323 @@ +// SPDX-License-Identifier: Apache-2.0 +// This file is generated from a similarly-named Perl script in the BoringSSL +// source tree. Do not edit by hand. + +#if defined(__x86_64__) && defined(HAVE_AVX) && \ + defined(HAVE_VAES) && defined(HAVE_VPCLMULQDQ) + +#define _ASM +#include <sys/asm_linkage.h> + +/* Windows userland links with OpenSSL */ +#if !defined (_WIN32) || defined (_KERNEL) + +.section .rodata +.balign 16 + + +.Lbswap_mask: +.quad 0x08090a0b0c0d0e0f, 0x0001020304050607 + + + + + + + + +.Lgfpoly: +.quad 1, 0xc200000000000000 + + +.Lgfpoly_and_internal_carrybit: +.quad 1, 0xc200000000000001 + +.balign 32 + +.Lctr_pattern: +.quad 0, 0 +.quad 1, 0 +.Linc_2blocks: +.quad 2, 0 +.quad 2, 0 + +ENTRY_ALIGN(gcm_init_vpclmulqdq_avx2, 32) +.cfi_startproc + +ENDBR + + + + + + vmovdqu (%rsi),%xmm3 + // KCF/ICP stores H in network byte order with the hi qword first + // so we need to swap all bytes, not the 2 qwords. + vmovdqu .Lbswap_mask(%rip),%xmm4 + vpshufb %xmm4,%xmm3,%xmm3 + + + + + + vpshufd $0xd3,%xmm3,%xmm0 + vpsrad $31,%xmm0,%xmm0 + vpaddq %xmm3,%xmm3,%xmm3 + vpand .Lgfpoly_and_internal_carrybit(%rip),%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + + vbroadcasti128 .Lgfpoly(%rip),%ymm6 + + + vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 + vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5 + vpclmulqdq $0x01,%xmm0,%xmm6,%xmm1 + vpshufd $0x4e,%xmm0,%xmm0 + vpxor %xmm0,%xmm1,%xmm1 + vpclmulqdq $0x01,%xmm1,%xmm6,%xmm0 + vpshufd $0x4e,%xmm1,%xmm1 + vpxor %xmm1,%xmm5,%xmm5 + vpxor %xmm0,%xmm5,%xmm5 + + + + vinserti128 $1,%xmm3,%ymm5,%ymm3 + vinserti128 $1,%xmm5,%ymm5,%ymm5 + + + vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 + vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 + vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 + vpshufd $0x4e,%ymm0,%ymm0 + vpxor %ymm0,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 + vpshufd $0x4e,%ymm1,%ymm1 + vpxor %ymm1,%ymm4,%ymm4 + vpxor %ymm0,%ymm4,%ymm4 + + + + vmovdqu %ymm3,96(%rdi) + vmovdqu %ymm4,64(%rdi) + + + + vpunpcklqdq %ymm3,%ymm4,%ymm0 + vpunpckhqdq %ymm3,%ymm4,%ymm1 + vpxor %ymm1,%ymm0,%ymm0 + vmovdqu %ymm0,128+32(%rdi) + + + vpclmulqdq $0x00,%ymm5,%ymm4,%ymm0 + vpclmulqdq $0x01,%ymm5,%ymm4,%ymm1 + vpclmulqdq $0x10,%ymm5,%ymm4,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 + vpshufd $0x4e,%ymm0,%ymm0 + vpxor %ymm0,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x11,%ymm5,%ymm4,%ymm3 + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 + vpshufd $0x4e,%ymm1,%ymm1 + vpxor %ymm1,%ymm3,%ymm3 + vpxor %ymm0,%ymm3,%ymm3 + + vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 + vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 + vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 + vpshufd $0x4e,%ymm0,%ymm0 + vpxor %ymm0,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 + vpshufd $0x4e,%ymm1,%ymm1 + vpxor %ymm1,%ymm4,%ymm4 + vpxor %ymm0,%ymm4,%ymm4 + + vmovdqu %ymm3,32(%rdi) + vmovdqu %ymm4,0(%rdi) + + + + vpunpcklqdq %ymm3,%ymm4,%ymm0 + vpunpckhqdq %ymm3,%ymm4,%ymm1 + vpxor %ymm1,%ymm0,%ymm0 + vmovdqu %ymm0,128(%rdi) + + vzeroupper + RET + +.cfi_endproc +SET_SIZE(gcm_init_vpclmulqdq_avx2) +ENTRY_ALIGN(gcm_gmult_vpclmulqdq_avx2, 32) +.cfi_startproc + +ENDBR + + + + vmovdqu (%rdi),%xmm0 + vmovdqu .Lbswap_mask(%rip),%xmm1 + vmovdqu 128-16(%rsi),%xmm2 + vmovdqu .Lgfpoly(%rip),%xmm3 + vpshufb %xmm1,%xmm0,%xmm0 + + vpclmulqdq $0x00,%xmm2,%xmm0,%xmm4 + vpclmulqdq $0x01,%xmm2,%xmm0,%xmm5 + vpclmulqdq $0x10,%xmm2,%xmm0,%xmm6 + vpxor %xmm6,%xmm5,%xmm5 + vpclmulqdq $0x01,%xmm4,%xmm3,%xmm6 + vpshufd $0x4e,%xmm4,%xmm4 + vpxor %xmm4,%xmm5,%xmm5 + vpxor %xmm6,%xmm5,%xmm5 + vpclmulqdq $0x11,%xmm2,%xmm0,%xmm0 + vpclmulqdq $0x01,%xmm5,%xmm3,%xmm4 + vpshufd $0x4e,%xmm5,%xmm5 + vpxor %xmm5,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 + + + vpshufb %xmm1,%xmm0,%xmm0 + vmovdqu %xmm0,(%rdi) + + + RET + +.cfi_endproc +SET_SIZE(gcm_gmult_vpclmulqdq_avx2) +ENTRY_ALIGN(gcm_ghash_vpclmulqdq_avx2, 32) +.cfi_startproc + +ENDBR + + + + + + + vmovdqu .Lbswap_mask(%rip),%xmm6 + vmovdqu .Lgfpoly(%rip),%xmm7 + + + vmovdqu (%rdi),%xmm5 + vpshufb %xmm6,%xmm5,%xmm5 + + + cmpq $32,%rcx + jb .Lghash_lastblock + + + + vinserti128 $1,%xmm6,%ymm6,%ymm6 + vinserti128 $1,%xmm7,%ymm7,%ymm7 + + cmpq $127,%rcx + jbe .Lghash_loop_1x + + + vmovdqu 128(%rsi),%ymm8 + vmovdqu 128+32(%rsi),%ymm9 +.Lghash_loop_4x: + + vmovdqu 0(%rdx),%ymm1 + vpshufb %ymm6,%ymm1,%ymm1 + vmovdqu 0(%rsi),%ymm2 + vpxor %ymm5,%ymm1,%ymm1 + vpclmulqdq $0x00,%ymm2,%ymm1,%ymm3 + vpclmulqdq $0x11,%ymm2,%ymm1,%ymm5 + vpunpckhqdq %ymm1,%ymm1,%ymm0 + vpxor %ymm1,%ymm0,%ymm0 + vpclmulqdq $0x00,%ymm8,%ymm0,%ymm4 + + vmovdqu 32(%rdx),%ymm1 + vpshufb %ymm6,%ymm1,%ymm1 + vmovdqu 32(%rsi),%ymm2 + vpclmulqdq $0x00,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm3,%ymm3 + vpclmulqdq $0x11,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm5,%ymm5 + vpunpckhqdq %ymm1,%ymm1,%ymm0 + vpxor %ymm1,%ymm0,%ymm0 + vpclmulqdq $0x10,%ymm8,%ymm0,%ymm0 + vpxor %ymm0,%ymm4,%ymm4 + + vmovdqu 64(%rdx),%ymm1 + vpshufb %ymm6,%ymm1,%ymm1 + vmovdqu 64(%rsi),%ymm2 + vpclmulqdq $0x00,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm3,%ymm3 + vpclmulqdq $0x11,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm5,%ymm5 + vpunpckhqdq %ymm1,%ymm1,%ymm0 + vpxor %ymm1,%ymm0,%ymm0 + vpclmulqdq $0x00,%ymm9,%ymm0,%ymm0 + vpxor %ymm0,%ymm4,%ymm4 + + + vmovdqu 96(%rdx),%ymm1 + vpshufb %ymm6,%ymm1,%ymm1 + vmovdqu 96(%rsi),%ymm2 + vpclmulqdq $0x00,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm3,%ymm3 + vpclmulqdq $0x11,%ymm2,%ymm1,%ymm0 + vpxor %ymm0,%ymm5,%ymm5 + vpunpckhqdq %ymm1,%ymm1,%ymm0 + vpxor %ymm1,%ymm0,%ymm0 + vpclmulqdq $0x10,%ymm9,%ymm0,%ymm0 + vpxor %ymm0,%ymm4,%ymm4 + + vpxor %ymm3,%ymm4,%ymm4 + vpxor %ymm5,%ymm4,%ymm4 + + + vbroadcasti128 .Lgfpoly(%rip),%ymm2 + vpclmulqdq $0x01,%ymm3,%ymm2,%ymm0 + vpshufd $0x4e,%ymm3,%ymm3 + vpxor %ymm3,%ymm4,%ymm4 + vpxor %ymm0,%ymm4,%ymm4 + + vpclmulqdq $0x01,%ymm4,%ymm2,%ymm0 + vpshufd $0x4e,%ymm4,%ymm4 + vpxor %ymm4,%ymm5,%ymm5 + vpxor %ymm0,%ymm5,%ymm5 + vextracti128 $1,%ymm5,%xmm0 + vpxor %xmm0,%xmm5,%xmm5 + + subq $-128,%rdx + addq $-128,%rcx + cmpq $127,%rcx + ja .Lghash_loop_4x + + + cmpq $32,%rcx + jb .Lghash_loop_1x_done +.Lghash_loop_1x: + vmovdqu (%rdx),%ymm0 + vpshufb %ymm6,%ymm0,%ymm0 + vpxor %ymm0,%ymm5,%ymm5 + vmovdqu 128-32(%rsi),%ymm0 + vpclmulqdq $0x00,%ymm0,%ymm5,%ymm1 + vpclmulqdq $0x01,%ymm0,%ymm5,%ymm2 + vpclmulqdq $0x10,%ymm0,%ymm5,%ymm3 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x01,%ymm1,%ymm7,%ymm3 + vpshufd $0x4e,%ymm1,%ymm1 + vpxor %ymm1,%ymm2,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x11,%ymm0,%ymm5,%ymm5 + vpclmulqdq $0x01,%ymm2,%ymm7,%ymm1 + vpshufd $0x4e,%ymm2,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpxor %ymm1,%ymm5,%ymm5 + + vextracti128 $1,%ymm5,%xmm0 + vpxor %xmm0,%xmm5,%xmm5 + addq $32,%rdx + subq $32,%rcx + cmpq $32,%rcx + jae .Lghash_loop_1x +.Lghash_loop_1x_done: + + +.Lghash_lastblock: + testq %rcx,%rcx + jz .Lghash_done + vmovdqu (%rdx),%xmm0 + vpshufb %xmm6,%xmm0,%xmm0 + vpxor %xmm0,%xmm5,%xmm5 + vmovdqu 128-16(%rsi),%xmm0 + vpclmulqdq $0x00,%xmm0,%xmm5,%xmm1 + vpclmulqdq $0x01,%xmm0,%xmm5,%xmm2 + vpclmulqdq $0x10,%xmm0,%xmm5,%xmm3 + vpxor %xmm3,%xmm2,%xmm2 + vpclmulqdq $0x01,%xmm1,%xmm7,%xmm3 + vpshufd $0x4e,%xmm1,%xmm1 + vpxor %xmm1,%xmm2,%xmm2 + vpxor %xmm3,%xmm2,%xmm2 + vpclmulqdq $0x11,%xmm0,%xmm5,%xmm5 + vpclmulqdq $0x01,%xmm2,%xmm7,%xmm1 + vpshufd $0x4e,%xmm2,%xmm2 + vpxor %xmm2,%xmm5,%xmm5 + vpxor %xmm1,%xmm5,%xmm5 + + +.Lghash_done: + + vpshufb %xmm6,%xmm5,%xmm5 + vmovdqu %xmm5,(%rdi) + + vzeroupper + RET + +.cfi_endproc +SET_SIZE(gcm_ghash_vpclmulqdq_avx2) +ENTRY_ALIGN(aes_gcm_enc_update_vaes_avx2, 32) +.cfi_startproc + +ENDBR + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-16 + + movq 16(%rsp),%r12 +#ifdef BORINGSSL_DISPATCH_TEST +.extern BORINGSSL_function_hit +.hidden BORINGSSL_function_hit + movb $1,BORINGSSL_function_hit+6(%rip) +#endif + vbroadcasti128 .Lbswap_mask(%rip),%ymm0 + + + + vmovdqu (%r12),%xmm1 + vpshufb %xmm0,%xmm1,%xmm1 + vbroadcasti128 (%r8),%ymm11 + vpshufb %ymm0,%ymm11,%ymm11 + + + + movl 504(%rcx),%r10d // ICP has a larger offset for rounds. + leal -24(,%r10,4),%r10d // ICP uses 10,12,14 not 9,11,13 for rounds. + + + + + leaq 96(%rcx,%r10,4),%r11 + vbroadcasti128 (%rcx),%ymm9 + vbroadcasti128 (%r11),%ymm10 + + + vpaddd .Lctr_pattern(%rip),%ymm11,%ymm11 + + + + cmpq $127,%rdx + jbe .Lcrypt_loop_4x_done__func1 + + vmovdqu 128(%r9),%ymm7 + vmovdqu 128+32(%r9),%ymm8 + + + + vmovdqu .Linc_2blocks(%rip),%ymm2 + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm14 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm15 + vpaddd %ymm2,%ymm11,%ymm11 + + + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + vpxor %ymm9,%ymm14,%ymm14 + vpxor %ymm9,%ymm15,%ymm15 + + leaq 16(%rcx),%rax +.Lvaesenc_loop_first_4_vecs__func1: + vbroadcasti128 (%rax),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + addq $16,%rax + cmpq %rax,%r11 + jne .Lvaesenc_loop_first_4_vecs__func1 + vpxor 0(%rdi),%ymm10,%ymm2 + vpxor 32(%rdi),%ymm10,%ymm3 + vpxor 64(%rdi),%ymm10,%ymm5 + vpxor 96(%rdi),%ymm10,%ymm6 + vaesenclast %ymm2,%ymm12,%ymm12 + vaesenclast %ymm3,%ymm13,%ymm13 + vaesenclast %ymm5,%ymm14,%ymm14 + vaesenclast %ymm6,%ymm15,%ymm15 + vmovdqu %ymm12,0(%rsi) + vmovdqu %ymm13,32(%rsi) + vmovdqu %ymm14,64(%rsi) + vmovdqu %ymm15,96(%rsi) + + subq $-128,%rdi + addq $-128,%rdx + cmpq $127,%rdx + jbe .Lghash_last_ciphertext_4x__func1 +.balign 16 +.Lcrypt_loop_4x__func1: + + + + + vmovdqu .Linc_2blocks(%rip),%ymm2 + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm14 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm15 + vpaddd %ymm2,%ymm11,%ymm11 + + + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + vpxor %ymm9,%ymm14,%ymm14 + vpxor %ymm9,%ymm15,%ymm15 + + cmpl $24,%r10d + jl .Laes128__func1 + je .Laes192__func1 + + vbroadcasti128 -208(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vbroadcasti128 -192(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + +.Laes192__func1: + vbroadcasti128 -176(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vbroadcasti128 -160(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + +.Laes128__func1: + prefetcht0 512(%rdi) + prefetcht0 512+64(%rdi) + + vmovdqu 0(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 0(%r9),%ymm4 + vpxor %ymm1,%ymm3,%ymm3 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm7,%ymm2,%ymm6 + + vbroadcasti128 -144(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vbroadcasti128 -128(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vmovdqu 32(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 32(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm7,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -112(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vmovdqu 64(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 64(%r9),%ymm4 + + vbroadcasti128 -96(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + + vbroadcasti128 -80(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + + vmovdqu 96(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + + vbroadcasti128 -64(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vmovdqu 96(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -48(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm1,%ymm6,%ymm6 + + + vbroadcasti128 .Lgfpoly(%rip),%ymm4 + vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 + vpshufd $0x4e,%ymm5,%ymm5 + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -32(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 + vpshufd $0x4e,%ymm6,%ymm6 + vpxor %ymm6,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + + vbroadcasti128 -16(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vextracti128 $1,%ymm1,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + + + subq $-128,%rsi + vpxor 0(%rdi),%ymm10,%ymm2 + vpxor 32(%rdi),%ymm10,%ymm3 + vpxor 64(%rdi),%ymm10,%ymm5 + vpxor 96(%rdi),%ymm10,%ymm6 + vaesenclast %ymm2,%ymm12,%ymm12 + vaesenclast %ymm3,%ymm13,%ymm13 + vaesenclast %ymm5,%ymm14,%ymm14 + vaesenclast %ymm6,%ymm15,%ymm15 + vmovdqu %ymm12,0(%rsi) + vmovdqu %ymm13,32(%rsi) + vmovdqu %ymm14,64(%rsi) + vmovdqu %ymm15,96(%rsi) + + subq $-128,%rdi + + addq $-128,%rdx + cmpq $127,%rdx + ja .Lcrypt_loop_4x__func1 +.Lghash_last_ciphertext_4x__func1: + + vmovdqu 0(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 0(%r9),%ymm4 + vpxor %ymm1,%ymm3,%ymm3 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm7,%ymm2,%ymm6 + + vmovdqu 32(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 32(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm7,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vmovdqu 64(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 64(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + + vmovdqu 96(%rsi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 96(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm1,%ymm6,%ymm6 + + + vbroadcasti128 .Lgfpoly(%rip),%ymm4 + vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 + vpshufd $0x4e,%ymm5,%ymm5 + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm2,%ymm6,%ymm6 + + vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 + vpshufd $0x4e,%ymm6,%ymm6 + vpxor %ymm6,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + vextracti128 $1,%ymm1,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + + subq $-128,%rsi +.Lcrypt_loop_4x_done__func1: + + testq %rdx,%rdx + jz .Ldone__func1 + + + + + + leaq 128(%r9),%r8 + subq %rdx,%r8 + + + vpxor %xmm5,%xmm5,%xmm5 + vpxor %xmm6,%xmm6,%xmm6 + vpxor %xmm7,%xmm7,%xmm7 + + cmpq $64,%rdx + jb .Llessthan64bytes__func1 + + + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + leaq 16(%rcx),%rax +.Lvaesenc_loop_tail_1__func1: + vbroadcasti128 (%rax),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + addq $16,%rax + cmpq %rax,%r11 + jne .Lvaesenc_loop_tail_1__func1 + vaesenclast %ymm10,%ymm12,%ymm12 + vaesenclast %ymm10,%ymm13,%ymm13 + + + vmovdqu 0(%rdi),%ymm2 + vmovdqu 32(%rdi),%ymm3 + vpxor %ymm2,%ymm12,%ymm12 + vpxor %ymm3,%ymm13,%ymm13 + vmovdqu %ymm12,0(%rsi) + vmovdqu %ymm13,32(%rsi) + + + vpshufb %ymm0,%ymm12,%ymm12 + vpshufb %ymm0,%ymm13,%ymm13 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + vmovdqu 32(%r8),%ymm3 + vpclmulqdq $0x00,%ymm2,%ymm12,%ymm5 + vpclmulqdq $0x01,%ymm2,%ymm12,%ymm6 + vpclmulqdq $0x10,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm2,%ymm12,%ymm7 + vpclmulqdq $0x00,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm7,%ymm7 + + addq $64,%r8 + addq $64,%rdi + addq $64,%rsi + subq $64,%rdx + jz .Lreduce__func1 + + vpxor %xmm1,%xmm1,%xmm1 + + +.Llessthan64bytes__func1: + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + leaq 16(%rcx),%rax +.Lvaesenc_loop_tail_2__func1: + vbroadcasti128 (%rax),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + addq $16,%rax + cmpq %rax,%r11 + jne .Lvaesenc_loop_tail_2__func1 + vaesenclast %ymm10,%ymm12,%ymm12 + vaesenclast %ymm10,%ymm13,%ymm13 + + + + + cmpq $32,%rdx + jb .Lxor_one_block__func1 + je .Lxor_two_blocks__func1 + +.Lxor_three_blocks__func1: + vmovdqu 0(%rdi),%ymm2 + vmovdqu 32(%rdi),%xmm3 + vpxor %ymm2,%ymm12,%ymm12 + vpxor %xmm3,%xmm13,%xmm13 + vmovdqu %ymm12,0(%rsi) + vmovdqu %xmm13,32(%rsi) + + vpshufb %ymm0,%ymm12,%ymm12 + vpshufb %xmm0,%xmm13,%xmm13 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + vmovdqu 32(%r8),%xmm3 + vpclmulqdq $0x00,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm7,%ymm7 + jmp .Lghash_mul_one_vec_unreduced__func1 + +.Lxor_two_blocks__func1: + vmovdqu (%rdi),%ymm2 + vpxor %ymm2,%ymm12,%ymm12 + vmovdqu %ymm12,(%rsi) + vpshufb %ymm0,%ymm12,%ymm12 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + jmp .Lghash_mul_one_vec_unreduced__func1 + +.Lxor_one_block__func1: + vmovdqu (%rdi),%xmm2 + vpxor %xmm2,%xmm12,%xmm12 + vmovdqu %xmm12,(%rsi) + vpshufb %xmm0,%xmm12,%xmm12 + vpxor %xmm1,%xmm12,%xmm12 + vmovdqu (%r8),%xmm2 + +.Lghash_mul_one_vec_unreduced__func1: + vpclmulqdq $0x00,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm7,%ymm7 + +.Lreduce__func1: + + vbroadcasti128 .Lgfpoly(%rip),%ymm2 + vpclmulqdq $0x01,%ymm5,%ymm2,%ymm3 + vpshufd $0x4e,%ymm5,%ymm5 + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm3,%ymm6,%ymm6 + vpclmulqdq $0x01,%ymm6,%ymm2,%ymm3 + vpshufd $0x4e,%ymm6,%ymm6 + vpxor %ymm6,%ymm7,%ymm7 + vpxor %ymm3,%ymm7,%ymm7 + vextracti128 $1,%ymm7,%xmm1 + vpxor %xmm7,%xmm1,%xmm1 + +.Ldone__func1: + + vpshufb %xmm0,%xmm1,%xmm1 + vmovdqu %xmm1,(%r12) + + vzeroupper + popq %r12 +.cfi_adjust_cfa_offset -8 +.cfi_restore %r12 + RET + +.cfi_endproc +SET_SIZE(aes_gcm_enc_update_vaes_avx2) +ENTRY_ALIGN(aes_gcm_dec_update_vaes_avx2, 32) +.cfi_startproc + +ENDBR + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-16 + + movq 16(%rsp),%r12 + vbroadcasti128 .Lbswap_mask(%rip),%ymm0 + + + + vmovdqu (%r12),%xmm1 + vpshufb %xmm0,%xmm1,%xmm1 + vbroadcasti128 (%r8),%ymm11 + vpshufb %ymm0,%ymm11,%ymm11 + + + + movl 504(%rcx),%r10d // ICP has a larger offset for rounds. + leal -24(,%r10,4),%r10d // ICP uses 10,12,14 not 9,11,13 for rounds. + + + + + leaq 96(%rcx,%r10,4),%r11 + vbroadcasti128 (%rcx),%ymm9 + vbroadcasti128 (%r11),%ymm10 + + + vpaddd .Lctr_pattern(%rip),%ymm11,%ymm11 + + + + cmpq $127,%rdx + jbe .Lcrypt_loop_4x_done__func2 + + vmovdqu 128(%r9),%ymm7 + vmovdqu 128+32(%r9),%ymm8 +.balign 16 +.Lcrypt_loop_4x__func2: + + + + + vmovdqu .Linc_2blocks(%rip),%ymm2 + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm14 + vpaddd %ymm2,%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm15 + vpaddd %ymm2,%ymm11,%ymm11 + + + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + vpxor %ymm9,%ymm14,%ymm14 + vpxor %ymm9,%ymm15,%ymm15 + + cmpl $24,%r10d + jl .Laes128__func2 + je .Laes192__func2 + + vbroadcasti128 -208(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vbroadcasti128 -192(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + +.Laes192__func2: + vbroadcasti128 -176(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vbroadcasti128 -160(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + +.Laes128__func2: + prefetcht0 512(%rdi) + prefetcht0 512+64(%rdi) + + vmovdqu 0(%rdi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 0(%r9),%ymm4 + vpxor %ymm1,%ymm3,%ymm3 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm7,%ymm2,%ymm6 + + vbroadcasti128 -144(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vbroadcasti128 -128(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vmovdqu 32(%rdi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 32(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm7,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -112(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vmovdqu 64(%rdi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + vmovdqu 64(%r9),%ymm4 + + vbroadcasti128 -96(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + + vbroadcasti128 -80(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + + vmovdqu 96(%rdi),%ymm3 + vpshufb %ymm0,%ymm3,%ymm3 + + vbroadcasti128 -64(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vmovdqu 96(%r9),%ymm4 + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm5,%ymm5 + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm2 + vpxor %ymm2,%ymm1,%ymm1 + vpunpckhqdq %ymm3,%ymm3,%ymm2 + vpxor %ymm3,%ymm2,%ymm2 + vpclmulqdq $0x10,%ymm8,%ymm2,%ymm2 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -48(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm1,%ymm6,%ymm6 + + + vbroadcasti128 .Lgfpoly(%rip),%ymm4 + vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 + vpshufd $0x4e,%ymm5,%ymm5 + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm2,%ymm6,%ymm6 + + vbroadcasti128 -32(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + + vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 + vpshufd $0x4e,%ymm6,%ymm6 + vpxor %ymm6,%ymm1,%ymm1 + vpxor %ymm2,%ymm1,%ymm1 + + vbroadcasti128 -16(%r11),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + vaesenc %ymm2,%ymm14,%ymm14 + vaesenc %ymm2,%ymm15,%ymm15 + + vextracti128 $1,%ymm1,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + + + + vpxor 0(%rdi),%ymm10,%ymm2 + vpxor 32(%rdi),%ymm10,%ymm3 + vpxor 64(%rdi),%ymm10,%ymm5 + vpxor 96(%rdi),%ymm10,%ymm6 + vaesenclast %ymm2,%ymm12,%ymm12 + vaesenclast %ymm3,%ymm13,%ymm13 + vaesenclast %ymm5,%ymm14,%ymm14 + vaesenclast %ymm6,%ymm15,%ymm15 + vmovdqu %ymm12,0(%rsi) + vmovdqu %ymm13,32(%rsi) + vmovdqu %ymm14,64(%rsi) + vmovdqu %ymm15,96(%rsi) + + subq $-128,%rdi + subq $-128,%rsi + addq $-128,%rdx + cmpq $127,%rdx + ja .Lcrypt_loop_4x__func2 +.Lcrypt_loop_4x_done__func2: + + testq %rdx,%rdx + jz .Ldone__func2 + + + + + + leaq 128(%r9),%r8 + subq %rdx,%r8 + + + vpxor %xmm5,%xmm5,%xmm5 + vpxor %xmm6,%xmm6,%xmm6 + vpxor %xmm7,%xmm7,%xmm7 + + cmpq $64,%rdx + jb .Llessthan64bytes__func2 + + + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + leaq 16(%rcx),%rax +.Lvaesenc_loop_tail_1__func2: + vbroadcasti128 (%rax),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + addq $16,%rax + cmpq %rax,%r11 + jne .Lvaesenc_loop_tail_1__func2 + vaesenclast %ymm10,%ymm12,%ymm12 + vaesenclast %ymm10,%ymm13,%ymm13 + + + vmovdqu 0(%rdi),%ymm2 + vmovdqu 32(%rdi),%ymm3 + vpxor %ymm2,%ymm12,%ymm12 + vpxor %ymm3,%ymm13,%ymm13 + vmovdqu %ymm12,0(%rsi) + vmovdqu %ymm13,32(%rsi) + + + vpshufb %ymm0,%ymm2,%ymm12 + vpshufb %ymm0,%ymm3,%ymm13 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + vmovdqu 32(%r8),%ymm3 + vpclmulqdq $0x00,%ymm2,%ymm12,%ymm5 + vpclmulqdq $0x01,%ymm2,%ymm12,%ymm6 + vpclmulqdq $0x10,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm2,%ymm12,%ymm7 + vpclmulqdq $0x00,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm3,%ymm13,%ymm4 + vpxor %ymm4,%ymm7,%ymm7 + + addq $64,%r8 + addq $64,%rdi + addq $64,%rsi + subq $64,%rdx + jz .Lreduce__func2 + + vpxor %xmm1,%xmm1,%xmm1 + + +.Llessthan64bytes__func2: + vpshufb %ymm0,%ymm11,%ymm12 + vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 + vpshufb %ymm0,%ymm11,%ymm13 + vpxor %ymm9,%ymm12,%ymm12 + vpxor %ymm9,%ymm13,%ymm13 + leaq 16(%rcx),%rax +.Lvaesenc_loop_tail_2__func2: + vbroadcasti128 (%rax),%ymm2 + vaesenc %ymm2,%ymm12,%ymm12 + vaesenc %ymm2,%ymm13,%ymm13 + addq $16,%rax + cmpq %rax,%r11 + jne .Lvaesenc_loop_tail_2__func2 + vaesenclast %ymm10,%ymm12,%ymm12 + vaesenclast %ymm10,%ymm13,%ymm13 + + + + + cmpq $32,%rdx + jb .Lxor_one_block__func2 + je .Lxor_two_blocks__func2 + +.Lxor_three_blocks__func2: + vmovdqu 0(%rdi),%ymm2 + vmovdqu 32(%rdi),%xmm3 + vpxor %ymm2,%ymm12,%ymm12 + vpxor %xmm3,%xmm13,%xmm13 + vmovdqu %ymm12,0(%rsi) + vmovdqu %xmm13,32(%rsi) + + vpshufb %ymm0,%ymm2,%ymm12 + vpshufb %xmm0,%xmm3,%xmm13 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + vmovdqu 32(%r8),%xmm3 + vpclmulqdq $0x00,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%xmm3,%xmm13,%xmm4 + vpxor %ymm4,%ymm7,%ymm7 + jmp .Lghash_mul_one_vec_unreduced__func2 + +.Lxor_two_blocks__func2: + vmovdqu (%rdi),%ymm2 + vpxor %ymm2,%ymm12,%ymm12 + vmovdqu %ymm12,(%rsi) + vpshufb %ymm0,%ymm2,%ymm12 + vpxor %ymm1,%ymm12,%ymm12 + vmovdqu (%r8),%ymm2 + jmp .Lghash_mul_one_vec_unreduced__func2 + +.Lxor_one_block__func2: + vmovdqu (%rdi),%xmm2 + vpxor %xmm2,%xmm12,%xmm12 + vmovdqu %xmm12,(%rsi) + vpshufb %xmm0,%xmm2,%xmm12 + vpxor %xmm1,%xmm12,%xmm12 + vmovdqu (%r8),%xmm2 + +.Lghash_mul_one_vec_unreduced__func2: + vpclmulqdq $0x00,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm5,%ymm5 + vpclmulqdq $0x01,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x10,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm6,%ymm6 + vpclmulqdq $0x11,%ymm2,%ymm12,%ymm4 + vpxor %ymm4,%ymm7,%ymm7 + +.Lreduce__func2: + + vbroadcasti128 .Lgfpoly(%rip),%ymm2 + vpclmulqdq $0x01,%ymm5,%ymm2,%ymm3 + vpshufd $0x4e,%ymm5,%ymm5 + vpxor %ymm5,%ymm6,%ymm6 + vpxor %ymm3,%ymm6,%ymm6 + vpclmulqdq $0x01,%ymm6,%ymm2,%ymm3 + vpshufd $0x4e,%ymm6,%ymm6 + vpxor %ymm6,%ymm7,%ymm7 + vpxor %ymm3,%ymm7,%ymm7 + vextracti128 $1,%ymm7,%xmm1 + vpxor %xmm7,%xmm1,%xmm1 + +.Ldone__func2: + + vpshufb %xmm0,%xmm1,%xmm1 + vmovdqu %xmm1,(%r12) + + vzeroupper + popq %r12 +.cfi_adjust_cfa_offset -8 +.cfi_restore %r12 + RET + +.cfi_endproc +SET_SIZE(aes_gcm_dec_update_vaes_avx2) + +#endif /* !_WIN32 || _KERNEL */ + +/* Mark the stack non-executable. */ +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif + +#endif /* defined(__x86_64__) && defined(HAVE_AVX) && defined(HAVE_AES) ... */ diff --git a/sys/contrib/openzfs/module/icp/include/modes/modes.h b/sys/contrib/openzfs/module/icp/include/modes/modes.h index ca734cf4f045..de11d9eafafb 100644 --- a/sys/contrib/openzfs/module/icp/include/modes/modes.h +++ b/sys/contrib/openzfs/module/icp/include/modes/modes.h @@ -42,7 +42,7 @@ extern "C" { */ #if defined(__x86_64__) && defined(HAVE_AVX) && \ defined(HAVE_AES) && defined(HAVE_PCLMULQDQ) -#define CAN_USE_GCM_ASM +#define CAN_USE_GCM_ASM (HAVE_VAES && HAVE_VPCLMULQDQ ? 2 : 1) extern boolean_t gcm_avx_can_use_movbe; #endif @@ -129,6 +129,15 @@ typedef struct ccm_ctx { #define ccm_copy_to ccm_common.cc_copy_to #define ccm_flags ccm_common.cc_flags +#ifdef CAN_USE_GCM_ASM +typedef enum gcm_impl { + GCM_IMPL_GENERIC = 0, + GCM_IMPL_AVX, + GCM_IMPL_AVX2, + GCM_IMPL_MAX, +} gcm_impl; +#endif + /* * gcm_tag_len: Length of authentication tag. * @@ -174,7 +183,7 @@ typedef struct gcm_ctx { uint64_t gcm_len_a_len_c[2]; uint8_t *gcm_pt_buf; #ifdef CAN_USE_GCM_ASM - boolean_t gcm_use_avx; + enum gcm_impl impl; #endif } gcm_ctx_t; diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/kmod_core.c b/sys/contrib/openzfs/module/os/freebsd/zfs/kmod_core.c index c114db14a916..b218c0da8125 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/kmod_core.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/kmod_core.c @@ -112,7 +112,6 @@ static int zfs__fini(void); static void zfs_shutdown(void *, int); static eventhandler_tag zfs_shutdown_event_tag; -static eventhandler_tag zfs_mountroot_event_tag; #define ZFS_MIN_KSTACK_PAGES 4 @@ -311,9 +310,6 @@ zfs_modevent(module_t mod, int type, void *unused __unused) zfs_shutdown_event_tag = EVENTHANDLER_REGISTER( shutdown_post_sync, zfs_shutdown, NULL, SHUTDOWN_PRI_FIRST); - zfs_mountroot_event_tag = EVENTHANDLER_REGISTER( - mountroot, spa_boot_init, NULL, - SI_ORDER_ANY); } return (err); case MOD_UNLOAD: @@ -322,9 +318,6 @@ zfs_modevent(module_t mod, int type, void *unused __unused) if (zfs_shutdown_event_tag != NULL) EVENTHANDLER_DEREGISTER(shutdown_post_sync, zfs_shutdown_event_tag); - if (zfs_mountroot_event_tag != NULL) - EVENTHANDLER_DEREGISTER(mountroot, - zfs_mountroot_event_tag); } return (err); case MOD_SHUTDOWN: diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_ctldir.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_ctldir.c index 61d0bb26d1e5..a222c5de4a2a 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_ctldir.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_ctldir.c @@ -494,7 +494,7 @@ zfsctl_common_getattr(vnode_t *vp, vattr_t *vap) vap->va_uid = 0; vap->va_gid = 0; - vap->va_rdev = 0; + vap->va_rdev = NODEV; /* * We are a purely virtual object, so we have no * blocksize or allocated blocks. @@ -674,6 +674,7 @@ zfsctl_root_readdir(struct vop_readdir_args *ap) zfs_uio_t uio; int *eofp = ap->a_eofflag; off_t dots_offset; + ssize_t orig_resid; int error; zfs_uio_init(&uio, ap->a_uio); @@ -688,16 +689,16 @@ zfsctl_root_readdir(struct vop_readdir_args *ap) * count to return is 0. */ if (zfs_uio_offset(&uio) == 3 * sizeof (entry)) { + if (eofp != NULL) + *eofp = 1; return (0); } + orig_resid = zfs_uio_resid(&uio); error = sfs_readdir_common(zfsvfs->z_root, ZFSCTL_INO_ROOT, ap, &uio, &dots_offset); - if (error != 0) { - if (error == ENAMETOOLONG) /* ran out of destination space */ - error = 0; - return (error); - } + if (error != 0) + goto err; if (zfs_uio_offset(&uio) != dots_offset) return (SET_ERROR(EINVAL)); @@ -710,8 +711,11 @@ zfsctl_root_readdir(struct vop_readdir_args *ap) entry.d_reclen = sizeof (entry); error = vfs_read_dirent(ap, &entry, zfs_uio_offset(&uio)); if (error != 0) { - if (error == ENAMETOOLONG) - error = 0; +err: + if (error == ENAMETOOLONG) { + error = orig_resid == zfs_uio_resid(&uio) ? + EINVAL : 0; + } return (SET_ERROR(error)); } if (eofp != NULL) @@ -1056,17 +1060,21 @@ zfsctl_snapdir_readdir(struct vop_readdir_args *ap) zfs_uio_t uio; int *eofp = ap->a_eofflag; off_t dots_offset; + ssize_t orig_resid; int error; zfs_uio_init(&uio, ap->a_uio); + orig_resid = zfs_uio_resid(&uio); ASSERT3S(vp->v_type, ==, VDIR); error = sfs_readdir_common(ZFSCTL_INO_ROOT, ZFSCTL_INO_SNAPDIR, ap, &uio, &dots_offset); if (error != 0) { - if (error == ENAMETOOLONG) /* ran out of destination space */ - error = 0; + if (error == ENAMETOOLONG) { /* ran out of destination space */ + error = orig_resid == zfs_uio_resid(&uio) ? + EINVAL : 0; + } return (error); } @@ -1084,9 +1092,13 @@ zfsctl_snapdir_readdir(struct vop_readdir_args *ap) dsl_pool_config_exit(dmu_objset_pool(zfsvfs->z_os), FTAG); if (error != 0) { if (error == ENOENT) { - if (eofp != NULL) - *eofp = 1; - error = 0; + if (orig_resid == zfs_uio_resid(&uio)) { + error = EINVAL; + } else { + error = 0; + if (eofp != NULL) + *eofp = 1; + } } zfs_exit(zfsvfs, FTAG); return (error); @@ -1099,8 +1111,10 @@ zfsctl_snapdir_readdir(struct vop_readdir_args *ap) entry.d_reclen = sizeof (entry); error = vfs_read_dirent(ap, &entry, zfs_uio_offset(&uio)); if (error != 0) { - if (error == ENAMETOOLONG) - error = 0; + if (error == ENAMETOOLONG) { + error = orig_resid == zfs_uio_resid(&uio) ? + EINVAL : 0; + } zfs_exit(zfsvfs, FTAG); return (SET_ERROR(error)); } diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c index 1813c411b013..174141a5deab 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c @@ -1695,6 +1695,7 @@ zfs_readdir(vnode_t *vp, zfs_uio_t *uio, cred_t *cr, int *eofp, objset_t *os; caddr_t outbuf; size_t bufsize; + ssize_t orig_resid; zap_cursor_t zc; zap_attribute_t *zap; uint_t bytes_wanted; @@ -1735,7 +1736,7 @@ zfs_readdir(vnode_t *vp, zfs_uio_t *uio, cred_t *cr, int *eofp, /* * Quit if directory has been removed (posix) */ - if ((*eofp = zp->z_unlinked) != 0) { + if ((*eofp = (zp->z_unlinked != 0)) != 0) { zfs_exit(zfsvfs, FTAG); return (0); } @@ -1743,6 +1744,7 @@ zfs_readdir(vnode_t *vp, zfs_uio_t *uio, cred_t *cr, int *eofp, error = 0; os = zfsvfs->z_os; offset = zfs_uio_offset(uio); + orig_resid = zfs_uio_resid(uio); prefetch = zp->z_zn_prefetch; zap = zap_attribute_long_alloc(); @@ -1922,7 +1924,7 @@ update: kmem_free(outbuf, bufsize); if (error == ENOENT) - error = 0; + error = orig_resid == zfs_uio_resid(uio) ? EINVAL : 0; ZFS_ACCESSTIME_STAMP(zfsvfs, zp); @@ -2013,7 +2015,7 @@ zfs_getattr(vnode_t *vp, vattr_t *vap, int flags, cred_t *cr) if (vp->v_type == VBLK || vp->v_type == VCHR) vap->va_rdev = zfs_cmpldev(rdev); else - vap->va_rdev = 0; + vap->va_rdev = NODEV; vap->va_gen = zp->z_gen; vap->va_flags = 0; /* FreeBSD: Reset chflags(2) flags. */ vap->va_filerev = zp->z_seq; diff --git a/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c b/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c index 265dfd55fc4d..0dd2ecd7fd8d 100644 --- a/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c +++ b/sys/contrib/openzfs/module/os/freebsd/zfs/zvol_os.c @@ -31,7 +31,7 @@ * Copyright (c) 2012, 2017 by Delphix. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright (c) 2014 Integros [integros.com] - * Copyright (c) 2024, Klara, Inc. + * Copyright (c) 2024, 2025, Klara, Inc. */ /* Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org> */ @@ -196,7 +196,6 @@ DECLARE_GEOM_CLASS(zfs_zvol_class, zfs_zvol); static int zvol_geom_open(struct g_provider *pp, int flag, int count); static int zvol_geom_close(struct g_provider *pp, int flag, int count); -static void zvol_geom_destroy(zvol_state_t *zv); static int zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace); static void zvol_geom_bio_start(struct bio *bp); static int zvol_geom_bio_getattr(struct bio *bp); @@ -226,25 +225,14 @@ zvol_geom_open(struct g_provider *pp, int flag, int count) } retry: - rw_enter(&zvol_state_lock, ZVOL_RW_READER); - /* - * Obtain a copy of private under zvol_state_lock to make sure either - * the result of zvol free code setting private to NULL is observed, - * or the zv is protected from being freed because of the positive - * zv_open_count. - */ - zv = pp->private; - if (zv == NULL) { - rw_exit(&zvol_state_lock); - err = SET_ERROR(ENXIO); - goto out_locked; - } + zv = atomic_load_ptr(&pp->private); + if (zv == NULL) + return (SET_ERROR(ENXIO)); mutex_enter(&zv->zv_state_lock); if (zv->zv_zso->zso_dying || zv->zv_flags & ZVOL_REMOVING) { - rw_exit(&zvol_state_lock); err = SET_ERROR(ENXIO); - goto out_zv_locked; + goto out_locked; } ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM); @@ -257,8 +245,24 @@ retry: drop_suspend = B_TRUE; if (!rw_tryenter(&zv->zv_suspend_lock, ZVOL_RW_READER)) { mutex_exit(&zv->zv_state_lock); + + /* + * Removal may happen while the locks are down, so + * we can't trust zv any longer; we have to start over. + */ + zv = atomic_load_ptr(&pp->private); + if (zv == NULL) + return (SET_ERROR(ENXIO)); + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); mutex_enter(&zv->zv_state_lock); + + if (zv->zv_zso->zso_dying || + zv->zv_flags & ZVOL_REMOVING) { + err = SET_ERROR(ENXIO); + goto out_locked; + } + /* Check to see if zv_suspend_lock is needed. */ if (zv->zv_open_count != 0) { rw_exit(&zv->zv_suspend_lock); @@ -266,7 +270,6 @@ retry: } } } - rw_exit(&zvol_state_lock); ASSERT(MUTEX_HELD(&zv->zv_state_lock)); @@ -294,7 +297,7 @@ retry: if (drop_namespace) mutex_exit(&spa_namespace_lock); if (err) - goto out_zv_locked; + goto out_locked; pp->mediasize = zv->zv_volsize; pp->stripeoffset = 0; pp->stripesize = zv->zv_volblocksize; @@ -329,9 +332,8 @@ out_opened: zvol_last_close(zv); wakeup(zv); } -out_zv_locked: - mutex_exit(&zv->zv_state_lock); out_locked: + mutex_exit(&zv->zv_state_lock); if (drop_suspend) rw_exit(&zv->zv_suspend_lock); return (err); @@ -345,12 +347,9 @@ zvol_geom_close(struct g_provider *pp, int flag, int count) boolean_t drop_suspend = B_TRUE; int new_open_count; - rw_enter(&zvol_state_lock, ZVOL_RW_READER); - zv = pp->private; - if (zv == NULL) { - rw_exit(&zvol_state_lock); + zv = atomic_load_ptr(&pp->private); + if (zv == NULL) return (SET_ERROR(ENXIO)); - } mutex_enter(&zv->zv_state_lock); if (zv->zv_flags & ZVOL_EXCL) { @@ -377,6 +376,15 @@ zvol_geom_close(struct g_provider *pp, int flag, int count) mutex_exit(&zv->zv_state_lock); rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); mutex_enter(&zv->zv_state_lock); + + /* + * Unlike in zvol_geom_open(), we don't check if + * removal started here, because we might be one of the + * openers that needs to be thrown out! If we're the + * last, we need to call zvol_last_close() below to + * finish cleanup. So, no special treatment for us. + */ + /* Check to see if zv_suspend_lock is needed. */ new_open_count = zv->zv_open_count - count; if (new_open_count != 0) { @@ -387,7 +395,6 @@ zvol_geom_close(struct g_provider *pp, int flag, int count) } else { drop_suspend = B_FALSE; } - rw_exit(&zvol_state_lock); ASSERT(MUTEX_HELD(&zv->zv_state_lock)); @@ -408,20 +415,6 @@ zvol_geom_close(struct g_provider *pp, int flag, int count) return (0); } -static void -zvol_geom_destroy(zvol_state_t *zv) -{ - struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; - struct g_provider *pp = zsg->zsg_provider; - - ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM); - - g_topology_assert(); - - zsg->zsg_provider = NULL; - g_wither_geom(pp->geom, ENXIO); -} - void zvol_wait_close(zvol_state_t *zv) { @@ -454,7 +447,7 @@ zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace) ("Unsupported access request to %s (acr=%d, acw=%d, ace=%d).", pp->name, acr, acw, ace)); - if (pp->private == NULL) { + if (atomic_load_ptr(&pp->private) == NULL) { if (acr <= 0 && acw <= 0 && ace <= 0) return (0); return (pp->error); @@ -921,25 +914,14 @@ zvol_cdev_open(struct cdev *dev, int flags, int fmt, struct thread *td) boolean_t drop_suspend = B_FALSE; retry: - rw_enter(&zvol_state_lock, ZVOL_RW_READER); - /* - * Obtain a copy of si_drv2 under zvol_state_lock to make sure either - * the result of zvol free code setting si_drv2 to NULL is observed, - * or the zv is protected from being freed because of the positive - * zv_open_count. - */ - zv = dev->si_drv2; - if (zv == NULL) { - rw_exit(&zvol_state_lock); - err = SET_ERROR(ENXIO); - goto out_locked; - } + zv = atomic_load_ptr(&dev->si_drv2); + if (zv == NULL) + return (SET_ERROR(ENXIO)); mutex_enter(&zv->zv_state_lock); - if (zv->zv_zso->zso_dying) { - rw_exit(&zvol_state_lock); + if (zv->zv_zso->zso_dying || zv->zv_flags & ZVOL_REMOVING) { err = SET_ERROR(ENXIO); - goto out_zv_locked; + goto out_locked; } ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_DEV); @@ -954,6 +936,13 @@ retry: mutex_exit(&zv->zv_state_lock); rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); mutex_enter(&zv->zv_state_lock); + + if (unlikely(zv->zv_flags & ZVOL_REMOVING)) { + /* Removal started while locks were down. */ + err = SET_ERROR(ENXIO); + goto out_locked; + } + /* Check to see if zv_suspend_lock is needed. */ if (zv->zv_open_count != 0) { rw_exit(&zv->zv_suspend_lock); @@ -961,7 +950,6 @@ retry: } } } - rw_exit(&zvol_state_lock); ASSERT(MUTEX_HELD(&zv->zv_state_lock)); @@ -989,7 +977,7 @@ retry: if (drop_namespace) mutex_exit(&spa_namespace_lock); if (err) - goto out_zv_locked; + goto out_locked; } ASSERT(MUTEX_HELD(&zv->zv_state_lock)); @@ -1016,9 +1004,8 @@ out_opened: zvol_last_close(zv); wakeup(zv); } -out_zv_locked: - mutex_exit(&zv->zv_state_lock); out_locked: + mutex_exit(&zv->zv_state_lock); if (drop_suspend) rw_exit(&zv->zv_suspend_lock); return (err); @@ -1030,12 +1017,9 @@ zvol_cdev_close(struct cdev *dev, int flags, int fmt, struct thread *td) zvol_state_t *zv; boolean_t drop_suspend = B_TRUE; - rw_enter(&zvol_state_lock, ZVOL_RW_READER); - zv = dev->si_drv2; - if (zv == NULL) { - rw_exit(&zvol_state_lock); + zv = atomic_load_ptr(&dev->si_drv2); + if (zv == NULL) return (SET_ERROR(ENXIO)); - } mutex_enter(&zv->zv_state_lock); if (zv->zv_flags & ZVOL_EXCL) { @@ -1060,6 +1044,15 @@ zvol_cdev_close(struct cdev *dev, int flags, int fmt, struct thread *td) mutex_exit(&zv->zv_state_lock); rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); mutex_enter(&zv->zv_state_lock); + + /* + * Unlike in zvol_cdev_open(), we don't check if + * removal started here, because we might be one of the + * openers that needs to be thrown out! If we're the + * last, we need to call zvol_last_close() below to + * finish cleanup. So, no special treatment for us. + */ + /* Check to see if zv_suspend_lock is needed. */ if (zv->zv_open_count != 1) { rw_exit(&zv->zv_suspend_lock); @@ -1069,7 +1062,6 @@ zvol_cdev_close(struct cdev *dev, int flags, int fmt, struct thread *td) } else { drop_suspend = B_FALSE; } - rw_exit(&zvol_state_lock); ASSERT(MUTEX_HELD(&zv->zv_state_lock)); @@ -1101,7 +1093,8 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data, int error; boolean_t sync; - zv = dev->si_drv2; + zv = atomic_load_ptr(&dev->si_drv2); + ASSERT3P(zv, !=, NULL); error = 0; KASSERT(zv->zv_open_count > 0, @@ -1162,6 +1155,7 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data, *(off_t *)data = 0; break; case DIOCGATTR: { + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); spa_t *spa = dmu_objset_spa(zv->zv_objset); struct diocgattr_arg *arg = (struct diocgattr_arg *)data; uint64_t refd, avail, usedobjs, availobjs; @@ -1186,6 +1180,7 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data, arg->value.off = refd / DEV_BSIZE; } else error = SET_ERROR(ENOIOCTL); + rw_exit(&zv->zv_suspend_lock); break; } case FIOSEEKHOLE: @@ -1196,10 +1191,12 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data, hole = (cmd == FIOSEEKHOLE); noff = *off; + rw_enter(&zv->zv_suspend_lock, ZVOL_RW_READER); lr = zfs_rangelock_enter(&zv->zv_rangelock, 0, UINT64_MAX, RL_READER); error = dmu_offset_next(zv->zv_objset, ZVOL_OBJ, hole, &noff); zfs_rangelock_exit(lr); + rw_exit(&zv->zv_suspend_lock); *off = noff; break; } @@ -1400,42 +1397,65 @@ zvol_alloc(const char *name, uint64_t volsize, uint64_t volblocksize, * Remove minor node for the specified volume. */ void -zvol_os_free(zvol_state_t *zv) +zvol_os_remove_minor(zvol_state_t *zv) { - ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock)); - ASSERT(!MUTEX_HELD(&zv->zv_state_lock)); + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); ASSERT0(zv->zv_open_count); + ASSERT0(atomic_read(&zv->zv_suspend_ref)); + ASSERT(zv->zv_flags & ZVOL_REMOVING); - ZFS_LOG(1, "ZVOL %s destroyed.", zv->zv_name); - - rw_destroy(&zv->zv_suspend_lock); - zfs_rangelock_fini(&zv->zv_rangelock); + struct zvol_state_os *zso = zv->zv_zso; + zv->zv_zso = NULL; if (zv->zv_volmode == ZFS_VOLMODE_GEOM) { - struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; - struct g_provider *pp __maybe_unused = zsg->zsg_provider; - - ASSERT0P(pp->private); + struct zvol_state_geom *zsg = &zso->zso_geom; + struct g_provider *pp = zsg->zsg_provider; + atomic_store_ptr(&pp->private, NULL); + mutex_exit(&zv->zv_state_lock); g_topology_lock(); - zvol_geom_destroy(zv); + g_wither_geom(pp->geom, ENXIO); g_topology_unlock(); } else if (zv->zv_volmode == ZFS_VOLMODE_DEV) { - struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev; + struct zvol_state_dev *zsd = &zso->zso_dev; struct cdev *dev = zsd->zsd_cdev; + if (dev != NULL) + atomic_store_ptr(&dev->si_drv2, NULL); + mutex_exit(&zv->zv_state_lock); + if (dev != NULL) { - ASSERT0P(dev->si_drv2); destroy_dev(dev); knlist_clear(&zsd->zsd_selinfo.si_note, 0); knlist_destroy(&zsd->zsd_selinfo.si_note); } } + kmem_free(zso, sizeof (struct zvol_state_os)); + + mutex_enter(&zv->zv_state_lock); +} + +void +zvol_os_free(zvol_state_t *zv) +{ + ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock)); + ASSERT(!MUTEX_HELD(&zv->zv_state_lock)); + ASSERT0(zv->zv_open_count); + ASSERT0P(zv->zv_zso); + + ASSERT0P(zv->zv_objset); + ASSERT0P(zv->zv_zilog); + ASSERT0P(zv->zv_dn); + + ZFS_LOG(1, "ZVOL %s destroyed.", zv->zv_name); + + rw_destroy(&zv->zv_suspend_lock); + zfs_rangelock_fini(&zv->zv_rangelock); + mutex_destroy(&zv->zv_state_lock); cv_destroy(&zv->zv_removing_cv); dataset_kstats_destroy(&zv->zv_kstat); - kmem_free(zv->zv_zso, sizeof (struct zvol_state_os)); kmem_free(zv, sizeof (zvol_state_t)); zvol_minors--; } @@ -1538,28 +1558,6 @@ out_doi: return (error); } -void -zvol_os_clear_private(zvol_state_t *zv) -{ - ASSERT(RW_LOCK_HELD(&zvol_state_lock)); - if (zv->zv_volmode == ZFS_VOLMODE_GEOM) { - struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom; - struct g_provider *pp = zsg->zsg_provider; - - if (pp->private == NULL) /* already cleared */ - return; - - pp->private = NULL; - ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock)); - } else if (zv->zv_volmode == ZFS_VOLMODE_DEV) { - struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev; - struct cdev *dev = zsd->zsd_cdev; - - if (dev != NULL) - dev->si_drv2 = NULL; - } -} - int zvol_os_update_volsize(zvol_state_t *zv, uint64_t volsize) { diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c index a73acdad34ae..bac166fcd89e 100644 --- a/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c +++ b/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c @@ -22,7 +22,7 @@ /* * Copyright (c) 2012, 2020 by Delphix. All rights reserved. * Copyright (c) 2024, Rob Norris <robn@despairlabs.com> - * Copyright (c) 2024, Klara, Inc. + * Copyright (c) 2024, 2025, Klara, Inc. */ #include <sys/dataset_kstats.h> @@ -679,28 +679,19 @@ zvol_open(struct block_device *bdev, fmode_t flag) retry: #endif - rw_enter(&zvol_state_lock, RW_READER); - /* - * Obtain a copy of private_data under the zvol_state_lock to make - * sure that either the result of zvol free code path setting - * disk->private_data to NULL is observed, or zvol_os_free() - * is not called on this zv because of the positive zv_open_count. - */ + #ifdef HAVE_BLK_MODE_T - zv = disk->private_data; + zv = atomic_load_ptr(&disk->private_data); #else - zv = bdev->bd_disk->private_data; + zv = atomic_load_ptr(&bdev->bd_disk->private_data); #endif if (zv == NULL) { - rw_exit(&zvol_state_lock); return (-SET_ERROR(ENXIO)); } mutex_enter(&zv->zv_state_lock); - if (unlikely(zv->zv_flags & ZVOL_REMOVING)) { mutex_exit(&zv->zv_state_lock); - rw_exit(&zvol_state_lock); return (-SET_ERROR(ENXIO)); } @@ -712,8 +703,28 @@ retry: if (zv->zv_open_count == 0) { if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) { mutex_exit(&zv->zv_state_lock); + + /* + * Removal may happen while the locks are down, so + * we can't trust zv any longer; we have to start over. + */ +#ifdef HAVE_BLK_MODE_T + zv = atomic_load_ptr(&disk->private_data); +#else + zv = atomic_load_ptr(&bdev->bd_disk->private_data); +#endif + if (zv == NULL) + return (-SET_ERROR(ENXIO)); + rw_enter(&zv->zv_suspend_lock, RW_READER); mutex_enter(&zv->zv_state_lock); + + if (unlikely(zv->zv_flags & ZVOL_REMOVING)) { + mutex_exit(&zv->zv_state_lock); + rw_exit(&zv->zv_suspend_lock); + return (-SET_ERROR(ENXIO)); + } + /* check to see if zv_suspend_lock is needed */ if (zv->zv_open_count != 0) { rw_exit(&zv->zv_suspend_lock); @@ -724,7 +735,6 @@ retry: drop_suspend = B_TRUE; } } - rw_exit(&zvol_state_lock); ASSERT(MUTEX_HELD(&zv->zv_state_lock)); @@ -821,11 +831,11 @@ zvol_release(struct gendisk *disk, fmode_t unused) #if !defined(HAVE_BLOCK_DEVICE_OPERATIONS_RELEASE_1ARG) (void) unused; #endif - zvol_state_t *zv; boolean_t drop_suspend = B_TRUE; - rw_enter(&zvol_state_lock, RW_READER); - zv = disk->private_data; + zvol_state_t *zv = atomic_load_ptr(&disk->private_data); + if (zv == NULL) + return; mutex_enter(&zv->zv_state_lock); ASSERT3U(zv->zv_open_count, >, 0); @@ -839,6 +849,15 @@ zvol_release(struct gendisk *disk, fmode_t unused) mutex_exit(&zv->zv_state_lock); rw_enter(&zv->zv_suspend_lock, RW_READER); mutex_enter(&zv->zv_state_lock); + + /* + * Unlike in zvol_open(), we don't check if removal + * started here, because we might be one of the openers + * that needs to be thrown out! If we're the last, we + * need to call zvol_last_close() below to finish + * cleanup. So, no special treatment for us. + */ + /* check to see if zv_suspend_lock is needed */ if (zv->zv_open_count != 1) { rw_exit(&zv->zv_suspend_lock); @@ -848,7 +867,6 @@ zvol_release(struct gendisk *disk, fmode_t unused) } else { drop_suspend = B_FALSE; } - rw_exit(&zvol_state_lock); ASSERT(MUTEX_HELD(&zv->zv_state_lock)); @@ -868,9 +886,10 @@ static int zvol_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) { - zvol_state_t *zv = bdev->bd_disk->private_data; int error = 0; + zvol_state_t *zv = atomic_load_ptr(&bdev->bd_disk->private_data); + ASSERT3P(zv, !=, NULL); ASSERT3U(zv->zv_open_count, >, 0); switch (cmd) { @@ -923,9 +942,8 @@ zvol_check_events(struct gendisk *disk, unsigned int clearing) { unsigned int mask = 0; - rw_enter(&zvol_state_lock, RW_READER); + zvol_state_t *zv = atomic_load_ptr(&disk->private_data); - zvol_state_t *zv = disk->private_data; if (zv != NULL) { mutex_enter(&zv->zv_state_lock); mask = zv->zv_changed ? DISK_EVENT_MEDIA_CHANGE : 0; @@ -933,17 +951,14 @@ zvol_check_events(struct gendisk *disk, unsigned int clearing) mutex_exit(&zv->zv_state_lock); } - rw_exit(&zvol_state_lock); - return (mask); } static int zvol_revalidate_disk(struct gendisk *disk) { - rw_enter(&zvol_state_lock, RW_READER); + zvol_state_t *zv = atomic_load_ptr(&disk->private_data); - zvol_state_t *zv = disk->private_data; if (zv != NULL) { mutex_enter(&zv->zv_state_lock); set_capacity(zv->zv_zso->zvo_disk, @@ -951,8 +966,6 @@ zvol_revalidate_disk(struct gendisk *disk) mutex_exit(&zv->zv_state_lock); } - rw_exit(&zvol_state_lock); - return (0); } @@ -971,16 +984,6 @@ zvol_os_update_volsize(zvol_state_t *zv, uint64_t volsize) return (0); } -void -zvol_os_clear_private(zvol_state_t *zv) -{ - /* - * Cleared while holding zvol_state_lock as a writer - * which will prevent zvol_open() from opening it. - */ - zv->zv_zso->zvo_disk->private_data = NULL; -} - /* * Provide a simple virtual geometry for legacy compatibility. For devices * smaller than 1 MiB a small head and sector count is used to allow very @@ -990,9 +993,10 @@ zvol_os_clear_private(zvol_state_t *zv) static int zvol_getgeo(struct block_device *bdev, struct hd_geometry *geo) { - zvol_state_t *zv = bdev->bd_disk->private_data; sector_t sectors; + zvol_state_t *zv = atomic_load_ptr(&bdev->bd_disk->private_data); + ASSERT3P(zv, !=, NULL); ASSERT3U(zv->zv_open_count, >, 0); sectors = get_capacity(zv->zv_zso->zvo_disk); @@ -1417,53 +1421,70 @@ out_kmem: return (ret); } -/* - * Cleanup then free a zvol_state_t which was created by zvol_alloc(). - * At this time, the structure is not opened by anyone, is taken off - * the zvol_state_list, and has its private data set to NULL. - * The zvol_state_lock is dropped. - * - * This function may take many milliseconds to complete (e.g. we've seen - * it take over 256ms), due to the calls to "blk_cleanup_queue" and - * "del_gendisk". Thus, consumers need to be careful to account for this - * latency when calling this function. - */ void -zvol_os_free(zvol_state_t *zv) +zvol_os_remove_minor(zvol_state_t *zv) { - - ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock)); - ASSERT(!MUTEX_HELD(&zv->zv_state_lock)); + ASSERT(MUTEX_HELD(&zv->zv_state_lock)); ASSERT0(zv->zv_open_count); - ASSERT0P(zv->zv_zso->zvo_disk->private_data); + ASSERT0(atomic_read(&zv->zv_suspend_ref)); + ASSERT(zv->zv_flags & ZVOL_REMOVING); - rw_destroy(&zv->zv_suspend_lock); - zfs_rangelock_fini(&zv->zv_rangelock); + struct zvol_state_os *zso = zv->zv_zso; + zv->zv_zso = NULL; + + /* Clearing private_data will make new callers return immediately. */ + atomic_store_ptr(&zso->zvo_disk->private_data, NULL); + + /* + * Drop the state lock before calling del_gendisk(). There may be + * callers waiting to acquire it, but del_gendisk() will block until + * they exit, which would deadlock. + */ + mutex_exit(&zv->zv_state_lock); - del_gendisk(zv->zv_zso->zvo_disk); + del_gendisk(zso->zvo_disk); #if defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS) && \ (defined(HAVE_BLK_ALLOC_DISK) || defined(HAVE_BLK_ALLOC_DISK_2ARG)) #if defined(HAVE_BLK_CLEANUP_DISK) - blk_cleanup_disk(zv->zv_zso->zvo_disk); + blk_cleanup_disk(zso->zvo_disk); #else - put_disk(zv->zv_zso->zvo_disk); + put_disk(zso->zvo_disk); #endif #else - blk_cleanup_queue(zv->zv_zso->zvo_queue); - put_disk(zv->zv_zso->zvo_disk); + blk_cleanup_queue(zso->zvo_queue); + put_disk(zso->zvo_disk); #endif - if (zv->zv_zso->use_blk_mq) - blk_mq_free_tag_set(&zv->zv_zso->tag_set); + if (zso->use_blk_mq) + blk_mq_free_tag_set(&zso->tag_set); + + ida_simple_remove(&zvol_ida, MINOR(zso->zvo_dev) >> ZVOL_MINOR_BITS); - ida_simple_remove(&zvol_ida, - MINOR(zv->zv_zso->zvo_dev) >> ZVOL_MINOR_BITS); + kmem_free(zso, sizeof (struct zvol_state_os)); + + mutex_enter(&zv->zv_state_lock); +} + +void +zvol_os_free(zvol_state_t *zv) +{ + + ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock)); + ASSERT(!MUTEX_HELD(&zv->zv_state_lock)); + ASSERT0(zv->zv_open_count); + ASSERT0P(zv->zv_zso); + + ASSERT0P(zv->zv_objset); + ASSERT0P(zv->zv_zilog); + ASSERT0P(zv->zv_dn); + + rw_destroy(&zv->zv_suspend_lock); + zfs_rangelock_fini(&zv->zv_rangelock); cv_destroy(&zv->zv_removing_cv); mutex_destroy(&zv->zv_state_lock); dataset_kstats_destroy(&zv->zv_kstat); - kmem_free(zv->zv_zso, sizeof (struct zvol_state_os)); kmem_free(zv, sizeof (zvol_state_t)); } diff --git a/sys/contrib/openzfs/module/zcommon/simd_stat.c b/sys/contrib/openzfs/module/zcommon/simd_stat.c index 11e2080ff9f2..007ae9e4fbbc 100644 --- a/sys/contrib/openzfs/module/zcommon/simd_stat.c +++ b/sys/contrib/openzfs/module/zcommon/simd_stat.c @@ -118,6 +118,10 @@ simd_stat_kstat_data(char *buf, size_t size, void *data) "pclmulqdq", zfs_pclmulqdq_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, "movbe", zfs_movbe_available()); + off += SIMD_STAT_PRINT(simd_stat_kstat_payload, + "vaes", zfs_vaes_available()); + off += SIMD_STAT_PRINT(simd_stat_kstat_payload, + "vpclmulqdq", zfs_vpclmulqdq_available()); off += SIMD_STAT_PRINT(simd_stat_kstat_payload, "osxsave", boot_cpu_has(X86_FEATURE_OSXSAVE)); diff --git a/sys/contrib/openzfs/module/zfs/dbuf.c b/sys/contrib/openzfs/module/zfs/dbuf.c index 3d0f88b36336..7403f10d91b7 100644 --- a/sys/contrib/openzfs/module/zfs/dbuf.c +++ b/sys/contrib/openzfs/module/zfs/dbuf.c @@ -2557,12 +2557,13 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) /* * Due to our use of dn_nlevels below, this can only be called - * in open context, unless we are operating on the MOS. - * From syncing context, dn_nlevels may be different from the - * dn_nlevels used when dbuf was dirtied. + * in open context, unless we are operating on the MOS or it's + * a special object. From syncing context, dn_nlevels may be + * different from the dn_nlevels used when dbuf was dirtied. */ ASSERT(db->db_objset == dmu_objset_pool(db->db_objset)->dp_meta_objset || + DMU_OBJECT_IS_SPECIAL(db->db.db_object) || txg != spa_syncing_txg(dmu_objset_spa(db->db_objset))); ASSERT(db->db_blkid != DMU_BONUS_BLKID); ASSERT0(db->db_level); diff --git a/sys/contrib/openzfs/module/zfs/dsl_deadlist.c b/sys/contrib/openzfs/module/zfs/dsl_deadlist.c index 475db3c89508..41ac72bf1c16 100644 --- a/sys/contrib/openzfs/module/zfs/dsl_deadlist.c +++ b/sys/contrib/openzfs/module/zfs/dsl_deadlist.c @@ -1049,7 +1049,8 @@ dsl_livelist_iterate(void *arg, const blkptr_t *bp, boolean_t bp_freed, ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(&found->le_bp)); ASSERT3U(BP_GET_CHECKSUM(bp), ==, BP_GET_CHECKSUM(&found->le_bp)); - ASSERT3U(BP_GET_BIRTH(bp), ==, BP_GET_BIRTH(&found->le_bp)); + ASSERT3U(BP_GET_PHYSICAL_BIRTH(bp), ==, + BP_GET_PHYSICAL_BIRTH(&found->le_bp)); } if (bp_freed) { if (found == NULL) { diff --git a/sys/contrib/openzfs/module/zfs/multilist.c b/sys/contrib/openzfs/module/zfs/multilist.c index 7b85d19e19ee..46fb79269310 100644 --- a/sys/contrib/openzfs/module/zfs/multilist.c +++ b/sys/contrib/openzfs/module/zfs/multilist.c @@ -81,7 +81,7 @@ multilist_create_impl(multilist_t *ml, size_t size, size_t offset, ml->ml_num_sublists = num; ml->ml_index_func = index_func; - ml->ml_sublists = kmem_zalloc(sizeof (multilist_sublist_t) * + ml->ml_sublists = vmem_zalloc(sizeof (multilist_sublist_t) * ml->ml_num_sublists, KM_SLEEP); ASSERT3P(ml->ml_sublists, !=, NULL); @@ -134,7 +134,7 @@ multilist_destroy(multilist_t *ml) } ASSERT3P(ml->ml_sublists, !=, NULL); - kmem_free(ml->ml_sublists, + vmem_free(ml->ml_sublists, sizeof (multilist_sublist_t) * ml->ml_num_sublists); ml->ml_num_sublists = 0; diff --git a/sys/contrib/openzfs/module/zfs/spa_config.c b/sys/contrib/openzfs/module/zfs/spa_config.c index 7d4d06659146..cf28955b0c50 100644 --- a/sys/contrib/openzfs/module/zfs/spa_config.c +++ b/sys/contrib/openzfs/module/zfs/spa_config.c @@ -48,18 +48,17 @@ /* * Pool configuration repository. * - * Pool configuration is stored as a packed nvlist on the filesystem. By - * default, all pools are stored in /etc/zfs/zpool.cache and loaded on boot - * (when the ZFS module is loaded). Pools can also have the 'cachefile' - * property set that allows them to be stored in an alternate location until - * the control of external software. + * Pool configuration is stored as a packed nvlist on the filesystem. When + * pools are imported they are added to the /etc/zfs/zpool.cache file and + * removed from it when exported. For each cache file, we have a single nvlist + * which holds all the configuration information. Pools can also have the + * 'cachefile' property set which allows this config to be stored in an + * alternate location under the control of external software. * - * For each cache file, we have a single nvlist which holds all the - * configuration information. When the module loads, we read this information - * from /etc/zfs/zpool.cache and populate the SPA namespace. This namespace is - * maintained independently in spa.c. Whenever the namespace is modified, or - * the configuration of a pool is changed, we call spa_write_cachefile(), which - * walks through all the active pools and writes the configuration to disk. + * The kernel independantly maintains an AVL tree of imported pools. See the + * "SPA locking" comment in spa.c. Whenever a pool configuration is modified + * we call spa_write_cachefile() which walks through all the active pools and + * writes the updated configuration to to /etc/zfs/zpool.cache file. */ static uint64_t spa_config_generation = 1; @@ -69,94 +68,6 @@ static uint64_t spa_config_generation = 1; * userland pools when doing testing. */ char *spa_config_path = (char *)ZPOOL_CACHE; -#ifdef _KERNEL -static int zfs_autoimport_disable = B_TRUE; -#endif - -/* - * Called when the module is first loaded, this routine loads the configuration - * file into the SPA namespace. It does not actually open or load the pools; it - * only populates the namespace. - */ -void -spa_config_load(void) -{ - void *buf = NULL; - nvlist_t *nvlist, *child; - nvpair_t *nvpair; - char *pathname; - zfs_file_t *fp; - zfs_file_attr_t zfa; - uint64_t fsize; - int err; - -#ifdef _KERNEL - if (zfs_autoimport_disable) - return; -#endif - - /* - * Open the configuration file. - */ - pathname = kmem_alloc(MAXPATHLEN, KM_SLEEP); - - (void) snprintf(pathname, MAXPATHLEN, "%s", spa_config_path); - - err = zfs_file_open(pathname, O_RDONLY, 0, &fp); - -#ifdef __FreeBSD__ - if (err) - err = zfs_file_open(ZPOOL_CACHE_BOOT, O_RDONLY, 0, &fp); -#endif - kmem_free(pathname, MAXPATHLEN); - - if (err) - return; - - if (zfs_file_getattr(fp, &zfa)) - goto out; - - fsize = zfa.zfa_size; - buf = kmem_alloc(fsize, KM_SLEEP); - - /* - * Read the nvlist from the file. - */ - if (zfs_file_read(fp, buf, fsize, NULL) < 0) - goto out; - - /* - * Unpack the nvlist. - */ - if (nvlist_unpack(buf, fsize, &nvlist, KM_SLEEP) != 0) - goto out; - - /* - * Iterate over all elements in the nvlist, creating a new spa_t for - * each one with the specified configuration. - */ - mutex_enter(&spa_namespace_lock); - nvpair = NULL; - while ((nvpair = nvlist_next_nvpair(nvlist, nvpair)) != NULL) { - if (nvpair_type(nvpair) != DATA_TYPE_NVLIST) - continue; - - child = fnvpair_value_nvlist(nvpair); - - if (spa_lookup(nvpair_name(nvpair)) != NULL) - continue; - (void) spa_add(nvpair_name(nvpair), child, NULL); - } - mutex_exit(&spa_namespace_lock); - - nvlist_free(nvlist); - -out: - if (buf != NULL) - kmem_free(buf, fsize); - - zfs_file_close(fp); -} static int spa_config_remove(spa_config_dirent_t *dp) @@ -623,7 +534,6 @@ spa_config_update(spa_t *spa, int what) spa_config_update(spa, SPA_CONFIG_UPDATE_VDEVS); } -EXPORT_SYMBOL(spa_config_load); EXPORT_SYMBOL(spa_all_configs); EXPORT_SYMBOL(spa_config_set); EXPORT_SYMBOL(spa_config_generate); @@ -634,8 +544,3 @@ EXPORT_SYMBOL(spa_config_update); ZFS_MODULE_PARAM(zfs_spa, spa_, config_path, STRING, ZMOD_RD, "SPA config file (/etc/zfs/zpool.cache)"); #endif - -#ifdef _KERNEL -ZFS_MODULE_PARAM(zfs, zfs_, autoimport_disable, INT, ZMOD_RW, - "Disable pool import at module load"); -#endif diff --git a/sys/contrib/openzfs/module/zfs/spa_misc.c b/sys/contrib/openzfs/module/zfs/spa_misc.c index cce772eae598..dceafbc27556 100644 --- a/sys/contrib/openzfs/module/zfs/spa_misc.c +++ b/sys/contrib/openzfs/module/zfs/spa_misc.c @@ -2548,13 +2548,6 @@ spa_name_compare(const void *a1, const void *a2) } void -spa_boot_init(void *unused) -{ - (void) unused; - spa_config_load(); -} - -void spa_init(spa_mode_t mode) { mutex_init(&spa_namespace_lock, NULL, MUTEX_DEFAULT, NULL); @@ -2607,7 +2600,6 @@ spa_init(spa_mode_t mode) chksum_init(); zpool_prop_init(); zpool_feature_init(); - spa_config_load(); vdev_prop_init(); l2arc_start(); scan_init(); diff --git a/sys/contrib/openzfs/module/zfs/zil.c b/sys/contrib/openzfs/module/zfs/zil.c index 31b59c55f17b..0307df55aa21 100644 --- a/sys/contrib/openzfs/module/zfs/zil.c +++ b/sys/contrib/openzfs/module/zfs/zil.c @@ -819,34 +819,37 @@ zil_lwb_vdev_compare(const void *x1, const void *x2) * we choose them here and later make the block allocation match. */ static lwb_t * -zil_alloc_lwb(zilog_t *zilog, int sz, blkptr_t *bp, boolean_t slog, - uint64_t txg, lwb_state_t state) +zil_alloc_lwb(zilog_t *zilog, blkptr_t *bp, int min_sz, int sz, + boolean_t slog, uint64_t txg) { lwb_t *lwb; lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); + lwb->lwb_flags = 0; lwb->lwb_zilog = zilog; if (bp) { lwb->lwb_blk = *bp; - lwb->lwb_slim = (BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_ZILOG2); + if (BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_ZILOG2) + lwb->lwb_flags |= LWB_FLAG_SLIM; sz = BP_GET_LSIZE(bp); + lwb->lwb_min_sz = sz; } else { BP_ZERO(&lwb->lwb_blk); - lwb->lwb_slim = (spa_version(zilog->zl_spa) >= - SPA_VERSION_SLIM_ZIL); + if (spa_version(zilog->zl_spa) >= SPA_VERSION_SLIM_ZIL) + lwb->lwb_flags |= LWB_FLAG_SLIM; + lwb->lwb_min_sz = min_sz; } - lwb->lwb_slog = slog; + if (slog) + lwb->lwb_flags |= LWB_FLAG_SLOG; lwb->lwb_error = 0; - if (lwb->lwb_slim) { - lwb->lwb_nmax = sz; - lwb->lwb_nused = lwb->lwb_nfilled = sizeof (zil_chain_t); - } else { - lwb->lwb_nmax = sz - sizeof (zil_chain_t); - lwb->lwb_nused = lwb->lwb_nfilled = 0; - } + /* + * Buffer allocation and capacity setup will be done in + * zil_lwb_write_open() when the LWB is opened for ITX assignment. + */ + lwb->lwb_nmax = lwb->lwb_nused = lwb->lwb_nfilled = 0; lwb->lwb_sz = sz; - lwb->lwb_state = state; - lwb->lwb_buf = zio_buf_alloc(sz); + lwb->lwb_buf = NULL; + lwb->lwb_state = LWB_STATE_NEW; lwb->lwb_child_zio = NULL; lwb->lwb_write_zio = NULL; lwb->lwb_root_zio = NULL; @@ -857,8 +860,6 @@ zil_alloc_lwb(zilog_t *zilog, int sz, blkptr_t *bp, boolean_t slog, mutex_enter(&zilog->zl_lock); list_insert_tail(&zilog->zl_lwb_list, lwb); - if (state != LWB_STATE_NEW) - zilog->zl_last_lwb_opened = lwb; mutex_exit(&zilog->zl_lock); return (lwb); @@ -878,7 +879,7 @@ zil_free_lwb(zilog_t *zilog, lwb_t *lwb) VERIFY(list_is_empty(&lwb->lwb_itxs)); VERIFY(list_is_empty(&lwb->lwb_waiters)); ASSERT(avl_is_empty(&lwb->lwb_vdev_tree)); - ASSERT(!MUTEX_HELD(&lwb->lwb_vdev_lock)); + ASSERT(!MUTEX_HELD(&lwb->lwb_lock)); /* * Clear the zilog's field to indicate this lwb is no longer @@ -1019,7 +1020,7 @@ zil_create(zilog_t *zilog) } error = zio_alloc_zil(zilog->zl_spa, zilog->zl_os, txg, &blk, - ZIL_MIN_BLKSZ, &slog); + ZIL_MIN_BLKSZ, ZIL_MIN_BLKSZ, &slog, B_TRUE); if (error == 0) zil_init_log_chain(zilog, &blk); } @@ -1028,7 +1029,7 @@ zil_create(zilog_t *zilog) * Allocate a log write block (lwb) for the first log block. */ if (error == 0) - lwb = zil_alloc_lwb(zilog, 0, &blk, slog, txg, LWB_STATE_NEW); + lwb = zil_alloc_lwb(zilog, &blk, 0, 0, slog, txg); /* * If we just allocated the first log block, commit our transaction @@ -1324,10 +1325,12 @@ zil_check_log_chain(dsl_pool_t *dp, dsl_dataset_t *ds, void *tx) * zil_commit() is racing with spa_sync(). */ static void -zil_commit_waiter_skip(zil_commit_waiter_t *zcw) +zil_commit_waiter_done(zil_commit_waiter_t *zcw, int err) { mutex_enter(&zcw->zcw_lock); ASSERT3B(zcw->zcw_done, ==, B_FALSE); + zcw->zcw_lwb = NULL; + zcw->zcw_error = err; zcw->zcw_done = B_TRUE; cv_broadcast(&zcw->zcw_cv); mutex_exit(&zcw->zcw_lock); @@ -1389,7 +1392,7 @@ zil_lwb_add_block(lwb_t *lwb, const blkptr_t *bp) if (zil_nocacheflush) return; - mutex_enter(&lwb->lwb_vdev_lock); + mutex_enter(&lwb->lwb_lock); for (i = 0; i < ndvas; i++) { zvsearch.zv_vdev = DVA_GET_VDEV(&bp->blk_dva[i]); if (avl_find(t, &zvsearch, &where) == NULL) { @@ -1398,7 +1401,7 @@ zil_lwb_add_block(lwb_t *lwb, const blkptr_t *bp) avl_insert(t, zv, where); } } - mutex_exit(&lwb->lwb_vdev_lock); + mutex_exit(&lwb->lwb_lock); } static void @@ -1415,12 +1418,12 @@ zil_lwb_flush_defer(lwb_t *lwb, lwb_t *nlwb) /* * While 'lwb' is at a point in its lifetime where lwb_vdev_tree does - * not need the protection of lwb_vdev_lock (it will only be modified + * not need the protection of lwb_lock (it will only be modified * while holding zilog->zl_lock) as its writes and those of its * children have all completed. The younger 'nlwb' may be waiting on * future writes to additional vdevs. */ - mutex_enter(&nlwb->lwb_vdev_lock); + mutex_enter(&nlwb->lwb_lock); /* * Tear down the 'lwb' vdev tree, ensuring that entries which do not * exist in 'nlwb' are moved to it, freeing any would-be duplicates. @@ -1434,7 +1437,7 @@ zil_lwb_flush_defer(lwb_t *lwb, lwb_t *nlwb) kmem_free(zv, sizeof (*zv)); } } - mutex_exit(&nlwb->lwb_vdev_lock); + mutex_exit(&nlwb->lwb_lock); } void @@ -1491,10 +1494,6 @@ zil_lwb_flush_vdevs_done(zio_t *zio) zil_itx_destroy(itx, 0); while ((zcw = list_remove_head(&lwb->lwb_waiters)) != NULL) { - mutex_enter(&zcw->zcw_lock); - - ASSERT3P(zcw->zcw_lwb, ==, lwb); - zcw->zcw_lwb = NULL; /* * We expect any ZIO errors from child ZIOs to have been * propagated "up" to this specific LWB's root ZIO, in @@ -1509,14 +1508,7 @@ zil_lwb_flush_vdevs_done(zio_t *zio) * errors not being handled correctly here. See the * comment above the call to "zio_flush" for details. */ - - zcw->zcw_zio_error = zio->io_error; - - ASSERT3B(zcw->zcw_done, ==, B_FALSE); - zcw->zcw_done = B_TRUE; - cv_broadcast(&zcw->zcw_cv); - - mutex_exit(&zcw->zcw_lock); + zil_commit_waiter_done(zcw, zio->io_error); } uint64_t txg = lwb->lwb_issued_txg; @@ -1588,7 +1580,7 @@ zil_lwb_write_done(zio_t *zio) avl_tree_t *t = &lwb->lwb_vdev_tree; void *cookie = NULL; zil_vdev_node_t *zv; - lwb_t *nlwb; + lwb_t *nlwb = NULL; ASSERT3S(spa_config_held(spa, SCL_STATE, RW_READER), !=, 0); @@ -1608,9 +1600,11 @@ zil_lwb_write_done(zio_t *zio) * its write ZIO a parent this ZIO. In such case we can not defer * our flushes or below may be a race between the done callbacks. */ - nlwb = list_next(&zilog->zl_lwb_list, lwb); - if (nlwb && nlwb->lwb_state != LWB_STATE_ISSUED) - nlwb = NULL; + if (!(lwb->lwb_flags & LWB_FLAG_CRASHED)) { + nlwb = list_next(&zilog->zl_lwb_list, lwb); + if (nlwb && nlwb->lwb_state != LWB_STATE_ISSUED) + nlwb = NULL; + } mutex_exit(&zilog->zl_lock); if (avl_numnodes(t) == 0) @@ -1624,12 +1618,17 @@ zil_lwb_write_done(zio_t *zio) * written out. * * Additionally, we don't perform any further error handling at - * this point (e.g. setting "zcw_zio_error" appropriately), as - * we expect that to occur in "zil_lwb_flush_vdevs_done" (thus, - * we expect any error seen here, to have been propagated to - * that function). + * this point (e.g. setting "zcw_error" appropriately), as we + * expect that to occur in "zil_lwb_flush_vdevs_done" (thus, we + * expect any error seen here, to have been propagated to that + * function). + * + * Note that we treat a "crashed" LWB as though it was in error, + * even if it did appear to succeed, because we've already + * signaled error and cleaned up waiters and committers in + * zil_crash(); we just want to clean up and get out of here. */ - if (zio->io_error != 0) { + if (zio->io_error != 0 || (lwb->lwb_flags & LWB_FLAG_CRASHED)) { while ((zv = avl_destroy_nodes(t, &cookie)) != NULL) kmem_free(zv, sizeof (*zv)); return; @@ -1742,10 +1741,26 @@ zil_lwb_write_open(zilog_t *zilog, lwb_t *lwb) return; } + mutex_enter(&lwb->lwb_lock); mutex_enter(&zilog->zl_lock); lwb->lwb_state = LWB_STATE_OPENED; zilog->zl_last_lwb_opened = lwb; mutex_exit(&zilog->zl_lock); + mutex_exit(&lwb->lwb_lock); + + /* + * Allocate buffer and set up LWB capacities. + */ + ASSERT0P(lwb->lwb_buf); + ASSERT3U(lwb->lwb_sz, >, 0); + lwb->lwb_buf = zio_buf_alloc(lwb->lwb_sz); + if (lwb->lwb_flags & LWB_FLAG_SLIM) { + lwb->lwb_nmax = lwb->lwb_sz; + lwb->lwb_nused = lwb->lwb_nfilled = sizeof (zil_chain_t); + } else { + lwb->lwb_nmax = lwb->lwb_sz - sizeof (zil_chain_t); + lwb->lwb_nused = lwb->lwb_nfilled = 0; + } } /* @@ -1762,6 +1777,8 @@ static uint_t zil_lwb_plan(zilog_t *zilog, uint64_t size, uint_t *minsize) { uint_t md = zilog->zl_max_block_size - sizeof (zil_chain_t); + uint_t waste = zil_max_waste_space(zilog); + waste = MAX(waste, zilog->zl_cur_max); if (size <= md) { /* @@ -1772,9 +1789,10 @@ zil_lwb_plan(zilog_t *zilog, uint64_t size, uint_t *minsize) } else if (size > 8 * md) { /* * Big bursts use maximum blocks. The first block size - * is hard to predict, but it does not really matter. + * is hard to predict, but we need at least enough space + * to make reasonable progress. */ - *minsize = 0; + *minsize = waste; return (md); } @@ -1787,57 +1805,52 @@ zil_lwb_plan(zilog_t *zilog, uint64_t size, uint_t *minsize) uint_t s = size; uint_t n = DIV_ROUND_UP(s, md - sizeof (lr_write_t)); uint_t chunk = DIV_ROUND_UP(s, n); - uint_t waste = zil_max_waste_space(zilog); - waste = MAX(waste, zilog->zl_cur_max); if (chunk <= md - waste) { *minsize = MAX(s - (md - waste) * (n - 1), waste); return (chunk); } else { - *minsize = 0; + *minsize = waste; return (md); } } /* * Try to predict next block size based on previous history. Make prediction - * sufficient for 7 of 8 previous bursts. Don't try to save if the saving is - * less then 50%, extra writes may cost more, but we don't want single spike - * to badly affect our predictions. + * sufficient for 7 of 8 previous bursts, but don't try to save if the saving + * is less then 50%. Extra writes may cost more, but we don't want single + * spike to badly affect our predictions. */ -static uint_t -zil_lwb_predict(zilog_t *zilog) +static void +zil_lwb_predict(zilog_t *zilog, uint64_t *min_predict, uint64_t *max_predict) { - uint_t m, o; + uint_t m1 = 0, m2 = 0, o; - /* If we are in the middle of a burst, take it into account also. */ - if (zilog->zl_cur_size > 0) { - o = zil_lwb_plan(zilog, zilog->zl_cur_size, &m); - } else { + /* If we are in the middle of a burst, take it as another data point. */ + if (zilog->zl_cur_size > 0) + o = zil_lwb_plan(zilog, zilog->zl_cur_size, &m1); + else o = UINT_MAX; - m = 0; - } - /* Find minimum optimal size. We don't need to go below that. */ - for (int i = 0; i < ZIL_BURSTS; i++) - o = MIN(o, zilog->zl_prev_opt[i]); - - /* Find two biggest minimal first block sizes above the optimal. */ - uint_t m1 = MAX(m, o), m2 = o; + /* Find two largest minimal first block sizes. */ for (int i = 0; i < ZIL_BURSTS; i++) { - m = zilog->zl_prev_min[i]; - if (m >= m1) { + uint_t cur = zilog->zl_prev_min[i]; + if (cur >= m1) { m2 = m1; - m1 = m; - } else if (m > m2) { - m2 = m; + m1 = cur; + } else if (cur > m2) { + m2 = cur; } } - /* - * If second minimum size gives 50% saving -- use it. It may cost us - * one additional write later, but the space saving is just too big. - */ - return ((m1 < m2 * 2) ? m1 : m2); + /* Minimum should guarantee progress in most cases. */ + *min_predict = (m1 < m2 * 2) ? m1 : m2; + + /* Maximum doesn't need to go below the minimum optimal size. */ + for (int i = 0; i < ZIL_BURSTS; i++) + o = MIN(o, zilog->zl_prev_opt[i]); + m1 = MAX(m1, o); + m2 = MAX(m2, o); + *max_predict = (m1 < m2 * 2) ? m1 : m2; } /* @@ -1845,12 +1858,13 @@ zil_lwb_predict(zilog_t *zilog) * Has to be called under zl_issuer_lock to chain more lwbs. */ static lwb_t * -zil_lwb_write_close(zilog_t *zilog, lwb_t *lwb, lwb_state_t state) +zil_lwb_write_close(zilog_t *zilog, lwb_t *lwb) { - uint64_t blksz, plan, plan2; + uint64_t minbs, maxbs; ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock)); ASSERT3S(lwb->lwb_state, ==, LWB_STATE_OPENED); + membar_producer(); lwb->lwb_state = LWB_STATE_CLOSED; /* @@ -1875,27 +1889,34 @@ zil_lwb_write_close(zilog_t *zilog, lwb_t *lwb, lwb_state_t state) * Try to predict what can it be and plan for the worst case. */ uint_t m; - plan = zil_lwb_plan(zilog, zilog->zl_cur_left, &m); + maxbs = zil_lwb_plan(zilog, zilog->zl_cur_left, &m); + minbs = m; if (zilog->zl_parallel) { - plan2 = zil_lwb_plan(zilog, zilog->zl_cur_left + - zil_lwb_predict(zilog), &m); - if (plan < plan2) - plan = plan2; + uint64_t minp, maxp; + zil_lwb_predict(zilog, &minp, &maxp); + maxp = zil_lwb_plan(zilog, zilog->zl_cur_left + maxp, + &m); + if (maxbs < maxp) + maxbs = maxp; } } else { /* * The previous burst is done and we can only predict what * will come next. */ - plan = zil_lwb_predict(zilog); + zil_lwb_predict(zilog, &minbs, &maxbs); } - blksz = plan + sizeof (zil_chain_t); - blksz = P2ROUNDUP_TYPED(blksz, ZIL_MIN_BLKSZ, uint64_t); - blksz = MIN(blksz, zilog->zl_max_block_size); - DTRACE_PROBE3(zil__block__size, zilog_t *, zilog, uint64_t, blksz, - uint64_t, plan); - return (zil_alloc_lwb(zilog, blksz, NULL, 0, 0, state)); + minbs += sizeof (zil_chain_t); + maxbs += sizeof (zil_chain_t); + minbs = P2ROUNDUP_TYPED(minbs, ZIL_MIN_BLKSZ, uint64_t); + maxbs = P2ROUNDUP_TYPED(maxbs, ZIL_MIN_BLKSZ, uint64_t); + maxbs = MIN(maxbs, zilog->zl_max_block_size); + minbs = MIN(minbs, maxbs); + DTRACE_PROBE3(zil__block__size, zilog_t *, zilog, uint64_t, minbs, + uint64_t, maxbs); + + return (zil_alloc_lwb(zilog, NULL, minbs, maxbs, 0, 0)); } /* @@ -1944,14 +1965,16 @@ zil_lwb_write_issue(zilog_t *zilog, lwb_t *lwb) mutex_exit(&zilog->zl_lock); next_lwb: - if (lwb->lwb_slim) + if (lwb->lwb_flags & LWB_FLAG_SLIM) zilc = (zil_chain_t *)lwb->lwb_buf; else zilc = (zil_chain_t *)(lwb->lwb_buf + lwb->lwb_nmax); - int wsz = lwb->lwb_sz; + uint64_t alloc_size = BP_GET_LSIZE(&lwb->lwb_blk); + int wsz = alloc_size; if (lwb->lwb_error == 0) { abd_t *lwb_abd = abd_get_from_buf(lwb->lwb_buf, lwb->lwb_sz); - if (!lwb->lwb_slog || zilog->zl_cur_size <= zil_slog_bulk) + if (!(lwb->lwb_flags & LWB_FLAG_SLOG) || + zilog->zl_cur_size <= zil_slog_bulk) prio = ZIO_PRIORITY_SYNC_WRITE; else prio = ZIO_PRIORITY_ASYNC_WRITE; @@ -1959,16 +1982,17 @@ next_lwb: ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_SEQ]); lwb->lwb_write_zio = zio_rewrite(lwb->lwb_root_zio, spa, 0, - &lwb->lwb_blk, lwb_abd, lwb->lwb_sz, zil_lwb_write_done, + &lwb->lwb_blk, lwb_abd, alloc_size, zil_lwb_write_done, lwb, prio, ZIO_FLAG_CANFAIL, &zb); zil_lwb_add_block(lwb, &lwb->lwb_blk); - if (lwb->lwb_slim) { + if (lwb->lwb_flags & LWB_FLAG_SLIM) { /* For Slim ZIL only write what is used. */ wsz = P2ROUNDUP_TYPED(lwb->lwb_nused, ZIL_MIN_BLKSZ, int); - ASSERT3S(wsz, <=, lwb->lwb_sz); - zio_shrink(lwb->lwb_write_zio, wsz); + ASSERT3S(wsz, <=, alloc_size); + if (wsz < alloc_size) + zio_shrink(lwb->lwb_write_zio, wsz); wsz = lwb->lwb_write_zio->io_size; } memset(lwb->lwb_buf + lwb->lwb_nused, 0, wsz - lwb->lwb_nused); @@ -2004,13 +2028,53 @@ next_lwb: BP_ZERO(bp); error = lwb->lwb_error; if (error == 0) { - error = zio_alloc_zil(spa, zilog->zl_os, txg, bp, nlwb->lwb_sz, - &slog); + /* + * Allocation flexibility depends on LWB state: + * if NEW: allow range allocation and larger sizes; + * if OPENED: use fixed predetermined allocation size; + * if CLOSED + Slim: allocate precisely for actual usage. + */ + boolean_t flexible = (nlwb->lwb_state == LWB_STATE_NEW); + if (flexible) { + /* We need to prevent opening till we update lwb_sz. */ + mutex_enter(&nlwb->lwb_lock); + flexible = (nlwb->lwb_state == LWB_STATE_NEW); + if (!flexible) + mutex_exit(&nlwb->lwb_lock); /* We lost. */ + } + boolean_t closed_slim = (nlwb->lwb_state == LWB_STATE_CLOSED && + (lwb->lwb_flags & LWB_FLAG_SLIM)); + + uint64_t min_size, max_size; + if (closed_slim) { + /* This transition is racy, but only one way. */ + membar_consumer(); + min_size = max_size = P2ROUNDUP_TYPED(nlwb->lwb_nused, + ZIL_MIN_BLKSZ, uint64_t); + } else if (flexible) { + min_size = nlwb->lwb_min_sz; + max_size = nlwb->lwb_sz; + } else { + min_size = max_size = nlwb->lwb_sz; + } + + error = zio_alloc_zil(spa, zilog->zl_os, txg, bp, + min_size, max_size, &slog, flexible); + if (error == 0) { + if (closed_slim) + ASSERT3U(BP_GET_LSIZE(bp), ==, max_size); + else if (flexible) + nlwb->lwb_sz = BP_GET_LSIZE(bp); + else + ASSERT3U(BP_GET_LSIZE(bp), ==, nlwb->lwb_sz); + } + if (flexible) + mutex_exit(&nlwb->lwb_lock); } if (error == 0) { ASSERT3U(BP_GET_BIRTH(bp), ==, txg); - BP_SET_CHECKSUM(bp, nlwb->lwb_slim ? ZIO_CHECKSUM_ZILOG2 : - ZIO_CHECKSUM_ZILOG); + BP_SET_CHECKSUM(bp, (nlwb->lwb_flags & LWB_FLAG_SLIM) ? + ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG); bp->blk_cksum = lwb->lwb_blk.blk_cksum; bp->blk_cksum.zc_word[ZIL_ZC_SEQ]++; } @@ -2039,14 +2103,15 @@ next_lwb: if (nlwb) { nlwb->lwb_blk = *bp; nlwb->lwb_error = error; - nlwb->lwb_slog = slog; + if (slog) + nlwb->lwb_flags |= LWB_FLAG_SLOG; nlwb->lwb_alloc_txg = txg; if (nlwb->lwb_state != LWB_STATE_READY) nlwb = NULL; } mutex_exit(&zilog->zl_lock); - if (lwb->lwb_slog) { + if (lwb->lwb_flags & LWB_FLAG_SLOG) { ZIL_STAT_BUMP(zilog, zil_itx_metaslab_slog_count); ZIL_STAT_INCR(zilog, zil_itx_metaslab_slog_bytes, lwb->lwb_nused); @@ -2220,7 +2285,6 @@ zil_lwb_assign(zilog_t *zilog, lwb_t *lwb, itx_t *itx, list_t *ilwbs) ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock)); ASSERT3P(lwb, !=, NULL); - ASSERT3P(lwb->lwb_buf, !=, NULL); zil_lwb_write_open(zilog, lwb); @@ -2262,9 +2326,10 @@ cont: (dlen % max_log_data == 0 || lwb_sp < reclen + dlen % max_log_data))) { list_insert_tail(ilwbs, lwb); - lwb = zil_lwb_write_close(zilog, lwb, LWB_STATE_OPENED); + lwb = zil_lwb_write_close(zilog, lwb); if (lwb == NULL) return (NULL); + zil_lwb_write_open(zilog, lwb); lwb_sp = lwb->lwb_nmax - lwb->lwb_nused; } @@ -2554,7 +2619,7 @@ zil_itxg_clean(void *arg) * called) we will hit this case. */ if (itx->itx_lr.lrc_txtype == TX_COMMIT) - zil_commit_waiter_skip(itx->itx_private); + zil_commit_waiter_done(itx->itx_private, 0); zil_itx_destroy(itx, 0); } @@ -2742,6 +2807,7 @@ zil_crash_clean(zilog_t *zilog, uint64_t synced_txg) } /* This LWB is from the past, so we can clean it up now. */ + ASSERT(lwb->lwb_flags & LWB_FLAG_CRASHED); list_remove(&zilog->zl_lwb_crash_list, lwb); if (lwb->lwb_buf != NULL) zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); @@ -2981,7 +3047,7 @@ zil_prune_commit_list(zilog_t *zilog) * never any itx's for it to wait on), so it's * safe to skip this waiter and mark it done. */ - zil_commit_waiter_skip(itx->itx_private); + zil_commit_waiter_done(itx->itx_private, 0); } else { zil_commit_waiter_link_lwb(itx->itx_private, last_lwb); } @@ -3212,15 +3278,21 @@ zil_process_commit_list(zilog_t *zilog, zil_commit_waiter_t *zcw, list_t *ilwbs) * "next" lwb on-disk. When this happens, we must stall * the ZIL write pipeline; see the comment within * zil_commit_writer_stall() for more details. + * + * ESHUTDOWN has to be handled carefully here. If we get it, + * then the pool suspended and zil_crash() was called, so we + * need to stop trying and just get an error back to the + * callers. */ int err = 0; while ((lwb = list_remove_head(ilwbs)) != NULL) { - err = zil_lwb_write_issue(zilog, lwb); - if (err != 0) - break; + if (err == 0) + err = zil_lwb_write_issue(zilog, lwb); } - if (err == 0) + if (err != ESHUTDOWN) err = zil_commit_writer_stall(zilog); + if (err == ESHUTDOWN) + err = SET_ERROR(EIO); /* * Additionally, we have to signal and mark the "nolwb" @@ -3230,7 +3302,7 @@ zil_process_commit_list(zilog_t *zilog, zil_commit_waiter_t *zcw, list_t *ilwbs) */ zil_commit_waiter_t *zcw; while ((zcw = list_remove_head(&nolwb_waiters)) != NULL) - zil_commit_waiter_skip(zcw); + zil_commit_waiter_done(zcw, err); /* * And finally, we have to destroy the itx's that @@ -3238,7 +3310,7 @@ zil_process_commit_list(zilog_t *zilog, zil_commit_waiter_t *zcw, list_t *ilwbs) * the itx's callback if one exists for the itx. */ while ((itx = list_remove_head(&nolwb_itxs)) != NULL) - zil_itx_destroy(itx, 0); + zil_itx_destroy(itx, err); } else { ASSERT(list_is_empty(&nolwb_waiters)); ASSERT3P(lwb, !=, NULL); @@ -3292,17 +3364,17 @@ zil_process_commit_list(zilog_t *zilog, zil_commit_waiter_t *zcw, list_t *ilwbs) (!zilog->zl_parallel || zilog->zl_suspend > 0)) { zil_burst_done(zilog); list_insert_tail(ilwbs, lwb); - lwb = zil_lwb_write_close(zilog, lwb, LWB_STATE_NEW); + lwb = zil_lwb_write_close(zilog, lwb); if (lwb == NULL) { int err = 0; while ((lwb = list_remove_head(ilwbs)) != NULL) { - err = zil_lwb_write_issue(zilog, lwb); - if (err != 0) - break; + if (err == 0) + err = zil_lwb_write_issue( + zilog, lwb); } - if (err == 0) - zil_commit_writer_stall(zilog); + if (err != ESHUTDOWN) + (void) zil_commit_writer_stall(zilog); } } } @@ -3470,7 +3542,7 @@ zil_commit_waiter_timeout(zilog_t *zilog, zil_commit_waiter_t *zcw) * hasn't been issued. */ zil_burst_done(zilog); - lwb_t *nlwb = zil_lwb_write_close(zilog, lwb, LWB_STATE_NEW); + lwb_t *nlwb = zil_lwb_write_close(zilog, lwb); ASSERT3S(lwb->lwb_state, ==, LWB_STATE_CLOSED); @@ -3546,7 +3618,7 @@ zil_commit_waiter(zilog_t *zilog, zil_commit_waiter_t *zcw) * commit itxs. When this occurs, the commit waiters linked * off of these commit itxs will not be committed to an * lwb. Additionally, these commit waiters will not be - * marked done until zil_commit_waiter_skip() is called via + * marked done until zil_commit_waiter_done() is called via * zil_itxg_clean(). * * Thus, it's possible for this commit waiter (i.e. the @@ -3624,7 +3696,7 @@ zil_alloc_commit_waiter(void) list_link_init(&zcw->zcw_node); zcw->zcw_lwb = NULL; zcw->zcw_done = B_FALSE; - zcw->zcw_zio_error = 0; + zcw->zcw_error = 0; return (zcw); } @@ -3728,6 +3800,9 @@ zil_crash(zilog_t *zilog) */ for (lwb_t *lwb = list_head(&zilog->zl_lwb_crash_list); lwb != NULL; lwb = list_next(&zilog->zl_lwb_crash_list, lwb)) { + ASSERT(!(lwb->lwb_flags & LWB_FLAG_CRASHED)); + lwb->lwb_flags |= LWB_FLAG_CRASHED; + itx_t *itx; while ((itx = list_remove_head(&lwb->lwb_itxs)) != NULL) zil_itx_destroy(itx, EIO); @@ -3736,7 +3811,7 @@ zil_crash(zilog_t *zilog) while ((zcw = list_remove_head(&lwb->lwb_waiters)) != NULL) { mutex_enter(&zcw->zcw_lock); zcw->zcw_lwb = NULL; - zcw->zcw_zio_error = EIO; + zcw->zcw_error = EIO; zcw->zcw_done = B_TRUE; cv_broadcast(&zcw->zcw_cv); mutex_exit(&zcw->zcw_lock); @@ -4014,7 +4089,7 @@ zil_commit_impl(zilog_t *zilog, uint64_t foid) zil_commit_waiter(zilog, zcw); int err = 0; - if (zcw->zcw_zio_error != 0) { + if (zcw->zcw_error != 0) { /* * If there was an error writing out the ZIL blocks that * this thread is waiting on, then we fallback to @@ -4149,7 +4224,7 @@ zil_lwb_cons(void *vbuf, void *unused, int kmflag) offsetof(zil_commit_waiter_t, zcw_node)); avl_create(&lwb->lwb_vdev_tree, zil_lwb_vdev_compare, sizeof (zil_vdev_node_t), offsetof(zil_vdev_node_t, zv_node)); - mutex_init(&lwb->lwb_vdev_lock, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&lwb->lwb_lock, NULL, MUTEX_DEFAULT, NULL); return (0); } @@ -4158,7 +4233,7 @@ zil_lwb_dest(void *vbuf, void *unused) { (void) unused; lwb_t *lwb = vbuf; - mutex_destroy(&lwb->lwb_vdev_lock); + mutex_destroy(&lwb->lwb_lock); avl_destroy(&lwb->lwb_vdev_tree); list_destroy(&lwb->lwb_waiters); list_destroy(&lwb->lwb_itxs); @@ -4381,7 +4456,7 @@ zil_close(zilog_t *zilog) if (lwb != NULL) { ASSERT(list_is_empty(&zilog->zl_lwb_list)); ASSERT3S(lwb->lwb_state, ==, LWB_STATE_NEW); - zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); + ASSERT0P(lwb->lwb_buf); zil_free_lwb(zilog, lwb); } mutex_exit(&zilog->zl_lock); @@ -4472,16 +4547,16 @@ zil_suspend(const char *osname, void **cookiep) cv_wait(&zilog->zl_cv_suspend, &zilog->zl_lock); mutex_exit(&zilog->zl_lock); - if (cookiep == NULL) + if (zilog->zl_restart_txg > 0) { + /* ZIL crashed while we were waiting. */ + zil_resume(os); + error = SET_ERROR(EBUSY); + } else if (cookiep == NULL) zil_resume(os); else *cookiep = os; - if (zilog->zl_restart_txg > 0) - /* ZIL crashed while we were waiting. */ - return (SET_ERROR(EBUSY)); - - return (0); + return (error); } /* diff --git a/sys/contrib/openzfs/module/zfs/zio.c b/sys/contrib/openzfs/module/zfs/zio.c index 3f0ddb63249d..4cf8912d4269 100644 --- a/sys/contrib/openzfs/module/zfs/zio.c +++ b/sys/contrib/openzfs/module/zfs/zio.c @@ -4434,12 +4434,15 @@ zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp) */ int zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp, - uint64_t size, boolean_t *slog) + uint64_t min_size, uint64_t max_size, boolean_t *slog, + boolean_t allow_larger) { int error; zio_alloc_list_t io_alloc_list; + uint64_t alloc_size = 0; ASSERT(txg > spa_syncing_txg(spa)); + ASSERT3U(min_size, <=, max_size); metaslab_trace_init(&io_alloc_list); @@ -4448,7 +4451,7 @@ zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp, * Fill in the obvious ones before calling into metaslab_alloc(). */ BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); - BP_SET_PSIZE(new_bp, size); + BP_SET_PSIZE(new_bp, max_size); BP_SET_LEVEL(new_bp, 0); /* @@ -4463,43 +4466,51 @@ zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp, ZIOSTAT_BUMP(ziostat_total_allocations); /* Try log class (dedicated slog devices) first */ - error = metaslab_alloc(spa, spa_log_class(spa), size, new_bp, 1, - txg, NULL, flags, &io_alloc_list, allocator, NULL); + error = metaslab_alloc_range(spa, spa_log_class(spa), min_size, + max_size, new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator, + NULL, &alloc_size); *slog = (error == 0); /* Try special_embedded_log class (reserved on special vdevs) */ if (error != 0) { - error = metaslab_alloc(spa, spa_special_embedded_log_class(spa), - size, new_bp, 1, txg, NULL, flags, &io_alloc_list, - allocator, NULL); + error = metaslab_alloc_range(spa, + spa_special_embedded_log_class(spa), min_size, max_size, + new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator, + NULL, &alloc_size); } /* Try special class (general special vdev allocation) */ if (error != 0) { - error = metaslab_alloc(spa, spa_special_class(spa), size, - new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator, - NULL); + error = metaslab_alloc_range(spa, spa_special_class(spa), + min_size, max_size, new_bp, 1, txg, NULL, flags, + &io_alloc_list, allocator, NULL, &alloc_size); } /* Try embedded_log class (reserved on normal vdevs) */ if (error != 0) { - error = metaslab_alloc(spa, spa_embedded_log_class(spa), size, - new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator, - NULL); + error = metaslab_alloc_range(spa, spa_embedded_log_class(spa), + min_size, max_size, new_bp, 1, txg, NULL, flags, + &io_alloc_list, allocator, NULL, &alloc_size); } /* Finally fall back to normal class */ if (error != 0) { ZIOSTAT_BUMP(ziostat_alloc_class_fallbacks); - error = metaslab_alloc(spa, spa_normal_class(spa), size, - new_bp, 1, txg, NULL, flags, &io_alloc_list, allocator, - NULL); + error = metaslab_alloc_range(spa, spa_normal_class(spa), + min_size, max_size, new_bp, 1, txg, NULL, flags, + &io_alloc_list, allocator, NULL, &alloc_size); } metaslab_trace_fini(&io_alloc_list); if (error == 0) { - BP_SET_LSIZE(new_bp, size); - BP_SET_PSIZE(new_bp, size); + if (!allow_larger) + alloc_size = MIN(alloc_size, max_size); + else if (max_size <= SPA_OLD_MAXBLOCKSIZE) + alloc_size = MIN(alloc_size, SPA_OLD_MAXBLOCKSIZE); + alloc_size = P2ALIGN_TYPED(alloc_size, ZIL_MIN_BLKSZ, uint64_t); + + BP_SET_LSIZE(new_bp, alloc_size); + BP_SET_PSIZE(new_bp, alloc_size); BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); BP_SET_CHECKSUM(new_bp, spa_version(spa) >= SPA_VERSION_SLIM_ZIL @@ -4527,8 +4538,8 @@ zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp, } } else { zfs_dbgmsg("%s: zil block allocation failure: " - "size %llu, error %d", spa_name(spa), (u_longlong_t)size, - error); + "min_size %llu, max_size %llu, error %d", spa_name(spa), + (u_longlong_t)min_size, (u_longlong_t)max_size, error); } return (error); diff --git a/sys/contrib/openzfs/module/zfs/zvol.c b/sys/contrib/openzfs/module/zfs/zvol.c index 29f51e230a37..2fd3e1c37045 100644 --- a/sys/contrib/openzfs/module/zfs/zvol.c +++ b/sys/contrib/openzfs/module/zfs/zvol.c @@ -38,25 +38,36 @@ * Copyright 2014 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2016 Actifio, Inc. All rights reserved. * Copyright (c) 2012, 2019 by Delphix. All rights reserved. - * Copyright (c) 2024, Klara, Inc. + * Copyright (c) 2024, 2025, Klara, Inc. */ /* * Note on locking of zvol state structures. * - * These structures are used to maintain internal state used to emulate block - * devices on top of zvols. In particular, management of device minor number - * operations - create, remove, rename, and set_snapdev - involves access to - * these structures. The zvol_state_lock is primarily used to protect the - * zvol_state_list. The zv->zv_state_lock is used to protect the contents - * of the zvol_state_t structures, as well as to make sure that when the - * time comes to remove the structure from the list, it is not in use, and - * therefore, it can be taken off zvol_state_list and freed. + * zvol_state_t represents the connection between a single dataset + * (DMU_OST_ZVOL) and the device "minor" (some OS-specific representation of a + * "disk" or "device" or "volume", eg, a /dev/zdXX node, a GEOM object, etc). * - * The zv_suspend_lock was introduced to allow for suspending I/O to a zvol, - * e.g. for the duration of receive and rollback operations. This lock can be - * held for significant periods of time. Given that it is undesirable to hold - * mutexes for long periods of time, the following lock ordering applies: + * The global zvol_state_lock is used to protect access to zvol_state_list and + * zvol_htable, which are the primary way to obtain a zvol_state_t from a name. + * It should not be used for anything not name-relateds, and you should avoid + * sleeping or waiting while its held. See zvol_find_by_name(), zvol_insert(), + * zvol_remove(). + * + * The zv_state_lock is used to protect the contents of the associated + * zvol_state_t. Most of the zvol_state_t is dedicated to control and + * configuration; almost none of it is needed for data operations (that is, + * read, write, flush) so this lock is rarely taken during general IO. It + * should be released quickly; you should avoid sleeping or waiting while its + * held. + * + * zv_suspend_lock is used to suspend IO/data operations to a zvol. The read + * half should held for the duration of an IO operation. The write half should + * be taken when something to wait for IO to complete and the block further IO, + * eg for the duration of receive and rollback operations. This lock can be + * held for long periods of time. + * + * Thus, the following lock ordering appies. * - take zvol_state_lock if necessary, to protect zvol_state_list * - take zv_suspend_lock if necessary, by the code path in question * - take zv_state_lock to protect zvol_state_t @@ -67,9 +78,8 @@ * these operations are serialized per pool. Consequently, we can be certain * that for a given zvol, there is only one operation at a time in progress. * That is why one can be sure that first, zvol_state_t for a given zvol is - * allocated and placed on zvol_state_list, and then other minor operations - * for this zvol are going to proceed in the order of issue. - * + * allocated and placed on zvol_state_list, and then other minor operations for + * this zvol are going to proceed in the order of issue. */ #include <sys/dataset_kstats.h> @@ -1570,184 +1580,156 @@ zvol_create_minors_impl(zvol_task_t *task) } /* - * Remove minors for specified dataset including children and snapshots. - */ - -/* - * Remove the minor for a given zvol. This will do it all: - * - flag the zvol for removal, so new requests are rejected - * - wait until outstanding requests are completed - * - remove it from lists - * - free it - * It's also usable as a taskq task, and smells nice too. + * Remove minors for specified dataset and, optionally, its children and + * snapshots. */ static void -zvol_remove_minor_task(void *arg) -{ - zvol_state_t *zv = (zvol_state_t *)arg; - - ASSERT(!RW_LOCK_HELD(&zvol_state_lock)); - ASSERT(!MUTEX_HELD(&zv->zv_state_lock)); - - mutex_enter(&zv->zv_state_lock); - while (zv->zv_open_count > 0 || atomic_read(&zv->zv_suspend_ref)) { - zv->zv_flags |= ZVOL_REMOVING; - cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock); - } - mutex_exit(&zv->zv_state_lock); - - rw_enter(&zvol_state_lock, RW_WRITER); - mutex_enter(&zv->zv_state_lock); - - zvol_remove(zv); - zvol_os_clear_private(zv); - - mutex_exit(&zv->zv_state_lock); - rw_exit(&zvol_state_lock); - - zvol_os_free(zv); -} - -static void -zvol_free_task(void *arg) -{ - zvol_os_free(arg); -} - -static void zvol_remove_minors_impl(zvol_task_t *task) { zvol_state_t *zv, *zv_next; const char *name = task ? task->zt_name1 : NULL; int namelen = ((name) ? strlen(name) : 0); - taskqid_t t; - list_t delay_list, free_list; + boolean_t children = task ? !!task->zt_value : B_TRUE; if (zvol_inhibit_dev) return; - list_create(&delay_list, sizeof (zvol_state_t), - offsetof(zvol_state_t, zv_next)); - list_create(&free_list, sizeof (zvol_state_t), - offsetof(zvol_state_t, zv_next)); + /* + * We collect up zvols that we want to remove on a separate list, so + * that we don't have to hold zvol_state_lock for the whole time. + * + * We can't remove them from the global lists until we're completely + * done with them, because that would make them appear to ZFS-side ops + * that they don't exist, and the name might be reused, which can't be + * good. + */ + list_t remove_list; + list_create(&remove_list, sizeof (zvol_state_t), + offsetof(zvol_state_t, zv_remove_node)); - rw_enter(&zvol_state_lock, RW_WRITER); + rw_enter(&zvol_state_lock, RW_READER); for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { zv_next = list_next(&zvol_state_list, zv); mutex_enter(&zv->zv_state_lock); + if (zv->zv_flags & ZVOL_REMOVING) { + /* Another thread is handling shutdown, skip it. */ + mutex_exit(&zv->zv_state_lock); + continue; + } + + /* + * This zvol should be removed if: + * - no name was offered (ie removing all at shutdown); or + * - name matches exactly; or + * - we were asked to remove children, and + * - the start of the name matches, and + * - there is a '/' immediately after the matched name; or + * - there is a '@' immediately after the matched name + */ if (name == NULL || strcmp(zv->zv_name, name) == 0 || - (strncmp(zv->zv_name, name, namelen) == 0 && + (children && strncmp(zv->zv_name, name, namelen) == 0 && (zv->zv_name[namelen] == '/' || zv->zv_name[namelen] == '@'))) { - /* - * By holding zv_state_lock here, we guarantee that no - * one is currently using this zv - */ /* - * If in use, try to throw everyone off and try again - * later. + * Matched, so mark it removal. We want to take the + * write half of the suspend lock to make sure that + * the zvol is not suspended, and give any data ops + * chance to finish. */ - if (zv->zv_open_count > 0 || - atomic_read(&zv->zv_suspend_ref)) { - zv->zv_flags |= ZVOL_REMOVING; - t = taskq_dispatch( - zv->zv_objset->os_spa->spa_zvol_taskq, - zvol_remove_minor_task, zv, TQ_SLEEP); - if (t == TASKQID_INVALID) { - /* - * Couldn't create the task, so we'll - * do it in place once the loop is - * finished. - */ - list_insert_head(&delay_list, zv); - } + mutex_exit(&zv->zv_state_lock); + rw_enter(&zv->zv_suspend_lock, RW_WRITER); + mutex_enter(&zv->zv_state_lock); + + if (zv->zv_flags & ZVOL_REMOVING) { + /* Another thread has taken it, let them. */ mutex_exit(&zv->zv_state_lock); + rw_exit(&zv->zv_suspend_lock); continue; } - zvol_remove(zv); - /* - * Cleared while holding zvol_state_lock as a writer - * which will prevent zvol_open() from opening it. + * Mark it and unlock. New entries will see the flag + * and return ENXIO. */ - zvol_os_clear_private(zv); - - /* Drop zv_state_lock before zvol_free() */ + zv->zv_flags |= ZVOL_REMOVING; mutex_exit(&zv->zv_state_lock); + rw_exit(&zv->zv_suspend_lock); - /* Try parallel zv_free, if failed do it in place */ - t = taskq_dispatch(system_taskq, zvol_free_task, zv, - TQ_SLEEP); - if (t == TASKQID_INVALID) - list_insert_head(&free_list, zv); - } else { + /* Put it on the list for the next stage. */ + list_insert_head(&remove_list, zv); + } else mutex_exit(&zv->zv_state_lock); - } } - rw_exit(&zvol_state_lock); - /* Wait for zvols that we couldn't create a remove task for */ - while ((zv = list_remove_head(&delay_list)) != NULL) - zvol_remove_minor_task(zv); - - /* Free any that we couldn't free in parallel earlier */ - while ((zv = list_remove_head(&free_list)) != NULL) - zvol_os_free(zv); -} - -/* Remove minor for this specific volume only */ -static int -zvol_remove_minor_impl(const char *name) -{ - zvol_state_t *zv = NULL, *zv_next; - - if (zvol_inhibit_dev) - return (0); + rw_exit(&zvol_state_lock); - rw_enter(&zvol_state_lock, RW_WRITER); + /* Didn't match any, nothing to do! */ + if (list_is_empty(&remove_list)) { + if (task) + task->zt_error = SET_ERROR(ENOENT); + return; + } - for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { - zv_next = list_next(&zvol_state_list, zv); + /* Actually shut them all down. */ + for (zv = list_head(&remove_list); zv != NULL; zv = zv_next) { + zv_next = list_next(&remove_list, zv); mutex_enter(&zv->zv_state_lock); - if (strcmp(zv->zv_name, name) == 0) - /* Found, leave the the loop with zv_lock held */ - break; - mutex_exit(&zv->zv_state_lock); - } - - if (zv == NULL) { - rw_exit(&zvol_state_lock); - return (SET_ERROR(ENOENT)); - } - ASSERT(MUTEX_HELD(&zv->zv_state_lock)); + /* + * Still open or suspended, just wait. This can happen if, for + * example, we managed to acquire zv_state_lock in the moments + * where zvol_open() or zvol_release() are trading locks to + * call zvol_first_open() or zvol_last_close(). + */ + while (zv->zv_open_count > 0 || + atomic_read(&zv->zv_suspend_ref)) + cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock); - if (zv->zv_open_count > 0 || atomic_read(&zv->zv_suspend_ref)) { /* - * In use, so try to throw everyone off, then wait - * until finished. + * No users, shut down the OS side. This may not remove the + * minor from view immediately, depending on the kernel + * specifics, but it will ensure that it is unusable and that + * this zvol_state_t can never again be reached from an OS-side + * operation. */ - zv->zv_flags |= ZVOL_REMOVING; + zvol_os_remove_minor(zv); mutex_exit(&zv->zv_state_lock); + + /* Remove it from the name lookup lists */ + rw_enter(&zvol_state_lock, RW_WRITER); + zvol_remove(zv); rw_exit(&zvol_state_lock); - zvol_remove_minor_task(zv); - return (0); } - zvol_remove(zv); - zvol_os_clear_private(zv); + /* + * Our own references on remove_list is the last one, free them and + * we're done. + */ + while ((zv = list_remove_head(&remove_list)) != NULL) + zvol_os_free(zv); - mutex_exit(&zv->zv_state_lock); - rw_exit(&zvol_state_lock); + list_destroy(&remove_list); +} - zvol_os_free(zv); +/* Remove minor for this specific volume only */ +static int +zvol_remove_minor_impl(const char *name) +{ + if (zvol_inhibit_dev) + return (0); - return (0); + zvol_task_t task; + memset(&task, 0, sizeof (zvol_task_t)); + strlcpy(task.zt_name1, name, sizeof (task.zt_name1)); + task.zt_value = B_FALSE; + + zvol_remove_minors_impl(&task); + + return (task.zt_error); } /* @@ -2067,6 +2049,7 @@ zvol_remove_minors(spa_t *spa, const char *name, boolean_t async) task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); task->zt_op = ZVOL_ASYNC_REMOVE_MINORS; strlcpy(task->zt_name1, name, sizeof (task->zt_name1)); + task->zt_value = B_TRUE; id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); if ((async == B_FALSE) && (id != TASKQID_INVALID)) taskq_wait_id(spa->spa_zvol_taskq, id); @@ -2188,14 +2171,6 @@ zvol_fini_impl(void) zvol_remove_minors_impl(NULL); - /* - * The call to "zvol_remove_minors_impl" may dispatch entries to - * the system_taskq, but it doesn't wait for those entries to - * complete before it returns. Thus, we must wait for all of the - * removals to finish, before we can continue. - */ - taskq_wait_outstanding(system_taskq, 0); - kmem_free(zvol_htable, ZVOL_HT_SIZE * sizeof (struct hlist_head)); list_destroy(&zvol_state_list); rw_destroy(&zvol_state_lock); diff --git a/sys/contrib/openzfs/module/zstd/zfs_zstd.c b/sys/contrib/openzfs/module/zstd/zfs_zstd.c index 391216d6e263..3db196953f74 100644 --- a/sys/contrib/openzfs/module/zstd/zfs_zstd.c +++ b/sys/contrib/openzfs/module/zstd/zfs_zstd.c @@ -876,9 +876,9 @@ static void __init zstd_mempool_init(void) { zstd_mempool_cctx = - kmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP); + vmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP); zstd_mempool_dctx = - kmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP); + vmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP); for (int i = 0; i < ZSTD_POOL_MAX; i++) { mutex_init(&zstd_mempool_cctx[i].barrier, NULL, @@ -924,8 +924,8 @@ zstd_mempool_deinit(void) release_pool(&zstd_mempool_dctx[i]); } - kmem_free(zstd_mempool_dctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool)); - kmem_free(zstd_mempool_cctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool)); + vmem_free(zstd_mempool_dctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool)); + vmem_free(zstd_mempool_cctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool)); zstd_mempool_dctx = NULL; zstd_mempool_cctx = NULL; } diff --git a/sys/contrib/openzfs/scripts/spdxcheck.pl b/sys/contrib/openzfs/scripts/spdxcheck.pl index 88f5a235d70c..cdab5368f19c 100755 --- a/sys/contrib/openzfs/scripts/spdxcheck.pl +++ b/sys/contrib/openzfs/scripts/spdxcheck.pl @@ -190,6 +190,7 @@ my @path_license_tags = ( ['BSD-2-Clause OR GPL-2.0-only', 'CDDL-1.0'], 'module/icp' => ['Apache-2.0', 'CDDL-1.0'], + 'contrib/icp' => ['Apache-2.0', 'CDDL-1.0'], # Python bindings are always Apache-2.0 'contrib/pyzfs' => ['Apache-2.0'], diff --git a/sys/contrib/openzfs/tests/runfiles/common.run b/sys/contrib/openzfs/tests/runfiles/common.run index 131845f5ed40..2da46458289a 100644 --- a/sys/contrib/openzfs/tests/runfiles/common.run +++ b/sys/contrib/openzfs/tests/runfiles/common.run @@ -1093,7 +1093,7 @@ tests = ['zvol_misc_002_pos', 'zvol_misc_hierarchy', 'zvol_misc_rename_inuse', tags = ['functional', 'zvol', 'zvol_misc'] [tests/functional/zvol/zvol_stress] -tests = ['zvol_stress'] +tests = ['zvol_stress', 'zvol_stress_destroy'] tags = ['functional', 'zvol', 'zvol_stress'] [tests/functional/zvol/zvol_swap] diff --git a/sys/contrib/openzfs/tests/zfs-tests/cmd/crypto_test.c b/sys/contrib/openzfs/tests/zfs-tests/cmd/crypto_test.c index e08003f80464..cbebd33e0bf6 100644 --- a/sys/contrib/openzfs/tests/zfs-tests/cmd/crypto_test.c +++ b/sys/contrib/openzfs/tests/zfs-tests/cmd/crypto_test.c @@ -529,6 +529,8 @@ static const char *aes_gcm_impl[][2] = { { "aesni", "pclmulqdq" }, { "x86_64", "avx" }, { "aesni", "avx" }, + { "x86_64", "avx2" }, + { "aesni", "avx2" }, }; /* signature of function to call after setting implementation params */ diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am index b8b8bbe45a42..41e7b45ef4ec 100644 --- a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am @@ -2244,6 +2244,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/zvol/zvol_stress/cleanup.ksh \ functional/zvol/zvol_stress/setup.ksh \ functional/zvol/zvol_stress/zvol_stress.ksh \ + functional/zvol/zvol_stress/zvol_stress_destroy.ksh \ functional/zvol/zvol_swap/cleanup.ksh \ functional/zvol/zvol_swap/setup.ksh \ functional/zvol/zvol_swap/zvol_swap_001_pos.ksh \ diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_stress/zvol_stress_destroy.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_stress/zvol_stress_destroy.ksh new file mode 100755 index 000000000000..669b59fac01f --- /dev/null +++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/zvol/zvol_stress/zvol_stress_destroy.ksh @@ -0,0 +1,66 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2025, Klara, Inc. +# + +. $STF_SUITE/include/libtest.shlib + +verify_runnable "global" + +typeset -i nzvols=1000 +typeset -i parallel=$(( $(get_num_cpus) * 2 )) + +function cleanup { + for zvol in $(zfs list -Ho name -t vol) ; do + log_must_busy zfs destroy $zvol + done +} + +log_onexit cleanup + +log_assert "stress test concurrent zvol create/destroy" + +function destroy_zvols_until { + typeset cond=$1 + while true ; do + IFS='' zfs list -Ho name -t vol | read -r -d '' zvols + if [[ -n $zvols ]] ; then + echo $zvols | xargs -n 1 -P $parallel zfs destroy + fi + if ! $cond ; then + break + fi + done +} + +( seq $nzvols | \ + xargs -P $parallel -I % zfs create -s -V 1G $TESTPOOL/testvol% ) & +cpid=$! +sleep 1 + +destroy_zvols_until "kill -0 $cpid" +destroy_zvols_until "false" + +log_pass "stress test done" diff --git a/sys/crypto/ccp/ccp.c b/sys/crypto/ccp/ccp.c index 7db9a27ab059..c3d40f6e99ac 100644 --- a/sys/crypto/ccp/ccp.c +++ b/sys/crypto/ccp/ccp.c @@ -79,7 +79,7 @@ static struct pciid { { 0x15df1022, "AMD CCP-5a" }, }; -static struct random_source random_ccp = { +static const struct random_source random_ccp = { .rs_ident = "AMD CCP TRNG", .rs_source = RANDOM_PURE_CCP, .rs_read = random_ccp_read, diff --git a/sys/dev/acpica/acpi_powerres.c b/sys/dev/acpica/acpi_powerres.c index 29d1690f1bdd..0a8b67a5fa84 100644 --- a/sys/dev/acpica/acpi_powerres.c +++ b/sys/dev/acpica/acpi_powerres.c @@ -76,6 +76,13 @@ struct acpi_powerconsumer { /* Device which is powered */ ACPI_HANDLE ac_consumer; int ac_state; + + struct { + bool prx_has; + size_t prx_count; + ACPI_HANDLE *prx_deps; + } ac_prx[ACPI_D_STATE_COUNT]; + TAILQ_ENTRY(acpi_powerconsumer) ac_link; TAILQ_HEAD(,acpi_powerreference) ac_references; }; @@ -96,9 +103,7 @@ static TAILQ_HEAD(acpi_powerconsumer_list, acpi_powerconsumer) ACPI_SERIAL_DECL(powerres, "ACPI power resources"); static ACPI_STATUS acpi_pwr_register_consumer(ACPI_HANDLE consumer); -#ifdef notyet static ACPI_STATUS acpi_pwr_deregister_consumer(ACPI_HANDLE consumer); -#endif /* notyet */ static ACPI_STATUS acpi_pwr_register_resource(ACPI_HANDLE res); #ifdef notyet static ACPI_STATUS acpi_pwr_deregister_resource(ACPI_HANDLE res); @@ -112,6 +117,8 @@ static struct acpi_powerresource *acpi_pwr_find_resource(ACPI_HANDLE res); static struct acpi_powerconsumer *acpi_pwr_find_consumer(ACPI_HANDLE consumer); +static ACPI_STATUS acpi_pwr_infer_state(struct acpi_powerconsumer *pc); +static ACPI_STATUS acpi_pwr_get_state_locked(ACPI_HANDLE consumer, int *state); /* * Register a power resource. @@ -222,6 +229,84 @@ acpi_pwr_deregister_resource(ACPI_HANDLE res) #endif /* notyet */ /* + * Evaluate the _PRx (power resources each D-state depends on). This also + * populates the acpi_powerresources queue with the power resources discovered + * during this step. + * + * ACPI 7.3.8 - 7.3.11 guarantee that _PRx will return the same data each + * time they are evaluated. + * + * If this function fails, acpi_pwr_deregister_consumer() must be called on the + * power consumer to free already allocated memory. + */ +static ACPI_STATUS +acpi_pwr_get_power_resources(ACPI_HANDLE consumer, struct acpi_powerconsumer *pc) +{ + ACPI_INTEGER status; + ACPI_STRING reslist_name; + ACPI_HANDLE reslist_handle; + ACPI_STRING reslist_names[] = {"_PR0", "_PR1", "_PR2", "_PR3"}; + ACPI_BUFFER reslist; + ACPI_OBJECT *reslist_object; + ACPI_OBJECT *dep; + ACPI_HANDLE *res; + + ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__); + ACPI_SERIAL_ASSERT(powerres); + + MPASS(consumer != NULL); + + for (int state = ACPI_STATE_D0; state <= ACPI_STATE_D3_HOT; state++) { + pc->ac_prx[state].prx_has = false; + pc->ac_prx[state].prx_count = 0; + pc->ac_prx[state].prx_deps = NULL; + + reslist_name = reslist_names[state - ACPI_STATE_D0]; + if (ACPI_FAILURE(AcpiGetHandle(consumer, reslist_name, &reslist_handle))) + continue; + + reslist.Pointer = NULL; + reslist.Length = ACPI_ALLOCATE_BUFFER; + status = AcpiEvaluateObjectTyped(reslist_handle, NULL, NULL, &reslist, + ACPI_TYPE_PACKAGE); + if (ACPI_FAILURE(status) || reslist.Pointer == NULL) + /* + * ACPI_ALLOCATE_BUFFER entails everything will be freed on error + * by AcpiEvaluateObjectTyped. + */ + continue; + + reslist_object = (ACPI_OBJECT *)reslist.Pointer; + pc->ac_prx[state].prx_has = true; + pc->ac_prx[state].prx_count = reslist_object->Package.Count; + + if (reslist_object->Package.Count == 0) { + AcpiOsFree(reslist_object); + continue; + } + + pc->ac_prx[state].prx_deps = mallocarray(pc->ac_prx[state].prx_count, + sizeof(*pc->ac_prx[state].prx_deps), M_ACPIPWR, M_NOWAIT); + if (pc->ac_prx[state].prx_deps == NULL) { + AcpiOsFree(reslist_object); + return_ACPI_STATUS (AE_NO_MEMORY); + } + + for (size_t i = 0; i < reslist_object->Package.Count; i++) { + dep = &reslist_object->Package.Elements[i]; + res = dep->Reference.Handle; + pc->ac_prx[state].prx_deps[i] = res; + + /* It's fine to attempt to register the same resource twice. */ + acpi_pwr_register_resource(res); + } + AcpiOsFree(reslist_object); + } + + return_ACPI_STATUS (AE_OK); +} + +/* * Register a power consumer. * * It's OK to call this if we already know about the consumer. @@ -229,6 +314,7 @@ acpi_pwr_deregister_resource(ACPI_HANDLE res) static ACPI_STATUS acpi_pwr_register_consumer(ACPI_HANDLE consumer) { + ACPI_INTEGER status; struct acpi_powerconsumer *pc; ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__); @@ -239,14 +325,30 @@ acpi_pwr_register_consumer(ACPI_HANDLE consumer) return_ACPI_STATUS (AE_OK); /* Allocate a new power consumer */ - if ((pc = malloc(sizeof(*pc), M_ACPIPWR, M_NOWAIT)) == NULL) + if ((pc = malloc(sizeof(*pc), M_ACPIPWR, M_NOWAIT | M_ZERO)) == NULL) return_ACPI_STATUS (AE_NO_MEMORY); TAILQ_INSERT_HEAD(&acpi_powerconsumers, pc, ac_link); TAILQ_INIT(&pc->ac_references); pc->ac_consumer = consumer; - /* XXX we should try to find its current state */ - pc->ac_state = ACPI_STATE_UNKNOWN; + /* + * Get all its power resource dependencies, if it has _PRx. We do this now + * as an opportunity to populate the acpi_powerresources queue. + * + * If this fails, immediately deregister it. + */ + status = acpi_pwr_get_power_resources(consumer, pc); + if (ACPI_FAILURE(status)) { + ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, + "failed to get power resources for %s\n", + acpi_name(consumer))); + acpi_pwr_deregister_consumer(consumer); + return_ACPI_STATUS (status); + } + + /* Find its initial state. */ + if (ACPI_FAILURE(acpi_pwr_get_state_locked(consumer, &pc->ac_state))) + pc->ac_state = ACPI_STATE_UNKNOWN; ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "registered power consumer %s\n", acpi_name(consumer))); @@ -254,7 +356,6 @@ acpi_pwr_register_consumer(ACPI_HANDLE consumer) return_ACPI_STATUS (AE_OK); } -#ifdef notyet /* * Deregister a power consumer. * @@ -279,6 +380,9 @@ acpi_pwr_deregister_consumer(ACPI_HANDLE consumer) /* Pull the consumer off the list and free it */ TAILQ_REMOVE(&acpi_powerconsumers, pc, ac_link); + for (size_t i = 0; i < sizeof(pc->ac_prx) / sizeof(*pc->ac_prx); i++) + if (pc->ac_prx[i].prx_deps != NULL) + free(pc->ac_prx[i].prx_deps, M_ACPIPWR); free(pc, M_ACPIPWR); ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "deregistered power consumer %s\n", @@ -286,10 +390,139 @@ acpi_pwr_deregister_consumer(ACPI_HANDLE consumer) return_ACPI_STATUS (AE_OK); } -#endif /* notyet */ /* - * Set a power consumer to a particular power state. + * The _PSC control method isn't required if it's possible to infer the D-state + * from the _PRx control methods. (See 7.3.6.) + * We can infer that a given D-state has been achieved when all the dependencies + * are in the ON state. + */ +static ACPI_STATUS +acpi_pwr_infer_state(struct acpi_powerconsumer *pc) +{ + ACPI_HANDLE *res; + uint32_t on; + bool all_on = false; + + ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__); + ACPI_SERIAL_ASSERT(powerres); + + /* It is important we go from the hottest to the coldest state. */ + for ( + pc->ac_state = ACPI_STATE_D0; + pc->ac_state <= ACPI_STATE_D3_HOT && !all_on; + pc->ac_state++ + ) { + MPASS(pc->ac_state <= sizeof(pc->ac_prx) / sizeof(*pc->ac_prx)); + + if (!pc->ac_prx[pc->ac_state].prx_has) + continue; + + all_on = true; + + for (size_t i = 0; i < pc->ac_prx[pc->ac_state].prx_count; i++) { + res = pc->ac_prx[pc->ac_state].prx_deps[i]; + /* If failure, better to assume D-state is hotter than colder. */ + if (ACPI_FAILURE(acpi_GetInteger(res, "_STA", &on))) + continue; + if (on == 0) { + all_on = false; + break; + } + } + } + + MPASS(pc->ac_state != ACPI_STATE_D0); + + /* + * If none of the power resources required for the shallower D-states are + * on, then we can assume it is unpowered (i.e. D3cold). A device is not + * required to support D3cold however; in that case, _PR3 is not explicitly + * provided. Those devices should default to D3hot instead. + * + * See comments of first row of table 7.1 in ACPI spec. + */ + if (!all_on) + pc->ac_state = pc->ac_prx[ACPI_STATE_D3_HOT].prx_has ? + ACPI_STATE_D3_COLD : ACPI_STATE_D3_HOT; + else + pc->ac_state--; + + return_ACPI_STATUS (AE_OK); +} + +static ACPI_STATUS +acpi_pwr_get_state_locked(ACPI_HANDLE consumer, int *state) +{ + struct acpi_powerconsumer *pc; + ACPI_HANDLE method_handle; + ACPI_STATUS status; + ACPI_BUFFER result; + ACPI_OBJECT *object = NULL; + + ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__); + ACPI_SERIAL_ASSERT(powerres); + + if (consumer == NULL) + return_ACPI_STATUS (AE_NOT_FOUND); + + if ((pc = acpi_pwr_find_consumer(consumer)) == NULL) { + if (ACPI_FAILURE(status = acpi_pwr_register_consumer(consumer))) + goto out; + if ((pc = acpi_pwr_find_consumer(consumer)) == NULL) + panic("acpi added power consumer but can't find it"); + } + + status = AcpiGetHandle(consumer, "_PSC", &method_handle); + if (ACPI_FAILURE(status)) { + ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "no _PSC object - %s\n", + AcpiFormatException(status))); + status = acpi_pwr_infer_state(pc); + if (ACPI_FAILURE(status)) { + ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "couldn't infer D-state - %s\n", + AcpiFormatException(status))); + pc->ac_state = ACPI_STATE_UNKNOWN; + } + goto out; + } + + result.Pointer = NULL; + result.Length = ACPI_ALLOCATE_BUFFER; + status = AcpiEvaluateObjectTyped(method_handle, NULL, NULL, &result, ACPI_TYPE_INTEGER); + if (ACPI_FAILURE(status) || result.Pointer == NULL) { + ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "failed to get state with _PSC - %s\n", + AcpiFormatException(status))); + pc->ac_state = ACPI_STATE_UNKNOWN; + goto out; + } + + object = (ACPI_OBJECT *)result.Pointer; + pc->ac_state = ACPI_STATE_D0 + object->Integer.Value; + status = AE_OK; + +out: + if (object != NULL) + AcpiOsFree(object); + *state = pc->ac_state; + return_ACPI_STATUS (status); +} + +/* + * Get a power consumer's D-state. + */ +ACPI_STATUS +acpi_pwr_get_state(ACPI_HANDLE consumer, int *state) +{ + ACPI_STATUS res; + + ACPI_SERIAL_BEGIN(powerres); + res = acpi_pwr_get_state_locked(consumer, state); + ACPI_SERIAL_END(powerres); + return (res); +} + +/* + * Set a power consumer to a particular D-state. */ ACPI_STATUS acpi_pwr_switch_consumer(ACPI_HANDLE consumer, int state) @@ -300,6 +533,7 @@ acpi_pwr_switch_consumer(ACPI_HANDLE consumer, int state) ACPI_OBJECT *reslist_object; ACPI_STATUS status; char *method_name, *reslist_name = NULL; + int new_state; ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__); @@ -501,8 +735,28 @@ acpi_pwr_switch_consumer(ACPI_HANDLE consumer, int state) } } - /* Transition was successful */ - pc->ac_state = state; + /* + * Make sure the transition succeeded. If getting new state failed, + * just assume the new state is what we wanted. This was the behaviour + * before we were checking D-states. + */ + if (ACPI_FAILURE(acpi_pwr_get_state_locked(consumer, &new_state))) { + printf("%s: failed to get new D-state\n", __func__); + pc->ac_state = state; + } else { + if (new_state != state) + printf("%s: new power state %s is not the one requested %s\n", + __func__, acpi_d_state_to_str(new_state), + acpi_d_state_to_str(state)); + pc->ac_state = new_state; + } + + /* + * We consider the transition successful even if the state we got doesn't + * reflect what we set it to. This is because we weren't previously + * checking the new state at all, so there might exist buggy platforms on + * which suspend would otherwise succeed if we failed here. + */ status = AE_OK; out: diff --git a/sys/dev/acpica/acpivar.h b/sys/dev/acpica/acpivar.h index 6887f080311d..7495a010432b 100644 --- a/sys/dev/acpica/acpivar.h +++ b/sys/dev/acpica/acpivar.h @@ -490,6 +490,7 @@ EVENTHANDLER_DECLARE(acpi_video_event, acpi_event_handler_t); /* Device power control. */ ACPI_STATUS acpi_pwr_wake_enable(ACPI_HANDLE consumer, int enable); +ACPI_STATUS acpi_pwr_get_state(ACPI_HANDLE consumer, int *state); ACPI_STATUS acpi_pwr_switch_consumer(ACPI_HANDLE consumer, int state); acpi_pwr_for_sleep_t acpi_device_pwr_for_sleep; int acpi_set_powerstate(device_t child, int state); diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.c b/sys/dev/bnxt/bnxt_re/qplib_res.c index 69661c67708c..f527af031176 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_res.c +++ b/sys/dev/bnxt/bnxt_re/qplib_res.c @@ -875,7 +875,7 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, dpi->umdbr = umaddr; switch (type) { case BNXT_QPLIB_DPI_TYPE_KERNEL: - /* priviledged dbr was already mapped just initialize it. */ + /* privileged dbr was already mapped just initialize it. */ dpi->umdbr = dpit->ucreg.bar_base + dpit->ucreg.offset + bit_num * PAGE_SIZE; dpi->dbr = dpit->priv_db; @@ -1150,7 +1150,7 @@ int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res) } ucreg->bar_reg = ioremap(ucreg->bar_base, ucreg->len); if (!ucreg->bar_reg) { - dev_err(&res->pdev->dev, "priviledged dpi map failed!\n"); + dev_err(&res->pdev->dev, "privileged dpi map failed!\n"); return -ENOMEM; } diff --git a/sys/dev/e1000/e1000_phy.c b/sys/dev/e1000/e1000_phy.c index c34897e3b31a..634f48171c3e 100644 --- a/sys/dev/e1000/e1000_phy.c +++ b/sys/dev/e1000/e1000_phy.c @@ -1707,10 +1707,9 @@ s32 e1000_setup_copper_link_generic(struct e1000_hw *hw) * autonegotiation. */ ret_val = e1000_copper_link_autoneg(hw); - if (ret_val && !hw->mac.forced_speed_duplex) + if (ret_val) return ret_val; - } - if (!hw->mac.autoneg || (ret_val && hw->mac.forced_speed_duplex)) { + } else { /* PHY will be set to 10H, 10F, 100H or 100F * depending on user settings. */ diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c index f0ef6051fab1..9c5ae2806f75 100644 --- a/sys/dev/e1000/if_em.c +++ b/sys/dev/e1000/if_em.c @@ -2000,18 +2000,7 @@ em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) (sc->hw.phy.media_type == e1000_media_type_internal_serdes)) { if (sc->hw.mac.type == e1000_82545) fiber_type = IFM_1000_LX; - switch (sc->link_speed) { - case 10: - ifmr->ifm_active |= IFM_10_FL; - break; - case 100: - ifmr->ifm_active |= IFM_100_FX; - break; - case 1000: - default: - ifmr->ifm_active |= fiber_type | IFM_FDX; - break; - } + ifmr->ifm_active |= fiber_type | IFM_FDX; } else { switch (sc->link_speed) { case 10: @@ -2024,12 +2013,11 @@ em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) ifmr->ifm_active |= IFM_1000_T; break; } + if (sc->link_duplex == FULL_DUPLEX) + ifmr->ifm_active |= IFM_FDX; + else + ifmr->ifm_active |= IFM_HDX; } - - if (sc->link_duplex == FULL_DUPLEX) - ifmr->ifm_active |= IFM_FDX; - else - ifmr->ifm_active |= IFM_HDX; } /********************************************************************* @@ -2063,26 +2051,6 @@ em_if_media_change(if_ctx_t ctx) sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; break; case IFM_100_TX: - sc->hw.mac.autoneg = DO_AUTO_NEG; - if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) { - sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; - sc->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; - } else { - sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; - sc->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; - } - break; - case IFM_10_T: - sc->hw.mac.autoneg = DO_AUTO_NEG; - if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) { - sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; - sc->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; - } else { - sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; - sc->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; - } - break; - case IFM_100_FX: sc->hw.mac.autoneg = false; sc->hw.phy.autoneg_advertised = 0; if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) @@ -2090,7 +2058,7 @@ em_if_media_change(if_ctx_t ctx) else sc->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; break; - case IFM_10_FL: + case IFM_10_T: sc->hw.mac.autoneg = false; sc->hw.phy.autoneg_advertised = 0; if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) diff --git a/sys/dev/fdt/fdt_common.c b/sys/dev/fdt/fdt_common.c index 1fea4c6f1392..f43551c6310e 100644 --- a/sys/dev/fdt/fdt_common.c +++ b/sys/dev/fdt/fdt_common.c @@ -62,8 +62,6 @@ SYSCTL_NODE(_hw, OID_AUTO, fdt, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Flattened Device Tree"); -struct fdt_ic_list fdt_ic_list_head = SLIST_HEAD_INITIALIZER(fdt_ic_list_head); - static int fdt_get_range_by_busaddr(phandle_t node, u_long addr, u_long *base, u_long *size) diff --git a/sys/dev/fdt/fdt_common.h b/sys/dev/fdt/fdt_common.h index ece54290a6ad..f597233f9771 100644 --- a/sys/dev/fdt/fdt_common.h +++ b/sys/dev/fdt/fdt_common.h @@ -59,13 +59,6 @@ struct fdt_fixup_entry { extern struct fdt_fixup_entry fdt_fixup_table[]; #endif -extern SLIST_HEAD(fdt_ic_list, fdt_ic) fdt_ic_list_head; -struct fdt_ic { - SLIST_ENTRY(fdt_ic) fdt_ics; - ihandle_t iph; - device_t dev; -}; - #if defined(FDT_DTB_STATIC) extern u_char fdt_static_dtb; #endif diff --git a/sys/dev/gpio/acpi_gpiobus.c b/sys/dev/gpio/acpi_gpiobus.c index 170f23615416..0d2455cab399 100644 --- a/sys/dev/gpio/acpi_gpiobus.c +++ b/sys/dev/gpio/acpi_gpiobus.c @@ -37,6 +37,7 @@ #include <dev/gpio/gpiobusvar.h> #include <dev/gpio/acpi_gpiobusvar.h> #include <dev/gpio/gpiobus_internal.h> +#include <sys/sbuf.h> #include "gpiobus_if.h" @@ -52,12 +53,11 @@ struct acpi_gpiobus_ctx { struct acpi_gpiobus_ivar { - struct gpiobus_ivar gpiobus; /* Must come first */ - ACPI_HANDLE dev_handle; /* ACPI handle for bus */ - uint32_t flags; + struct gpiobus_ivar gpiobus; + ACPI_HANDLE handle; }; -static uint32_t +uint32_t acpi_gpiobus_convflags(ACPI_RESOURCE_GPIO *gpio_res) { uint32_t flags = 0; @@ -150,70 +150,24 @@ acpi_gpiobus_enumerate_res(ACPI_RESOURCE *res, void *context) return (AE_OK); } -static struct acpi_gpiobus_ivar * -acpi_gpiobus_setup_devinfo(device_t bus, device_t child, - ACPI_RESOURCE_GPIO *gpio_res) -{ - struct acpi_gpiobus_ivar *devi; - - devi = malloc(sizeof(*devi), M_DEVBUF, M_NOWAIT | M_ZERO); - if (devi == NULL) - return (NULL); - resource_list_init(&devi->gpiobus.rl); - - devi->flags = acpi_gpiobus_convflags(gpio_res); - if (acpi_quirks & ACPI_Q_AEI_NOPULL) - devi->flags &= ~GPIO_PIN_PULLUP; - - devi->gpiobus.npins = 1; - if (gpiobus_alloc_ivars(&devi->gpiobus) != 0) { - free(devi, M_DEVBUF); - return (NULL); - } - - for (int i = 0; i < devi->gpiobus.npins; i++) - devi->gpiobus.pins[i] = gpio_res->PinTable[i]; - - return (devi); -} - static ACPI_STATUS acpi_gpiobus_enumerate_aei(ACPI_RESOURCE *res, void *context) { ACPI_RESOURCE_GPIO *gpio_res = &res->Data.Gpio; - struct acpi_gpiobus_ctx *ctx = context; - device_t bus = ctx->sc->sc_busdev; - device_t child; - struct acpi_gpiobus_ivar *devi; + uint32_t *npins = context, *pins = npins + 1; - /* Check that we have a GpioInt object. */ + /* + * Check that we have a GpioInt object. + * Note that according to the spec this + * should always be the case. + */ if (res->Type != ACPI_RESOURCE_TYPE_GPIO) return (AE_OK); if (gpio_res->ConnectionType != ACPI_RESOURCE_GPIO_TYPE_INT) return (AE_OK); - /* Add a child. */ - child = device_add_child_ordered(bus, 0, "gpio_aei", DEVICE_UNIT_ANY); - if (child == NULL) - return (AE_OK); - devi = acpi_gpiobus_setup_devinfo(bus, child, gpio_res); - if (devi == NULL) { - device_delete_child(bus, child); - return (AE_OK); - } - device_set_ivars(child, devi); - - for (int i = 0; i < devi->gpiobus.npins; i++) { - if (GPIOBUS_PIN_SETFLAGS(bus, child, 0, devi->flags & - ~GPIO_INTR_MASK)) { - device_delete_child(bus, child); - return (AE_OK); - } - } - - /* Pass ACPI information to children. */ - devi->dev_handle = ctx->dev_handle; - + for (int i = 0; i < gpio_res->PinTableLength; i++) + pins[(*npins)++] = gpio_res->PinTable[i]; return (AE_OK); } @@ -296,6 +250,63 @@ err: return (AE_BAD_PARAMETER); } +static void +acpi_gpiobus_attach_aei(struct acpi_gpiobus_softc *sc, ACPI_HANDLE handle) +{ + struct acpi_gpiobus_ivar *devi; + ACPI_HANDLE aei_handle; + device_t child; + uint32_t *pins; + ACPI_STATUS status; + int err; + + status = AcpiGetHandle(handle, "_AEI", &aei_handle); + if (ACPI_FAILURE(status)) + return; + + /* pins[0] specifies the length of the array. */ + pins = mallocarray(sc->super_sc.sc_npins + 1, + sizeof(uint32_t), M_DEVBUF, M_WAITOK); + pins[0] = 0; + + status = AcpiWalkResources(handle, "_AEI", + acpi_gpiobus_enumerate_aei, pins); + if (ACPI_FAILURE(status)) { + device_printf(sc->super_sc.sc_busdev, + "Failed to enumerate AEI resources\n"); + free(pins, M_DEVBUF); + return; + } + + child = BUS_ADD_CHILD(sc->super_sc.sc_busdev, 0, "gpio_aei", + DEVICE_UNIT_ANY); + if (child == NULL) { + device_printf(sc->super_sc.sc_busdev, + "Failed to add gpio_aei child\n"); + free(pins, M_DEVBUF); + return; + } + + devi = device_get_ivars(child); + devi->gpiobus.npins = pins[0]; + devi->handle = aei_handle; + + err = gpiobus_alloc_ivars(&devi->gpiobus); + if (err != 0) { + device_printf(sc->super_sc.sc_busdev, + "Failed to allocate gpio_aei ivars\n"); + device_delete_child(sc->super_sc.sc_busdev, child); + free(pins, M_DEVBUF); + return; + } + + for (int i = 0; i < pins[0]; i++) + devi->gpiobus.pins[i] = pins[i + 1]; + free(pins, M_DEVBUF); + + bus_attach_children(sc->super_sc.sc_busdev); +} + static int acpi_gpiobus_probe(device_t dev) { @@ -353,13 +364,8 @@ acpi_gpiobus_attach(device_t dev) if (ACPI_FAILURE(status)) device_printf(dev, "Failed to enumerate GPIO resources\n"); - /* Look for AEI children */ - status = AcpiWalkResources(handle, "_AEI", acpi_gpiobus_enumerate_aei, - &ctx); - - if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) - device_printf(dev, "Failed to enumerate AEI resources\n"); - + /* Look for AEI child */ + acpi_gpiobus_attach_aei(sc, handle); return (0); } @@ -390,10 +396,7 @@ acpi_gpiobus_read_ivar(device_t dev, device_t child, int which, switch (which) { case ACPI_GPIOBUS_IVAR_HANDLE: - *result = (uintptr_t)devi->dev_handle; - break; - case ACPI_GPIOBUS_IVAR_FLAGS: - *result = (uintptr_t)devi->flags; + *result = (uintptr_t)devi->handle; break; default: return (gpiobus_read_ivar(dev, child, which, result)); @@ -402,6 +405,28 @@ acpi_gpiobus_read_ivar(device_t dev, device_t child, int which, return (0); } +static device_t +acpi_gpiobus_add_child(device_t dev, u_int order, const char *name, int unit) +{ + return (gpiobus_add_child_common(dev, order, name, unit, + sizeof(struct acpi_gpiobus_ivar))); +} + +static int +acpi_gpiobus_child_location(device_t bus, device_t child, struct sbuf *sb) +{ + struct acpi_gpiobus_ivar *devi; + int err; + + err = gpiobus_child_location(bus, child, sb); + if (err != 0) + return (err); + + devi = device_get_ivars(child); + sbuf_printf(sb, " handle=%s", acpi_name(devi->handle)); + return (0); +} + static device_method_t acpi_gpiobus_methods[] = { /* Device interface */ DEVMETHOD(device_probe, acpi_gpiobus_probe), @@ -410,6 +435,8 @@ static device_method_t acpi_gpiobus_methods[] = { /* Bus interface */ DEVMETHOD(bus_read_ivar, acpi_gpiobus_read_ivar), + DEVMETHOD(bus_add_child, acpi_gpiobus_add_child), + DEVMETHOD(bus_child_location, acpi_gpiobus_child_location), DEVMETHOD_END }; diff --git a/sys/dev/gpio/acpi_gpiobusvar.h b/sys/dev/gpio/acpi_gpiobusvar.h index f8d502eab9d1..288e8bd0f2af 100644 --- a/sys/dev/gpio/acpi_gpiobusvar.h +++ b/sys/dev/gpio/acpi_gpiobusvar.h @@ -33,16 +33,16 @@ #include <contrib/dev/acpica/include/acpi.h> enum acpi_gpiobus_ivars { - ACPI_GPIOBUS_IVAR_HANDLE = 10600, - ACPI_GPIOBUS_IVAR_FLAGS, + ACPI_GPIOBUS_IVAR_HANDLE = 10600 }; #define ACPI_GPIOBUS_ACCESSOR(var, ivar, type) \ __BUS_ACCESSOR(acpi_gpiobus, var, ACPI_GPIOBUS, ivar, type) ACPI_GPIOBUS_ACCESSOR(handle, HANDLE, ACPI_HANDLE) -ACPI_GPIOBUS_ACCESSOR(flags, FLAGS, uint32_t) #undef ACPI_GPIOBUS_ACCESSOR +uint32_t acpi_gpiobus_convflags(ACPI_RESOURCE_GPIO *); + #endif /* __ACPI_GPIOBUS_H__ */ diff --git a/sys/dev/gpio/gpioaei.c b/sys/dev/gpio/gpioaei.c index ecae8ccaf2fa..7b97277aaf61 100644 --- a/sys/dev/gpio/gpioaei.c +++ b/sys/dev/gpio/gpioaei.c @@ -45,13 +45,21 @@ enum gpio_aei_type { ACPI_AEI_TYPE_EVT }; -struct gpio_aei_softc { - ACPI_HANDLE handle; - enum gpio_aei_type type; - int pin; +struct gpio_aei_ctx { + SLIST_ENTRY(gpio_aei_ctx) next; struct resource * intr_res; - int intr_rid; void * intr_cookie; + ACPI_HANDLE handle; + gpio_pin_t gpio; + uint32_t pin; + int intr_rid; + enum gpio_aei_type type; +}; + +struct gpio_aei_softc { + SLIST_HEAD(, gpio_aei_ctx) aei_ctx; + ACPI_HANDLE dev_handle; + device_t dev; }; static int @@ -65,69 +73,157 @@ gpio_aei_probe(device_t dev) static void gpio_aei_intr(void * arg) { - struct gpio_aei_softc * sc = arg; + struct gpio_aei_ctx * ctx = arg; /* Ask ACPI to run the appropriate _EVT, _Exx or _Lxx method. */ - if (sc->type == ACPI_AEI_TYPE_EVT) - acpi_SetInteger(sc->handle, NULL, sc->pin); + if (ctx->type == ACPI_AEI_TYPE_EVT) + acpi_SetInteger(ctx->handle, NULL, ctx->pin); else - AcpiEvaluateObject(sc->handle, NULL, NULL, NULL); + AcpiEvaluateObject(ctx->handle, NULL, NULL, NULL); +} + +static ACPI_STATUS +gpio_aei_enumerate(ACPI_RESOURCE * res, void * context) +{ + ACPI_RESOURCE_GPIO * gpio_res = &res->Data.Gpio; + struct gpio_aei_softc * sc = context; + uint32_t flags, maxpin; + device_t busdev; + int err; + + /* + * Check that we have a GpioInt object. + * Note that according to the spec this + * should always be the case. + */ + if (res->Type != ACPI_RESOURCE_TYPE_GPIO) + return (AE_OK); + if (gpio_res->ConnectionType != ACPI_RESOURCE_GPIO_TYPE_INT) + return (AE_OK); + + flags = acpi_gpiobus_convflags(gpio_res); + if (acpi_quirks & ACPI_Q_AEI_NOPULL) + flags &= ~GPIO_PIN_PULLUP; + + err = GPIO_PIN_MAX(acpi_get_device(sc->dev_handle), &maxpin); + if (err != 0) + return (AE_ERROR); + + busdev = GPIO_GET_BUS(acpi_get_device(sc->dev_handle)); + for (int i = 0; i < gpio_res->PinTableLength; i++) { + struct gpio_aei_ctx * ctx; + uint32_t pin = gpio_res->PinTable[i]; + + if (__predict_false(pin > maxpin)) { + device_printf(sc->dev, + "Invalid pin 0x%x, max: 0x%x (bad ACPI tables?)\n", + pin, maxpin); + continue; + } + + ctx = malloc(sizeof(struct gpio_aei_ctx), M_DEVBUF, M_WAITOK); + ctx->type = ACPI_AEI_TYPE_UNKNOWN; + if (pin <= 255) { + char objname[5]; /* "_EXX" or "_LXX" */ + sprintf(objname, "_%c%02X", + (flags & GPIO_INTR_EDGE_MASK) ? 'E' : 'L', pin); + if (ACPI_SUCCESS(AcpiGetHandle(sc->dev_handle, objname, + &ctx->handle))) + ctx->type = ACPI_AEI_TYPE_ELX; + } + + if (ctx->type == ACPI_AEI_TYPE_UNKNOWN) { + if (ACPI_SUCCESS(AcpiGetHandle(sc->dev_handle, "_EVT", + &ctx->handle))) + ctx->type = ACPI_AEI_TYPE_EVT; + else { + device_printf(sc->dev, + "AEI Device type is unknown for pin 0x%x\n", + pin); + + free(ctx, M_DEVBUF); + continue; + } + } + + err = gpio_pin_get_by_bus_pinnum(busdev, pin, &ctx->gpio); + if (err != 0) { + device_printf(sc->dev, "Cannot acquire pin 0x%x\n", + pin); + + free(ctx, M_DEVBUF); + continue; + } + + err = gpio_pin_setflags(ctx->gpio, flags & ~GPIO_INTR_MASK); + if (err != 0) { + device_printf(sc->dev, + "Cannot set pin flags for pin 0x%x\n", pin); + + gpio_pin_release(ctx->gpio); + free(ctx, M_DEVBUF); + continue; + } + + ctx->intr_rid = 0; + ctx->intr_res = gpio_alloc_intr_resource(sc->dev, + &ctx->intr_rid, RF_ACTIVE, ctx->gpio, + flags & GPIO_INTR_MASK); + if (ctx->intr_res == NULL) { + device_printf(sc->dev, + "Cannot allocate an IRQ for pin 0x%x\n", pin); + + gpio_pin_release(ctx->gpio); + free(ctx, M_DEVBUF); + continue; + } + + err = bus_setup_intr(sc->dev, ctx->intr_res, INTR_TYPE_MISC | + INTR_MPSAFE | INTR_EXCL | INTR_SLEEPABLE, NULL, + gpio_aei_intr, ctx, &ctx->intr_cookie); + if (err != 0) { + device_printf(sc->dev, + "Cannot set up an IRQ for pin 0x%x\n", pin); + + bus_release_resource(sc->dev, ctx->intr_res); + gpio_pin_release(ctx->gpio); + free(ctx, M_DEVBUF); + continue; + } + + ctx->pin = pin; + SLIST_INSERT_HEAD(&sc->aei_ctx, ctx, next); + } + + return (AE_OK); } static int gpio_aei_attach(device_t dev) { struct gpio_aei_softc * sc = device_get_softc(dev); - gpio_pin_t pin; - uint32_t flags; ACPI_HANDLE handle; - int err; + ACPI_STATUS status; /* This is us. */ device_set_desc(dev, "ACPI Event Information Device"); - /* Store parameters needed by gpio_aei_intr. */ handle = acpi_gpiobus_get_handle(dev); - if (gpio_pin_get_by_child_index(dev, 0, &pin) != 0) { - device_printf(dev, "Unable to get the input pin\n"); + status = AcpiGetParent(handle, &sc->dev_handle); + if (ACPI_FAILURE(status)) { + device_printf(dev, "Cannot get parent of %s\n", + acpi_name(handle)); return (ENXIO); } - sc->type = ACPI_AEI_TYPE_UNKNOWN; - sc->pin = pin->pin; - - flags = acpi_gpiobus_get_flags(dev); - if (pin->pin <= 255) { - char objname[5]; /* "_EXX" or "_LXX" */ - sprintf(objname, "_%c%02X", - (flags & GPIO_INTR_EDGE_MASK) ? 'E' : 'L', pin->pin); - if (ACPI_SUCCESS(AcpiGetHandle(handle, objname, &sc->handle))) - sc->type = ACPI_AEI_TYPE_ELX; - } - if (sc->type == ACPI_AEI_TYPE_UNKNOWN) { - if (ACPI_SUCCESS(AcpiGetHandle(handle, "_EVT", &sc->handle))) - sc->type = ACPI_AEI_TYPE_EVT; - } - - if (sc->type == ACPI_AEI_TYPE_UNKNOWN) { - device_printf(dev, "ACPI Event Information Device type is unknown"); - return (ENOTSUP); - } + SLIST_INIT(&sc->aei_ctx); + sc->dev = dev; - /* Set up the interrupt. */ - if ((sc->intr_res = gpio_alloc_intr_resource(dev, &sc->intr_rid, - RF_ACTIVE, pin, flags & GPIO_INTR_MASK)) == NULL) { - device_printf(dev, "Cannot allocate an IRQ\n"); - return (ENOTSUP); - } - err = bus_setup_intr(dev, sc->intr_res, INTR_TYPE_MISC | INTR_MPSAFE | - INTR_EXCL | INTR_SLEEPABLE, NULL, gpio_aei_intr, sc, - &sc->intr_cookie); - if (err != 0) { - device_printf(dev, "Cannot set up IRQ\n"); - bus_release_resource(dev, SYS_RES_IRQ, sc->intr_rid, - sc->intr_res); - return (err); + status = AcpiWalkResources(sc->dev_handle, "_AEI", + gpio_aei_enumerate, sc); + if (ACPI_FAILURE(status)) { + device_printf(dev, "Failed to enumerate AEI resources\n"); + return (ENXIO); } return (0); @@ -137,9 +233,15 @@ static int gpio_aei_detach(device_t dev) { struct gpio_aei_softc * sc = device_get_softc(dev); + struct gpio_aei_ctx * ctx, * tctx; + + SLIST_FOREACH_SAFE(ctx, &sc->aei_ctx, next, tctx) { + bus_teardown_intr(dev, ctx->intr_res, ctx->intr_cookie); + bus_release_resource(dev, ctx->intr_res); + gpio_pin_release(ctx->gpio); + free(ctx, M_DEVBUF); + } - bus_teardown_intr(dev, sc->intr_res, sc->intr_cookie); - bus_release_resource(dev, SYS_RES_IRQ, sc->intr_rid, sc->intr_res); return (0); } diff --git a/sys/dev/gpio/gpiobus.c b/sys/dev/gpio/gpiobus.c index c25c41f43042..5f1f6532a79b 100644 --- a/sys/dev/gpio/gpiobus.c +++ b/sys/dev/gpio/gpiobus.c @@ -57,7 +57,6 @@ static int gpiobus_suspend(device_t); static int gpiobus_resume(device_t); static void gpiobus_probe_nomatch(device_t, device_t); static int gpiobus_print_child(device_t, device_t); -static int gpiobus_child_location(device_t, device_t, struct sbuf *); static device_t gpiobus_add_child(device_t, u_int, const char *, int); static void gpiobus_hinted_child(device_t, const char *, int); @@ -662,7 +661,7 @@ gpiobus_print_child(device_t dev, device_t child) return (retval); } -static int +int gpiobus_child_location(device_t bus, device_t child, struct sbuf *sb) { struct gpiobus_ivar *devi; @@ -674,16 +673,19 @@ gpiobus_child_location(device_t bus, device_t child, struct sbuf *sb) return (0); } -static device_t -gpiobus_add_child(device_t dev, u_int order, const char *name, int unit) +device_t +gpiobus_add_child_common(device_t dev, u_int order, const char *name, int unit, + size_t ivars_size) { device_t child; struct gpiobus_ivar *devi; + KASSERT(ivars_size >= sizeof(struct gpiobus_ivar), + ("child ivars must include gpiobus_ivar as their first member")); child = device_add_child_ordered(dev, order, name, unit); if (child == NULL) return (child); - devi = malloc(sizeof(struct gpiobus_ivar), M_DEVBUF, M_NOWAIT | M_ZERO); + devi = malloc(ivars_size, M_DEVBUF, M_NOWAIT | M_ZERO); if (devi == NULL) { device_delete_child(dev, child); return (NULL); @@ -694,6 +696,13 @@ gpiobus_add_child(device_t dev, u_int order, const char *name, int unit) return (child); } +static device_t +gpiobus_add_child(device_t dev, u_int order, const char *name, int unit) +{ + return (gpiobus_add_child_common(dev, order, name, unit, + sizeof(struct gpiobus_ivar))); +} + static void gpiobus_child_deleted(device_t dev, device_t child) { diff --git a/sys/dev/gpio/gpiobus_internal.h b/sys/dev/gpio/gpiobus_internal.h index de3f57663132..c198e5f79989 100644 --- a/sys/dev/gpio/gpiobus_internal.h +++ b/sys/dev/gpio/gpiobus_internal.h @@ -42,6 +42,8 @@ void gpiobus_free_ivars(struct gpiobus_ivar *); int gpiobus_read_ivar(device_t, device_t, int, uintptr_t *); int gpiobus_acquire_pin(device_t, uint32_t); void gpiobus_release_pin(device_t, uint32_t); +int gpiobus_child_location(device_t, device_t, struct sbuf *); +device_t gpiobus_add_child_common(device_t, u_int, const char *, int, size_t); extern driver_t gpiobus_driver; #endif diff --git a/sys/dev/gpio/ofw_gpiobus.c b/sys/dev/gpio/ofw_gpiobus.c index fc5fb03d6824..b12b78fac18c 100644 --- a/sys/dev/gpio/ofw_gpiobus.c +++ b/sys/dev/gpio/ofw_gpiobus.c @@ -451,28 +451,22 @@ ofw_gpiobus_add_child(device_t dev, u_int order, const char *name, int unit) device_t child; struct ofw_gpiobus_devinfo *devi; - child = device_add_child_ordered(dev, order, name, unit); + child = gpiobus_add_child_common(dev, order, name, unit, + sizeof(struct ofw_gpiobus_devinfo)); if (child == NULL) - return (child); - devi = malloc(sizeof(struct ofw_gpiobus_devinfo), M_DEVBUF, - M_NOWAIT | M_ZERO); - if (devi == NULL) { - device_delete_child(dev, child); - return (0); - } + return (NULL); /* * NULL all the OFW-related parts of the ivars for non-OFW * children. */ + devi = device_get_ivars(child); devi->opd_obdinfo.obd_node = -1; devi->opd_obdinfo.obd_name = NULL; devi->opd_obdinfo.obd_compat = NULL; devi->opd_obdinfo.obd_type = NULL; devi->opd_obdinfo.obd_model = NULL; - device_set_ivars(child, devi); - return (child); } diff --git a/sys/dev/hid/hidbus.c b/sys/dev/hid/hidbus.c index 96d36c8d191d..683449fca49c 100644 --- a/sys/dev/hid/hidbus.c +++ b/sys/dev/hid/hidbus.c @@ -65,7 +65,7 @@ struct hidbus_ivars { struct mtx *mtx; /* child intr mtx */ hid_intr_t *intr_handler; /* executed under mtx*/ void *intr_ctx; - unsigned int refcnt; /* protected by mtx */ + bool active; /* protected by mtx */ struct epoch_context epoch_ctx; CK_STAILQ_ENTRY(hidbus_ivars) link; }; @@ -398,7 +398,7 @@ hidbus_child_detached(device_t bus, device_t child) struct hidbus_softc *sc = device_get_softc(bus); struct hidbus_ivars *tlc = device_get_ivars(child); - KASSERT(tlc->refcnt == 0, ("Child device is running")); + KASSERT(!tlc->active, ("Child device is running")); tlc->mtx = &sc->mtx; tlc->intr_handler = NULL; tlc->flags &= ~HIDBUS_FLAG_CAN_POLL; @@ -423,7 +423,7 @@ hidbus_child_deleted(device_t bus, device_t child) struct hidbus_ivars *tlc = device_get_ivars(child); sx_xlock(&sc->sx); - KASSERT(tlc->refcnt == 0, ("Child device is running")); + KASSERT(!tlc->active, ("Child device is running")); CK_STAILQ_REMOVE(&sc->tlcs, tlc, hidbus_ivars, link); sx_unlock(&sc->sx); epoch_call(INPUT_EPOCH, hidbus_ivar_dtor, &tlc->epoch_ctx); @@ -572,7 +572,7 @@ hidbus_intr(void *context, void *buf, hid_size_t len) if (!HID_IN_POLLING_MODE()) epoch_enter_preempt(INPUT_EPOCH, &et); CK_STAILQ_FOREACH(tlc, &sc->tlcs, link) { - if (tlc->refcnt == 0 || tlc->intr_handler == NULL) + if (!tlc->active || tlc->intr_handler == NULL) continue; if (HID_IN_POLLING_MODE()) { if ((tlc->flags & HIDBUS_FLAG_CAN_POLL) != 0) @@ -602,21 +602,14 @@ hidbus_intr_start(device_t bus, device_t child) MPASS(bus == device_get_parent(child)); struct hidbus_softc *sc = device_get_softc(bus); struct hidbus_ivars *ivar = device_get_ivars(child); - struct hidbus_ivars *tlc; - bool refcnted = false; int error; if (sx_xlock_sig(&sc->sx) != 0) return (EINTR); - CK_STAILQ_FOREACH(tlc, &sc->tlcs, link) { - refcnted |= (tlc->refcnt != 0); - if (tlc == ivar) { - mtx_lock(tlc->mtx); - ++tlc->refcnt; - mtx_unlock(tlc->mtx); - } - } - error = refcnted ? 0 : hid_intr_start(bus); + mtx_lock(ivar->mtx); + ivar->active = true; + mtx_unlock(ivar->mtx); + error = hid_intr_start(bus); sx_unlock(&sc->sx); return (error); @@ -629,21 +622,17 @@ hidbus_intr_stop(device_t bus, device_t child) struct hidbus_softc *sc = device_get_softc(bus); struct hidbus_ivars *ivar = device_get_ivars(child); struct hidbus_ivars *tlc; - bool refcnted = false; + bool active = false; int error; if (sx_xlock_sig(&sc->sx) != 0) return (EINTR); - CK_STAILQ_FOREACH(tlc, &sc->tlcs, link) { - if (tlc == ivar) { - mtx_lock(tlc->mtx); - MPASS(tlc->refcnt != 0); - --tlc->refcnt; - mtx_unlock(tlc->mtx); - } - refcnted |= (tlc->refcnt != 0); - } - error = refcnted ? 0 : hid_intr_stop(bus); + mtx_lock(ivar->mtx); + ivar->active = false; + mtx_unlock(ivar->mtx); + CK_STAILQ_FOREACH(tlc, &sc->tlcs, link) + active |= tlc->active; + error = active ? 0 : hid_intr_stop(bus); sx_unlock(&sc->sx); return (error); diff --git a/sys/dev/hid/hidquirk.h b/sys/dev/hid/hidquirk.h index 4f8b8acbe201..f6fa9f88c6c9 100644 --- a/sys/dev/hid/hidquirk.h +++ b/sys/dev/hid/hidquirk.h @@ -50,6 +50,7 @@ HQ(IS_XBOX360GP), /* device is XBox 360 GamePad */ \ HQ(NOWRITE), /* device does not support writes */ \ HQ(IICHID_SAMPLING), /* IIC backend runs in sampling mode */ \ + HQ(NO_READAHEAD), /* Disable interrupt after one report */\ \ /* Various quirks */ \ HQ(HID_IGNORE), /* device should be ignored by hid class */ \ diff --git a/sys/dev/hid/hidraw.c b/sys/dev/hid/hidraw.c index 06f70070f61b..4855843cd265 100644 --- a/sys/dev/hid/hidraw.c +++ b/sys/dev/hid/hidraw.c @@ -85,6 +85,12 @@ SYSCTL_INT(_hw_hid_hidraw, OID_AUTO, debug, CTLFLAG_RWTUN, free((buf), M_DEVBUF); \ } +#ifdef HIDRAW_MAKE_UHID_ALIAS +#define HIDRAW_NAME "uhid" +#else +#define HIDRAW_NAME "hidraw" +#endif + struct hidraw_softc { device_t sc_dev; /* base device */ @@ -183,8 +189,8 @@ hidraw_identify(driver_t *driver, device_t parent) { device_t child; - if (device_find_child(parent, "hidraw", DEVICE_UNIT_ANY) == NULL) { - child = BUS_ADD_CHILD(parent, 0, "hidraw", + if (device_find_child(parent, HIDRAW_NAME, DEVICE_UNIT_ANY) == NULL) { + child = BUS_ADD_CHILD(parent, 0, HIDRAW_NAME, device_get_unit(parent)); if (child != NULL) hidbus_set_index(child, HIDRAW_INDEX); @@ -1050,7 +1056,7 @@ static device_method_t hidraw_methods[] = { }; static driver_t hidraw_driver = { - "hidraw", + HIDRAW_NAME, hidraw_methods, sizeof(struct hidraw_softc) }; diff --git a/sys/dev/hid/hkbd.c b/sys/dev/hid/hkbd.c index 5eff7557bc42..6255c42d3b62 100644 --- a/sys/dev/hid/hkbd.c +++ b/sys/dev/hid/hkbd.c @@ -95,14 +95,16 @@ #ifdef HID_DEBUG static int hkbd_debug = 0; +#endif static int hkbd_no_leds = 0; static SYSCTL_NODE(_hw_hid, OID_AUTO, hkbd, CTLFLAG_RW, 0, "USB keyboard"); +#ifdef HID_DEBUG SYSCTL_INT(_hw_hid_hkbd, OID_AUTO, debug, CTLFLAG_RWTUN, &hkbd_debug, 0, "Debug level"); +#endif SYSCTL_INT(_hw_hid_hkbd, OID_AUTO, no_leds, CTLFLAG_RWTUN, &hkbd_no_leds, 0, "Disables setting of keyboard leds"); -#endif #define INPUT_EPOCH global_epoch_preempt @@ -1596,8 +1598,16 @@ hkbd_ioctl_locked(keyboard_t *kbd, u_long cmd, caddr_t arg) sc->sc_state &= ~LOCK_MASK; sc->sc_state |= *(int *)arg; - /* set LEDs and quit */ - return (hkbd_ioctl_locked(kbd, KDSETLED, arg)); + /* + * Attempt to set the keyboard LEDs; ignore the return value + * intentionally. Note: Some hypervisors/emulators (e.g., QEMU, + * Parallels—at least as of the time of writing) may fail when + * setting LEDs. This can prevent kbdmux from attaching the + * keyboard, which in turn may block the console from accessing + * it. + */ + (void)hkbd_ioctl_locked(kbd, KDSETLED, arg); + return (0); case KDSETREPEAT: /* set keyboard repeat rate (new * interface) */ @@ -1766,10 +1776,8 @@ hkbd_set_leds(struct hkbd_softc *sc, uint8_t leds) SYSCONS_LOCK_ASSERT(); DPRINTF("leds=0x%02x\n", leds); -#ifdef HID_DEBUG if (hkbd_no_leds) return (0); -#endif memset(sc->sc_buffer, 0, HKBD_BUFFER_SIZE); @@ -1820,6 +1828,7 @@ hkbd_set_leds(struct hkbd_softc *sc, uint8_t leds) SYSCONS_UNLOCK(); error = hid_write(sc->sc_dev, buf, len); SYSCONS_LOCK(); + DPRINTF("error %d", error); return (error); } diff --git a/sys/dev/hid/ietp.c b/sys/dev/hid/ietp.c index 217585a7948b..73a5cb7414d4 100644 --- a/sys/dev/hid/ietp.c +++ b/sys/dev/hid/ietp.c @@ -102,6 +102,7 @@ struct ietp_softc { device_t dev; struct evdev_dev *evdev; + bool open; uint8_t report_id; hid_size_t report_len; @@ -217,13 +218,25 @@ static const struct evdev_methods ietp_evdev_methods = { static int ietp_ev_open(struct evdev_dev *evdev) { - return (hid_intr_start(evdev_get_softc(evdev))); + struct ietp_softc *sc = evdev_get_softc(evdev); + int error; + + error = hid_intr_start(sc->dev); + if (error == 0) + sc->open = true; + return (error); } static int ietp_ev_close(struct evdev_dev *evdev) { - return (hid_intr_stop(evdev_get_softc(evdev))); + struct ietp_softc *sc = evdev_get_softc(evdev); + int error; + + error = hid_intr_stop(sc->dev); + if (error == 0) + sc->open = false; + return (error); } static int @@ -275,7 +288,7 @@ ietp_attach(struct ietp_softc *sc) evdev_set_id(sc->evdev, hw->idBus, hw->idVendor, hw->idProduct, hw->idVersion); evdev_set_serial(sc->evdev, hw->serial); - evdev_set_methods(sc->evdev, sc->dev, &ietp_evdev_methods); + evdev_set_methods(sc->evdev, sc, &ietp_evdev_methods); evdev_set_flag(sc->evdev, EVDEV_FLAG_MT_STCOMPAT); evdev_set_flag(sc->evdev, EVDEV_FLAG_EXT_EPOCH); /* hidbus child */ @@ -584,11 +597,13 @@ ietp_iic_set_absolute_mode(device_t dev, bool enable) * Some ASUS touchpads need to be powered on to enter absolute mode. */ require_wakeup = false; - for (i = 0; i < nitems(special_fw); i++) { - if (sc->ic_type == special_fw[i].ic_type && - sc->product_id == special_fw[i].product_id) { - require_wakeup = true; - break; + if (!sc->open) { + for (i = 0; i < nitems(special_fw); i++) { + if (sc->ic_type == special_fw[i].ic_type && + sc->product_id == special_fw[i].product_id) { + require_wakeup = true; + break; + } } } diff --git a/sys/dev/hid/u2f.c b/sys/dev/hid/u2f.c new file mode 100644 index 000000000000..ac2eba7a499d --- /dev/null +++ b/sys/dev/hid/u2f.c @@ -0,0 +1,590 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2022-2023 Vladimir Kondratyev <wulf@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "opt_hid.h" + +#include <sys/param.h> +#ifdef COMPAT_FREEBSD32 +#include <sys/abi_compat.h> +#endif +#include <sys/bus.h> +#include <sys/conf.h> +#include <sys/fcntl.h> +#include <sys/filio.h> +#include <sys/ioccom.h> +#include <sys/kernel.h> +#include <sys/lock.h> +#include <sys/malloc.h> +#include <sys/module.h> +#include <sys/mutex.h> +#include <sys/poll.h> +#include <sys/priv.h> +#include <sys/proc.h> +#include <sys/selinfo.h> +#include <sys/sysctl.h> +#include <sys/systm.h> +#include <sys/uio.h> + +#include <dev/evdev/input.h> + +#define HID_DEBUG_VAR u2f_debug +#include <dev/hid/hid.h> +#include <dev/hid/hidbus.h> +#include <dev/hid/hidquirk.h> + +#include <dev/usb/usb_ioctl.h> + +#ifdef HID_DEBUG +static int u2f_debug = 0; +static SYSCTL_NODE(_hw_hid, OID_AUTO, u2f, CTLFLAG_RW, 0, + "FIDO/U2F authenticator"); +SYSCTL_INT(_hw_hid_u2f, OID_AUTO, debug, CTLFLAG_RWTUN, + &u2f_debug, 0, "Debug level"); +#endif + +#define U2F_MAX_REPORT_SIZE 64 + +/* A match on these entries will load u2f */ +static const struct hid_device_id u2f_devs[] = { + { HID_BUS(BUS_USB), HID_TLC(HUP_FIDO, HUF_U2FHID) }, +}; + +struct u2f_softc { + device_t sc_dev; /* base device */ + struct cdev *dev; + + struct mtx sc_mtx; /* hidbus private mutex */ + void *sc_rdesc; + hid_size_t sc_rdesc_size; + hid_size_t sc_isize; + hid_size_t sc_osize; + struct selinfo sc_rsel; + struct { /* driver state */ + bool open:1; /* device is open */ + bool aslp:1; /* waiting for device data in read() */ + bool sel:1; /* waiting for device data in poll() */ + bool data:1; /* input report is stored in sc_buf */ + int reserved:28; + } sc_state; + int sc_fflags; /* access mode for open lifetime */ + + uint8_t sc_buf[U2F_MAX_REPORT_SIZE]; +}; + +static d_open_t u2f_open; +static d_read_t u2f_read; +static d_write_t u2f_write; +static d_ioctl_t u2f_ioctl; +static d_poll_t u2f_poll; +static d_kqfilter_t u2f_kqfilter; + +static d_priv_dtor_t u2f_dtor; + +static struct cdevsw u2f_cdevsw = { + .d_version = D_VERSION, + .d_open = u2f_open, + .d_read = u2f_read, + .d_write = u2f_write, + .d_ioctl = u2f_ioctl, + .d_poll = u2f_poll, + .d_kqfilter = u2f_kqfilter, + .d_name = "u2f", +}; + +static hid_intr_t u2f_intr; + +static device_probe_t u2f_probe; +static device_attach_t u2f_attach; +static device_detach_t u2f_detach; + +static int u2f_kqread(struct knote *, long); +static void u2f_kqdetach(struct knote *); +static void u2f_notify(struct u2f_softc *); + +static struct filterops u2f_filterops_read = { + .f_isfd = 1, + .f_detach = u2f_kqdetach, + .f_event = u2f_kqread, +}; + +static int +u2f_probe(device_t dev) +{ + int error; + + error = HIDBUS_LOOKUP_DRIVER_INFO(dev, u2f_devs); + if (error != 0) + return (error); + + hidbus_set_desc(dev, "Authenticator"); + + return (BUS_PROBE_GENERIC); +} + +static int +u2f_attach(device_t dev) +{ + struct u2f_softc *sc = device_get_softc(dev); + struct hid_device_info *hw = __DECONST(struct hid_device_info *, + hid_get_device_info(dev)); + struct make_dev_args mda; + int error; + + sc->sc_dev = dev; + + error = hid_get_report_descr(dev, &sc->sc_rdesc, &sc->sc_rdesc_size); + if (error != 0) + return (ENXIO); + sc->sc_isize = hid_report_size_max(sc->sc_rdesc, sc->sc_rdesc_size, + hid_input, NULL); + if (sc->sc_isize > U2F_MAX_REPORT_SIZE) { + device_printf(dev, "Input report size too large. Truncate.\n"); + sc->sc_isize = U2F_MAX_REPORT_SIZE; + } + sc->sc_osize = hid_report_size_max(sc->sc_rdesc, sc->sc_rdesc_size, + hid_output, NULL); + if (sc->sc_osize > U2F_MAX_REPORT_SIZE) { + device_printf(dev, "Output report size too large. Truncate.\n"); + sc->sc_osize = U2F_MAX_REPORT_SIZE; + } + + mtx_init(&sc->sc_mtx, "u2f lock", NULL, MTX_DEF); + knlist_init_mtx(&sc->sc_rsel.si_note, &sc->sc_mtx); + + make_dev_args_init(&mda); + mda.mda_flags = MAKEDEV_WAITOK; + mda.mda_devsw = &u2f_cdevsw; + mda.mda_uid = UID_ROOT; + mda.mda_gid = GID_U2F; + mda.mda_mode = 0660; + mda.mda_si_drv1 = sc; + + error = make_dev_s(&mda, &sc->dev, "u2f/%d", device_get_unit(dev)); + if (error) { + device_printf(dev, "Can not create character device\n"); + u2f_detach(dev); + return (error); + } +#ifdef U2F_MAKE_UHID_ALIAS + (void)make_dev_alias(sc->dev, "uhid%d", device_get_unit(dev)); +#endif + + hid_add_dynamic_quirk(hw, HQ_NO_READAHEAD); + + hidbus_set_lock(dev, &sc->sc_mtx); + hidbus_set_intr(dev, u2f_intr, sc); + + return (0); +} + +static int +u2f_detach(device_t dev) +{ + struct u2f_softc *sc = device_get_softc(dev); + + DPRINTF("sc=%p\n", sc); + + if (sc->dev != NULL) { + mtx_lock(&sc->sc_mtx); + sc->dev->si_drv1 = NULL; + /* Wake everyone */ + u2f_notify(sc); + mtx_unlock(&sc->sc_mtx); + destroy_dev(sc->dev); + } + + hid_intr_stop(sc->sc_dev); + + knlist_clear(&sc->sc_rsel.si_note, 0); + knlist_destroy(&sc->sc_rsel.si_note); + seldrain(&sc->sc_rsel); + mtx_destroy(&sc->sc_mtx); + + return (0); +} + +void +u2f_intr(void *context, void *buf, hid_size_t len) +{ + struct u2f_softc *sc = context; + + mtx_assert(&sc->sc_mtx, MA_OWNED); + + DPRINTFN(5, "len=%d\n", len); + DPRINTFN(5, "data = %*D\n", len, buf, " "); + + if (sc->sc_state.data) + return; + + if (len > sc->sc_isize) + len = sc->sc_isize; + + bcopy(buf, sc->sc_buf, len); + + /* Make sure we don't process old data */ + if (len < sc->sc_isize) + bzero(sc->sc_buf + len, sc->sc_isize - len); + + sc->sc_state.data = true; + + u2f_notify(sc); +} + +static int +u2f_open(struct cdev *dev, int flag, int mode, struct thread *td) +{ + struct u2f_softc *sc = dev->si_drv1; + int error; + + if (sc == NULL) + return (ENXIO); + + DPRINTF("sc=%p\n", sc); + + mtx_lock(&sc->sc_mtx); + if (sc->sc_state.open) { + mtx_unlock(&sc->sc_mtx); + return (EBUSY); + } + sc->sc_state.open = true; + mtx_unlock(&sc->sc_mtx); + + error = devfs_set_cdevpriv(sc, u2f_dtor); + if (error != 0) { + mtx_lock(&sc->sc_mtx); + sc->sc_state.open = false; + mtx_unlock(&sc->sc_mtx); + return (error); + } + + /* Set up interrupt pipe. */ + sc->sc_state.data = false; + sc->sc_fflags = flag; + + return (0); +} + + +static void +u2f_dtor(void *data) +{ + struct u2f_softc *sc = data; + +#ifdef NOT_YET + /* Disable interrupts. */ + hid_intr_stop(sc->sc_dev); +#endif + + mtx_lock(&sc->sc_mtx); + sc->sc_state.open = false; + mtx_unlock(&sc->sc_mtx); +} + +static int +u2f_read(struct cdev *dev, struct uio *uio, int flag) +{ + uint8_t buf[U2F_MAX_REPORT_SIZE]; + struct u2f_softc *sc = dev->si_drv1; + size_t length = 0; + int error; + + DPRINTFN(1, "\n"); + + if (sc == NULL) + return (EIO); + + if (!sc->sc_state.data) + hid_intr_start(sc->sc_dev); + + mtx_lock(&sc->sc_mtx); + if (dev->si_drv1 == NULL) { + error = EIO; + goto exit; + } + + while (!sc->sc_state.data) { + if (flag & O_NONBLOCK) { + error = EWOULDBLOCK; + goto exit; + } + sc->sc_state.aslp = true; + DPRINTFN(5, "sleep on %p\n", &sc->sc_buf); + error = mtx_sleep(&sc->sc_buf, &sc->sc_mtx, PZERO | PCATCH, + "u2frd", 0); + DPRINTFN(5, "woke, error=%d\n", error); + if (dev->si_drv1 == NULL) + error = EIO; + if (error) { + sc->sc_state.aslp = false; + goto exit; + } + } + + if (sc->sc_state.data && uio->uio_resid > 0) { + length = min(uio->uio_resid, sc->sc_isize); + memcpy(buf, sc->sc_buf, length); + sc->sc_state.data = false; + } +exit: + mtx_unlock(&sc->sc_mtx); + if (length != 0) { + /* Copy the data to the user process. */ + DPRINTFN(5, "got %lu chars\n", (u_long)length); + error = uiomove(buf, length, uio); + } + + return (error); +} + +static int +u2f_write(struct cdev *dev, struct uio *uio, int flag) +{ + uint8_t buf[U2F_MAX_REPORT_SIZE]; + struct u2f_softc *sc = dev->si_drv1; + int error; + + DPRINTFN(1, "\n"); + + if (sc == NULL) + return (EIO); + + if (uio->uio_resid != sc->sc_osize) + return (EINVAL); + error = uiomove(buf, uio->uio_resid, uio); + if (error == 0) + error = hid_write(sc->sc_dev, buf, sc->sc_osize); + + return (error); +} + +#ifdef COMPAT_FREEBSD32 +static void +update_ugd32(const struct usb_gen_descriptor *ugd, + struct usb_gen_descriptor32 *ugd32) +{ + /* Don't update hgd_data pointer */ + CP(*ugd, *ugd32, ugd_lang_id); + CP(*ugd, *ugd32, ugd_maxlen); + CP(*ugd, *ugd32, ugd_actlen); + CP(*ugd, *ugd32, ugd_offset); + CP(*ugd, *ugd32, ugd_config_index); + CP(*ugd, *ugd32, ugd_string_index); + CP(*ugd, *ugd32, ugd_iface_index); + CP(*ugd, *ugd32, ugd_altif_index); + CP(*ugd, *ugd32, ugd_endpt_index); + CP(*ugd, *ugd32, ugd_report_type); + /* Don't update reserved */ +} +#endif + +static int +u2f_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, + struct thread *td) +{ +#ifdef COMPAT_FREEBSD32 + struct usb_gen_descriptor local_ugd; + struct usb_gen_descriptor32 *ugd32 = NULL; +#endif + struct u2f_softc *sc = dev->si_drv1; + struct usb_gen_descriptor *ugd = (struct usb_gen_descriptor *)addr; + uint32_t size; + + DPRINTFN(2, "cmd=%lx\n", cmd); + + if (sc == NULL) + return (EIO); + +#ifdef COMPAT_FREEBSD32 + switch (cmd) { + case USB_GET_REPORT_DESC32: + cmd = _IOC_NEWTYPE(cmd, struct usb_gen_descriptor); + ugd32 = (struct usb_gen_descriptor32 *)addr; + ugd = &local_ugd; + PTRIN_CP(*ugd32, *ugd, ugd_data); + CP(*ugd32, *ugd, ugd_lang_id); + CP(*ugd32, *ugd, ugd_maxlen); + CP(*ugd32, *ugd, ugd_actlen); + CP(*ugd32, *ugd, ugd_offset); + CP(*ugd32, *ugd, ugd_config_index); + CP(*ugd32, *ugd, ugd_string_index); + CP(*ugd32, *ugd, ugd_iface_index); + CP(*ugd32, *ugd, ugd_altif_index); + CP(*ugd32, *ugd, ugd_endpt_index); + CP(*ugd32, *ugd, ugd_report_type); + /* Don't copy reserved */ + break; + } +#endif + + /* fixed-length ioctls handling */ + switch (cmd) { + case FIONBIO: + /* All handled in the upper FS layer. */ + return (0); + + case USB_GET_REPORT_DESC: + size = MIN(sc->sc_rdesc_size, ugd->ugd_maxlen); + ugd->ugd_actlen = size; +#ifdef COMPAT_FREEBSD32 + if (ugd32 != NULL) + update_ugd32(ugd, ugd32); +#endif + if (ugd->ugd_data == NULL) + return (0); /* descriptor length only */ + + return (copyout(sc->sc_rdesc, ugd->ugd_data, size)); + + case USB_GET_DEVICEINFO: + return(hid_ioctl( + sc->sc_dev, USB_GET_DEVICEINFO, (uintptr_t)addr)); + } + + return (EINVAL); +} + +static int +u2f_poll(struct cdev *dev, int events, struct thread *td) +{ + struct u2f_softc *sc = dev->si_drv1; + int revents = 0; + bool start_intr = false; + + if (sc == NULL) + return (POLLHUP); + + if (events & (POLLOUT | POLLWRNORM) && (sc->sc_fflags & FWRITE)) + revents |= events & (POLLOUT | POLLWRNORM); + if (events & (POLLIN | POLLRDNORM) && (sc->sc_fflags & FREAD)) { + mtx_lock(&sc->sc_mtx); + if (sc->sc_state.data) + revents |= events & (POLLIN | POLLRDNORM); + else { + sc->sc_state.sel = true; + start_intr = true; + selrecord(td, &sc->sc_rsel); + } + mtx_unlock(&sc->sc_mtx); + if (start_intr) + hid_intr_start(sc->sc_dev); + } + + return (revents); +} + +static int +u2f_kqfilter(struct cdev *dev, struct knote *kn) +{ + struct u2f_softc *sc = dev->si_drv1; + + if (sc == NULL) + return (ENXIO); + + switch(kn->kn_filter) { + case EVFILT_READ: + if (sc->sc_fflags & FREAD) { + kn->kn_fop = &u2f_filterops_read; + break; + } + /* FALLTHROUGH */ + default: + return(EINVAL); + } + kn->kn_hook = sc; + + knlist_add(&sc->sc_rsel.si_note, kn, 0); + return (0); +} + +static int +u2f_kqread(struct knote *kn, long hint) +{ + struct u2f_softc *sc = kn->kn_hook; + int ret; + + mtx_assert(&sc->sc_mtx, MA_OWNED); + + if (sc->dev->si_drv1 == NULL) { + kn->kn_flags |= EV_EOF; + ret = 1; + } else { + ret = sc->sc_state.data ? 1 : 0; + if (!sc->sc_state.data) + hid_intr_start(sc->sc_dev); + } + + return (ret); +} + +static void +u2f_kqdetach(struct knote *kn) +{ + struct u2f_softc *sc = kn->kn_hook; + + knlist_remove(&sc->sc_rsel.si_note, kn, 0); +} + +static void +u2f_notify(struct u2f_softc *sc) +{ + mtx_assert(&sc->sc_mtx, MA_OWNED); + + if (sc->sc_state.aslp) { + sc->sc_state.aslp = false; + DPRINTFN(5, "waking %p\n", &sc->sc_buf); + wakeup(&sc->sc_buf); + } + if (sc->sc_state.sel) { + sc->sc_state.sel = false; + selwakeuppri(&sc->sc_rsel, PZERO); + } + KNOTE_LOCKED(&sc->sc_rsel.si_note, 0); +} + +static device_method_t u2f_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, u2f_probe), + DEVMETHOD(device_attach, u2f_attach), + DEVMETHOD(device_detach, u2f_detach), + + DEVMETHOD_END +}; + +static driver_t u2f_driver = { +#ifdef U2F_MAKE_UHID_ALIAS + "uhid", +#else + "u2f", +#endif + u2f_methods, + sizeof(struct u2f_softc) +}; + +DRIVER_MODULE(u2f, hidbus, u2f_driver, NULL, NULL); +MODULE_DEPEND(u2f, hidbus, 1, 1, 1); +MODULE_DEPEND(u2f, hid, 1, 1, 1); +MODULE_VERSION(u2f, 1); +HID_PNP_INFO(u2f_devs); diff --git a/sys/dev/ice/ice_bitops.h b/sys/dev/ice/ice_bitops.h index c480900596f4..a623f810c101 100644 --- a/sys/dev/ice/ice_bitops.h +++ b/sys/dev/ice/ice_bitops.h @@ -198,7 +198,7 @@ static inline void ice_zero_bitmap(ice_bitmap_t *bmp, u16 size) * ice_and_bitmap - bitwise AND 2 bitmaps and store result in dst bitmap * @dst: Destination bitmap that receive the result of the operation * @bmp1: The first bitmap to intersect - * @bmp2: The second bitmap to intersect wit the first + * @bmp2: The second bitmap to intersect with the first * @size: Size of the bitmaps in bits * * This function performs a bitwise AND on two "source" bitmaps of the same size @@ -237,7 +237,7 @@ ice_and_bitmap(ice_bitmap_t *dst, const ice_bitmap_t *bmp1, * ice_or_bitmap - bitwise OR 2 bitmaps and store result in dst bitmap * @dst: Destination bitmap that receive the result of the operation * @bmp1: The first bitmap to intersect - * @bmp2: The second bitmap to intersect wit the first + * @bmp2: The second bitmap to intersect with the first * @size: Size of the bitmaps in bits * * This function performs a bitwise OR on two "source" bitmaps of the same size diff --git a/sys/dev/ice/ice_lan_tx_rx.h b/sys/dev/ice/ice_lan_tx_rx.h index 693e0ca5efc6..eedacdab0216 100644 --- a/sys/dev/ice/ice_lan_tx_rx.h +++ b/sys/dev/ice/ice_lan_tx_rx.h @@ -630,7 +630,7 @@ enum ice_rxdid { ICE_RXDID_LAST = 63, }; -/* Recceive Flex descriptor Dword Index */ +/* Receive Flex descriptor Dword Index */ enum ice_flex_word { ICE_RX_FLEX_DWORD_0 = 0, ICE_RX_FLEX_DWORD_1, diff --git a/sys/dev/ice/ice_lib.h b/sys/dev/ice/ice_lib.h index 308b2bda2790..640bdf8fed7b 100644 --- a/sys/dev/ice/ice_lib.h +++ b/sys/dev/ice/ice_lib.h @@ -313,7 +313,7 @@ enum ice_dyn_idx_t { ICE_ITR_NONE = 3 /* ITR_NONE must not be used as an index */ }; -/* By convenction ITR0 is used for RX, and ITR1 is used for TX */ +/* By convention ITR0 is used for RX, and ITR1 is used for TX */ #define ICE_RX_ITR ICE_IDX_ITR0 #define ICE_TX_ITR ICE_IDX_ITR1 diff --git a/sys/dev/ice/ice_protocol_type.h b/sys/dev/ice/ice_protocol_type.h index 300d61bfb5d9..b90c25e6c427 100644 --- a/sys/dev/ice/ice_protocol_type.h +++ b/sys/dev/ice/ice_protocol_type.h @@ -143,7 +143,7 @@ enum ice_prot_id { ICE_PROT_LLDP_OF = 117, ICE_PROT_ARP_OF = 118, ICE_PROT_EAPOL_OF = 120, - ICE_PROT_META_ID = 255, /* when offset == metaddata */ + ICE_PROT_META_ID = 255, /* when offset == metadata */ ICE_PROT_INVALID = 255 /* when offset == ICE_FV_OFFSET_INVAL */ }; diff --git a/sys/dev/iicbus/iichid.c b/sys/dev/iicbus/iichid.c index 3f1d7a0cefba..fdb4816b8bd9 100644 --- a/sys/dev/iicbus/iichid.c +++ b/sys/dev/iicbus/iichid.c @@ -861,7 +861,8 @@ iichid_intr_start(device_t dev, device_t child __unused) sc = device_get_softc(dev); DPRINTF(sc, "iichid device open\n"); - iichid_set_power_state(sc, IICHID_PS_ON, IICHID_PS_NULL); + if (!sc->open) + iichid_set_power_state(sc, IICHID_PS_ON, IICHID_PS_NULL); return (0); } diff --git a/sys/dev/isci/scil/intel_sata.h b/sys/dev/isci/scil/intel_sata.h index 4cf4adf03e07..fdad5be9b083 100644 --- a/sys/dev/isci/scil/intel_sata.h +++ b/sys/dev/isci/scil/intel_sata.h @@ -61,7 +61,7 @@ * * @brief This file defines all of the SATA releated constants, enumerations, * and types. Please note that this file does not necessarily contain - * an exhaustive list of all contants and commands. + * an exhaustive list of all constants and commands. */ /** diff --git a/sys/dev/ixgbe/if_ix.c b/sys/dev/ixgbe/if_ix.c index 959afa79e7da..73c0fd1ab16f 100644 --- a/sys/dev/ixgbe/if_ix.c +++ b/sys/dev/ixgbe/if_ix.c @@ -45,7 +45,7 @@ /************************************************************************ * Driver version ************************************************************************/ -static const char ixgbe_driver_version[] = "4.0.1-k"; +static const char ixgbe_driver_version[] = "5.0.1-k"; /************************************************************************ * PCI Device ID Table @@ -144,6 +144,16 @@ static const pci_vendor_info_t ixgbe_vendor_info_array[] = "Intel(R) X540-T2 (Bypass)"), PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, "Intel(R) X520 82599 (Bypass)"), + PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_BACKPLANE, + "Intel(R) E610 (Backplane)"), + PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_SFP, + "Intel(R) E610 (SFP)"), + PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_2_5G_T, + "Intel(R) E610 (2.5 GbE)"), + PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_10G_T, + "Intel(R) E610 (10 GbE)"), + PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_SGMII, + "Intel(R) E610 (SGMII)"), /* required last entry */ PVID_END }; @@ -253,6 +263,10 @@ static int ixgbe_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS); static void ixgbe_handle_msf(void *); static void ixgbe_handle_mod(void *); static void ixgbe_handle_phy(void *); +static void ixgbe_handle_fw_event(void *); + +static int ixgbe_enable_lse(struct ixgbe_softc *sc); +static int ixgbe_disable_lse(struct ixgbe_softc *sc); /************************************************************************ * FreeBSD Device Interface Entry Points @@ -621,6 +635,7 @@ ixgbe_initialize_rss_mapping(struct ixgbe_softc *sc) case ixgbe_mac_X550: case ixgbe_mac_X550EM_x: case ixgbe_mac_X550EM_a: + case ixgbe_mac_E610: table_size = 512; break; default: @@ -902,6 +917,32 @@ ixgbe_initialize_transmit_units(if_ctx_t ctx) } /* ixgbe_initialize_transmit_units */ +static int +ixgbe_check_fw_api_version(struct ixgbe_softc *sc) +{ + struct ixgbe_hw *hw = &sc->hw; + if (hw->api_maj_ver > IXGBE_FW_API_VER_MAJOR) { + device_printf(sc->dev, + "The driver for the device stopped because the NVM " + "image is newer than expected. You must install the " + "most recent version of the network driver.\n"); + return (EOPNOTSUPP); + } else if (hw->api_maj_ver == IXGBE_FW_API_VER_MAJOR && + hw->api_min_ver > (IXGBE_FW_API_VER_MINOR + 2)) { + device_printf(sc->dev, + "The driver for the device detected a newer version of " + "the NVM image than expected. Please install the most " + "recent version of the network driver.\n"); + } else if (hw->api_maj_ver < IXGBE_FW_API_VER_MAJOR || + hw->api_min_ver < IXGBE_FW_API_VER_MINOR - 2) { + device_printf(sc->dev, + "The driver for the device detected an older version " + "of the NVM image than expected. " + "Please update the NVM image.\n"); + } + return (0); +} + /************************************************************************ * ixgbe_register ************************************************************************/ @@ -970,6 +1011,9 @@ ixgbe_if_attach_pre(if_ctx_t ctx) goto err_pci; } + if (hw->mac.type == ixgbe_mac_E610) + ixgbe_init_aci(hw); + if (hw->mac.ops.fw_recovery_mode && hw->mac.ops.fw_recovery_mode(hw)) { device_printf(dev, @@ -1058,6 +1102,12 @@ ixgbe_if_attach_pre(if_ctx_t ctx) break; } + /* Check the FW API version */ + if (hw->mac.type == ixgbe_mac_E610 && ixgbe_check_fw_api_version(sc)) { + error = EIO; + goto err_pci; + } + /* Most of the iflib initialization... */ iflib_set_mac(ctx, hw->mac.addr); @@ -1111,6 +1161,9 @@ err_pci: IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); ixgbe_free_pci_resources(ctx); + if (hw->mac.type == ixgbe_mac_E610) + ixgbe_shutdown_aci(hw); + return (error); } /* ixgbe_if_attach_pre */ @@ -1358,6 +1411,10 @@ ixgbe_add_media_types(if_ctx_t ctx) /* Media types with matching FreeBSD media defines */ if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) ifmedia_add(sc->media, IFM_ETHER | IFM_10G_T, 0, NULL); + if (layer & IXGBE_PHYSICAL_LAYER_5000BASE_T) + ifmedia_add(sc->media, IFM_ETHER | IFM_5000_T, 0, NULL); + if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_T) + ifmedia_add(sc->media, IFM_ETHER | IFM_2500_T, 0, NULL); if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) @@ -1459,6 +1516,7 @@ ixgbe_is_sfp(struct ixgbe_hw *hw) } case ixgbe_mac_X550EM_x: case ixgbe_mac_X550EM_a: + case ixgbe_mac_E610: if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) return (true); return (false); @@ -1525,6 +1583,15 @@ ixgbe_config_link(if_ctx_t ctx) IXGBE_LINK_SPEED_5GB_FULL); } + if (hw->mac.type == ixgbe_mac_E610) { + hw->phy.ops.init(hw); + err = ixgbe_enable_lse(sc); + if (err) + device_printf(sc->dev, + "Failed to enable Link Status Event, " + "error: %d", err); + } + if (hw->mac.ops.setup_link) err = hw->mac.ops.setup_link(hw, autoneg, sc->link_up); @@ -2158,14 +2225,15 @@ get_parent_info: ixgbe_set_pci_config_data_generic(hw, link); display: - device_printf(dev, "PCI Express Bus: Speed %s %s\n", - ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s" : + device_printf(dev, "PCI Express Bus: Speed %s Width %s\n", + ((hw->bus.speed == ixgbe_bus_speed_16000) ? "16.0GT/s" : + (hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s" : (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s" : (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s" : "Unknown"), - ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" : - (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" : - (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" : + ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "x8" : + (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "x4" : + (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "x1" : "Unknown")); if (bus_info_valid) { @@ -2372,14 +2440,17 @@ ixgbe_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr) ifmr->ifm_status |= IFM_ACTIVE; layer = sc->phy_layer; - if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || - layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || - layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || - layer & IXGBE_PHYSICAL_LAYER_10BASE_T) + if (layer & IXGBE_PHYSICAL_LAYERS_BASE_T_ALL) switch (sc->link_speed) { case IXGBE_LINK_SPEED_10GB_FULL: ifmr->ifm_active |= IFM_10G_T | IFM_FDX; break; + case IXGBE_LINK_SPEED_5GB_FULL: + ifmr->ifm_active |= IFM_5000_T | IFM_FDX; + break; + case IXGBE_LINK_SPEED_2_5GB_FULL: + ifmr->ifm_active |= IFM_2500_T | IFM_FDX; + break; case IXGBE_LINK_SPEED_1GB_FULL: ifmr->ifm_active |= IFM_1000_T | IFM_FDX; break; @@ -2390,15 +2461,6 @@ ixgbe_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr) ifmr->ifm_active |= IFM_10_T | IFM_FDX; break; } - if (hw->mac.type == ixgbe_mac_X550) - switch (sc->link_speed) { - case IXGBE_LINK_SPEED_5GB_FULL: - ifmr->ifm_active |= IFM_5000_T | IFM_FDX; - break; - case IXGBE_LINK_SPEED_2_5GB_FULL: - ifmr->ifm_active |= IFM_2500_T | IFM_FDX; - break; - } if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) switch (sc->link_speed) { @@ -2676,6 +2738,11 @@ ixgbe_msix_link(void *arg) sc->task_requests |= IXGBE_REQUEST_TASK_LSC; } + if (eicr & IXGBE_EICR_FW_EVENT) { + IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FW_EVENT); + sc->task_requests |= IXGBE_REQUEST_TASK_FWEVENT; + } + if (sc->hw.mac.type != ixgbe_mac_82598EB) { if ((sc->feat_en & IXGBE_FEATURE_FDIR) && (eicr & IXGBE_EICR_FLOW_DIR)) { @@ -2734,11 +2801,16 @@ ixgbe_msix_link(void *arg) /* Check for VF message */ if ((sc->feat_en & IXGBE_FEATURE_SRIOV) && - (eicr & IXGBE_EICR_MAILBOX)) + (eicr & IXGBE_EICR_MAILBOX)) { sc->task_requests |= IXGBE_REQUEST_TASK_MBX; + } } - if (ixgbe_is_sfp(hw)) { + /* + * On E610, the firmware handles PHY configuration, so + * there is no need to perform any SFP-specific tasks. + */ + if (hw->mac.type != ixgbe_mac_E610 && ixgbe_is_sfp(hw)) { /* Pluggable optics-related interrupt */ if (hw->mac.type >= ixgbe_mac_X540) eicr_mask = IXGBE_EICR_GPI_SDP0_X540; @@ -2985,7 +3057,13 @@ ixgbe_if_detach(if_ctx_t ctx) callout_drain(&sc->fw_mode_timer); + if (sc->hw.mac.type == ixgbe_mac_E610) { + ixgbe_disable_lse(sc); + ixgbe_shutdown_aci(&sc->hw); + } + ixgbe_free_pci_resources(ctx); + free(sc->mta, M_IXGBE); return (0); @@ -3404,6 +3482,7 @@ ixgbe_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type) case ixgbe_mac_X550: case ixgbe_mac_X550EM_x: case ixgbe_mac_X550EM_a: + case ixgbe_mac_E610: if (type == -1) { /* MISC IVAR */ index = (entry & 1) * 8; ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); @@ -3826,6 +3905,96 @@ ixgbe_handle_phy(void *context) } /* ixgbe_handle_phy */ /************************************************************************ + * ixgbe_enable_lse - enable link status events + * + * Sets mask and enables link status events + ************************************************************************/ +s32 ixgbe_enable_lse(struct ixgbe_softc *sc) +{ + s32 error; + + u16 mask = ~((u16)(IXGBE_ACI_LINK_EVENT_UPDOWN | + IXGBE_ACI_LINK_EVENT_MEDIA_NA | + IXGBE_ACI_LINK_EVENT_MODULE_QUAL_FAIL | + IXGBE_ACI_LINK_EVENT_PHY_FW_LOAD_FAIL)); + + error = ixgbe_configure_lse(&sc->hw, TRUE, mask); + if (error) + return (error); + + sc->lse_mask = mask; + return (IXGBE_SUCCESS); +} /* ixgbe_enable_lse */ + +/************************************************************************ + * ixgbe_disable_lse - disable link status events + ************************************************************************/ +s32 ixgbe_disable_lse(struct ixgbe_softc *sc) +{ + s32 error; + + error = ixgbe_configure_lse(&sc->hw, false, sc->lse_mask); + if (error) + return (error); + + sc->lse_mask = 0; + return (IXGBE_SUCCESS); +} /* ixgbe_disable_lse */ + +/************************************************************************ + * ixgbe_handle_fw_event - Tasklet for MSI-X Link Status Event interrupts + ************************************************************************/ +static void +ixgbe_handle_fw_event(void *context) +{ + if_ctx_t ctx = context; + struct ixgbe_softc *sc = iflib_get_softc(ctx); + struct ixgbe_hw *hw = &sc->hw; + struct ixgbe_aci_event event; + bool pending = false; + s32 error; + + event.buf_len = IXGBE_ACI_MAX_BUFFER_SIZE; + event.msg_buf = malloc(event.buf_len, M_IXGBE, M_ZERO | M_NOWAIT); + if (!event.msg_buf) { + device_printf(sc->dev, "Can not allocate buffer for " + "event message\n"); + return; + } + + do { + error = ixgbe_aci_get_event(hw, &event, &pending); + if (error) { + device_printf(sc->dev, "Error getting event from " + "FW:%d\n", error); + break; + } + + switch (le16toh(event.desc.opcode)) { + case ixgbe_aci_opc_get_link_status: + sc->task_requests |= IXGBE_REQUEST_TASK_LSC; + break; + + case ixgbe_aci_opc_temp_tca_event: + if (hw->adapter_stopped == FALSE) + ixgbe_if_stop(ctx); + device_printf(sc->dev, + "CRITICAL: OVER TEMP!! PHY IS SHUT DOWN!!\n"); + device_printf(sc->dev, "System shutdown required!\n"); + break; + + default: + device_printf(sc->dev, + "Unknown FW event captured, opcode=0x%04X\n", + le16toh(event.desc.opcode)); + break; + } + } while (pending); + + free(event.msg_buf, M_IXGBE); +} /* ixgbe_handle_fw_event */ + +/************************************************************************ * ixgbe_if_stop - Stop the hardware * * Disables all traffic on the adapter by issuing a @@ -3899,6 +4068,8 @@ ixgbe_if_update_admin_status(if_ctx_t ctx) } /* Handle task requests from msix_link() */ + if (sc->task_requests & IXGBE_REQUEST_TASK_FWEVENT) + ixgbe_handle_fw_event(ctx); if (sc->task_requests & IXGBE_REQUEST_TASK_MOD) ixgbe_handle_mod(ctx); if (sc->task_requests & IXGBE_REQUEST_TASK_MSF) @@ -3986,6 +4157,9 @@ ixgbe_if_enable_intr(if_ctx_t ctx) mask |= IXGBE_EICR_GPI_SDP0_X540; mask |= IXGBE_EIMS_ECC; break; + case ixgbe_mac_E610: + mask |= IXGBE_EIMS_FW_EVENT; + break; default: break; } @@ -4008,6 +4182,7 @@ ixgbe_if_enable_intr(if_ctx_t ctx) /* Don't autoclear Link */ mask &= ~IXGBE_EIMS_OTHER; mask &= ~IXGBE_EIMS_LSC; + mask &= ~IXGBE_EIMS_FW_EVENT; if (sc->feat_cap & IXGBE_FEATURE_SRIOV) mask &= ~IXGBE_EIMS_MAILBOX; IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); @@ -4026,7 +4201,7 @@ ixgbe_if_enable_intr(if_ctx_t ctx) } /* ixgbe_if_enable_intr */ /************************************************************************ - * ixgbe_disable_intr + * ixgbe_if_disable_intr ************************************************************************/ static void ixgbe_if_disable_intr(if_ctx_t ctx) @@ -4176,8 +4351,9 @@ ixgbe_intr(void *arg) /* External PHY interrupt */ if ((hw->phy.type == ixgbe_phy_x550em_ext_t) && - (eicr & IXGBE_EICR_GPI_SDP0_X540)) + (eicr & IXGBE_EICR_GPI_SDP0_X540)) { sc->task_requests |= IXGBE_REQUEST_TASK_PHY; + } return (FILTER_SCHEDULE_THREAD); } /* ixgbe_intr */ @@ -4219,7 +4395,7 @@ ixgbe_sysctl_flowcntl(SYSCTL_HANDLER_ARGS) int error, fc; sc = (struct ixgbe_softc *)arg1; - fc = sc->hw.fc.current_mode; + fc = sc->hw.fc.requested_mode; error = sysctl_handle_int(oidp, &fc, 0, req); if ((error) || (req->newptr == NULL)) @@ -4248,12 +4424,10 @@ ixgbe_set_flowcntl(struct ixgbe_softc *sc, int fc) case ixgbe_fc_rx_pause: case ixgbe_fc_tx_pause: case ixgbe_fc_full: - sc->hw.fc.requested_mode = fc; if (sc->num_rx_queues > 1) ixgbe_disable_rx_drop(sc); break; case ixgbe_fc_none: - sc->hw.fc.requested_mode = ixgbe_fc_none; if (sc->num_rx_queues > 1) ixgbe_enable_rx_drop(sc); break; @@ -4261,6 +4435,8 @@ ixgbe_set_flowcntl(struct ixgbe_softc *sc, int fc) return (EINVAL); } + sc->hw.fc.requested_mode = fc; + /* Don't autoneg if forcing a value */ sc->hw.fc.disable_fc_autoneg = true; ixgbe_fc_enable(&sc->hw); @@ -4978,6 +5154,9 @@ ixgbe_init_device_features(struct ixgbe_softc *sc) if (sc->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP) sc->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; break; + case ixgbe_mac_E610: + sc->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE; + break; default: break; } diff --git a/sys/dev/ixgbe/if_ixv.c b/sys/dev/ixgbe/if_ixv.c index 54b2c8c1dd68..8a1c1aae041d 100644 --- a/sys/dev/ixgbe/if_ixv.c +++ b/sys/dev/ixgbe/if_ixv.c @@ -68,6 +68,8 @@ static const pci_vendor_info_t ixv_vendor_info_array[] = "Intel(R) X552 Virtual Function"), PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, "Intel(R) X553 Virtual Function"), + PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_VF, + "Intel(R) E610 Virtual Function"), /* required last entry */ PVID_END }; @@ -1020,6 +1022,9 @@ ixv_identify_hardware(if_ctx_t ctx) case IXGBE_DEV_ID_X550EM_A_VF: hw->mac.type = ixgbe_mac_X550EM_a_vf; break; + case IXGBE_DEV_ID_E610_VF: + hw->mac.type = ixgbe_mac_E610_vf; + break; default: device_printf(dev, "unknown mac type\n"); hw->mac.type = ixgbe_mac_unknown; @@ -1955,6 +1960,7 @@ ixv_init_device_features(struct ixgbe_softc *sc) case ixgbe_mac_X550_vf: case ixgbe_mac_X550EM_x_vf: case ixgbe_mac_X550EM_a_vf: + case ixgbe_mac_E610_vf: sc->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; sc->feat_cap |= IXGBE_FEATURE_RSS; break; diff --git a/sys/dev/ixgbe/ixgbe.h b/sys/dev/ixgbe/ixgbe.h index 341d4ebfcebc..844064bf8543 100644 --- a/sys/dev/ixgbe/ixgbe.h +++ b/sys/dev/ixgbe/ixgbe.h @@ -86,6 +86,7 @@ #include "ixgbe_phy.h" #include "ixgbe_vf.h" #include "ixgbe_features.h" +#include "ixgbe_e610.h" /* Tunables */ @@ -195,6 +196,15 @@ CSUM_IP_UDP|CSUM_IP_TCP|CSUM_IP_SCTP| \ CSUM_IP6_UDP|CSUM_IP6_TCP|CSUM_IP6_SCTP) +/* All BASE-T Physical layers */ +#define IXGBE_PHYSICAL_LAYERS_BASE_T_ALL \ + (IXGBE_PHYSICAL_LAYER_10GBASE_T |\ + IXGBE_PHYSICAL_LAYER_5000BASE_T |\ + IXGBE_PHYSICAL_LAYER_2500BASE_T |\ + IXGBE_PHYSICAL_LAYER_1000BASE_T |\ + IXGBE_PHYSICAL_LAYER_100BASE_TX |\ + IXGBE_PHYSICAL_LAYER_10BASE_T) + #define IXGBE_CAPS (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | IFCAP_TSO | \ IFCAP_LRO | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | \ IFCAP_VLAN_HWCSUM | IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU | \ @@ -464,6 +474,7 @@ struct ixgbe_softc { /* Feature capable/enabled flags. See ixgbe_features.h */ u32 feat_cap; u32 feat_en; + u16 lse_mask; }; /* Precision Time Sync (IEEE 1588) defines */ diff --git a/sys/dev/ixgbe/ixgbe_api.c b/sys/dev/ixgbe/ixgbe_api.c index 4c50f10ed92e..f11f52a646e4 100644 --- a/sys/dev/ixgbe/ixgbe_api.c +++ b/sys/dev/ixgbe/ixgbe_api.c @@ -112,11 +112,15 @@ s32 ixgbe_init_shared_code(struct ixgbe_hw *hw) case ixgbe_mac_X550EM_a: status = ixgbe_init_ops_X550EM_a(hw); break; + case ixgbe_mac_E610: + status = ixgbe_init_ops_E610(hw); + break; case ixgbe_mac_82599_vf: case ixgbe_mac_X540_vf: case ixgbe_mac_X550_vf: case ixgbe_mac_X550EM_x_vf: case ixgbe_mac_X550EM_a_vf: + case ixgbe_mac_E610_vf: status = ixgbe_init_ops_vf(hw); break; default: @@ -240,6 +244,18 @@ s32 ixgbe_set_mac_type(struct ixgbe_hw *hw) hw->mac.type = ixgbe_mac_X550EM_a_vf; hw->mvals = ixgbe_mvals_X550EM_a; break; + case IXGBE_DEV_ID_E610_BACKPLANE: + case IXGBE_DEV_ID_E610_SFP: + case IXGBE_DEV_ID_E610_10G_T: + case IXGBE_DEV_ID_E610_2_5G_T: + case IXGBE_DEV_ID_E610_SGMII: + hw->mac.type = ixgbe_mac_E610; + hw->mvals = ixgbe_mvals_X550EM_a; + break; + case IXGBE_DEV_ID_E610_VF: + hw->mac.type = ixgbe_mac_E610_vf; + hw->mvals = ixgbe_mvals_X550EM_a; + break; default: ret_val = IXGBE_ERR_DEVICE_NOT_SUPPORTED; ERROR_REPORT2(IXGBE_ERROR_UNSUPPORTED, diff --git a/sys/dev/ixgbe/ixgbe_api.h b/sys/dev/ixgbe/ixgbe_api.h index b81510dacb95..2b4cec8d110e 100644 --- a/sys/dev/ixgbe/ixgbe_api.h +++ b/sys/dev/ixgbe/ixgbe_api.h @@ -48,6 +48,7 @@ extern s32 ixgbe_init_ops_X550(struct ixgbe_hw *hw); extern s32 ixgbe_init_ops_X550EM(struct ixgbe_hw *hw); extern s32 ixgbe_init_ops_X550EM_x(struct ixgbe_hw *hw); extern s32 ixgbe_init_ops_X550EM_a(struct ixgbe_hw *hw); +extern s32 ixgbe_init_ops_E610(struct ixgbe_hw *hw); extern s32 ixgbe_init_ops_vf(struct ixgbe_hw *hw); s32 ixgbe_set_mac_type(struct ixgbe_hw *hw); diff --git a/sys/dev/ixgbe/ixgbe_common.c b/sys/dev/ixgbe/ixgbe_common.c index df7ab90e72ab..bff022585a03 100644 --- a/sys/dev/ixgbe/ixgbe_common.c +++ b/sys/dev/ixgbe/ixgbe_common.c @@ -178,6 +178,7 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw) case IXGBE_DEV_ID_X550EM_A_SFP_N: case IXGBE_DEV_ID_X550EM_A_QSFP: case IXGBE_DEV_ID_X550EM_A_QSFP_N: + case IXGBE_DEV_ID_E610_SFP: supported = false; break; default: @@ -210,6 +211,8 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw) case IXGBE_DEV_ID_X550EM_A_10G_T: case IXGBE_DEV_ID_X550EM_A_1G_T: case IXGBE_DEV_ID_X550EM_A_1G_T_L: + case IXGBE_DEV_ID_E610_10G_T: + case IXGBE_DEV_ID_E610_2_5G_T: supported = true; break; default: @@ -616,7 +619,8 @@ s32 ixgbe_clear_hw_cntrs_generic(struct ixgbe_hw *hw) } } - if (hw->mac.type == ixgbe_mac_X550 || hw->mac.type == ixgbe_mac_X540) { + if (hw->mac.type == ixgbe_mac_X540 || + hw->mac.type == ixgbe_mac_X550) { if (hw->phy.id == 0) ixgbe_identify_phy(hw); hw->phy.ops.read_reg(hw, IXGBE_PCRC8ECL, @@ -1037,6 +1041,9 @@ void ixgbe_set_pci_config_data_generic(struct ixgbe_hw *hw, u16 link_status) case IXGBE_PCI_LINK_SPEED_8000: hw->bus.speed = ixgbe_bus_speed_8000; break; + case IXGBE_PCI_LINK_SPEED_16000: + hw->bus.speed = ixgbe_bus_speed_16000; + break; default: hw->bus.speed = ixgbe_bus_speed_unknown; break; @@ -1059,7 +1066,9 @@ s32 ixgbe_get_bus_info_generic(struct ixgbe_hw *hw) DEBUGFUNC("ixgbe_get_bus_info_generic"); /* Get the negotiated link width and speed from PCI config space */ - link_status = IXGBE_READ_PCIE_WORD(hw, IXGBE_PCI_LINK_STATUS); + link_status = IXGBE_READ_PCIE_WORD(hw, hw->mac.type == ixgbe_mac_E610 ? + IXGBE_PCI_LINK_STATUS_E610 : + IXGBE_PCI_LINK_STATUS); ixgbe_set_pci_config_data_generic(hw, link_status); @@ -1878,7 +1887,6 @@ static s32 ixgbe_get_eeprom_semaphore(struct ixgbe_hw *hw) DEBUGFUNC("ixgbe_get_eeprom_semaphore"); - /* Get SMBI software semaphore between device drivers first */ for (i = 0; i < timeout; i++) { /* @@ -3363,7 +3371,6 @@ s32 ixgbe_disable_sec_rx_path_generic(struct ixgbe_hw *hw) DEBUGFUNC("ixgbe_disable_sec_rx_path_generic"); - secrxreg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL); secrxreg |= IXGBE_SECRXCTRL_RX_DIS; IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, secrxreg); @@ -3692,6 +3699,10 @@ u16 ixgbe_get_pcie_msix_count_generic(struct ixgbe_hw *hw) pcie_offset = IXGBE_PCIE_MSIX_82599_CAPS; max_msix_count = IXGBE_MAX_MSIX_VECTORS_82599; break; + case ixgbe_mac_E610: + pcie_offset = IXGBE_PCIE_MSIX_E610_CAPS; + max_msix_count = IXGBE_MAX_MSIX_VECTORS_82599; + break; default: return msix_count; } @@ -4139,7 +4150,6 @@ s32 ixgbe_clear_vfta_generic(struct ixgbe_hw *hw) return IXGBE_SUCCESS; } - /** * ixgbe_toggle_txdctl_generic - Toggle VF's queues * @hw: pointer to hardware structure @@ -4323,7 +4333,8 @@ s32 ixgbe_check_mac_link_generic(struct ixgbe_hw *hw, ixgbe_link_speed *speed, break; case IXGBE_LINKS_SPEED_100_82599: *speed = IXGBE_LINK_SPEED_100_FULL; - if (hw->mac.type == ixgbe_mac_X550) { + if (hw->mac.type == ixgbe_mac_X550 || + hw->mac.type == ixgbe_mac_E610) { if (links_reg & IXGBE_LINKS_SPEED_NON_STD) *speed = IXGBE_LINK_SPEED_5GB_FULL; } @@ -5494,6 +5505,7 @@ void ixgbe_get_nvm_version(struct ixgbe_hw *hw, case ixgbe_mac_X550: case ixgbe_mac_X550EM_x: case ixgbe_mac_X550EM_a: + case ixgbe_mac_E610: /* version of eeprom section */ if (ixgbe_read_eeprom(hw, NVM_EEP_OFFSET_X540, &word)) word = NVM_VER_INVALID; @@ -5512,6 +5524,7 @@ void ixgbe_get_nvm_version(struct ixgbe_hw *hw, case ixgbe_mac_X550: case ixgbe_mac_X550EM_x: case ixgbe_mac_X550EM_a: + case ixgbe_mac_E610: /* intel phy firmware version */ if (ixgbe_read_eeprom(hw, NVM_EEP_PHY_OFF_X540, &word)) word = NVM_VER_INVALID; diff --git a/sys/dev/ixgbe/ixgbe_e610.c b/sys/dev/ixgbe/ixgbe_e610.c new file mode 100644 index 000000000000..95c6dca416c6 --- /dev/null +++ b/sys/dev/ixgbe/ixgbe_e610.c @@ -0,0 +1,5567 @@ +/****************************************************************************** + SPDX-License-Identifier: BSD-3-Clause + + Copyright (c) 2025, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ + +#include "ixgbe_type.h" +#include "ixgbe_e610.h" +#include "ixgbe_x550.h" +#include "ixgbe_common.h" +#include "ixgbe_phy.h" +#include "ixgbe_api.h" + +/** + * ixgbe_init_aci - initialization routine for Admin Command Interface + * @hw: pointer to the hardware structure + * + * Initialize the ACI lock. + */ +void ixgbe_init_aci(struct ixgbe_hw *hw) +{ + ixgbe_init_lock(&hw->aci.lock); +} + +/** + * ixgbe_shutdown_aci - shutdown routine for Admin Command Interface + * @hw: pointer to the hardware structure + * + * Destroy the ACI lock. + */ +void ixgbe_shutdown_aci(struct ixgbe_hw *hw) +{ + ixgbe_destroy_lock(&hw->aci.lock); +} + +/** + * ixgbe_should_retry_aci_send_cmd_execute - decide if ACI command should + * be resent + * @opcode: ACI opcode + * + * Check if ACI command should be sent again depending on the provided opcode. + * + * Return: true if the sending command routine should be repeated, + * otherwise false. + */ +static bool ixgbe_should_retry_aci_send_cmd_execute(u16 opcode) +{ + switch (opcode) { + case ixgbe_aci_opc_disable_rxen: + case ixgbe_aci_opc_get_phy_caps: + case ixgbe_aci_opc_get_link_status: + case ixgbe_aci_opc_get_link_topo: + return true; + } + + return false; +} + +/** + * ixgbe_aci_send_cmd_execute - execute sending FW Admin Command to FW Admin + * Command Interface + * @hw: pointer to the HW struct + * @desc: descriptor describing the command + * @buf: buffer to use for indirect commands (NULL for direct commands) + * @buf_size: size of buffer for indirect commands (0 for direct commands) + * + * Admin Command is sent using CSR by setting descriptor and buffer in specific + * registers. + * + * Return: the exit code of the operation. + * * - IXGBE_SUCCESS - success. + * * - IXGBE_ERR_ACI_DISABLED - CSR mechanism is not enabled. + * * - IXGBE_ERR_ACI_BUSY - CSR mechanism is busy. + * * - IXGBE_ERR_PARAM - buf_size is too big or + * invalid argument buf or buf_size. + * * - IXGBE_ERR_ACI_TIMEOUT - Admin Command X command timeout. + * * - IXGBE_ERR_ACI_ERROR - Admin Command X invalid state of HICR register or + * Admin Command failed because of bad opcode was returned or + * Admin Command failed with error Y. + */ +static s32 +ixgbe_aci_send_cmd_execute(struct ixgbe_hw *hw, struct ixgbe_aci_desc *desc, + void *buf, u16 buf_size) +{ + u32 hicr = 0, tmp_buf_size = 0, i = 0; + u32 *raw_desc = (u32 *)desc; + s32 status = IXGBE_SUCCESS; + bool valid_buf = false; + u32 *tmp_buf = NULL; + u16 opcode = 0; + + do { + hw->aci.last_status = IXGBE_ACI_RC_OK; + + /* It's necessary to check if mechanism is enabled */ + hicr = IXGBE_READ_REG(hw, PF_HICR); + if (!(hicr & PF_HICR_EN)) { + status = IXGBE_ERR_ACI_DISABLED; + break; + } + if (hicr & PF_HICR_C) { + hw->aci.last_status = IXGBE_ACI_RC_EBUSY; + status = IXGBE_ERR_ACI_BUSY; + break; + } + opcode = desc->opcode; + + if (buf_size > IXGBE_ACI_MAX_BUFFER_SIZE) { + status = IXGBE_ERR_PARAM; + break; + } + + if (buf) + desc->flags |= IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_BUF); + + /* Check if buf and buf_size are proper params */ + if (desc->flags & IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_BUF)) { + if ((buf && buf_size == 0) || + (buf == NULL && buf_size)) { + status = IXGBE_ERR_PARAM; + break; + } + if (buf && buf_size) + valid_buf = true; + } + + if (valid_buf == true) { + if (buf_size % 4 == 0) + tmp_buf_size = buf_size; + else + tmp_buf_size = (buf_size & (u16)(~0x03)) + 4; + + tmp_buf = (u32*)ixgbe_malloc(hw, tmp_buf_size); + if (!tmp_buf) + return IXGBE_ERR_OUT_OF_MEM; + + /* tmp_buf will be firstly filled with 0xFF and after + * that the content of buf will be written into it. + * This approach lets us use valid buf_size and + * prevents us from reading past buf area + * when buf_size mod 4 not equal to 0. + */ + memset(tmp_buf, 0xFF, tmp_buf_size); + memcpy(tmp_buf, buf, buf_size); + + if (tmp_buf_size > IXGBE_ACI_LG_BUF) + desc->flags |= + IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_LB); + + desc->datalen = IXGBE_CPU_TO_LE16(buf_size); + + if (desc->flags & IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_RD)) { + for (i = 0; i < tmp_buf_size / 4; i++) { + IXGBE_WRITE_REG(hw, PF_HIBA(i), + IXGBE_LE32_TO_CPU(tmp_buf[i])); + } + } + } + + /* Descriptor is written to specific registers */ + for (i = 0; i < IXGBE_ACI_DESC_SIZE_IN_DWORDS; i++) + IXGBE_WRITE_REG(hw, PF_HIDA(i), + IXGBE_LE32_TO_CPU(raw_desc[i])); + + /* SW has to set PF_HICR.C bit and clear PF_HICR.SV and + * PF_HICR_EV + */ + hicr = IXGBE_READ_REG(hw, PF_HICR); + hicr = (hicr | PF_HICR_C) & ~(PF_HICR_SV | PF_HICR_EV); + IXGBE_WRITE_REG(hw, PF_HICR, hicr); + + /* Wait for sync Admin Command response */ + for (i = 0; i < IXGBE_ACI_SYNC_RESPONSE_TIMEOUT; i += 1) { + hicr = IXGBE_READ_REG(hw, PF_HICR); + if ((hicr & PF_HICR_SV) || !(hicr & PF_HICR_C)) + break; + + msec_delay(1); + } + + /* Wait for async Admin Command response */ + if ((hicr & PF_HICR_SV) && (hicr & PF_HICR_C)) { + for (i = 0; i < IXGBE_ACI_ASYNC_RESPONSE_TIMEOUT; + i += 1) { + hicr = IXGBE_READ_REG(hw, PF_HICR); + if ((hicr & PF_HICR_EV) || !(hicr & PF_HICR_C)) + break; + + msec_delay(1); + } + } + + /* Read sync Admin Command response */ + if ((hicr & PF_HICR_SV)) { + for (i = 0; i < IXGBE_ACI_DESC_SIZE_IN_DWORDS; i++) { + raw_desc[i] = IXGBE_READ_REG(hw, PF_HIDA(i)); + raw_desc[i] = IXGBE_CPU_TO_LE32(raw_desc[i]); + } + } + + /* Read async Admin Command response */ + if ((hicr & PF_HICR_EV) && !(hicr & PF_HICR_C)) { + for (i = 0; i < IXGBE_ACI_DESC_SIZE_IN_DWORDS; i++) { + raw_desc[i] = IXGBE_READ_REG(hw, PF_HIDA_2(i)); + raw_desc[i] = IXGBE_CPU_TO_LE32(raw_desc[i]); + } + } + + /* Handle timeout and invalid state of HICR register */ + if (hicr & PF_HICR_C) { + status = IXGBE_ERR_ACI_TIMEOUT; + break; + } else if (!(hicr & PF_HICR_SV) && !(hicr & PF_HICR_EV)) { + status = IXGBE_ERR_ACI_ERROR; + break; + } + + /* For every command other than 0x0014 treat opcode mismatch + * as an error. Response to 0x0014 command read from HIDA_2 + * is a descriptor of an event which is expected to contain + * different opcode than the command. + */ + if (desc->opcode != opcode && + opcode != IXGBE_CPU_TO_LE16(ixgbe_aci_opc_get_fw_event)) { + status = IXGBE_ERR_ACI_ERROR; + break; + } + + if (desc->retval != IXGBE_ACI_RC_OK) { + hw->aci.last_status = (enum ixgbe_aci_err)desc->retval; + status = IXGBE_ERR_ACI_ERROR; + break; + } + + /* Write a response values to a buf */ + if (valid_buf && (desc->flags & + IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_BUF))) { + for (i = 0; i < tmp_buf_size / 4; i++) { + tmp_buf[i] = IXGBE_READ_REG(hw, PF_HIBA(i)); + tmp_buf[i] = IXGBE_CPU_TO_LE32(tmp_buf[i]); + } + memcpy(buf, tmp_buf, buf_size); + } + } while (0); + + if (tmp_buf) + ixgbe_free(hw, tmp_buf); + + return status; +} + +/** + * ixgbe_aci_send_cmd - send FW Admin Command to FW Admin Command Interface + * @hw: pointer to the HW struct + * @desc: descriptor describing the command + * @buf: buffer to use for indirect commands (NULL for direct commands) + * @buf_size: size of buffer for indirect commands (0 for direct commands) + * + * Helper function to send FW Admin Commands to the FW Admin Command Interface. + * + * Retry sending the FW Admin Command multiple times to the FW ACI + * if the EBUSY Admin Command error is returned. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_send_cmd(struct ixgbe_hw *hw, struct ixgbe_aci_desc *desc, + void *buf, u16 buf_size) +{ + struct ixgbe_aci_desc desc_cpy; + enum ixgbe_aci_err last_status; + bool is_cmd_for_retry; + u8 *buf_cpy = NULL; + s32 status; + u16 opcode; + u8 idx = 0; + + opcode = IXGBE_LE16_TO_CPU(desc->opcode); + is_cmd_for_retry = ixgbe_should_retry_aci_send_cmd_execute(opcode); + memset(&desc_cpy, 0, sizeof(desc_cpy)); + + if (is_cmd_for_retry) { + if (buf) { + buf_cpy = (u8 *)ixgbe_malloc(hw, buf_size); + if (!buf_cpy) + return IXGBE_ERR_OUT_OF_MEM; + } + memcpy(&desc_cpy, desc, sizeof(desc_cpy)); + } + + do { + ixgbe_acquire_lock(&hw->aci.lock); + status = ixgbe_aci_send_cmd_execute(hw, desc, buf, buf_size); + last_status = hw->aci.last_status; + ixgbe_release_lock(&hw->aci.lock); + + if (!is_cmd_for_retry || status == IXGBE_SUCCESS || + (last_status != IXGBE_ACI_RC_EBUSY && status != IXGBE_ERR_ACI_ERROR)) + break; + + if (buf) + memcpy(buf, buf_cpy, buf_size); + memcpy(desc, &desc_cpy, sizeof(desc_cpy)); + + msec_delay(IXGBE_ACI_SEND_DELAY_TIME_MS); + } while (++idx < IXGBE_ACI_SEND_MAX_EXECUTE); + + if (buf_cpy) + ixgbe_free(hw, buf_cpy); + + return status; +} + +/** + * ixgbe_aci_check_event_pending - check if there are any pending events + * @hw: pointer to the HW struct + * + * Determine if there are any pending events. + * + * Return: true if there are any currently pending events + * otherwise false. + */ +bool ixgbe_aci_check_event_pending(struct ixgbe_hw *hw) +{ + u32 ep_bit_mask; + u32 fwsts; + + ep_bit_mask = hw->bus.func ? GL_FWSTS_EP_PF1 : GL_FWSTS_EP_PF0; + + /* Check state of Event Pending (EP) bit */ + fwsts = IXGBE_READ_REG(hw, GL_FWSTS); + return (fwsts & ep_bit_mask) ? true : false; +} + +/** + * ixgbe_aci_get_event - get an event from ACI + * @hw: pointer to the HW struct + * @e: event information structure + * @pending: optional flag signaling that there are more pending events + * + * Obtain an event from ACI and return its content + * through 'e' using ACI command (0x0014). + * Provide information if there are more events + * to retrieve through 'pending'. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_get_event(struct ixgbe_hw *hw, struct ixgbe_aci_event *e, + bool *pending) +{ + struct ixgbe_aci_desc desc; + s32 status; + + if (!e || (!e->msg_buf && e->buf_len) || (e->msg_buf && !e->buf_len)) + return IXGBE_ERR_PARAM; + + ixgbe_acquire_lock(&hw->aci.lock); + + /* Check if there are any events pending */ + if (!ixgbe_aci_check_event_pending(hw)) { + status = IXGBE_ERR_ACI_NO_EVENTS; + goto aci_get_event_exit; + } + + /* Obtain pending event */ + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_get_fw_event); + status = ixgbe_aci_send_cmd_execute(hw, &desc, e->msg_buf, e->buf_len); + if (status) + goto aci_get_event_exit; + + /* Returned 0x0014 opcode indicates that no event was obtained */ + if (desc.opcode == IXGBE_CPU_TO_LE16(ixgbe_aci_opc_get_fw_event)) { + status = IXGBE_ERR_ACI_NO_EVENTS; + goto aci_get_event_exit; + } + + /* Determine size of event data */ + e->msg_len = MIN_T(u16, IXGBE_LE16_TO_CPU(desc.datalen), e->buf_len); + /* Write event descriptor to event info structure */ + memcpy(&e->desc, &desc, sizeof(e->desc)); + + /* Check if there are any further events pending */ + if (pending) { + *pending = ixgbe_aci_check_event_pending(hw); + } + +aci_get_event_exit: + ixgbe_release_lock(&hw->aci.lock); + + return status; +} + +/** + * ixgbe_fill_dflt_direct_cmd_desc - fill ACI descriptor with default values. + * @desc: pointer to the temp descriptor (non DMA mem) + * @opcode: the opcode can be used to decide which flags to turn off or on + * + * Helper function to fill the descriptor desc with default values + * and the provided opcode. + */ +void ixgbe_fill_dflt_direct_cmd_desc(struct ixgbe_aci_desc *desc, u16 opcode) +{ + /* zero out the desc */ + memset(desc, 0, sizeof(*desc)); + desc->opcode = IXGBE_CPU_TO_LE16(opcode); + desc->flags = IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_SI); +} + +/** + * ixgbe_aci_get_fw_ver - get the firmware version + * @hw: pointer to the HW struct + * + * Get the firmware version using ACI command (0x0001). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_get_fw_ver(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_get_ver *resp; + struct ixgbe_aci_desc desc; + s32 status; + + resp = &desc.params.get_ver; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_get_ver); + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + + if (!status) { + hw->fw_branch = resp->fw_branch; + hw->fw_maj_ver = resp->fw_major; + hw->fw_min_ver = resp->fw_minor; + hw->fw_patch = resp->fw_patch; + hw->fw_build = IXGBE_LE32_TO_CPU(resp->fw_build); + hw->api_branch = resp->api_branch; + hw->api_maj_ver = resp->api_major; + hw->api_min_ver = resp->api_minor; + hw->api_patch = resp->api_patch; + } + + return status; +} + +/** + * ixgbe_aci_send_driver_ver - send the driver version to firmware + * @hw: pointer to the HW struct + * @dv: driver's major, minor version + * + * Send the driver version to the firmware + * using the ACI command (0x0002). + * + * Return: the exit code of the operation. + * Returns IXGBE_ERR_PARAM, if dv is NULL. + */ +s32 ixgbe_aci_send_driver_ver(struct ixgbe_hw *hw, struct ixgbe_driver_ver *dv) +{ + struct ixgbe_aci_cmd_driver_ver *cmd; + struct ixgbe_aci_desc desc; + u16 len; + + cmd = &desc.params.driver_ver; + + if (!dv) + return IXGBE_ERR_PARAM; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_driver_ver); + + desc.flags |= IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_RD); + cmd->major_ver = dv->major_ver; + cmd->minor_ver = dv->minor_ver; + cmd->build_ver = dv->build_ver; + cmd->subbuild_ver = dv->subbuild_ver; + + len = 0; + while (len < sizeof(dv->driver_string) && + IS_ASCII(dv->driver_string[len]) && dv->driver_string[len]) + len++; + + return ixgbe_aci_send_cmd(hw, &desc, dv->driver_string, len); +} + +/** + * ixgbe_aci_req_res - request a common resource + * @hw: pointer to the HW struct + * @res: resource ID + * @access: access type + * @sdp_number: resource number + * @timeout: the maximum time in ms that the driver may hold the resource + * + * Requests a common resource using the ACI command (0x0008). + * Specifies the maximum time the driver may hold the resource. + * If the requested resource is currently occupied by some other driver, + * a busy return value is returned and the timeout field value indicates the + * maximum time the current owner has to free it. + * + * Return: the exit code of the operation. + */ +static s32 +ixgbe_aci_req_res(struct ixgbe_hw *hw, enum ixgbe_aci_res_ids res, + enum ixgbe_aci_res_access_type access, u8 sdp_number, + u32 *timeout) +{ + struct ixgbe_aci_cmd_req_res *cmd_resp; + struct ixgbe_aci_desc desc; + s32 status; + + cmd_resp = &desc.params.res_owner; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_req_res); + + cmd_resp->res_id = IXGBE_CPU_TO_LE16(res); + cmd_resp->access_type = IXGBE_CPU_TO_LE16(access); + cmd_resp->res_number = IXGBE_CPU_TO_LE32(sdp_number); + cmd_resp->timeout = IXGBE_CPU_TO_LE32(*timeout); + *timeout = 0; + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + + /* The completion specifies the maximum time in ms that the driver + * may hold the resource in the Timeout field. + * If the resource is held by some other driver, the command completes + * with a busy return value and the timeout field indicates the maximum + * time the current owner of the resource has to free it. + */ + if (!status || hw->aci.last_status == IXGBE_ACI_RC_EBUSY) + *timeout = IXGBE_LE32_TO_CPU(cmd_resp->timeout); + + return status; +} + +/** + * ixgbe_aci_release_res - release a common resource using ACI + * @hw: pointer to the HW struct + * @res: resource ID + * @sdp_number: resource number + * + * Release a common resource using ACI command (0x0009). + * + * Return: the exit code of the operation. + */ +static s32 +ixgbe_aci_release_res(struct ixgbe_hw *hw, enum ixgbe_aci_res_ids res, + u8 sdp_number) +{ + struct ixgbe_aci_cmd_req_res *cmd; + struct ixgbe_aci_desc desc; + + cmd = &desc.params.res_owner; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_release_res); + + cmd->res_id = IXGBE_CPU_TO_LE16(res); + cmd->res_number = IXGBE_CPU_TO_LE32(sdp_number); + + return ixgbe_aci_send_cmd(hw, &desc, NULL, 0); +} + +/** + * ixgbe_acquire_res - acquire the ownership of a resource + * @hw: pointer to the HW structure + * @res: resource ID + * @access: access type (read or write) + * @timeout: timeout in milliseconds + * + * Make an attempt to acquire the ownership of a resource using + * the ixgbe_aci_req_res to utilize ACI. + * In case if some other driver has previously acquired the resource and + * performed any necessary updates, the IXGBE_ERR_ACI_NO_WORK is returned, + * and the caller does not obtain the resource and has no further work to do. + * If needed, the function will poll until the current lock owner timeouts. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_acquire_res(struct ixgbe_hw *hw, enum ixgbe_aci_res_ids res, + enum ixgbe_aci_res_access_type access, u32 timeout) +{ +#define IXGBE_RES_POLLING_DELAY_MS 10 + u32 delay = IXGBE_RES_POLLING_DELAY_MS; + u32 res_timeout = timeout; + u32 retry_timeout = 0; + s32 status; + + status = ixgbe_aci_req_res(hw, res, access, 0, &res_timeout); + + /* A return code of IXGBE_ERR_ACI_NO_WORK means that another driver has + * previously acquired the resource and performed any necessary updates; + * in this case the caller does not obtain the resource and has no + * further work to do. + */ + if (status == IXGBE_ERR_ACI_NO_WORK) + goto ixgbe_acquire_res_exit; + + /* If necessary, poll until the current lock owner timeouts. + * Set retry_timeout to the timeout value reported by the FW in the + * response to the "Request Resource Ownership" (0x0008) Admin Command + * as it indicates the maximum time the current owner of the resource + * is allowed to hold it. + */ + retry_timeout = res_timeout; + while (status && retry_timeout && res_timeout) { + msec_delay(delay); + retry_timeout = (retry_timeout > delay) ? + retry_timeout - delay : 0; + status = ixgbe_aci_req_res(hw, res, access, 0, &res_timeout); + + if (status == IXGBE_ERR_ACI_NO_WORK) + /* lock free, but no work to do */ + break; + + if (!status) + /* lock acquired */ + break; + } + +ixgbe_acquire_res_exit: + return status; +} + +/** + * ixgbe_release_res - release a common resource + * @hw: pointer to the HW structure + * @res: resource ID + * + * Release a common resource using ixgbe_aci_release_res. + */ +void ixgbe_release_res(struct ixgbe_hw *hw, enum ixgbe_aci_res_ids res) +{ + u32 total_delay = 0; + s32 status; + + status = ixgbe_aci_release_res(hw, res, 0); + + /* There are some rare cases when trying to release the resource + * results in an admin command timeout, so handle them correctly. + */ + while ((status == IXGBE_ERR_ACI_TIMEOUT) && + (total_delay < IXGBE_ACI_RELEASE_RES_TIMEOUT)) { + msec_delay(1); + status = ixgbe_aci_release_res(hw, res, 0); + total_delay++; + } +} + +/** + * ixgbe_parse_common_caps - Parse common device/function capabilities + * @hw: pointer to the HW struct + * @caps: pointer to common capabilities structure + * @elem: the capability element to parse + * @prefix: message prefix for tracing capabilities + * + * Given a capability element, extract relevant details into the common + * capability structure. + * + * Return: true if the capability matches one of the common capability ids, + * false otherwise. + */ +static bool +ixgbe_parse_common_caps(struct ixgbe_hw *hw, struct ixgbe_hw_common_caps *caps, + struct ixgbe_aci_cmd_list_caps_elem *elem, + const char *prefix) +{ + u32 logical_id = IXGBE_LE32_TO_CPU(elem->logical_id); + u32 phys_id = IXGBE_LE32_TO_CPU(elem->phys_id); + u32 number = IXGBE_LE32_TO_CPU(elem->number); + u16 cap = IXGBE_LE16_TO_CPU(elem->cap); + bool found = true; + + UNREFERENCED_1PARAMETER(hw); + + switch (cap) { + case IXGBE_ACI_CAPS_VALID_FUNCTIONS: + caps->valid_functions = number; + break; + case IXGBE_ACI_CAPS_SRIOV: + caps->sr_iov_1_1 = (number == 1); + break; + case IXGBE_ACI_CAPS_VMDQ: + caps->vmdq = (number == 1); + break; + case IXGBE_ACI_CAPS_DCB: + caps->dcb = (number == 1); + caps->active_tc_bitmap = logical_id; + caps->maxtc = phys_id; + break; + case IXGBE_ACI_CAPS_RSS: + caps->rss_table_size = number; + caps->rss_table_entry_width = logical_id; + break; + case IXGBE_ACI_CAPS_RXQS: + caps->num_rxq = number; + caps->rxq_first_id = phys_id; + break; + case IXGBE_ACI_CAPS_TXQS: + caps->num_txq = number; + caps->txq_first_id = phys_id; + break; + case IXGBE_ACI_CAPS_MSIX: + caps->num_msix_vectors = number; + caps->msix_vector_first_id = phys_id; + break; + case IXGBE_ACI_CAPS_NVM_VER: + break; + case IXGBE_ACI_CAPS_NVM_MGMT: + caps->sec_rev_disabled = + (number & IXGBE_NVM_MGMT_SEC_REV_DISABLED) ? + true : false; + caps->update_disabled = + (number & IXGBE_NVM_MGMT_UPDATE_DISABLED) ? + true : false; + caps->nvm_unified_update = + (number & IXGBE_NVM_MGMT_UNIFIED_UPD_SUPPORT) ? + true : false; + caps->netlist_auth = + (number & IXGBE_NVM_MGMT_NETLIST_AUTH_SUPPORT) ? + true : false; + break; + case IXGBE_ACI_CAPS_MAX_MTU: + caps->max_mtu = number; + break; + case IXGBE_ACI_CAPS_PCIE_RESET_AVOIDANCE: + caps->pcie_reset_avoidance = (number > 0); + break; + case IXGBE_ACI_CAPS_POST_UPDATE_RESET_RESTRICT: + caps->reset_restrict_support = (number == 1); + break; + case IXGBE_ACI_CAPS_EXT_TOPO_DEV_IMG0: + case IXGBE_ACI_CAPS_EXT_TOPO_DEV_IMG1: + case IXGBE_ACI_CAPS_EXT_TOPO_DEV_IMG2: + case IXGBE_ACI_CAPS_EXT_TOPO_DEV_IMG3: + { + u8 index = cap - IXGBE_ACI_CAPS_EXT_TOPO_DEV_IMG0; + + caps->ext_topo_dev_img_ver_high[index] = number; + caps->ext_topo_dev_img_ver_low[index] = logical_id; + caps->ext_topo_dev_img_part_num[index] = + (phys_id & IXGBE_EXT_TOPO_DEV_IMG_PART_NUM_M) >> + IXGBE_EXT_TOPO_DEV_IMG_PART_NUM_S; + caps->ext_topo_dev_img_load_en[index] = + (phys_id & IXGBE_EXT_TOPO_DEV_IMG_LOAD_EN) != 0; + caps->ext_topo_dev_img_prog_en[index] = + (phys_id & IXGBE_EXT_TOPO_DEV_IMG_PROG_EN) != 0; + break; + } + case IXGBE_ACI_CAPS_OROM_RECOVERY_UPDATE: + caps->orom_recovery_update = (number == 1); + break; + case IXGBE_ACI_CAPS_NEXT_CLUSTER_ID: + caps->next_cluster_id_support = (number == 1); + DEBUGOUT2("%s: next_cluster_id_support = %d\n", + prefix, caps->next_cluster_id_support); + break; + default: + /* Not one of the recognized common capabilities */ + found = false; + } + + return found; +} + +/** + * ixgbe_hweight8 - count set bits among the 8 lowest bits + * @w: variable storing set bits to count + * + * Return: the number of set bits among the 8 lowest bits in the provided value. + */ +static u8 ixgbe_hweight8(u32 w) +{ + u8 hweight = 0, i; + + for (i = 0; i < 8; i++) + if (w & (1 << i)) + hweight++; + + return hweight; +} + +/** + * ixgbe_hweight32 - count set bits among the 32 lowest bits + * @w: variable storing set bits to count + * + * Return: the number of set bits among the 32 lowest bits in the + * provided value. + */ +static u8 ixgbe_hweight32(u32 w) +{ + u32 bitMask = 0x1, i; + u8 bitCnt = 0; + + for (i = 0; i < 32; i++) + { + if (w & bitMask) + bitCnt++; + + bitMask = bitMask << 0x1; + } + + return bitCnt; +} + +/** + * ixgbe_parse_valid_functions_cap - Parse IXGBE_ACI_CAPS_VALID_FUNCTIONS caps + * @hw: pointer to the HW struct + * @dev_p: pointer to device capabilities structure + * @cap: capability element to parse + * + * Parse IXGBE_ACI_CAPS_VALID_FUNCTIONS for device capabilities. + */ +static void +ixgbe_parse_valid_functions_cap(struct ixgbe_hw *hw, + struct ixgbe_hw_dev_caps *dev_p, + struct ixgbe_aci_cmd_list_caps_elem *cap) +{ + u32 number = IXGBE_LE32_TO_CPU(cap->number); + + UNREFERENCED_1PARAMETER(hw); + + dev_p->num_funcs = ixgbe_hweight32(number); +} + +/** + * ixgbe_parse_vf_dev_caps - Parse IXGBE_ACI_CAPS_VF device caps + * @hw: pointer to the HW struct + * @dev_p: pointer to device capabilities structure + * @cap: capability element to parse + * + * Parse IXGBE_ACI_CAPS_VF for device capabilities. + */ +static void ixgbe_parse_vf_dev_caps(struct ixgbe_hw *hw, + struct ixgbe_hw_dev_caps *dev_p, + struct ixgbe_aci_cmd_list_caps_elem *cap) +{ + u32 number = IXGBE_LE32_TO_CPU(cap->number); + + UNREFERENCED_1PARAMETER(hw); + + dev_p->num_vfs_exposed = number; +} + +/** + * ixgbe_parse_vsi_dev_caps - Parse IXGBE_ACI_CAPS_VSI device caps + * @hw: pointer to the HW struct + * @dev_p: pointer to device capabilities structure + * @cap: capability element to parse + * + * Parse IXGBE_ACI_CAPS_VSI for device capabilities. + */ +static void ixgbe_parse_vsi_dev_caps(struct ixgbe_hw *hw, + struct ixgbe_hw_dev_caps *dev_p, + struct ixgbe_aci_cmd_list_caps_elem *cap) +{ + u32 number = IXGBE_LE32_TO_CPU(cap->number); + + UNREFERENCED_1PARAMETER(hw); + + dev_p->num_vsi_allocd_to_host = number; +} + +/** + * ixgbe_parse_fdir_dev_caps - Parse IXGBE_ACI_CAPS_FD device caps + * @hw: pointer to the HW struct + * @dev_p: pointer to device capabilities structure + * @cap: capability element to parse + * + * Parse IXGBE_ACI_CAPS_FD for device capabilities. + */ +static void ixgbe_parse_fdir_dev_caps(struct ixgbe_hw *hw, + struct ixgbe_hw_dev_caps *dev_p, + struct ixgbe_aci_cmd_list_caps_elem *cap) +{ + u32 number = IXGBE_LE32_TO_CPU(cap->number); + + UNREFERENCED_1PARAMETER(hw); + + dev_p->num_flow_director_fltr = number; +} + +/** + * ixgbe_parse_dev_caps - Parse device capabilities + * @hw: pointer to the HW struct + * @dev_p: pointer to device capabilities structure + * @buf: buffer containing the device capability records + * @cap_count: the number of capabilities + * + * Helper device to parse device (0x000B) capabilities list. For + * capabilities shared between device and function, this relies on + * ixgbe_parse_common_caps. + * + * Loop through the list of provided capabilities and extract the relevant + * data into the device capabilities structured. + */ +static void ixgbe_parse_dev_caps(struct ixgbe_hw *hw, + struct ixgbe_hw_dev_caps *dev_p, + void *buf, u32 cap_count) +{ + struct ixgbe_aci_cmd_list_caps_elem *cap_resp; + u32 i; + + cap_resp = (struct ixgbe_aci_cmd_list_caps_elem *)buf; + + memset(dev_p, 0, sizeof(*dev_p)); + + for (i = 0; i < cap_count; i++) { + u16 cap = IXGBE_LE16_TO_CPU(cap_resp[i].cap); + bool found; + + found = ixgbe_parse_common_caps(hw, &dev_p->common_cap, + &cap_resp[i], "dev caps"); + + switch (cap) { + case IXGBE_ACI_CAPS_VALID_FUNCTIONS: + ixgbe_parse_valid_functions_cap(hw, dev_p, + &cap_resp[i]); + break; + case IXGBE_ACI_CAPS_VF: + ixgbe_parse_vf_dev_caps(hw, dev_p, &cap_resp[i]); + break; + case IXGBE_ACI_CAPS_VSI: + ixgbe_parse_vsi_dev_caps(hw, dev_p, &cap_resp[i]); + break; + case IXGBE_ACI_CAPS_FD: + ixgbe_parse_fdir_dev_caps(hw, dev_p, &cap_resp[i]); + break; + default: + /* Don't list common capabilities as unknown */ + if (!found) + break; + } + } + +} + +/** + * ixgbe_parse_vf_func_caps - Parse IXGBE_ACI_CAPS_VF function caps + * @hw: pointer to the HW struct + * @func_p: pointer to function capabilities structure + * @cap: pointer to the capability element to parse + * + * Extract function capabilities for IXGBE_ACI_CAPS_VF. + */ +static void ixgbe_parse_vf_func_caps(struct ixgbe_hw *hw, + struct ixgbe_hw_func_caps *func_p, + struct ixgbe_aci_cmd_list_caps_elem *cap) +{ + u32 logical_id = IXGBE_LE32_TO_CPU(cap->logical_id); + u32 number = IXGBE_LE32_TO_CPU(cap->number); + + UNREFERENCED_1PARAMETER(hw); + + func_p->num_allocd_vfs = number; + func_p->vf_base_id = logical_id; +} + +/** + * ixgbe_get_num_per_func - determine number of resources per PF + * @hw: pointer to the HW structure + * @max: value to be evenly split between each PF + * + * Determine the number of valid functions by going through the bitmap returned + * from parsing capabilities and use this to calculate the number of resources + * per PF based on the max value passed in. + * + * Return: the number of resources per PF or 0, if no PH are available. + */ +static u32 ixgbe_get_num_per_func(struct ixgbe_hw *hw, u32 max) +{ + u8 funcs; + +#define IXGBE_CAPS_VALID_FUNCS_M 0xFF + funcs = ixgbe_hweight8(hw->dev_caps.common_cap.valid_functions & + IXGBE_CAPS_VALID_FUNCS_M); + + if (!funcs) + return 0; + + return max / funcs; +} + +/** + * ixgbe_parse_vsi_func_caps - Parse IXGBE_ACI_CAPS_VSI function caps + * @hw: pointer to the HW struct + * @func_p: pointer to function capabilities structure + * @cap: pointer to the capability element to parse + * + * Extract function capabilities for IXGBE_ACI_CAPS_VSI. + */ +static void ixgbe_parse_vsi_func_caps(struct ixgbe_hw *hw, + struct ixgbe_hw_func_caps *func_p, + struct ixgbe_aci_cmd_list_caps_elem *cap) +{ + func_p->guar_num_vsi = ixgbe_get_num_per_func(hw, IXGBE_MAX_VSI); +} + +/** + * ixgbe_parse_func_caps - Parse function capabilities + * @hw: pointer to the HW struct + * @func_p: pointer to function capabilities structure + * @buf: buffer containing the function capability records + * @cap_count: the number of capabilities + * + * Helper function to parse function (0x000A) capabilities list. For + * capabilities shared between device and function, this relies on + * ixgbe_parse_common_caps. + * + * Loop through the list of provided capabilities and extract the relevant + * data into the function capabilities structured. + */ +static void ixgbe_parse_func_caps(struct ixgbe_hw *hw, + struct ixgbe_hw_func_caps *func_p, + void *buf, u32 cap_count) +{ + struct ixgbe_aci_cmd_list_caps_elem *cap_resp; + u32 i; + + cap_resp = (struct ixgbe_aci_cmd_list_caps_elem *)buf; + + memset(func_p, 0, sizeof(*func_p)); + + for (i = 0; i < cap_count; i++) { + u16 cap = IXGBE_LE16_TO_CPU(cap_resp[i].cap); + ixgbe_parse_common_caps(hw, &func_p->common_cap, + &cap_resp[i], "func caps"); + + switch (cap) { + case IXGBE_ACI_CAPS_VF: + ixgbe_parse_vf_func_caps(hw, func_p, &cap_resp[i]); + break; + case IXGBE_ACI_CAPS_VSI: + ixgbe_parse_vsi_func_caps(hw, func_p, &cap_resp[i]); + break; + default: + /* Don't list common capabilities as unknown */ + break; + } + } + +} + +/** + * ixgbe_aci_list_caps - query function/device capabilities + * @hw: pointer to the HW struct + * @buf: a buffer to hold the capabilities + * @buf_size: size of the buffer + * @cap_count: if not NULL, set to the number of capabilities reported + * @opc: capabilities type to discover, device or function + * + * Get the function (0x000A) or device (0x000B) capabilities description from + * firmware and store it in the buffer. + * + * If the cap_count pointer is not NULL, then it is set to the number of + * capabilities firmware will report. Note that if the buffer size is too + * small, it is possible the command will return IXGBE_ERR_OUT_OF_MEM. The + * cap_count will still be updated in this case. It is recommended that the + * buffer size be set to IXGBE_ACI_MAX_BUFFER_SIZE (the largest possible + * buffer that firmware could return) to avoid this. + * + * Return: the exit code of the operation. + * Exit code of IXGBE_ERR_OUT_OF_MEM means the buffer size is too small. + */ +s32 ixgbe_aci_list_caps(struct ixgbe_hw *hw, void *buf, u16 buf_size, + u32 *cap_count, enum ixgbe_aci_opc opc) +{ + struct ixgbe_aci_cmd_list_caps *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + cmd = &desc.params.get_cap; + + if (opc != ixgbe_aci_opc_list_func_caps && + opc != ixgbe_aci_opc_list_dev_caps) + return IXGBE_ERR_PARAM; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, opc); + status = ixgbe_aci_send_cmd(hw, &desc, buf, buf_size); + + if (cap_count) + *cap_count = IXGBE_LE32_TO_CPU(cmd->count); + + return status; +} + +/** + * ixgbe_discover_dev_caps - Read and extract device capabilities + * @hw: pointer to the hardware structure + * @dev_caps: pointer to device capabilities structure + * + * Read the device capabilities and extract them into the dev_caps structure + * for later use. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_discover_dev_caps(struct ixgbe_hw *hw, + struct ixgbe_hw_dev_caps *dev_caps) +{ + u32 status, cap_count = 0; + u8 *cbuf = NULL; + + cbuf = (u8*)ixgbe_malloc(hw, IXGBE_ACI_MAX_BUFFER_SIZE); + if (!cbuf) + return IXGBE_ERR_OUT_OF_MEM; + /* Although the driver doesn't know the number of capabilities the + * device will return, we can simply send a 4KB buffer, the maximum + * possible size that firmware can return. + */ + cap_count = IXGBE_ACI_MAX_BUFFER_SIZE / + sizeof(struct ixgbe_aci_cmd_list_caps_elem); + + status = ixgbe_aci_list_caps(hw, cbuf, IXGBE_ACI_MAX_BUFFER_SIZE, + &cap_count, + ixgbe_aci_opc_list_dev_caps); + if (!status) + ixgbe_parse_dev_caps(hw, dev_caps, cbuf, cap_count); + + if (cbuf) + ixgbe_free(hw, cbuf); + + return status; +} + +/** + * ixgbe_discover_func_caps - Read and extract function capabilities + * @hw: pointer to the hardware structure + * @func_caps: pointer to function capabilities structure + * + * Read the function capabilities and extract them into the func_caps structure + * for later use. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_discover_func_caps(struct ixgbe_hw *hw, + struct ixgbe_hw_func_caps *func_caps) +{ + u32 cap_count = 0; + u8 *cbuf = NULL; + s32 status; + + cbuf = (u8*)ixgbe_malloc(hw, IXGBE_ACI_MAX_BUFFER_SIZE); + if(!cbuf) + return IXGBE_ERR_OUT_OF_MEM; + /* Although the driver doesn't know the number of capabilities the + * device will return, we can simply send a 4KB buffer, the maximum + * possible size that firmware can return. + */ + cap_count = IXGBE_ACI_MAX_BUFFER_SIZE / + sizeof(struct ixgbe_aci_cmd_list_caps_elem); + + status = ixgbe_aci_list_caps(hw, cbuf, IXGBE_ACI_MAX_BUFFER_SIZE, + &cap_count, + ixgbe_aci_opc_list_func_caps); + if (!status) + ixgbe_parse_func_caps(hw, func_caps, cbuf, cap_count); + + if (cbuf) + ixgbe_free(hw, cbuf); + + return status; +} + +/** + * ixgbe_get_caps - get info about the HW + * @hw: pointer to the hardware structure + * + * Retrieve both device and function capabilities. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_get_caps(struct ixgbe_hw *hw) +{ + s32 status; + + status = ixgbe_discover_dev_caps(hw, &hw->dev_caps); + if (status) + return status; + + return ixgbe_discover_func_caps(hw, &hw->func_caps); +} + +/** + * ixgbe_aci_disable_rxen - disable RX + * @hw: pointer to the HW struct + * + * Request a safe disable of Receive Enable using ACI command (0x000C). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_disable_rxen(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_disable_rxen *cmd; + struct ixgbe_aci_desc desc; + + UNREFERENCED_1PARAMETER(hw); + + cmd = &desc.params.disable_rxen; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_disable_rxen); + + cmd->lport_num = (u8)hw->bus.func; + + return ixgbe_aci_send_cmd(hw, &desc, NULL, 0); +} + +/** + * ixgbe_aci_get_phy_caps - returns PHY capabilities + * @hw: pointer to the HW struct + * @qual_mods: report qualified modules + * @report_mode: report mode capabilities + * @pcaps: structure for PHY capabilities to be filled + * + * Returns the various PHY capabilities supported on the Port + * using ACI command (0x0600). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_get_phy_caps(struct ixgbe_hw *hw, bool qual_mods, u8 report_mode, + struct ixgbe_aci_cmd_get_phy_caps_data *pcaps) +{ + struct ixgbe_aci_cmd_get_phy_caps *cmd; + u16 pcaps_size = sizeof(*pcaps); + struct ixgbe_aci_desc desc; + s32 status; + + cmd = &desc.params.get_phy; + + if (!pcaps || (report_mode & ~IXGBE_ACI_REPORT_MODE_M)) + return IXGBE_ERR_PARAM; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_get_phy_caps); + + if (qual_mods) + cmd->param0 |= IXGBE_CPU_TO_LE16(IXGBE_ACI_GET_PHY_RQM); + + cmd->param0 |= IXGBE_CPU_TO_LE16(report_mode); + status = ixgbe_aci_send_cmd(hw, &desc, pcaps, pcaps_size); + + if (status == IXGBE_SUCCESS && + report_mode == IXGBE_ACI_REPORT_TOPO_CAP_MEDIA) { + hw->phy.phy_type_low = IXGBE_LE64_TO_CPU(pcaps->phy_type_low); + hw->phy.phy_type_high = IXGBE_LE64_TO_CPU(pcaps->phy_type_high); + memcpy(hw->link.link_info.module_type, &pcaps->module_type, + sizeof(hw->link.link_info.module_type)); + } + + return status; +} + +/** + * ixgbe_phy_caps_equals_cfg - check if capabilities match the PHY config + * @phy_caps: PHY capabilities + * @phy_cfg: PHY configuration + * + * Helper function to determine if PHY capabilities match PHY + * configuration + * + * Return: true if PHY capabilities match PHY configuration. + */ +bool +ixgbe_phy_caps_equals_cfg(struct ixgbe_aci_cmd_get_phy_caps_data *phy_caps, + struct ixgbe_aci_cmd_set_phy_cfg_data *phy_cfg) +{ + u8 caps_mask, cfg_mask; + + if (!phy_caps || !phy_cfg) + return false; + + /* These bits are not common between capabilities and configuration. + * Do not use them to determine equality. + */ + caps_mask = IXGBE_ACI_PHY_CAPS_MASK & ~(IXGBE_ACI_PHY_AN_MODE | + IXGBE_ACI_PHY_EN_MOD_QUAL); + cfg_mask = IXGBE_ACI_PHY_ENA_VALID_MASK & + ~IXGBE_ACI_PHY_ENA_AUTO_LINK_UPDT; + + if (phy_caps->phy_type_low != phy_cfg->phy_type_low || + phy_caps->phy_type_high != phy_cfg->phy_type_high || + ((phy_caps->caps & caps_mask) != (phy_cfg->caps & cfg_mask)) || + phy_caps->low_power_ctrl_an != phy_cfg->low_power_ctrl_an || + phy_caps->eee_cap != phy_cfg->eee_cap || + phy_caps->eeer_value != phy_cfg->eeer_value || + phy_caps->link_fec_options != phy_cfg->link_fec_opt) + return false; + + return true; +} + +/** + * ixgbe_copy_phy_caps_to_cfg - Copy PHY ability data to configuration data + * @caps: PHY ability structure to copy data from + * @cfg: PHY configuration structure to copy data to + * + * Helper function to copy data from PHY capabilities data structure + * to PHY configuration data structure + */ +void ixgbe_copy_phy_caps_to_cfg(struct ixgbe_aci_cmd_get_phy_caps_data *caps, + struct ixgbe_aci_cmd_set_phy_cfg_data *cfg) +{ + if (!caps || !cfg) + return; + + memset(cfg, 0, sizeof(*cfg)); + cfg->phy_type_low = caps->phy_type_low; + cfg->phy_type_high = caps->phy_type_high; + cfg->caps = caps->caps; + cfg->low_power_ctrl_an = caps->low_power_ctrl_an; + cfg->eee_cap = caps->eee_cap; + cfg->eeer_value = caps->eeer_value; + cfg->link_fec_opt = caps->link_fec_options; + cfg->module_compliance_enforcement = + caps->module_compliance_enforcement; +} + +/** + * ixgbe_aci_set_phy_cfg - set PHY configuration + * @hw: pointer to the HW struct + * @cfg: structure with PHY configuration data to be set + * + * Set the various PHY configuration parameters supported on the Port + * using ACI command (0x0601). + * One or more of the Set PHY config parameters may be ignored in an MFP + * mode as the PF may not have the privilege to set some of the PHY Config + * parameters. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_set_phy_cfg(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_set_phy_cfg_data *cfg) +{ + struct ixgbe_aci_desc desc; + s32 status; + + if (!cfg) + return IXGBE_ERR_PARAM; + + /* Ensure that only valid bits of cfg->caps can be turned on. */ + if (cfg->caps & ~IXGBE_ACI_PHY_ENA_VALID_MASK) { + cfg->caps &= IXGBE_ACI_PHY_ENA_VALID_MASK; + } + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_set_phy_cfg); + desc.flags |= IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_RD); + + status = ixgbe_aci_send_cmd(hw, &desc, cfg, sizeof(*cfg)); + + if (!status) + hw->phy.curr_user_phy_cfg = *cfg; + + return status; +} + +/** + * ixgbe_aci_set_link_restart_an - set up link and restart AN + * @hw: pointer to the HW struct + * @ena_link: if true: enable link, if false: disable link + * + * Function sets up the link and restarts the Auto-Negotiation over the link. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_set_link_restart_an(struct ixgbe_hw *hw, bool ena_link) +{ + struct ixgbe_aci_cmd_restart_an *cmd; + struct ixgbe_aci_desc desc; + + cmd = &desc.params.restart_an; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_restart_an); + + cmd->cmd_flags = IXGBE_ACI_RESTART_AN_LINK_RESTART; + if (ena_link) + cmd->cmd_flags |= IXGBE_ACI_RESTART_AN_LINK_ENABLE; + else + cmd->cmd_flags &= ~IXGBE_ACI_RESTART_AN_LINK_ENABLE; + + return ixgbe_aci_send_cmd(hw, &desc, NULL, 0); +} + +/** + * ixgbe_is_media_cage_present - check if media cage is present + * @hw: pointer to the HW struct + * + * Identify presence of media cage using the ACI command (0x06E0). + * + * Return: true if media cage is present, else false. If no cage, then + * media type is backplane or BASE-T. + */ +static bool ixgbe_is_media_cage_present(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_get_link_topo *cmd; + struct ixgbe_aci_desc desc; + + cmd = &desc.params.get_link_topo; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_get_link_topo); + + cmd->addr.topo_params.node_type_ctx = + (IXGBE_ACI_LINK_TOPO_NODE_CTX_PORT << + IXGBE_ACI_LINK_TOPO_NODE_CTX_S); + + /* set node type */ + cmd->addr.topo_params.node_type_ctx |= + (IXGBE_ACI_LINK_TOPO_NODE_TYPE_M & + IXGBE_ACI_LINK_TOPO_NODE_TYPE_CAGE); + + /* Node type cage can be used to determine if cage is present. If AQC + * returns error (ENOENT), then no cage present. If no cage present then + * connection type is backplane or BASE-T. + */ + return ixgbe_aci_get_netlist_node(hw, cmd, NULL, NULL); +} + +/** + * ixgbe_get_media_type_from_phy_type - Gets media type based on phy type + * @hw: pointer to the HW struct + * + * Try to identify the media type based on the phy type. + * If more than one media type, the ixgbe_media_type_unknown is returned. + * First, phy_type_low is checked, then phy_type_high. + * If none are identified, the ixgbe_media_type_unknown is returned + * + * Return: type of a media based on phy type in form of enum. + */ +static enum ixgbe_media_type +ixgbe_get_media_type_from_phy_type(struct ixgbe_hw *hw) +{ + struct ixgbe_link_status *hw_link_info; + + if (!hw) + return ixgbe_media_type_unknown; + + hw_link_info = &hw->link.link_info; + if (hw_link_info->phy_type_low && hw_link_info->phy_type_high) + /* If more than one media type is selected, report unknown */ + return ixgbe_media_type_unknown; + + if (hw_link_info->phy_type_low) { + /* 1G SGMII is a special case where some DA cable PHYs + * may show this as an option when it really shouldn't + * be since SGMII is meant to be between a MAC and a PHY + * in a backplane. Try to detect this case and handle it + */ + if (hw_link_info->phy_type_low == IXGBE_PHY_TYPE_LOW_1G_SGMII && + (hw_link_info->module_type[IXGBE_ACI_MOD_TYPE_IDENT] == + IXGBE_ACI_MOD_TYPE_BYTE1_SFP_PLUS_CU_ACTIVE || + hw_link_info->module_type[IXGBE_ACI_MOD_TYPE_IDENT] == + IXGBE_ACI_MOD_TYPE_BYTE1_SFP_PLUS_CU_PASSIVE)) + return ixgbe_media_type_da; + + switch (hw_link_info->phy_type_low) { + case IXGBE_PHY_TYPE_LOW_1000BASE_SX: + case IXGBE_PHY_TYPE_LOW_1000BASE_LX: + case IXGBE_PHY_TYPE_LOW_10GBASE_SR: + case IXGBE_PHY_TYPE_LOW_10GBASE_LR: + return ixgbe_media_type_fiber; + case IXGBE_PHY_TYPE_LOW_10G_SFI_AOC_ACC: + return ixgbe_media_type_fiber; + case IXGBE_PHY_TYPE_LOW_100BASE_TX: + case IXGBE_PHY_TYPE_LOW_1000BASE_T: + case IXGBE_PHY_TYPE_LOW_2500BASE_T: + case IXGBE_PHY_TYPE_LOW_5GBASE_T: + case IXGBE_PHY_TYPE_LOW_10GBASE_T: + return ixgbe_media_type_copper; + case IXGBE_PHY_TYPE_LOW_10G_SFI_DA: + return ixgbe_media_type_da; + case IXGBE_PHY_TYPE_LOW_1000BASE_KX: + case IXGBE_PHY_TYPE_LOW_2500BASE_KX: + case IXGBE_PHY_TYPE_LOW_2500BASE_X: + case IXGBE_PHY_TYPE_LOW_5GBASE_KR: + case IXGBE_PHY_TYPE_LOW_10GBASE_KR_CR1: + case IXGBE_PHY_TYPE_LOW_10G_SFI_C2C: + return ixgbe_media_type_backplane; + } + } else { + switch (hw_link_info->phy_type_high) { + case IXGBE_PHY_TYPE_HIGH_10BASE_T: + return ixgbe_media_type_copper; + } + } + return ixgbe_media_type_unknown; +} + +/** + * ixgbe_update_link_info - update status of the HW network link + * @hw: pointer to the HW struct + * + * Update the status of the HW network link. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_update_link_info(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_get_phy_caps_data *pcaps; + struct ixgbe_link_status *li; + s32 status; + + if (!hw) + return IXGBE_ERR_PARAM; + + li = &hw->link.link_info; + + status = ixgbe_aci_get_link_info(hw, true, NULL); + if (status) + return status; + + if (li->link_info & IXGBE_ACI_MEDIA_AVAILABLE) { + pcaps = (struct ixgbe_aci_cmd_get_phy_caps_data *) + ixgbe_malloc(hw, sizeof(*pcaps)); + if (!pcaps) + return IXGBE_ERR_OUT_OF_MEM; + + status = ixgbe_aci_get_phy_caps(hw, false, + IXGBE_ACI_REPORT_TOPO_CAP_MEDIA, + pcaps); + + if (status == IXGBE_SUCCESS) + memcpy(li->module_type, &pcaps->module_type, + sizeof(li->module_type)); + + ixgbe_free(hw, pcaps); + } + + return status; +} + +/** + * ixgbe_get_link_status - get status of the HW network link + * @hw: pointer to the HW struct + * @link_up: pointer to bool (true/false = linkup/linkdown) + * + * Variable link_up is true if link is up, false if link is down. + * The variable link_up is invalid if status is non zero. As a + * result of this call, link status reporting becomes enabled + * + * Return: the exit code of the operation. + */ +s32 ixgbe_get_link_status(struct ixgbe_hw *hw, bool *link_up) +{ + s32 status = IXGBE_SUCCESS; + + if (!hw || !link_up) + return IXGBE_ERR_PARAM; + + if (hw->link.get_link_info) { + status = ixgbe_update_link_info(hw); + if (status) { + return status; + } + } + + *link_up = hw->link.link_info.link_info & IXGBE_ACI_LINK_UP; + + return status; +} + +/** + * ixgbe_aci_get_link_info - get the link status + * @hw: pointer to the HW struct + * @ena_lse: enable/disable LinkStatusEvent reporting + * @link: pointer to link status structure - optional + * + * Get the current Link Status using ACI command (0x607). + * The current link can be optionally provided to update + * the status. + * + * Return: the link status of the adapter. + */ +s32 ixgbe_aci_get_link_info(struct ixgbe_hw *hw, bool ena_lse, + struct ixgbe_link_status *link) +{ + struct ixgbe_aci_cmd_get_link_status_data link_data = { 0 }; + struct ixgbe_aci_cmd_get_link_status *resp; + struct ixgbe_link_status *li_old, *li; + struct ixgbe_fc_info *hw_fc_info; + struct ixgbe_aci_desc desc; + bool tx_pause, rx_pause; + u8 cmd_flags; + s32 status; + + if (!hw) + return IXGBE_ERR_PARAM; + + li_old = &hw->link.link_info_old; + li = &hw->link.link_info; + hw_fc_info = &hw->fc; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_get_link_status); + cmd_flags = (ena_lse) ? IXGBE_ACI_LSE_ENA : IXGBE_ACI_LSE_DIS; + resp = &desc.params.get_link_status; + resp->cmd_flags = cmd_flags; + + status = ixgbe_aci_send_cmd(hw, &desc, &link_data, sizeof(link_data)); + + if (status != IXGBE_SUCCESS) + return status; + + /* save off old link status information */ + *li_old = *li; + + /* update current link status information */ + li->link_speed = IXGBE_LE16_TO_CPU(link_data.link_speed); + li->phy_type_low = IXGBE_LE64_TO_CPU(link_data.phy_type_low); + li->phy_type_high = IXGBE_LE64_TO_CPU(link_data.phy_type_high); + li->link_info = link_data.link_info; + li->link_cfg_err = link_data.link_cfg_err; + li->an_info = link_data.an_info; + li->ext_info = link_data.ext_info; + li->max_frame_size = IXGBE_LE16_TO_CPU(link_data.max_frame_size); + li->fec_info = link_data.cfg & IXGBE_ACI_FEC_MASK; + li->topo_media_conflict = link_data.topo_media_conflict; + li->pacing = link_data.cfg & (IXGBE_ACI_CFG_PACING_M | + IXGBE_ACI_CFG_PACING_TYPE_M); + + /* update fc info */ + tx_pause = !!(link_data.an_info & IXGBE_ACI_LINK_PAUSE_TX); + rx_pause = !!(link_data.an_info & IXGBE_ACI_LINK_PAUSE_RX); + if (tx_pause && rx_pause) + hw_fc_info->current_mode = ixgbe_fc_full; + else if (tx_pause) + hw_fc_info->current_mode = ixgbe_fc_tx_pause; + else if (rx_pause) + hw_fc_info->current_mode = ixgbe_fc_rx_pause; + else + hw_fc_info->current_mode = ixgbe_fc_none; + + li->lse_ena = !!(resp->cmd_flags & IXGBE_ACI_LSE_IS_ENABLED); + + /* save link status information */ + if (link) + *link = *li; + + /* flag cleared so calling functions don't call AQ again */ + hw->link.get_link_info = false; + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_aci_set_event_mask - set event mask + * @hw: pointer to the HW struct + * @port_num: port number of the physical function + * @mask: event mask to be set + * + * Set the event mask using ACI command (0x0613). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_set_event_mask(struct ixgbe_hw *hw, u8 port_num, u16 mask) +{ + struct ixgbe_aci_cmd_set_event_mask *cmd; + struct ixgbe_aci_desc desc; + + cmd = &desc.params.set_event_mask; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_set_event_mask); + + cmd->event_mask = IXGBE_CPU_TO_LE16(mask); + return ixgbe_aci_send_cmd(hw, &desc, NULL, 0); +} + +/** + * ixgbe_configure_lse - enable/disable link status events + * @hw: pointer to the HW struct + * @activate: bool value deciding if lse should be enabled nor disabled + * @mask: event mask to be set; a set bit means deactivation of the + * corresponding event + * + * Set the event mask and then enable or disable link status events + * + * Return: the exit code of the operation. + */ +s32 ixgbe_configure_lse(struct ixgbe_hw *hw, bool activate, u16 mask) +{ + s32 rc; + + rc = ixgbe_aci_set_event_mask(hw, (u8)hw->bus.func, mask); + if (rc) { + return rc; + } + + /* Enabling link status events generation by fw */ + rc = ixgbe_aci_get_link_info(hw, activate, NULL); + if (rc) { + return rc; + } + return IXGBE_SUCCESS; +} + +/** + * ixgbe_aci_get_netlist_node - get a node handle + * @hw: pointer to the hw struct + * @cmd: get_link_topo AQ structure + * @node_part_number: output node part number if node found + * @node_handle: output node handle parameter if node found + * + * Get the netlist node and assigns it to + * the provided handle using ACI command (0x06E0). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_get_netlist_node(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_get_link_topo *cmd, + u8 *node_part_number, u16 *node_handle) +{ + struct ixgbe_aci_desc desc; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_get_link_topo); + desc.params.get_link_topo = *cmd; + + if (ixgbe_aci_send_cmd(hw, &desc, NULL, 0)) + return IXGBE_ERR_NOT_SUPPORTED; + + if (node_handle) + *node_handle = + IXGBE_LE16_TO_CPU(desc.params.get_link_topo.addr.handle); + if (node_part_number) + *node_part_number = desc.params.get_link_topo.node_part_num; + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_find_netlist_node - find a node handle + * @hw: pointer to the hw struct + * @node_type_ctx: type of netlist node to look for + * @node_part_number: node part number to look for + * @node_handle: output parameter if node found - optional + * + * Find and return the node handle for a given node type and part number in the + * netlist. When found IXGBE_SUCCESS is returned, IXGBE_ERR_NOT_SUPPORTED + * otherwise. If @node_handle provided, it would be set to found node handle. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_find_netlist_node(struct ixgbe_hw *hw, u8 node_type_ctx, + u8 node_part_number, u16 *node_handle) +{ + struct ixgbe_aci_cmd_get_link_topo cmd; + u8 rec_node_part_number; + u16 rec_node_handle; + s32 status; + u8 idx; + + for (idx = 0; idx < IXGBE_MAX_NETLIST_SIZE; idx++) { + memset(&cmd, 0, sizeof(cmd)); + + cmd.addr.topo_params.node_type_ctx = + (node_type_ctx << IXGBE_ACI_LINK_TOPO_NODE_TYPE_S); + cmd.addr.topo_params.index = idx; + + status = ixgbe_aci_get_netlist_node(hw, &cmd, + &rec_node_part_number, + &rec_node_handle); + if (status) + return status; + + if (rec_node_part_number == node_part_number) { + if (node_handle) + *node_handle = rec_node_handle; + return IXGBE_SUCCESS; + } + } + + return IXGBE_ERR_NOT_SUPPORTED; +} + +/** + * ixgbe_aci_read_i2c - read I2C register value + * @hw: pointer to the hw struct + * @topo_addr: topology address for a device to communicate with + * @bus_addr: 7-bit I2C bus address + * @addr: I2C memory address (I2C offset) with up to 16 bits + * @params: I2C parameters: bit [7] - Repeated start, + * bits [6:5] data offset size, + * bit [4] - I2C address type, bits [3:0] - data size + * to read (0-16 bytes) + * @data: pointer to data (0 to 16 bytes) to be read from the I2C device + * + * Read the value of the I2C pin register using ACI command (0x06E2). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_read_i2c(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_link_topo_addr topo_addr, + u16 bus_addr, __le16 addr, u8 params, u8 *data) +{ + struct ixgbe_aci_desc desc = { 0 }; + struct ixgbe_aci_cmd_i2c *cmd; + u8 data_size; + s32 status; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_read_i2c); + cmd = &desc.params.read_write_i2c; + + if (!data) + return IXGBE_ERR_PARAM; + + data_size = (params & IXGBE_ACI_I2C_DATA_SIZE_M) >> + IXGBE_ACI_I2C_DATA_SIZE_S; + + cmd->i2c_bus_addr = IXGBE_CPU_TO_LE16(bus_addr); + cmd->topo_addr = topo_addr; + cmd->i2c_params = params; + cmd->i2c_addr = addr; + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + if (!status) { + struct ixgbe_aci_cmd_read_i2c_resp *resp; + u8 i; + + resp = &desc.params.read_i2c_resp; + for (i = 0; i < data_size; i++) { + *data = resp->i2c_data[i]; + data++; + } + } + + return status; +} + +/** + * ixgbe_aci_write_i2c - write a value to I2C register + * @hw: pointer to the hw struct + * @topo_addr: topology address for a device to communicate with + * @bus_addr: 7-bit I2C bus address + * @addr: I2C memory address (I2C offset) with up to 16 bits + * @params: I2C parameters: bit [4] - I2C address type, bits [3:0] - data size + * to write (0-7 bytes) + * @data: pointer to data (0 to 4 bytes) to be written to the I2C device + * + * Write a value to the I2C pin register using ACI command (0x06E3). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_write_i2c(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_link_topo_addr topo_addr, + u16 bus_addr, __le16 addr, u8 params, u8 *data) +{ + struct ixgbe_aci_desc desc = { 0 }; + struct ixgbe_aci_cmd_i2c *cmd; + u8 i, data_size; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_write_i2c); + cmd = &desc.params.read_write_i2c; + + data_size = (params & IXGBE_ACI_I2C_DATA_SIZE_M) >> + IXGBE_ACI_I2C_DATA_SIZE_S; + + /* data_size limited to 4 */ + if (data_size > 4) + return IXGBE_ERR_PARAM; + + cmd->i2c_bus_addr = IXGBE_CPU_TO_LE16(bus_addr); + cmd->topo_addr = topo_addr; + cmd->i2c_params = params; + cmd->i2c_addr = addr; + + for (i = 0; i < data_size; i++) { + cmd->i2c_data[i] = *data; + data++; + } + + return ixgbe_aci_send_cmd(hw, &desc, NULL, 0); +} + +/** + * ixgbe_aci_set_port_id_led - set LED value for the given port + * @hw: pointer to the HW struct + * @orig_mode: set LED original mode + * + * Set LED value for the given port (0x06E9) + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_set_port_id_led(struct ixgbe_hw *hw, bool orig_mode) +{ + struct ixgbe_aci_cmd_set_port_id_led *cmd; + struct ixgbe_aci_desc desc; + + cmd = &desc.params.set_port_id_led; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_set_port_id_led); + + cmd->lport_num = (u8)hw->bus.func; + cmd->lport_num_valid = IXGBE_ACI_PORT_ID_PORT_NUM_VALID; + + if (orig_mode) + cmd->ident_mode = IXGBE_ACI_PORT_IDENT_LED_ORIG; + else + cmd->ident_mode = IXGBE_ACI_PORT_IDENT_LED_BLINK; + + return ixgbe_aci_send_cmd(hw, &desc, NULL, 0); +} + +/** + * ixgbe_aci_set_gpio - set GPIO pin state + * @hw: pointer to the hw struct + * @gpio_ctrl_handle: GPIO controller node handle + * @pin_idx: IO Number of the GPIO that needs to be set + * @value: SW provide IO value to set in the LSB + * + * Set the GPIO pin state that is a part of the topology + * using ACI command (0x06EC). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_set_gpio(struct ixgbe_hw *hw, u16 gpio_ctrl_handle, u8 pin_idx, + bool value) +{ + struct ixgbe_aci_cmd_gpio *cmd; + struct ixgbe_aci_desc desc; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_set_gpio); + cmd = &desc.params.read_write_gpio; + cmd->gpio_ctrl_handle = IXGBE_CPU_TO_LE16(gpio_ctrl_handle); + cmd->gpio_num = pin_idx; + cmd->gpio_val = value ? 1 : 0; + + return ixgbe_aci_send_cmd(hw, &desc, NULL, 0); +} + +/** + * ixgbe_aci_get_gpio - get GPIO pin state + * @hw: pointer to the hw struct + * @gpio_ctrl_handle: GPIO controller node handle + * @pin_idx: IO Number of the GPIO that needs to be set + * @value: IO value read + * + * Get the value of a GPIO signal which is part of the topology + * using ACI command (0x06ED). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_get_gpio(struct ixgbe_hw *hw, u16 gpio_ctrl_handle, u8 pin_idx, + bool *value) +{ + struct ixgbe_aci_cmd_gpio *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_get_gpio); + cmd = &desc.params.read_write_gpio; + cmd->gpio_ctrl_handle = IXGBE_CPU_TO_LE16(gpio_ctrl_handle); + cmd->gpio_num = pin_idx; + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + if (status) + return status; + + *value = !!cmd->gpio_val; + return IXGBE_SUCCESS; +} + +/** + * ixgbe_aci_sff_eeprom - read/write SFF EEPROM + * @hw: pointer to the HW struct + * @lport: bits [7:0] = logical port, bit [8] = logical port valid + * @bus_addr: I2C bus address of the eeprom (typically 0xA0, 0=topo default) + * @mem_addr: I2C offset. lower 8 bits for address, 8 upper bits zero padding. + * @page: QSFP page + * @page_bank_ctrl: configuration of SFF/CMIS paging and banking control + * @data: pointer to data buffer to be read/written to the I2C device. + * @length: 1-16 for read, 1 for write. + * @write: 0 read, 1 for write. + * + * Read/write SFF EEPROM using ACI command (0x06EE). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_sff_eeprom(struct ixgbe_hw *hw, u16 lport, u8 bus_addr, + u16 mem_addr, u8 page, u8 page_bank_ctrl, u8 *data, + u8 length, bool write) +{ + struct ixgbe_aci_cmd_sff_eeprom *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + if (!data || (mem_addr & 0xff00)) + return IXGBE_ERR_PARAM; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_sff_eeprom); + cmd = &desc.params.read_write_sff_param; + desc.flags = IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_RD); + cmd->lport_num = (u8)(lport & 0xff); + cmd->lport_num_valid = (u8)((lport >> 8) & 0x01); + cmd->i2c_bus_addr = IXGBE_CPU_TO_LE16(((bus_addr >> 1) & + IXGBE_ACI_SFF_I2CBUS_7BIT_M) | + ((page_bank_ctrl << + IXGBE_ACI_SFF_PAGE_BANK_CTRL_S) & + IXGBE_ACI_SFF_PAGE_BANK_CTRL_M)); + cmd->i2c_offset = IXGBE_CPU_TO_LE16(mem_addr & 0xff); + cmd->module_page = page; + if (write) + cmd->i2c_bus_addr |= IXGBE_CPU_TO_LE16(IXGBE_ACI_SFF_IS_WRITE); + + status = ixgbe_aci_send_cmd(hw, &desc, data, length); + return status; +} + +/** + * ixgbe_aci_prog_topo_dev_nvm - program Topology Device NVM + * @hw: pointer to the hardware structure + * @topo_params: pointer to structure storing topology parameters for a device + * + * Program Topology Device NVM using ACI command (0x06F2). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_prog_topo_dev_nvm(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_link_topo_params *topo_params) +{ + struct ixgbe_aci_cmd_prog_topo_dev_nvm *cmd; + struct ixgbe_aci_desc desc; + + cmd = &desc.params.prog_topo_dev_nvm; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_prog_topo_dev_nvm); + + memcpy(&cmd->topo_params, topo_params, sizeof(*topo_params)); + + return ixgbe_aci_send_cmd(hw, &desc, NULL, 0); +} + +/** + * ixgbe_aci_read_topo_dev_nvm - read Topology Device NVM + * @hw: pointer to the hardware structure + * @topo_params: pointer to structure storing topology parameters for a device + * @start_address: byte offset in the topology device NVM + * @data: pointer to data buffer + * @data_size: number of bytes to be read from the topology device NVM + * Read Topology Device NVM (0x06F3) + * + * Read Topology of Device NVM using ACI command (0x06F3). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_read_topo_dev_nvm(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_link_topo_params *topo_params, + u32 start_address, u8 *data, u8 data_size) +{ + struct ixgbe_aci_cmd_read_topo_dev_nvm *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + if (!data || data_size == 0 || + data_size > IXGBE_ACI_READ_TOPO_DEV_NVM_DATA_READ_SIZE) + return IXGBE_ERR_PARAM; + + cmd = &desc.params.read_topo_dev_nvm; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_read_topo_dev_nvm); + + desc.datalen = IXGBE_CPU_TO_LE16(data_size); + memcpy(&cmd->topo_params, topo_params, sizeof(*topo_params)); + cmd->start_address = IXGBE_CPU_TO_LE32(start_address); + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + if (status) + return status; + + memcpy(data, cmd->data_read, data_size); + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_acquire_nvm - Generic request for acquiring the NVM ownership + * @hw: pointer to the HW structure + * @access: NVM access type (read or write) + * + * Request NVM ownership. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_acquire_nvm(struct ixgbe_hw *hw, + enum ixgbe_aci_res_access_type access) +{ + u32 fla; + + /* Skip if we are in blank NVM programming mode */ + fla = IXGBE_READ_REG(hw, GLNVM_FLA); + if ((fla & GLNVM_FLA_LOCKED_M) == 0) + return IXGBE_SUCCESS; + + return ixgbe_acquire_res(hw, IXGBE_NVM_RES_ID, access, + IXGBE_NVM_TIMEOUT); +} + +/** + * ixgbe_release_nvm - Generic request for releasing the NVM ownership + * @hw: pointer to the HW structure + * + * Release NVM ownership. + */ +void ixgbe_release_nvm(struct ixgbe_hw *hw) +{ + u32 fla; + + /* Skip if we are in blank NVM programming mode */ + fla = IXGBE_READ_REG(hw, GLNVM_FLA); + if ((fla & GLNVM_FLA_LOCKED_M) == 0) + return; + + ixgbe_release_res(hw, IXGBE_NVM_RES_ID); +} + + +/** + * ixgbe_aci_read_nvm - read NVM + * @hw: pointer to the HW struct + * @module_typeid: module pointer location in words from the NVM beginning + * @offset: byte offset from the module beginning + * @length: length of the section to be read (in bytes from the offset) + * @data: command buffer (size [bytes] = length) + * @last_command: tells if this is the last command in a series + * @read_shadow_ram: tell if this is a shadow RAM read + * + * Read the NVM using ACI command (0x0701). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_read_nvm(struct ixgbe_hw *hw, u16 module_typeid, u32 offset, + u16 length, void *data, bool last_command, + bool read_shadow_ram) +{ + struct ixgbe_aci_desc desc; + struct ixgbe_aci_cmd_nvm *cmd; + + cmd = &desc.params.nvm; + + if (offset > IXGBE_ACI_NVM_MAX_OFFSET) + return IXGBE_ERR_PARAM; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_nvm_read); + + if (!read_shadow_ram && module_typeid == IXGBE_ACI_NVM_START_POINT) + cmd->cmd_flags |= IXGBE_ACI_NVM_FLASH_ONLY; + + /* If this is the last command in a series, set the proper flag. */ + if (last_command) + cmd->cmd_flags |= IXGBE_ACI_NVM_LAST_CMD; + cmd->module_typeid = IXGBE_CPU_TO_LE16(module_typeid); + cmd->offset_low = IXGBE_CPU_TO_LE16(offset & 0xFFFF); + cmd->offset_high = (offset >> 16) & 0xFF; + cmd->length = IXGBE_CPU_TO_LE16(length); + + return ixgbe_aci_send_cmd(hw, &desc, data, length); +} + +/** + * ixgbe_aci_erase_nvm - erase NVM sector + * @hw: pointer to the HW struct + * @module_typeid: module pointer location in words from the NVM beginning + * + * Erase the NVM sector using the ACI command (0x0702). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_erase_nvm(struct ixgbe_hw *hw, u16 module_typeid) +{ + struct ixgbe_aci_desc desc; + struct ixgbe_aci_cmd_nvm *cmd; + s32 status; + __le16 len; + + /* read a length value from SR, so module_typeid is equal to 0 */ + /* calculate offset where module size is placed from bytes to words */ + /* set last command and read from SR values to true */ + status = ixgbe_aci_read_nvm(hw, 0, 2 * module_typeid + 2, 2, &len, true, + true); + if (status) + return status; + + cmd = &desc.params.nvm; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_nvm_erase); + + cmd->module_typeid = IXGBE_CPU_TO_LE16(module_typeid); + cmd->length = len; + cmd->offset_low = 0; + cmd->offset_high = 0; + + return ixgbe_aci_send_cmd(hw, &desc, NULL, 0); +} + +/** + * ixgbe_aci_update_nvm - update NVM + * @hw: pointer to the HW struct + * @module_typeid: module pointer location in words from the NVM beginning + * @offset: byte offset from the module beginning + * @length: length of the section to be written (in bytes from the offset) + * @data: command buffer (size [bytes] = length) + * @last_command: tells if this is the last command in a series + * @command_flags: command parameters + * + * Update the NVM using the ACI command (0x0703). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_update_nvm(struct ixgbe_hw *hw, u16 module_typeid, + u32 offset, u16 length, void *data, + bool last_command, u8 command_flags) +{ + struct ixgbe_aci_desc desc; + struct ixgbe_aci_cmd_nvm *cmd; + + cmd = &desc.params.nvm; + + /* In offset the highest byte must be zeroed. */ + if (offset & 0xFF000000) + return IXGBE_ERR_PARAM; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_nvm_write); + + cmd->cmd_flags |= command_flags; + + /* If this is the last command in a series, set the proper flag. */ + if (last_command) + cmd->cmd_flags |= IXGBE_ACI_NVM_LAST_CMD; + cmd->module_typeid = IXGBE_CPU_TO_LE16(module_typeid); + cmd->offset_low = IXGBE_CPU_TO_LE16(offset & 0xFFFF); + cmd->offset_high = (offset >> 16) & 0xFF; + cmd->length = IXGBE_CPU_TO_LE16(length); + + desc.flags |= IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_RD); + + return ixgbe_aci_send_cmd(hw, &desc, data, length); +} + +/** + * ixgbe_aci_read_nvm_cfg - read an NVM config block + * @hw: pointer to the HW struct + * @cmd_flags: NVM access admin command bits + * @field_id: field or feature ID + * @data: buffer for result + * @buf_size: buffer size + * @elem_count: pointer to count of elements read by FW + * + * Reads a single or multiple feature/field ID and data using ACI command + * (0x0704). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_read_nvm_cfg(struct ixgbe_hw *hw, u8 cmd_flags, + u16 field_id, void *data, u16 buf_size, + u16 *elem_count) +{ + struct ixgbe_aci_cmd_nvm_cfg *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + cmd = &desc.params.nvm_cfg; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_nvm_cfg_read); + + cmd->cmd_flags = cmd_flags; + cmd->id = IXGBE_CPU_TO_LE16(field_id); + + status = ixgbe_aci_send_cmd(hw, &desc, data, buf_size); + if (!status && elem_count) + *elem_count = IXGBE_LE16_TO_CPU(cmd->count); + + return status; +} + +/** + * ixgbe_aci_write_nvm_cfg - write an NVM config block + * @hw: pointer to the HW struct + * @cmd_flags: NVM access admin command bits + * @data: buffer for result + * @buf_size: buffer size + * @elem_count: count of elements to be written + * + * Writes a single or multiple feature/field ID and data using ACI command + * (0x0705). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_write_nvm_cfg(struct ixgbe_hw *hw, u8 cmd_flags, + void *data, u16 buf_size, u16 elem_count) +{ + struct ixgbe_aci_cmd_nvm_cfg *cmd; + struct ixgbe_aci_desc desc; + + cmd = &desc.params.nvm_cfg; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_nvm_cfg_write); + desc.flags |= IXGBE_CPU_TO_LE16(IXGBE_ACI_FLAG_RD); + + cmd->count = IXGBE_CPU_TO_LE16(elem_count); + cmd->cmd_flags = cmd_flags; + + return ixgbe_aci_send_cmd(hw, &desc, data, buf_size); +} + +/** + * ixgbe_nvm_validate_checksum - validate checksum + * @hw: pointer to the HW struct + * + * Verify NVM PFA checksum validity using ACI command (0x0706). + * If the checksum verification failed, IXGBE_ERR_NVM_CHECKSUM is returned. + * The function acquires and then releases the NVM ownership. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_nvm_validate_checksum(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_nvm_checksum *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + status = ixgbe_acquire_nvm(hw, IXGBE_RES_READ); + if (status) + return status; + + cmd = &desc.params.nvm_checksum; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_nvm_checksum); + cmd->flags = IXGBE_ACI_NVM_CHECKSUM_VERIFY; + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + + ixgbe_release_nvm(hw); + + if (!status) + if (IXGBE_LE16_TO_CPU(cmd->checksum) != + IXGBE_ACI_NVM_CHECKSUM_CORRECT) { + ERROR_REPORT1(IXGBE_ERROR_INVALID_STATE, + "Invalid Shadow Ram checksum"); + status = IXGBE_ERR_NVM_CHECKSUM; + } + + return status; +} + +/** + * ixgbe_nvm_recalculate_checksum - recalculate checksum + * @hw: pointer to the HW struct + * + * Recalculate NVM PFA checksum using ACI command (0x0706). + * The function acquires and then releases the NVM ownership. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_nvm_recalculate_checksum(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_nvm_checksum *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + status = ixgbe_acquire_nvm(hw, IXGBE_RES_WRITE); + if (status) + return status; + + cmd = &desc.params.nvm_checksum; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_nvm_checksum); + cmd->flags = IXGBE_ACI_NVM_CHECKSUM_RECALC; + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + + ixgbe_release_nvm(hw); + + return status; +} + +/** + * ixgbe_nvm_write_activate - NVM activate write + * @hw: pointer to the HW struct + * @cmd_flags: flags for write activate command + * @response_flags: response indicators from firmware + * + * Update the control word with the required banks' validity bits + * and dumps the Shadow RAM to flash using ACI command (0x0707). + * + * cmd_flags controls which banks to activate, the preservation level to use + * when activating the NVM bank, and whether an EMP reset is required for + * activation. + * + * Note that the 16bit cmd_flags value is split between two separate 1 byte + * flag values in the descriptor. + * + * On successful return of the firmware command, the response_flags variable + * is updated with the flags reported by firmware indicating certain status, + * such as whether EMP reset is enabled. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_nvm_write_activate(struct ixgbe_hw *hw, u16 cmd_flags, + u8 *response_flags) +{ + struct ixgbe_aci_desc desc; + struct ixgbe_aci_cmd_nvm *cmd; + s32 status; + + cmd = &desc.params.nvm; + ixgbe_fill_dflt_direct_cmd_desc(&desc, + ixgbe_aci_opc_nvm_write_activate); + + cmd->cmd_flags = LO_BYTE(cmd_flags); + cmd->offset_high = HI_BYTE(cmd_flags); + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + if (!status && response_flags) + *response_flags = cmd->cmd_flags; + + return status; +} + +/** + * ixgbe_get_flash_bank_offset - Get offset into requested flash bank + * @hw: pointer to the HW structure + * @bank: whether to read from the active or inactive flash bank + * @module: the module to read from + * + * Based on the module, lookup the module offset from the beginning of the + * flash. + * + * Return: the flash offset. Note that a value of zero is invalid and must be + * treated as an error. + */ +static u32 ixgbe_get_flash_bank_offset(struct ixgbe_hw *hw, + enum ixgbe_bank_select bank, + u16 module) +{ + struct ixgbe_bank_info *banks = &hw->flash.banks; + enum ixgbe_flash_bank active_bank; + bool second_bank_active; + u32 offset, size; + + switch (module) { + case E610_SR_1ST_NVM_BANK_PTR: + offset = banks->nvm_ptr; + size = banks->nvm_size; + active_bank = banks->nvm_bank; + break; + case E610_SR_1ST_OROM_BANK_PTR: + offset = banks->orom_ptr; + size = banks->orom_size; + active_bank = banks->orom_bank; + break; + case E610_SR_NETLIST_BANK_PTR: + offset = banks->netlist_ptr; + size = banks->netlist_size; + active_bank = banks->netlist_bank; + break; + default: + return 0; + } + + switch (active_bank) { + case IXGBE_1ST_FLASH_BANK: + second_bank_active = false; + break; + case IXGBE_2ND_FLASH_BANK: + second_bank_active = true; + break; + default: + return 0; + } + + /* The second flash bank is stored immediately following the first + * bank. Based on whether the 1st or 2nd bank is active, and whether + * we want the active or inactive bank, calculate the desired offset. + */ + switch (bank) { + case IXGBE_ACTIVE_FLASH_BANK: + return offset + (second_bank_active ? size : 0); + case IXGBE_INACTIVE_FLASH_BANK: + return offset + (second_bank_active ? 0 : size); + } + + return 0; +} + +/** + * ixgbe_read_flash_module - Read a word from one of the main NVM modules + * @hw: pointer to the HW structure + * @bank: which bank of the module to read + * @module: the module to read + * @offset: the offset into the module in bytes + * @data: storage for the word read from the flash + * @length: bytes of data to read + * + * Read data from the specified flash module. The bank parameter indicates + * whether or not to read from the active bank or the inactive bank of that + * module. + * + * The word will be read using flat NVM access, and relies on the + * hw->flash.banks data being setup by ixgbe_determine_active_flash_banks() + * during initialization. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_read_flash_module(struct ixgbe_hw *hw, + enum ixgbe_bank_select bank, + u16 module, u32 offset, u8 *data, u32 length) +{ + s32 status; + u32 start; + + start = ixgbe_get_flash_bank_offset(hw, bank, module); + if (!start) { + return IXGBE_ERR_PARAM; + } + + status = ixgbe_acquire_nvm(hw, IXGBE_RES_READ); + if (status) + return status; + + status = ixgbe_read_flat_nvm(hw, start + offset, &length, data, false); + + ixgbe_release_nvm(hw); + + return status; +} + +/** + * ixgbe_read_netlist_module - Read data from the netlist module area + * @hw: pointer to the HW structure + * @bank: whether to read from the active or inactive module + * @offset: offset into the netlist to read from + * @data: storage for returned word value + * + * Read a word from the specified netlist bank. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_read_netlist_module(struct ixgbe_hw *hw, + enum ixgbe_bank_select bank, + u32 offset, u16 *data) +{ + __le16 data_local; + s32 status; + + status = ixgbe_read_flash_module(hw, bank, E610_SR_NETLIST_BANK_PTR, + offset * sizeof(u16), + (u8 *)&data_local, + sizeof(u16)); + if (!status) + *data = IXGBE_LE16_TO_CPU(data_local); + + return status; +} + +/** + * ixgbe_read_nvm_module - Read from the active main NVM module + * @hw: pointer to the HW structure + * @bank: whether to read from active or inactive NVM module + * @offset: offset into the NVM module to read, in words + * @data: storage for returned word value + * + * Read the specified word from the active NVM module. This includes the CSS + * header at the start of the NVM module. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_read_nvm_module(struct ixgbe_hw *hw, + enum ixgbe_bank_select bank, + u32 offset, u16 *data) +{ + __le16 data_local; + s32 status; + + status = ixgbe_read_flash_module(hw, bank, E610_SR_1ST_NVM_BANK_PTR, + offset * sizeof(u16), + (u8 *)&data_local, + sizeof(u16)); + if (!status) + *data = IXGBE_LE16_TO_CPU(data_local); + + return status; +} + +/** + * ixgbe_get_nvm_css_hdr_len - Read the CSS header length from the + * NVM CSS header + * @hw: pointer to the HW struct + * @bank: whether to read from the active or inactive flash bank + * @hdr_len: storage for header length in words + * + * Read the CSS header length from the NVM CSS header and add the + * Authentication header size, and then convert to words. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_get_nvm_css_hdr_len(struct ixgbe_hw *hw, + enum ixgbe_bank_select bank, + u32 *hdr_len) +{ + u16 hdr_len_l, hdr_len_h; + u32 hdr_len_dword; + s32 status; + + status = ixgbe_read_nvm_module(hw, bank, IXGBE_NVM_CSS_HDR_LEN_L, + &hdr_len_l); + if (status) + return status; + + status = ixgbe_read_nvm_module(hw, bank, IXGBE_NVM_CSS_HDR_LEN_H, + &hdr_len_h); + if (status) + return status; + + /* CSS header length is in DWORD, so convert to words and add + * authentication header size + */ + hdr_len_dword = hdr_len_h << 16 | hdr_len_l; + *hdr_len = (hdr_len_dword * 2) + IXGBE_NVM_AUTH_HEADER_LEN; + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_read_nvm_sr_copy - Read a word from the Shadow RAM copy in the NVM bank + * @hw: pointer to the HW structure + * @bank: whether to read from the active or inactive NVM module + * @offset: offset into the Shadow RAM copy to read, in words + * @data: storage for returned word value + * + * Read the specified word from the copy of the Shadow RAM found in the + * specified NVM module. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_read_nvm_sr_copy(struct ixgbe_hw *hw, + enum ixgbe_bank_select bank, + u32 offset, u16 *data) +{ + u32 hdr_len; + s32 status; + + status = ixgbe_get_nvm_css_hdr_len(hw, bank, &hdr_len); + if (status) + return status; + + hdr_len = ROUND_UP(hdr_len, 32); + + return ixgbe_read_nvm_module(hw, bank, hdr_len + offset, data); +} + +/** + * ixgbe_get_nvm_minsrevs - Get the minsrevs values from flash + * @hw: pointer to the HW struct + * @minsrevs: structure to store NVM and OROM minsrev values + * + * Read the Minimum Security Revision TLV and extract + * the revision values from the flash image + * into a readable structure for processing. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_get_nvm_minsrevs(struct ixgbe_hw *hw, + struct ixgbe_minsrev_info *minsrevs) +{ + struct ixgbe_aci_cmd_nvm_minsrev data; + s32 status; + u16 valid; + + status = ixgbe_acquire_nvm(hw, IXGBE_RES_READ); + if (status) + return status; + + status = ixgbe_aci_read_nvm(hw, IXGBE_ACI_NVM_MINSREV_MOD_ID, + 0, sizeof(data), &data, + true, false); + + ixgbe_release_nvm(hw); + + if (status) + return status; + + valid = IXGBE_LE16_TO_CPU(data.validity); + + /* Extract NVM minimum security revision */ + if (valid & IXGBE_ACI_NVM_MINSREV_NVM_VALID) { + u16 minsrev_l = IXGBE_LE16_TO_CPU(data.nvm_minsrev_l); + u16 minsrev_h = IXGBE_LE16_TO_CPU(data.nvm_minsrev_h); + + minsrevs->nvm = minsrev_h << 16 | minsrev_l; + minsrevs->nvm_valid = true; + } + + /* Extract the OROM minimum security revision */ + if (valid & IXGBE_ACI_NVM_MINSREV_OROM_VALID) { + u16 minsrev_l = IXGBE_LE16_TO_CPU(data.orom_minsrev_l); + u16 minsrev_h = IXGBE_LE16_TO_CPU(data.orom_minsrev_h); + + minsrevs->orom = minsrev_h << 16 | minsrev_l; + minsrevs->orom_valid = true; + } + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_update_nvm_minsrevs - Update minsrevs TLV data in flash + * @hw: pointer to the HW struct + * @minsrevs: minimum security revision information + * + * Update the NVM or Option ROM minimum security revision fields in the PFA + * area of the flash. Reads the minsrevs->nvm_valid and minsrevs->orom_valid + * fields to determine what update is being requested. If the valid bit is not + * set for that module, then the associated minsrev will be left as is. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_update_nvm_minsrevs(struct ixgbe_hw *hw, + struct ixgbe_minsrev_info *minsrevs) +{ + struct ixgbe_aci_cmd_nvm_minsrev data; + s32 status; + + if (!minsrevs->nvm_valid && !minsrevs->orom_valid) { + return IXGBE_ERR_PARAM; + } + + status = ixgbe_acquire_nvm(hw, IXGBE_RES_WRITE); + if (status) + return status; + + /* Get current data */ + status = ixgbe_aci_read_nvm(hw, IXGBE_ACI_NVM_MINSREV_MOD_ID, 0, + sizeof(data), &data, true, false); + if (status) + goto exit_release_res; + + if (minsrevs->nvm_valid) { + data.nvm_minsrev_l = IXGBE_CPU_TO_LE16(minsrevs->nvm & 0xFFFF); + data.nvm_minsrev_h = IXGBE_CPU_TO_LE16(minsrevs->nvm >> 16); + data.validity |= + IXGBE_CPU_TO_LE16(IXGBE_ACI_NVM_MINSREV_NVM_VALID); + } + + if (minsrevs->orom_valid) { + data.orom_minsrev_l = IXGBE_CPU_TO_LE16(minsrevs->orom & 0xFFFF); + data.orom_minsrev_h = IXGBE_CPU_TO_LE16(minsrevs->orom >> 16); + data.validity |= + IXGBE_CPU_TO_LE16(IXGBE_ACI_NVM_MINSREV_OROM_VALID); + } + + /* Update flash data */ + status = ixgbe_aci_update_nvm(hw, IXGBE_ACI_NVM_MINSREV_MOD_ID, 0, + sizeof(data), &data, false, + IXGBE_ACI_NVM_SPECIAL_UPDATE); + if (status) + goto exit_release_res; + + /* Dump the Shadow RAM to the flash */ + status = ixgbe_nvm_write_activate(hw, 0, NULL); + +exit_release_res: + ixgbe_release_nvm(hw); + + return status; +} + +/** + * ixgbe_get_nvm_srev - Read the security revision from the NVM CSS header + * @hw: pointer to the HW struct + * @bank: whether to read from the active or inactive flash bank + * @srev: storage for security revision + * + * Read the security revision out of the CSS header of the active NVM module + * bank. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_get_nvm_srev(struct ixgbe_hw *hw, + enum ixgbe_bank_select bank, u32 *srev) +{ + u16 srev_l, srev_h; + s32 status; + + status = ixgbe_read_nvm_module(hw, bank, IXGBE_NVM_CSS_SREV_L, &srev_l); + if (status) + return status; + + status = ixgbe_read_nvm_module(hw, bank, IXGBE_NVM_CSS_SREV_H, &srev_h); + if (status) + return status; + + *srev = srev_h << 16 | srev_l; + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_get_nvm_ver_info - Read NVM version information + * @hw: pointer to the HW struct + * @bank: whether to read from the active or inactive flash bank + * @nvm: pointer to NVM info structure + * + * Read the NVM EETRACK ID and map version of the main NVM image bank, filling + * in the nvm info structure. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_get_nvm_ver_info(struct ixgbe_hw *hw, + enum ixgbe_bank_select bank, + struct ixgbe_nvm_info *nvm) +{ + u16 eetrack_lo, eetrack_hi, ver; + s32 status; + + status = ixgbe_read_nvm_sr_copy(hw, bank, + E610_SR_NVM_DEV_STARTER_VER, &ver); + if (status) { + return status; + } + + nvm->major = (ver & E610_NVM_VER_HI_MASK) >> E610_NVM_VER_HI_SHIFT; + nvm->minor = (ver & E610_NVM_VER_LO_MASK) >> E610_NVM_VER_LO_SHIFT; + + status = ixgbe_read_nvm_sr_copy(hw, bank, E610_SR_NVM_EETRACK_LO, + &eetrack_lo); + if (status) { + return status; + } + status = ixgbe_read_nvm_sr_copy(hw, bank, E610_SR_NVM_EETRACK_HI, + &eetrack_hi); + if (status) { + return status; + } + + nvm->eetrack = (eetrack_hi << 16) | eetrack_lo; + + status = ixgbe_get_nvm_srev(hw, bank, &nvm->srev); + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_get_inactive_nvm_ver - Read Option ROM version from the inactive bank + * @hw: pointer to the HW structure + * @nvm: storage for Option ROM version information + * + * Reads the NVM EETRACK ID, Map version, and security revision of the + * inactive NVM bank. Used to access version data for a pending update that + * has not yet been activated. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_get_inactive_nvm_ver(struct ixgbe_hw *hw, struct ixgbe_nvm_info *nvm) +{ + return ixgbe_get_nvm_ver_info(hw, IXGBE_INACTIVE_FLASH_BANK, nvm); +} + +/** + * ixgbe_get_active_nvm_ver - Read Option ROM version from the active bank + * @hw: pointer to the HW structure + * @nvm: storage for Option ROM version information + * + * Reads the NVM EETRACK ID, Map version, and security revision of the + * active NVM bank. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_get_active_nvm_ver(struct ixgbe_hw *hw, struct ixgbe_nvm_info *nvm) +{ + return ixgbe_get_nvm_ver_info(hw, IXGBE_ACTIVE_FLASH_BANK, nvm); +} + +/** + * ixgbe_get_netlist_info + * @hw: pointer to the HW struct + * @bank: whether to read from the active or inactive flash bank + * @netlist: pointer to netlist version info structure + * + * Get the netlist version information from the requested bank. Reads the Link + * Topology section to find the Netlist ID block and extract the relevant + * information into the netlist version structure. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_get_netlist_info(struct ixgbe_hw *hw, + enum ixgbe_bank_select bank, + struct ixgbe_netlist_info *netlist) +{ + u16 module_id, length, node_count, i; + u16 *id_blk; + s32 status; + + status = ixgbe_read_netlist_module(hw, bank, IXGBE_NETLIST_TYPE_OFFSET, + &module_id); + if (status) + return status; + + if (module_id != IXGBE_NETLIST_LINK_TOPO_MOD_ID) { + return IXGBE_ERR_NVM; + } + + status = ixgbe_read_netlist_module(hw, bank, IXGBE_LINK_TOPO_MODULE_LEN, + &length); + if (status) + return status; + + /* sanity check that we have at least enough words to store the + * netlist ID block + */ + if (length < IXGBE_NETLIST_ID_BLK_SIZE) { + return IXGBE_ERR_NVM; + } + + status = ixgbe_read_netlist_module(hw, bank, IXGBE_LINK_TOPO_NODE_COUNT, + &node_count); + if (status) + return status; + node_count &= IXGBE_LINK_TOPO_NODE_COUNT_M; + + id_blk = (u16 *)ixgbe_calloc(hw, IXGBE_NETLIST_ID_BLK_SIZE, + sizeof(*id_blk)); + if (!id_blk) + return IXGBE_ERR_NO_SPACE; + + /* Read out the entire Netlist ID Block at once. */ + status = ixgbe_read_flash_module(hw, bank, E610_SR_NETLIST_BANK_PTR, + IXGBE_NETLIST_ID_BLK_OFFSET(node_count) * sizeof(u16), + (u8 *)id_blk, + IXGBE_NETLIST_ID_BLK_SIZE * sizeof(u16)); + if (status) + goto exit_error; + + for (i = 0; i < IXGBE_NETLIST_ID_BLK_SIZE; i++) + id_blk[i] = IXGBE_LE16_TO_CPU(((__le16 *)id_blk)[i]); + + netlist->major = id_blk[IXGBE_NETLIST_ID_BLK_MAJOR_VER_HIGH] << 16 | + id_blk[IXGBE_NETLIST_ID_BLK_MAJOR_VER_LOW]; + netlist->minor = id_blk[IXGBE_NETLIST_ID_BLK_MINOR_VER_HIGH] << 16 | + id_blk[IXGBE_NETLIST_ID_BLK_MINOR_VER_LOW]; + netlist->type = id_blk[IXGBE_NETLIST_ID_BLK_TYPE_HIGH] << 16 | + id_blk[IXGBE_NETLIST_ID_BLK_TYPE_LOW]; + netlist->rev = id_blk[IXGBE_NETLIST_ID_BLK_REV_HIGH] << 16 | + id_blk[IXGBE_NETLIST_ID_BLK_REV_LOW]; + netlist->cust_ver = id_blk[IXGBE_NETLIST_ID_BLK_CUST_VER]; + /* Read the left most 4 bytes of SHA */ + netlist->hash = id_blk[IXGBE_NETLIST_ID_BLK_SHA_HASH_WORD(15)] << 16 | + id_blk[IXGBE_NETLIST_ID_BLK_SHA_HASH_WORD(14)]; + +exit_error: + ixgbe_free(hw, id_blk); + + return status; +} + +/** + * ixgbe_get_inactive_netlist_ver + * @hw: pointer to the HW struct + * @netlist: pointer to netlist version info structure + * + * Read the netlist version data from the inactive netlist bank. Used to + * extract version data of a pending flash update in order to display the + * version data. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_get_inactive_netlist_ver(struct ixgbe_hw *hw, + struct ixgbe_netlist_info *netlist) +{ + return ixgbe_get_netlist_info(hw, IXGBE_INACTIVE_FLASH_BANK, netlist); +} + +/** + * ixgbe_read_sr_pointer - Read the value of a Shadow RAM pointer word + * @hw: pointer to the HW structure + * @offset: the word offset of the Shadow RAM word to read + * @pointer: pointer value read from Shadow RAM + * + * Read the given Shadow RAM word, and convert it to a pointer value specified + * in bytes. This function assumes the specified offset is a valid pointer + * word. + * + * Each pointer word specifies whether it is stored in word size or 4KB + * sector size by using the highest bit. The reported pointer value will be in + * bytes, intended for flat NVM reads. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_read_sr_pointer(struct ixgbe_hw *hw, u16 offset, u32 *pointer) +{ + s32 status; + u16 value; + + status = ixgbe_read_ee_aci_E610(hw, offset, &value); + if (status) + return status; + + /* Determine if the pointer is in 4KB or word units */ + if (value & IXGBE_SR_NVM_PTR_4KB_UNITS) + *pointer = (value & ~IXGBE_SR_NVM_PTR_4KB_UNITS) * 4 * 1024; + else + *pointer = value * 2; + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_read_sr_area_size - Read an area size from a Shadow RAM word + * @hw: pointer to the HW structure + * @offset: the word offset of the Shadow RAM to read + * @size: size value read from the Shadow RAM + * + * Read the given Shadow RAM word, and convert it to an area size value + * specified in bytes. This function assumes the specified offset is a valid + * area size word. + * + * Each area size word is specified in 4KB sector units. This function reports + * the size in bytes, intended for flat NVM reads. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_read_sr_area_size(struct ixgbe_hw *hw, u16 offset, u32 *size) +{ + s32 status; + u16 value; + + status = ixgbe_read_ee_aci_E610(hw, offset, &value); + if (status) + return status; + + /* Area sizes are always specified in 4KB units */ + *size = value * 4 * 1024; + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_discover_flash_size - Discover the available flash size. + * @hw: pointer to the HW struct + * + * The device flash could be up to 16MB in size. However, it is possible that + * the actual size is smaller. Use bisection to determine the accessible size + * of flash memory. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_discover_flash_size(struct ixgbe_hw *hw) +{ + u32 min_size = 0, max_size = IXGBE_ACI_NVM_MAX_OFFSET + 1; + s32 status; + + status = ixgbe_acquire_nvm(hw, IXGBE_RES_READ); + if (status) + return status; + + while ((max_size - min_size) > 1) { + u32 offset = (max_size + min_size) / 2; + u32 len = 1; + u8 data; + + status = ixgbe_read_flat_nvm(hw, offset, &len, &data, false); + if (status == IXGBE_ERR_ACI_ERROR && + hw->aci.last_status == IXGBE_ACI_RC_EINVAL) { + status = IXGBE_SUCCESS; + max_size = offset; + } else if (!status) { + min_size = offset; + } else { + /* an unexpected error occurred */ + goto err_read_flat_nvm; + } + } + + hw->flash.flash_size = max_size; + +err_read_flat_nvm: + ixgbe_release_nvm(hw); + + return status; +} + +/** + * ixgbe_determine_active_flash_banks - Discover active bank for each module + * @hw: pointer to the HW struct + * + * Read the Shadow RAM control word and determine which banks are active for + * the NVM, OROM, and Netlist modules. Also read and calculate the associated + * pointer and size. These values are then cached into the ixgbe_flash_info + * structure for later use in order to calculate the correct offset to read + * from the active module. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_determine_active_flash_banks(struct ixgbe_hw *hw) +{ + struct ixgbe_bank_info *banks = &hw->flash.banks; + u16 ctrl_word; + s32 status; + + status = ixgbe_read_ee_aci_E610(hw, E610_SR_NVM_CTRL_WORD, &ctrl_word); + if (status) { + return status; + } + + /* Check that the control word indicates validity */ + if ((ctrl_word & IXGBE_SR_CTRL_WORD_1_M) >> IXGBE_SR_CTRL_WORD_1_S != + IXGBE_SR_CTRL_WORD_VALID) { + return IXGBE_ERR_CONFIG; + } + + if (!(ctrl_word & IXGBE_SR_CTRL_WORD_NVM_BANK)) + banks->nvm_bank = IXGBE_1ST_FLASH_BANK; + else + banks->nvm_bank = IXGBE_2ND_FLASH_BANK; + + if (!(ctrl_word & IXGBE_SR_CTRL_WORD_OROM_BANK)) + banks->orom_bank = IXGBE_1ST_FLASH_BANK; + else + banks->orom_bank = IXGBE_2ND_FLASH_BANK; + + if (!(ctrl_word & IXGBE_SR_CTRL_WORD_NETLIST_BANK)) + banks->netlist_bank = IXGBE_1ST_FLASH_BANK; + else + banks->netlist_bank = IXGBE_2ND_FLASH_BANK; + + status = ixgbe_read_sr_pointer(hw, E610_SR_1ST_NVM_BANK_PTR, + &banks->nvm_ptr); + if (status) { + return status; + } + + status = ixgbe_read_sr_area_size(hw, E610_SR_NVM_BANK_SIZE, + &banks->nvm_size); + if (status) { + return status; + } + + status = ixgbe_read_sr_pointer(hw, E610_SR_1ST_OROM_BANK_PTR, + &banks->orom_ptr); + if (status) { + return status; + } + + status = ixgbe_read_sr_area_size(hw, E610_SR_OROM_BANK_SIZE, + &banks->orom_size); + if (status) { + return status; + } + + status = ixgbe_read_sr_pointer(hw, E610_SR_NETLIST_BANK_PTR, + &banks->netlist_ptr); + if (status) { + return status; + } + + status = ixgbe_read_sr_area_size(hw, E610_SR_NETLIST_BANK_SIZE, + &banks->netlist_size); + if (status) { + return status; + } + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_init_nvm - initializes NVM setting + * @hw: pointer to the HW struct + * + * Read and populate NVM settings such as Shadow RAM size, + * max_timeout, and blank_nvm_mode + * + * Return: the exit code of the operation. + */ +s32 ixgbe_init_nvm(struct ixgbe_hw *hw) +{ + struct ixgbe_flash_info *flash = &hw->flash; + u32 fla, gens_stat, status; + u8 sr_size; + + /* The SR size is stored regardless of the NVM programming mode + * as the blank mode may be used in the factory line. + */ + gens_stat = IXGBE_READ_REG(hw, GLNVM_GENS); + sr_size = (gens_stat & GLNVM_GENS_SR_SIZE_M) >> GLNVM_GENS_SR_SIZE_S; + + /* Switching to words (sr_size contains power of 2) */ + flash->sr_words = BIT(sr_size) * IXGBE_SR_WORDS_IN_1KB; + + /* Check if we are in the normal or blank NVM programming mode */ + fla = IXGBE_READ_REG(hw, GLNVM_FLA); + if (fla & GLNVM_FLA_LOCKED_M) { /* Normal programming mode */ + flash->blank_nvm_mode = false; + } else { + /* Blank programming mode */ + flash->blank_nvm_mode = true; + return IXGBE_ERR_NVM_BLANK_MODE; + } + + status = ixgbe_discover_flash_size(hw); + if (status) { + return status; + } + + status = ixgbe_determine_active_flash_banks(hw); + if (status) { + return status; + } + + status = ixgbe_get_nvm_ver_info(hw, IXGBE_ACTIVE_FLASH_BANK, + &flash->nvm); + if (status) { + return status; + } + + /* read the netlist version information */ + status = ixgbe_get_netlist_info(hw, IXGBE_ACTIVE_FLASH_BANK, + &flash->netlist); + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_sanitize_operate - Clear the user data + * @hw: pointer to the HW struct + * + * Clear user data from NVM using ACI command (0x070C). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_sanitize_operate(struct ixgbe_hw *hw) +{ + s32 status; + u8 values; + + u8 cmd_flags = IXGBE_ACI_SANITIZE_REQ_OPERATE | + IXGBE_ACI_SANITIZE_OPERATE_SUBJECT_CLEAR; + + status = ixgbe_sanitize_nvm(hw, cmd_flags, &values); + if (status) + return status; + if ((!(values & IXGBE_ACI_SANITIZE_OPERATE_HOST_CLEAN_DONE) && + !(values & IXGBE_ACI_SANITIZE_OPERATE_BMC_CLEAN_DONE)) || + ((values & IXGBE_ACI_SANITIZE_OPERATE_HOST_CLEAN_DONE) && + !(values & IXGBE_ACI_SANITIZE_OPERATE_HOST_CLEAN_SUCCESS)) || + ((values & IXGBE_ACI_SANITIZE_OPERATE_BMC_CLEAN_DONE) && + !(values & IXGBE_ACI_SANITIZE_OPERATE_BMC_CLEAN_SUCCESS))) + return IXGBE_ERR_ACI_ERROR; + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_sanitize_nvm - Sanitize NVM + * @hw: pointer to the HW struct + * @cmd_flags: flag to the ACI command + * @values: values returned from the command + * + * Sanitize NVM using ACI command (0x070C). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_sanitize_nvm(struct ixgbe_hw *hw, u8 cmd_flags, u8 *values) +{ + struct ixgbe_aci_desc desc; + struct ixgbe_aci_cmd_nvm_sanitization *cmd; + s32 status; + + cmd = &desc.params.nvm_sanitization; + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_nvm_sanitization); + cmd->cmd_flags = cmd_flags; + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + if (values) + *values = cmd->values; + + return status; +} + +/** + * ixgbe_read_sr_word_aci - Reads Shadow RAM via ACI + * @hw: pointer to the HW structure + * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF) + * @data: word read from the Shadow RAM + * + * Reads one 16 bit word from the Shadow RAM using ixgbe_read_flat_nvm. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_read_sr_word_aci(struct ixgbe_hw *hw, u16 offset, u16 *data) +{ + u32 bytes = sizeof(u16); + __le16 data_local; + s32 status; + + status = ixgbe_read_flat_nvm(hw, offset * sizeof(u16), &bytes, + (u8 *)&data_local, true); + if (status) + return status; + + *data = IXGBE_LE16_TO_CPU(data_local); + return IXGBE_SUCCESS; +} + +/** + * ixgbe_read_sr_buf_aci - Reads Shadow RAM buf via ACI + * @hw: pointer to the HW structure + * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF) + * @words: (in) number of words to read; (out) number of words actually read + * @data: words read from the Shadow RAM + * + * Reads 16 bit words (data buf) from the Shadow RAM. Ownership of the NVM is + * taken before reading the buffer and later released. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_read_sr_buf_aci(struct ixgbe_hw *hw, u16 offset, u16 *words, + u16 *data) +{ + u32 bytes = *words * 2, i; + s32 status; + + status = ixgbe_read_flat_nvm(hw, offset * 2, &bytes, (u8 *)data, true); + + *words = bytes / 2; + + for (i = 0; i < *words; i++) + data[i] = IXGBE_LE16_TO_CPU(((__le16 *)data)[i]); + + return status; +} + +/** + * ixgbe_read_flat_nvm - Read portion of NVM by flat offset + * @hw: pointer to the HW struct + * @offset: offset from beginning of NVM + * @length: (in) number of bytes to read; (out) number of bytes actually read + * @data: buffer to return data in (sized to fit the specified length) + * @read_shadow_ram: if true, read from shadow RAM instead of NVM + * + * Reads a portion of the NVM, as a flat memory space. This function correctly + * breaks read requests across Shadow RAM sectors, prevents Shadow RAM size + * from being exceeded in case of Shadow RAM read requests and ensures that no + * single read request exceeds the maximum 4KB read for a single admin command. + * + * Returns a status code on failure. Note that the data pointer may be + * partially updated if some reads succeed before a failure. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_read_flat_nvm(struct ixgbe_hw *hw, u32 offset, u32 *length, + u8 *data, bool read_shadow_ram) +{ + u32 inlen = *length; + u32 bytes_read = 0; + bool last_cmd; + s32 status; + + *length = 0; + + /* Verify the length of the read if this is for the Shadow RAM */ + if (read_shadow_ram && ((offset + inlen) > + (hw->eeprom.word_size * 2u))) { + return IXGBE_ERR_PARAM; + } + + do { + u32 read_size, sector_offset; + + /* ixgbe_aci_read_nvm cannot read more than 4KB at a time. + * Additionally, a read from the Shadow RAM may not cross over + * a sector boundary. Conveniently, the sector size is also 4KB. + */ + sector_offset = offset % IXGBE_ACI_MAX_BUFFER_SIZE; + read_size = MIN_T(u32, + IXGBE_ACI_MAX_BUFFER_SIZE - sector_offset, + inlen - bytes_read); + + last_cmd = !(bytes_read + read_size < inlen); + + /* ixgbe_aci_read_nvm takes the length as a u16. Our read_size + * is calculated using a u32, but the IXGBE_ACI_MAX_BUFFER_SIZE + * maximum size guarantees that it will fit within the 2 bytes. + */ + status = ixgbe_aci_read_nvm(hw, IXGBE_ACI_NVM_START_POINT, + offset, (u16)read_size, + data + bytes_read, last_cmd, + read_shadow_ram); + if (status) + break; + + bytes_read += read_size; + offset += read_size; + } while (!last_cmd); + + *length = bytes_read; + return status; +} + +/** + * ixgbe_check_sr_access_params - verify params for Shadow RAM R/W operations. + * @hw: pointer to the HW structure + * @offset: offset in words from module start + * @words: number of words to access + * + * Check if all the parameters are valid + * before performing any Shadow RAM read/write operations. + * + * Return: the exit code of the operation. + * * - IXGBE_SUCCESS - success. + * * - IXGBE_ERR_PARAM - NVM error: offset beyond SR limit or + * NVM error: tried to access more words then the set limit or + * NVM error: cannot spread over two sectors. + */ +static s32 ixgbe_check_sr_access_params(struct ixgbe_hw *hw, u32 offset, + u16 words) +{ + if ((offset + words) > hw->eeprom.word_size) { + return IXGBE_ERR_PARAM; + } + + if (words > IXGBE_SR_SECTOR_SIZE_IN_WORDS) { + /* We can access only up to 4KB (one sector), + * in one Admin Command write + */ + return IXGBE_ERR_PARAM; + } + + if (((offset + (words - 1)) / IXGBE_SR_SECTOR_SIZE_IN_WORDS) != + (offset / IXGBE_SR_SECTOR_SIZE_IN_WORDS)) { + /* A single access cannot spread over two sectors */ + return IXGBE_ERR_PARAM; + } + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_write_sr_word_aci - Writes Shadow RAM word + * @hw: pointer to the HW structure + * @offset: offset of the Shadow RAM word to write + * @data: word to write to the Shadow RAM + * + * Writes a 16 bit word to the Shadow RAM using the admin command. + * NVM ownership must be acquired before calling this function and released + * by a caller. To commit SR to NVM update checksum function should be called. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_write_sr_word_aci(struct ixgbe_hw *hw, u32 offset, const u16 *data) +{ + __le16 data_local = IXGBE_CPU_TO_LE16(*data); + s32 status; + + status = ixgbe_check_sr_access_params(hw, offset, 1); + if (!status) + status = ixgbe_aci_update_nvm(hw, 0, BYTES_PER_WORD * offset, + BYTES_PER_WORD, &data_local, + false, 0); + + return status; +} + +/** + * ixgbe_write_sr_buf_aci - Writes Shadow RAM buf + * @hw: pointer to the HW structure + * @offset: offset of the Shadow RAM buffer to write + * @words: number of words to write + * @data: words to write to the Shadow RAM + * + * Writes a 16 bit word to the Shadow RAM using the admin command. + * NVM ownership must be acquired before calling this function and released + * by a caller. To commit SR to NVM update checksum function should be called. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_write_sr_buf_aci(struct ixgbe_hw *hw, u32 offset, u16 words, + const u16 *data) +{ + __le16 *data_local; + s32 status; + void *vmem; + u32 i; + + vmem = ixgbe_calloc(hw, words, sizeof(u16)); + if (!vmem) + return IXGBE_ERR_OUT_OF_MEM; + data_local = (__le16 *)vmem; + + for (i = 0; i < words; i++) + data_local[i] = IXGBE_CPU_TO_LE16(data[i]); + + /* Here we will only write one buffer as the size of the modules + * mirrored in the Shadow RAM is always less than 4K. + */ + status = ixgbe_check_sr_access_params(hw, offset, words); + if (!status) + status = ixgbe_aci_update_nvm(hw, 0, BYTES_PER_WORD * offset, + BYTES_PER_WORD * words, + data_local, false, 0); + + ixgbe_free(hw, vmem); + + return status; +} + +/** + * ixgbe_aci_alternate_write - write to alternate structure + * @hw: pointer to the hardware structure + * @reg_addr0: address of first dword to be written + * @reg_val0: value to be written under 'reg_addr0' + * @reg_addr1: address of second dword to be written + * @reg_val1: value to be written under 'reg_addr1' + * + * Write one or two dwords to alternate structure using ACI command (0x0900). + * Fields are indicated by 'reg_addr0' and 'reg_addr1' register numbers. + * + * Return: 0 on success and error code on failure. + */ +s32 ixgbe_aci_alternate_write(struct ixgbe_hw *hw, u32 reg_addr0, + u32 reg_val0, u32 reg_addr1, u32 reg_val1) +{ + struct ixgbe_aci_cmd_read_write_alt_direct *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + cmd = &desc.params.read_write_alt_direct; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_write_alt_direct); + cmd->dword0_addr = IXGBE_CPU_TO_LE32(reg_addr0); + cmd->dword1_addr = IXGBE_CPU_TO_LE32(reg_addr1); + cmd->dword0_value = IXGBE_CPU_TO_LE32(reg_val0); + cmd->dword1_value = IXGBE_CPU_TO_LE32(reg_val1); + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + + return status; +} + +/** + * ixgbe_aci_alternate_read - read from alternate structure + * @hw: pointer to the hardware structure + * @reg_addr0: address of first dword to be read + * @reg_val0: pointer for data read from 'reg_addr0' + * @reg_addr1: address of second dword to be read + * @reg_val1: pointer for data read from 'reg_addr1' + * + * Read one or two dwords from alternate structure using ACI command (0x0902). + * Fields are indicated by 'reg_addr0' and 'reg_addr1' register numbers. + * If 'reg_val1' pointer is not passed then only register at 'reg_addr0' + * is read. + * + * Return: 0 on success and error code on failure. + */ +s32 ixgbe_aci_alternate_read(struct ixgbe_hw *hw, u32 reg_addr0, + u32 *reg_val0, u32 reg_addr1, u32 *reg_val1) +{ + struct ixgbe_aci_cmd_read_write_alt_direct *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + cmd = &desc.params.read_write_alt_direct; + + if (!reg_val0) + return IXGBE_ERR_PARAM; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_read_alt_direct); + cmd->dword0_addr = IXGBE_CPU_TO_LE32(reg_addr0); + cmd->dword1_addr = IXGBE_CPU_TO_LE32(reg_addr1); + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + + if (status == IXGBE_SUCCESS) { + *reg_val0 = IXGBE_LE32_TO_CPU(cmd->dword0_value); + + if (reg_val1) + *reg_val1 = IXGBE_LE32_TO_CPU(cmd->dword1_value); + } + + return status; +} + +/** + * ixgbe_aci_alternate_write_done - check if writing to alternate structure + * is done + * @hw: pointer to the HW structure. + * @bios_mode: indicates whether the command is executed by UEFI or legacy BIOS + * @reset_needed: indicates the SW should trigger GLOBAL reset + * + * Indicates to the FW that alternate structures have been changed. + * + * Return: 0 on success and error code on failure. + */ +s32 ixgbe_aci_alternate_write_done(struct ixgbe_hw *hw, u8 bios_mode, + bool *reset_needed) +{ + struct ixgbe_aci_cmd_done_alt_write *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + cmd = &desc.params.done_alt_write; + + if (!reset_needed) + return IXGBE_ERR_PARAM; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_done_alt_write); + cmd->flags = bios_mode; + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + if (!status) + *reset_needed = (IXGBE_LE16_TO_CPU(cmd->flags) & + IXGBE_ACI_RESP_RESET_NEEDED) != 0; + + return status; +} + +/** + * ixgbe_aci_alternate_clear - clear alternate structure + * @hw: pointer to the HW structure. + * + * Clear the alternate structures of the port from which the function + * is called. + * + * Return: 0 on success and error code on failure. + */ +s32 ixgbe_aci_alternate_clear(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_desc desc; + s32 status; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, + ixgbe_aci_opc_clear_port_alt_write); + + status = ixgbe_aci_send_cmd(hw, &desc, NULL, 0); + + return status; +} + +/** + * ixgbe_aci_get_internal_data - get internal FW/HW data + * @hw: pointer to the hardware structure + * @cluster_id: specific cluster to dump + * @table_id: table ID within cluster + * @start: index of line in the block to read + * @buf: dump buffer + * @buf_size: dump buffer size + * @ret_buf_size: return buffer size (returned by FW) + * @ret_next_cluster: next cluster to read (returned by FW) + * @ret_next_table: next block to read (returned by FW) + * @ret_next_index: next index to read (returned by FW) + * + * Get internal FW/HW data using ACI command (0xFF08) for debug purposes. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_get_internal_data(struct ixgbe_hw *hw, u16 cluster_id, + u16 table_id, u32 start, void *buf, + u16 buf_size, u16 *ret_buf_size, + u16 *ret_next_cluster, u16 *ret_next_table, + u32 *ret_next_index) +{ + struct ixgbe_aci_cmd_debug_dump_internals *cmd; + struct ixgbe_aci_desc desc; + s32 status; + + cmd = &desc.params.debug_dump; + + if (buf_size == 0 || !buf) + return IXGBE_ERR_PARAM; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, + ixgbe_aci_opc_debug_dump_internals); + + cmd->cluster_id = IXGBE_CPU_TO_LE16(cluster_id); + cmd->table_id = IXGBE_CPU_TO_LE16(table_id); + cmd->idx = IXGBE_CPU_TO_LE32(start); + + status = ixgbe_aci_send_cmd(hw, &desc, buf, buf_size); + + if (!status) { + if (ret_buf_size) + *ret_buf_size = IXGBE_LE16_TO_CPU(desc.datalen); + if (ret_next_cluster) + *ret_next_cluster = IXGBE_LE16_TO_CPU(cmd->cluster_id); + if (ret_next_table) + *ret_next_table = IXGBE_LE16_TO_CPU(cmd->table_id); + if (ret_next_index) + *ret_next_index = IXGBE_LE32_TO_CPU(cmd->idx); + } + + return status; +} + +/** + * ixgbe_validate_nvm_rw_reg - Check that an NVM access request is valid + * @cmd: NVM access command structure + * + * Validates that an NVM access structure is request to read or write a valid + * register offset. First validates that the module and flags are correct, and + * then ensures that the register offset is one of the accepted registers. + * + * Return: 0 if the register access is valid, out of range error code otherwise. + */ +static s32 +ixgbe_validate_nvm_rw_reg(struct ixgbe_nvm_access_cmd *cmd) +{ + u16 i; + + switch (cmd->offset) { + case GL_HICR: + case GL_HICR_EN: /* Note, this register is read only */ + case GL_FWSTS: + case GL_MNG_FWSM: + case GLNVM_GENS: + case GLNVM_FLA: + case GL_FWRESETCNT: + return 0; + default: + break; + } + + for (i = 0; i <= GL_HIDA_MAX_INDEX; i++) + if (cmd->offset == (u32)GL_HIDA(i)) + return 0; + + for (i = 0; i <= GL_HIBA_MAX_INDEX; i++) + if (cmd->offset == (u32)GL_HIBA(i)) + return 0; + + /* All other register offsets are not valid */ + return IXGBE_ERR_OUT_OF_RANGE; +} + +/** + * ixgbe_nvm_access_read - Handle an NVM read request + * @hw: pointer to the HW struct + * @cmd: NVM access command to process + * @data: storage for the register value read + * + * Process an NVM access request to read a register. + * + * Return: 0 if the register read is valid and successful, + * out of range error code otherwise. + */ +static s32 ixgbe_nvm_access_read(struct ixgbe_hw *hw, + struct ixgbe_nvm_access_cmd *cmd, + struct ixgbe_nvm_access_data *data) +{ + s32 status; + + /* Always initialize the output data, even on failure */ + memset(&data->regval, 0, cmd->data_size); + + /* Make sure this is a valid read/write access request */ + status = ixgbe_validate_nvm_rw_reg(cmd); + if (status) + return status; + + DEBUGOUT1("NVM access: reading register %08x\n", cmd->offset); + + /* Read the register and store the contents in the data field */ + data->regval = IXGBE_READ_REG(hw, cmd->offset); + + return 0; +} + +/** + * ixgbe_nvm_access_write - Handle an NVM write request + * @hw: pointer to the HW struct + * @cmd: NVM access command to process + * @data: NVM access data to write + * + * Process an NVM access request to write a register. + * + * Return: 0 if the register write is valid and successful, + * out of range error code otherwise. + */ +static s32 ixgbe_nvm_access_write(struct ixgbe_hw *hw, + struct ixgbe_nvm_access_cmd *cmd, + struct ixgbe_nvm_access_data *data) +{ + s32 status; + + /* Make sure this is a valid read/write access request */ + status = ixgbe_validate_nvm_rw_reg(cmd); + if (status) + return status; + + /* Reject requests to write to read-only registers */ + switch (cmd->offset) { + case GL_HICR_EN: + return IXGBE_ERR_OUT_OF_RANGE; + default: + break; + } + + DEBUGOUT2("NVM access: writing register %08x with value %08x\n", + cmd->offset, data->regval); + + /* Write the data field to the specified register */ + IXGBE_WRITE_REG(hw, cmd->offset, data->regval); + + return 0; +} + +/** + * ixgbe_handle_nvm_access - Handle an NVM access request + * @hw: pointer to the HW struct + * @cmd: NVM access command info + * @data: pointer to read or return data + * + * Process an NVM access request. Read the command structure information and + * determine if it is valid. If not, report an error indicating the command + * was invalid. + * + * For valid commands, perform the necessary function, copying the data into + * the provided data buffer. + * + * Return: 0 if the nvm access request is valid and successful, + * error code otherwise. + */ +s32 ixgbe_handle_nvm_access(struct ixgbe_hw *hw, + struct ixgbe_nvm_access_cmd *cmd, + struct ixgbe_nvm_access_data *data) +{ + switch (cmd->command) { + case IXGBE_NVM_CMD_READ: + return ixgbe_nvm_access_read(hw, cmd, data); + case IXGBE_NVM_CMD_WRITE: + return ixgbe_nvm_access_write(hw, cmd, data); + default: + return IXGBE_ERR_PARAM; + } +} + +/** + * ixgbe_aci_set_health_status_config - Configure FW health events + * @hw: pointer to the HW struct + * @event_source: type of diagnostic events to enable + * + * Configure the health status event types that the firmware will send to this + * PF using ACI command (0xFF20). The supported event types are: PF-specific, + * all PFs, and global. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_aci_set_health_status_config(struct ixgbe_hw *hw, u8 event_source) +{ + struct ixgbe_aci_cmd_set_health_status_config *cmd; + struct ixgbe_aci_desc desc; + + cmd = &desc.params.set_health_status_config; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, + ixgbe_aci_opc_set_health_status_config); + + cmd->event_source = event_source; + + return ixgbe_aci_send_cmd(hw, &desc, NULL, 0); +} + +/** + * ixgbe_init_ops_E610 - Inits func ptrs and MAC type + * @hw: pointer to hardware structure + * + * Initialize the function pointers and assign the MAC type for E610. + * Does not touch the hardware. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_init_ops_E610(struct ixgbe_hw *hw) +{ + struct ixgbe_eeprom_info *eeprom = &hw->eeprom; + struct ixgbe_mac_info *mac = &hw->mac; + struct ixgbe_phy_info *phy = &hw->phy; + s32 ret_val; + + ret_val = ixgbe_init_ops_X550(hw); + + /* MAC */ + mac->ops.reset_hw = ixgbe_reset_hw_E610; + mac->ops.start_hw = ixgbe_start_hw_E610; + mac->ops.get_media_type = ixgbe_get_media_type_E610; + mac->ops.get_supported_physical_layer = + ixgbe_get_supported_physical_layer_E610; + mac->ops.get_san_mac_addr = NULL; + mac->ops.set_san_mac_addr = NULL; + mac->ops.get_wwn_prefix = NULL; + mac->ops.setup_link = ixgbe_setup_link_E610; + mac->ops.check_link = ixgbe_check_link_E610; + mac->ops.get_link_capabilities = ixgbe_get_link_capabilities_E610; + mac->ops.setup_fc = ixgbe_setup_fc_E610; + mac->ops.fc_autoneg = ixgbe_fc_autoneg_E610; + mac->ops.set_fw_drv_ver = ixgbe_set_fw_drv_ver_E610; + mac->ops.disable_rx = ixgbe_disable_rx_E610; + mac->ops.setup_eee = ixgbe_setup_eee_E610; + mac->ops.fw_recovery_mode = ixgbe_fw_recovery_mode_E610; + mac->ops.fw_rollback_mode = ixgbe_fw_rollback_mode_E610; + mac->ops.get_fw_tsam_mode = ixgbe_get_fw_tsam_mode_E610; + mac->ops.get_fw_version = ixgbe_aci_get_fw_ver; + mac->ops.get_nvm_version = ixgbe_get_active_nvm_ver; + mac->ops.get_thermal_sensor_data = NULL; + mac->ops.init_thermal_sensor_thresh = NULL; + + /* PHY */ + phy->ops.init = ixgbe_init_phy_ops_E610; + phy->ops.identify = ixgbe_identify_phy_E610; + phy->eee_speeds_supported = IXGBE_LINK_SPEED_10_FULL | + IXGBE_LINK_SPEED_100_FULL | + IXGBE_LINK_SPEED_1GB_FULL; + phy->eee_speeds_advertised = phy->eee_speeds_supported; + + /* Additional ops overrides for e610 to go here */ + eeprom->ops.init_params = ixgbe_init_eeprom_params_E610; + eeprom->ops.read = ixgbe_read_ee_aci_E610; + eeprom->ops.read_buffer = ixgbe_read_ee_aci_buffer_E610; + eeprom->ops.write = ixgbe_write_ee_aci_E610; + eeprom->ops.write_buffer = ixgbe_write_ee_aci_buffer_E610; + eeprom->ops.calc_checksum = ixgbe_calc_eeprom_checksum_E610; + eeprom->ops.update_checksum = ixgbe_update_eeprom_checksum_E610; + eeprom->ops.validate_checksum = ixgbe_validate_eeprom_checksum_E610; + eeprom->ops.read_pba_string = ixgbe_read_pba_string_E610; + + /* Initialize bus function number */ + hw->mac.ops.set_lan_id(hw); + + return ret_val; +} + +/** + * ixgbe_reset_hw_E610 - Perform hardware reset + * @hw: pointer to hardware structure + * + * Resets the hardware by resetting the transmit and receive units, masks + * and clears all interrupts, and perform a reset. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_reset_hw_E610(struct ixgbe_hw *hw) +{ + u32 swfw_mask = hw->phy.phy_semaphore_mask; + u32 ctrl, i; + s32 status; + + DEBUGFUNC("ixgbe_reset_hw_E610"); + + /* Call adapter stop to disable tx/rx and clear interrupts */ + status = hw->mac.ops.stop_adapter(hw); + if (status != IXGBE_SUCCESS) + goto reset_hw_out; + + /* flush pending Tx transactions */ + ixgbe_clear_tx_pending(hw); + + status = hw->phy.ops.init(hw); + if (status != IXGBE_SUCCESS) + DEBUGOUT1("Failed to initialize PHY ops, STATUS = %d\n", + status); +mac_reset_top: + status = hw->mac.ops.acquire_swfw_sync(hw, swfw_mask); + if (status != IXGBE_SUCCESS) { + ERROR_REPORT2(IXGBE_ERROR_CAUTION, + "semaphore failed with %d", status); + return IXGBE_ERR_SWFW_SYNC; + } + ctrl = IXGBE_CTRL_RST; + ctrl |= IXGBE_READ_REG(hw, IXGBE_CTRL); + IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl); + IXGBE_WRITE_FLUSH(hw); + hw->mac.ops.release_swfw_sync(hw, swfw_mask); + + /* Poll for reset bit to self-clear indicating reset is complete */ + for (i = 0; i < 10; i++) { + usec_delay(1); + ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL); + if (!(ctrl & IXGBE_CTRL_RST_MASK)) + break; + } + + if (ctrl & IXGBE_CTRL_RST_MASK) { + status = IXGBE_ERR_RESET_FAILED; + ERROR_REPORT1(IXGBE_ERROR_POLLING, + "Reset polling failed to complete.\n"); + } + msec_delay(100); + + /* + * Double resets are required for recovery from certain error + * conditions. Between resets, it is necessary to stall to allow time + * for any pending HW events to complete. + */ + if (hw->mac.flags & IXGBE_FLAGS_DOUBLE_RESET_REQUIRED) { + hw->mac.flags &= ~IXGBE_FLAGS_DOUBLE_RESET_REQUIRED; + goto mac_reset_top; + } + + /* Set the Rx packet buffer size. */ + IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(0), 384 << IXGBE_RXPBSIZE_SHIFT); + + /* Store the permanent mac address */ + hw->mac.ops.get_mac_addr(hw, hw->mac.perm_addr); + + /* + * Store MAC address from RAR0, clear receive address registers, and + * clear the multicast table. Also reset num_rar_entries to 128, + * since we modify this value when programming the SAN MAC address. + */ + hw->mac.num_rar_entries = 128; + hw->mac.ops.init_rx_addrs(hw); + +reset_hw_out: + return status; +} + +/** + * ixgbe_start_hw_E610 - Prepare hardware for Tx/Rx + * @hw: pointer to hardware structure + * + * Gets firmware version and if API version matches it + * starts the hardware using the generic start_hw function + * and the generation start_hw function. + * Then performs revision-specific operations, if any. + **/ +s32 ixgbe_start_hw_E610(struct ixgbe_hw *hw) +{ + s32 ret_val = IXGBE_SUCCESS; + + ret_val = hw->mac.ops.get_fw_version(hw); + if (ret_val) + goto out; + + ret_val = ixgbe_start_hw_generic(hw); + if (ret_val != IXGBE_SUCCESS) + goto out; + + ixgbe_start_hw_gen2(hw); + +out: + return ret_val; +} + +/** + * ixgbe_get_media_type_E610 - Gets media type + * @hw: pointer to the HW struct + * + * In order to get the media type, the function gets PHY + * capabilities and later on use them to identify the PHY type + * checking phy_type_high and phy_type_low. + * + * Return: the type of media in form of ixgbe_media_type enum + * or ixgbe_media_type_unknown in case of an error. + */ +enum ixgbe_media_type ixgbe_get_media_type_E610(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_get_phy_caps_data pcaps; + u64 phy_mask = 0; + s32 rc; + u8 i; + + rc = ixgbe_update_link_info(hw); + if (rc) { + return ixgbe_media_type_unknown; + } + + /* If there is no link but PHY (dongle) is available SW should use + * Get PHY Caps admin command instead of Get Link Status, find most + * significant bit that is set in PHY types reported by the command + * and use it to discover media type. + */ + if (!(hw->link.link_info.link_info & IXGBE_ACI_LINK_UP) && + (hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE)) { + /* Get PHY Capabilities */ + rc = ixgbe_aci_get_phy_caps(hw, false, + IXGBE_ACI_REPORT_TOPO_CAP_MEDIA, + &pcaps); + if (rc) { + return ixgbe_media_type_unknown; + } + + /* Check if there is some bit set in phy_type_high */ + for (i = 64; i > 0; i--) { + phy_mask = (u64)((u64)1 << (i - 1)); + if ((pcaps.phy_type_high & phy_mask) != 0) { + /* If any bit is set treat it as PHY type */ + hw->link.link_info.phy_type_high = phy_mask; + hw->link.link_info.phy_type_low = 0; + break; + } + phy_mask = 0; + } + + /* If nothing found in phy_type_high search in phy_type_low */ + if (phy_mask == 0) { + for (i = 64; i > 0; i--) { + phy_mask = (u64)((u64)1 << (i - 1)); + if ((pcaps.phy_type_low & phy_mask) != 0) { + /* If any bit is set treat it as PHY type */ + hw->link.link_info.phy_type_high = 0; + hw->link.link_info.phy_type_low = phy_mask; + break; + } + } + } + + } + + /* Based on link status or search above try to discover media type */ + hw->phy.media_type = ixgbe_get_media_type_from_phy_type(hw); + + return hw->phy.media_type; +} + +/** + * ixgbe_get_supported_physical_layer_E610 - Returns physical layer type + * @hw: pointer to hardware structure + * + * Determines physical layer capabilities of the current configuration. + * + * Return: the exit code of the operation. + **/ +u64 ixgbe_get_supported_physical_layer_E610(struct ixgbe_hw *hw) +{ + u64 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN; + struct ixgbe_aci_cmd_get_phy_caps_data pcaps; + u64 phy_type; + s32 rc; + + rc = ixgbe_aci_get_phy_caps(hw, false, IXGBE_ACI_REPORT_TOPO_CAP_MEDIA, + &pcaps); + if (rc) + return IXGBE_PHYSICAL_LAYER_UNKNOWN; + + phy_type = IXGBE_LE64_TO_CPU(pcaps.phy_type_low); + if(phy_type & IXGBE_PHY_TYPE_LOW_10GBASE_T) + physical_layer |= IXGBE_PHYSICAL_LAYER_10GBASE_T; + if(phy_type & IXGBE_PHY_TYPE_LOW_1000BASE_T) + physical_layer |= IXGBE_PHYSICAL_LAYER_1000BASE_T; + if(phy_type & IXGBE_PHY_TYPE_LOW_100BASE_TX) + physical_layer |= IXGBE_PHYSICAL_LAYER_100BASE_TX; + if(phy_type & IXGBE_PHY_TYPE_LOW_10GBASE_LR) + physical_layer |= IXGBE_PHYSICAL_LAYER_10GBASE_LR; + if(phy_type & IXGBE_PHY_TYPE_LOW_10GBASE_SR) + physical_layer |= IXGBE_PHYSICAL_LAYER_10GBASE_SR; + if(phy_type & IXGBE_PHY_TYPE_LOW_1000BASE_KX) + physical_layer |= IXGBE_PHYSICAL_LAYER_1000BASE_KX; + if(phy_type & IXGBE_PHY_TYPE_LOW_10GBASE_KR_CR1) + physical_layer |= IXGBE_PHYSICAL_LAYER_10GBASE_KR; + if(phy_type & IXGBE_PHY_TYPE_LOW_1000BASE_SX) + physical_layer |= IXGBE_PHYSICAL_LAYER_1000BASE_SX; + if(phy_type & IXGBE_PHY_TYPE_LOW_2500BASE_KX) + physical_layer |= IXGBE_PHYSICAL_LAYER_2500BASE_KX; + if(phy_type & IXGBE_PHY_TYPE_LOW_2500BASE_T) + physical_layer |= IXGBE_PHYSICAL_LAYER_2500BASE_T; + if(phy_type & IXGBE_PHY_TYPE_LOW_5GBASE_T) + physical_layer |= IXGBE_PHYSICAL_LAYER_5000BASE_T; + + phy_type = IXGBE_LE64_TO_CPU(pcaps.phy_type_high); + if(phy_type & IXGBE_PHY_TYPE_HIGH_10BASE_T) + physical_layer |= IXGBE_PHYSICAL_LAYER_10BASE_T; + + return physical_layer; +} + +/** + * ixgbe_setup_link_E610 - Set up link + * @hw: pointer to hardware structure + * @speed: new link speed + * @autoneg_wait: true when waiting for completion is needed + * + * Set up the link with the specified speed. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_setup_link_E610(struct ixgbe_hw *hw, ixgbe_link_speed speed, + bool autoneg_wait) +{ + /* Simply request FW to perform proper PHY setup */ + return hw->phy.ops.setup_link_speed(hw, speed, autoneg_wait); +} + +/** + * ixgbe_check_link_E610 - Determine link and speed status + * @hw: pointer to hardware structure + * @speed: pointer to link speed + * @link_up: true when link is up + * @link_up_wait_to_complete: bool used to wait for link up or not + * + * Determine if the link is up and the current link speed + * using ACI command (0x0607). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_check_link_E610(struct ixgbe_hw *hw, ixgbe_link_speed *speed, + bool *link_up, bool link_up_wait_to_complete) +{ + s32 rc; + u32 i; + + if (!speed || !link_up) + return IXGBE_ERR_PARAM; + + /* Set get_link_info flag to ensure that fresh + * link information will be obtained from FW + * by sending Get Link Status admin command. */ + hw->link.get_link_info = true; + + /* Update link information in adapter context. */ + rc = ixgbe_get_link_status(hw, link_up); + if (rc) + return rc; + + /* Wait for link up if it was requested. */ + if (link_up_wait_to_complete && *link_up == false) { + for (i = 0; i < hw->mac.max_link_up_time; i++) { + msec_delay(100); + hw->link.get_link_info = true; + rc = ixgbe_get_link_status(hw, link_up); + if (rc) + return rc; + if (*link_up) + break; + } + } + + /* Use link information in adapter context updated by the call + * to ixgbe_get_link_status() to determine current link speed. + * Link speed information is valid only when link up was + * reported by FW. */ + if (*link_up) { + switch (hw->link.link_info.link_speed) { + case IXGBE_ACI_LINK_SPEED_10MB: + *speed = IXGBE_LINK_SPEED_10_FULL; + break; + case IXGBE_ACI_LINK_SPEED_100MB: + *speed = IXGBE_LINK_SPEED_100_FULL; + break; + case IXGBE_ACI_LINK_SPEED_1000MB: + *speed = IXGBE_LINK_SPEED_1GB_FULL; + break; + case IXGBE_ACI_LINK_SPEED_2500MB: + *speed = IXGBE_LINK_SPEED_2_5GB_FULL; + break; + case IXGBE_ACI_LINK_SPEED_5GB: + *speed = IXGBE_LINK_SPEED_5GB_FULL; + break; + case IXGBE_ACI_LINK_SPEED_10GB: + *speed = IXGBE_LINK_SPEED_10GB_FULL; + break; + default: + *speed = IXGBE_LINK_SPEED_UNKNOWN; + break; + } + } else { + *speed = IXGBE_LINK_SPEED_UNKNOWN; + } + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_get_link_capabilities_E610 - Determine link capabilities + * @hw: pointer to hardware structure + * @speed: pointer to link speed + * @autoneg: true when autoneg or autotry is enabled + * + * Determine speed and AN parameters of a link. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_get_link_capabilities_E610(struct ixgbe_hw *hw, + ixgbe_link_speed *speed, + bool *autoneg) +{ + if (!speed || !autoneg) + return IXGBE_ERR_PARAM; + + *autoneg = true; + *speed = hw->phy.speeds_supported; + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_cfg_phy_fc - Configure PHY Flow Control (FC) data based on FC mode + * @hw: pointer to hardware structure + * @cfg: PHY configuration data to set FC mode + * @req_mode: FC mode to configure + * + * Configures PHY Flow Control according to the provided configuration. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_cfg_phy_fc(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_set_phy_cfg_data *cfg, + enum ixgbe_fc_mode req_mode) +{ + struct ixgbe_aci_cmd_get_phy_caps_data* pcaps = NULL; + s32 status = IXGBE_SUCCESS; + u8 pause_mask = 0x0; + + if (!cfg) + return IXGBE_ERR_PARAM; + + switch (req_mode) { + case ixgbe_fc_auto: + { + pcaps = (struct ixgbe_aci_cmd_get_phy_caps_data *) + ixgbe_malloc(hw, sizeof(*pcaps)); + if (!pcaps) { + status = IXGBE_ERR_OUT_OF_MEM; + goto out; + } + + /* Query the value of FC that both the NIC and the attached + * media can do. */ + status = ixgbe_aci_get_phy_caps(hw, false, + IXGBE_ACI_REPORT_TOPO_CAP_MEDIA, pcaps); + if (status) + goto out; + + pause_mask |= pcaps->caps & IXGBE_ACI_PHY_EN_TX_LINK_PAUSE; + pause_mask |= pcaps->caps & IXGBE_ACI_PHY_EN_RX_LINK_PAUSE; + + break; + } + case ixgbe_fc_full: + pause_mask |= IXGBE_ACI_PHY_EN_TX_LINK_PAUSE; + pause_mask |= IXGBE_ACI_PHY_EN_RX_LINK_PAUSE; + break; + case ixgbe_fc_rx_pause: + pause_mask |= IXGBE_ACI_PHY_EN_RX_LINK_PAUSE; + break; + case ixgbe_fc_tx_pause: + pause_mask |= IXGBE_ACI_PHY_EN_TX_LINK_PAUSE; + break; + default: + break; + } + + /* clear the old pause settings */ + cfg->caps &= ~(IXGBE_ACI_PHY_EN_TX_LINK_PAUSE | + IXGBE_ACI_PHY_EN_RX_LINK_PAUSE); + + /* set the new capabilities */ + cfg->caps |= pause_mask; + +out: + if (pcaps) + ixgbe_free(hw, pcaps); + return status; +} + +/** + * ixgbe_setup_fc_E610 - Set up flow control + * @hw: pointer to hardware structure + * + * Set up flow control. This has to be done during init time. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_setup_fc_E610(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_get_phy_caps_data pcaps = { 0 }; + struct ixgbe_aci_cmd_set_phy_cfg_data cfg = { 0 }; + s32 status; + + /* Get the current PHY config */ + status = ixgbe_aci_get_phy_caps(hw, false, + IXGBE_ACI_REPORT_ACTIVE_CFG, &pcaps); + if (status) + return status; + + ixgbe_copy_phy_caps_to_cfg(&pcaps, &cfg); + + /* Configure the set PHY data */ + status = ixgbe_cfg_phy_fc(hw, &cfg, hw->fc.requested_mode); + if (status) + return status; + + /* If the capabilities have changed, then set the new config */ + if (cfg.caps != pcaps.caps) { + cfg.caps |= IXGBE_ACI_PHY_ENA_AUTO_LINK_UPDT; + + status = ixgbe_aci_set_phy_cfg(hw, &cfg); + if (status) + return status; + } + + return status; +} + +/** + * ixgbe_fc_autoneg_E610 - Configure flow control + * @hw: pointer to hardware structure + * + * Configure Flow Control. + */ +void ixgbe_fc_autoneg_E610(struct ixgbe_hw *hw) +{ + s32 status; + + /* Get current link status. + * Current FC mode will be stored in the hw context. */ + status = ixgbe_aci_get_link_info(hw, false, NULL); + if (status) { + goto out; + } + + /* Check if the link is up */ + if (!(hw->link.link_info.link_info & IXGBE_ACI_LINK_UP)) { + status = IXGBE_ERR_FC_NOT_NEGOTIATED; + goto out; + } + + /* Check if auto-negotiation has completed */ + if (!(hw->link.link_info.an_info & IXGBE_ACI_AN_COMPLETED)) { + status = IXGBE_ERR_FC_NOT_NEGOTIATED; + goto out; + } + +out: + if (status == IXGBE_SUCCESS) { + hw->fc.fc_was_autonegged = true; + } else { + hw->fc.fc_was_autonegged = false; + hw->fc.current_mode = hw->fc.requested_mode; + } +} + +/** + * ixgbe_set_fw_drv_ver_E610 - Send driver version to FW + * @hw: pointer to the HW structure + * @maj: driver version major number + * @minor: driver version minor number + * @build: driver version build number + * @sub: driver version sub build number + * @len: length of driver_ver string + * @driver_ver: driver string + * + * Send driver version number to Firmware using ACI command (0x0002). + * + * Return: the exit code of the operation. + * IXGBE_SUCCESS - OK + * IXGBE_ERR_PARAM - incorrect parameters were given + * IXGBE_ERR_ACI_ERROR - encountered an error during sending the command + * IXGBE_ERR_ACI_TIMEOUT - a timeout occurred + * IXGBE_ERR_OUT_OF_MEM - ran out of memory + */ +s32 ixgbe_set_fw_drv_ver_E610(struct ixgbe_hw *hw, u8 maj, u8 minor, u8 build, + u8 sub, u16 len, const char *driver_ver) +{ + size_t limited_len = min(len, (u16)IXGBE_DRV_VER_STR_LEN_E610); + struct ixgbe_driver_ver dv; + + DEBUGFUNC("ixgbe_set_fw_drv_ver_E610"); + + if (!len || !driver_ver) + return IXGBE_ERR_PARAM; + + dv.major_ver = maj; + dv.minor_ver = minor; + dv.build_ver = build; + dv.subbuild_ver = sub; + + memset(dv.driver_string, 0, IXGBE_DRV_VER_STR_LEN_E610); + memcpy(dv.driver_string, driver_ver, limited_len); + + return ixgbe_aci_send_driver_ver(hw, &dv); +} + +/** + * ixgbe_disable_rx_E610 - Disable RX unit + * @hw: pointer to hardware structure + * + * Disable RX DMA unit on E610 with use of ACI command (0x000C). + * + * Return: the exit code of the operation. + */ +void ixgbe_disable_rx_E610(struct ixgbe_hw *hw) +{ + u32 rxctrl; + + DEBUGFUNC("ixgbe_disable_rx_E610"); + + rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); + if (rxctrl & IXGBE_RXCTRL_RXEN) { + u32 pfdtxgswc; + s32 status; + + pfdtxgswc = IXGBE_READ_REG(hw, IXGBE_PFDTXGSWC); + if (pfdtxgswc & IXGBE_PFDTXGSWC_VT_LBEN) { + pfdtxgswc &= ~IXGBE_PFDTXGSWC_VT_LBEN; + IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, pfdtxgswc); + hw->mac.set_lben = true; + } else { + hw->mac.set_lben = false; + } + + status = ixgbe_aci_disable_rxen(hw); + + /* If we fail - disable RX using register write */ + if (status) { + rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); + if (rxctrl & IXGBE_RXCTRL_RXEN) { + rxctrl &= ~IXGBE_RXCTRL_RXEN; + IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl); + } + } + } +} + +/** + * ixgbe_setup_eee_E610 - Enable/disable EEE support + * @hw: pointer to the HW structure + * @enable_eee: boolean flag to enable EEE + * + * Enables/disable EEE based on enable_eee flag. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_setup_eee_E610(struct ixgbe_hw *hw, bool enable_eee) +{ + struct ixgbe_aci_cmd_get_phy_caps_data phy_caps = { 0 }; + struct ixgbe_aci_cmd_set_phy_cfg_data phy_cfg = { 0 }; + u16 eee_cap = 0; + s32 status; + + status = ixgbe_aci_get_phy_caps(hw, false, + IXGBE_ACI_REPORT_ACTIVE_CFG, &phy_caps); + if (status != IXGBE_SUCCESS) + return status; + + ixgbe_copy_phy_caps_to_cfg(&phy_caps, &phy_cfg); + + phy_cfg.caps |= IXGBE_ACI_PHY_ENA_LINK; + phy_cfg.caps |= IXGBE_ACI_PHY_ENA_AUTO_LINK_UPDT; + + if (enable_eee) { + if (phy_caps.phy_type_low & IXGBE_PHY_TYPE_LOW_100BASE_TX) + eee_cap |= IXGBE_ACI_PHY_EEE_EN_100BASE_TX; + if (phy_caps.phy_type_low & IXGBE_PHY_TYPE_LOW_1000BASE_T) + eee_cap |= IXGBE_ACI_PHY_EEE_EN_1000BASE_T; + if (phy_caps.phy_type_low & IXGBE_PHY_TYPE_LOW_1000BASE_KX) + eee_cap |= IXGBE_ACI_PHY_EEE_EN_1000BASE_KX; + if (phy_caps.phy_type_low & IXGBE_PHY_TYPE_LOW_10GBASE_T) + eee_cap |= IXGBE_ACI_PHY_EEE_EN_10GBASE_T; + if (phy_caps.phy_type_low & IXGBE_PHY_TYPE_LOW_10GBASE_KR_CR1) + eee_cap |= IXGBE_ACI_PHY_EEE_EN_10GBASE_KR; + if (phy_caps.phy_type_high & IXGBE_PHY_TYPE_HIGH_10BASE_T) + eee_cap |= IXGBE_ACI_PHY_EEE_EN_10BASE_T; + } + + /* Set EEE capability for particular PHY types */ + phy_cfg.eee_cap = IXGBE_CPU_TO_LE16(eee_cap); + + status = ixgbe_aci_set_phy_cfg(hw, &phy_cfg); + + return status; +} + +/** + * ixgbe_fw_recovery_mode_E610 - Check FW NVM recovery mode + * @hw: pointer to hardware structure + * + * Checks FW NVM recovery mode by + * reading the value of the dedicated register. + * + * Return: true if FW is in recovery mode, otherwise false. + */ +bool ixgbe_fw_recovery_mode_E610(struct ixgbe_hw *hw) +{ + u32 fwsm = IXGBE_READ_REG(hw, GL_MNG_FWSM); + + return !!(fwsm & GL_MNG_FWSM_FW_MODES_RECOVERY_M); +} + +/** + * ixgbe_fw_rollback_mode_E610 - Check FW NVM Rollback + * @hw: pointer to hardware structure + * + * Checks FW NVM Rollback mode by reading the + * value of the dedicated register. + * + * Return: true if FW is in Rollback mode, otherwise false. + */ +bool ixgbe_fw_rollback_mode_E610(struct ixgbe_hw *hw) +{ + u32 fwsm = IXGBE_READ_REG(hw, GL_MNG_FWSM); + + return !!(fwsm & GL_MNG_FWSM_FW_MODES_ROLLBACK_M); +} + +/** + * ixgbe_get_fw_tsam_mode_E610 - Check FW NVM Thermal Sensor Autonomous Mode + * @hw: pointer to hardware structure + * + * Checks Thermal Sensor Autonomous Mode by reading the + * value of the dedicated register. + * + * Return: true if FW is in TSAM, otherwise false. + */ +bool ixgbe_get_fw_tsam_mode_E610(struct ixgbe_hw *hw) +{ + u32 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM_X550EM_a); + + return !!(fwsm & IXGBE_FWSM_TS_ENABLED); +} + +/** + * ixgbe_init_phy_ops_E610 - PHY specific init + * @hw: pointer to hardware structure + * + * Initialize any function pointers that were not able to be + * set during init_shared_code because the PHY type was not known. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_init_phy_ops_E610(struct ixgbe_hw *hw) +{ + struct ixgbe_mac_info *mac = &hw->mac; + struct ixgbe_phy_info *phy = &hw->phy; + s32 ret_val; + + phy->ops.identify_sfp = ixgbe_identify_module_E610; + phy->ops.read_reg = NULL; /* PHY reg access is not required */ + phy->ops.write_reg = NULL; + phy->ops.read_reg_mdi = NULL; + phy->ops.write_reg_mdi = NULL; + phy->ops.setup_link = ixgbe_setup_phy_link_E610; + phy->ops.get_firmware_version = ixgbe_get_phy_firmware_version_E610; + phy->ops.read_i2c_byte = NULL; /* disabled for E610 */ + phy->ops.write_i2c_byte = NULL; /* disabled for E610 */ + phy->ops.read_i2c_sff8472 = ixgbe_read_i2c_sff8472_E610; + phy->ops.read_i2c_eeprom = ixgbe_read_i2c_eeprom_E610; + phy->ops.write_i2c_eeprom = ixgbe_write_i2c_eeprom_E610; + phy->ops.i2c_bus_clear = NULL; /* do not use generic implementation */ + phy->ops.check_overtemp = ixgbe_check_overtemp_E610; + if (mac->ops.get_media_type(hw) == ixgbe_media_type_copper) + phy->ops.set_phy_power = ixgbe_set_phy_power_E610; + else + phy->ops.set_phy_power = NULL; + phy->ops.enter_lplu = ixgbe_enter_lplu_E610; + phy->ops.handle_lasi = NULL; /* no implementation for E610 */ + phy->ops.read_i2c_byte_unlocked = NULL; /* disabled for E610 */ + phy->ops.write_i2c_byte_unlocked = NULL; /* disabled for E610 */ + + /* TODO: Set functions pointers based on device ID */ + + /* Identify the PHY */ + ret_val = phy->ops.identify(hw); + if (ret_val != IXGBE_SUCCESS) + return ret_val; + + /* TODO: Set functions pointers based on PHY type */ + + return ret_val; +} + +/** + * ixgbe_identify_phy_E610 - Identify PHY + * @hw: pointer to hardware structure + * + * Determine PHY type, supported speeds and PHY ID. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_identify_phy_E610(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_get_phy_caps_data pcaps; + s32 rc; + + /* Set PHY type */ + hw->phy.type = ixgbe_phy_fw; + + rc = ixgbe_aci_get_phy_caps(hw, false, IXGBE_ACI_REPORT_TOPO_CAP_MEDIA, + &pcaps); + if (rc) + return rc; + + if (!(pcaps.module_compliance_enforcement & + IXGBE_ACI_MOD_ENFORCE_STRICT_MODE)) { + /* Handle lenient mode */ + rc = ixgbe_aci_get_phy_caps(hw, false, + IXGBE_ACI_REPORT_TOPO_CAP_NO_MEDIA, + &pcaps); + if (rc) + return rc; + } + + /* Determine supported speeds */ + hw->phy.speeds_supported = IXGBE_LINK_SPEED_UNKNOWN; + + if (pcaps.phy_type_high & IXGBE_PHY_TYPE_HIGH_10BASE_T || + pcaps.phy_type_high & IXGBE_PHY_TYPE_HIGH_10M_SGMII) + hw->phy.speeds_supported |= IXGBE_LINK_SPEED_10_FULL; + if (pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_100BASE_TX || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_100M_SGMII || + pcaps.phy_type_high & IXGBE_PHY_TYPE_HIGH_100M_USXGMII) + hw->phy.speeds_supported |= IXGBE_LINK_SPEED_100_FULL; + if (pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_1000BASE_T || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_1000BASE_SX || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_1000BASE_LX || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_1000BASE_KX || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_1G_SGMII || + pcaps.phy_type_high & IXGBE_PHY_TYPE_HIGH_1G_USXGMII) + hw->phy.speeds_supported |= IXGBE_LINK_SPEED_1GB_FULL; + if (pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_10GBASE_T || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_10G_SFI_DA || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_10GBASE_SR || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_10GBASE_LR || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_10GBASE_KR_CR1 || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_10G_SFI_AOC_ACC || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_10G_SFI_C2C || + pcaps.phy_type_high & IXGBE_PHY_TYPE_HIGH_10G_USXGMII) + hw->phy.speeds_supported |= IXGBE_LINK_SPEED_10GB_FULL; + + /* 2.5 and 5 Gbps link speeds must be excluded from the + * auto-negotiation set used during driver initialization due to + * compatibility issues with certain switches. Those issues do not + * exist in case of E610 2.5G SKU device (0x57b1). + */ + if (!hw->phy.autoneg_advertised && + hw->device_id != IXGBE_DEV_ID_E610_2_5G_T) + hw->phy.autoneg_advertised = hw->phy.speeds_supported; + + if (pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_2500BASE_T || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_2500BASE_X || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_2500BASE_KX || + pcaps.phy_type_high & IXGBE_PHY_TYPE_HIGH_2500M_SGMII || + pcaps.phy_type_high & IXGBE_PHY_TYPE_HIGH_2500M_USXGMII) + hw->phy.speeds_supported |= IXGBE_LINK_SPEED_2_5GB_FULL; + + if (!hw->phy.autoneg_advertised && + hw->device_id == IXGBE_DEV_ID_E610_2_5G_T) + hw->phy.autoneg_advertised = hw->phy.speeds_supported; + + if (pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_5GBASE_T || + pcaps.phy_type_low & IXGBE_PHY_TYPE_LOW_5GBASE_KR || + pcaps.phy_type_high & IXGBE_PHY_TYPE_HIGH_5G_USXGMII) + hw->phy.speeds_supported |= IXGBE_LINK_SPEED_5GB_FULL; + + /* Set PHY ID */ + memcpy(&hw->phy.id, pcaps.phy_id_oui, sizeof(u32)); + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_identify_module_E610 - Identify SFP module type + * @hw: pointer to hardware structure + * + * Identify the SFP module type. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_identify_module_E610(struct ixgbe_hw *hw) +{ + bool media_available; + u8 module_type; + s32 rc; + + rc = ixgbe_update_link_info(hw); + if (rc) + goto err; + + media_available = + (hw->link.link_info.link_info & + IXGBE_ACI_MEDIA_AVAILABLE) ? true : false; + + if (media_available) { + hw->phy.sfp_type = ixgbe_sfp_type_unknown; + + /* Get module type from hw context updated by ixgbe_update_link_info() */ + module_type = hw->link.link_info.module_type[IXGBE_ACI_MOD_TYPE_IDENT]; + + if ((module_type & IXGBE_ACI_MOD_TYPE_BYTE1_SFP_PLUS_CU_PASSIVE) || + (module_type & IXGBE_ACI_MOD_TYPE_BYTE1_SFP_PLUS_CU_ACTIVE)) { + hw->phy.sfp_type = ixgbe_sfp_type_da_cu; + } else if (module_type & IXGBE_ACI_MOD_TYPE_BYTE1_10G_BASE_SR) { + hw->phy.sfp_type = ixgbe_sfp_type_sr; + } else if ((module_type & IXGBE_ACI_MOD_TYPE_BYTE1_10G_BASE_LR) || + (module_type & IXGBE_ACI_MOD_TYPE_BYTE1_10G_BASE_LRM)) { + hw->phy.sfp_type = ixgbe_sfp_type_lr; + } + rc = IXGBE_SUCCESS; + } else { + hw->phy.sfp_type = ixgbe_sfp_type_not_present; + rc = IXGBE_ERR_SFP_NOT_PRESENT; + } +err: + return rc; +} + +/** + * ixgbe_setup_phy_link_E610 - Sets up firmware-controlled PHYs + * @hw: pointer to hardware structure + * + * Set the parameters for the firmware-controlled PHYs. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_setup_phy_link_E610(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_get_phy_caps_data pcaps; + struct ixgbe_aci_cmd_set_phy_cfg_data pcfg; + u8 rmode = IXGBE_ACI_REPORT_TOPO_CAP_MEDIA; + u64 sup_phy_type_low, sup_phy_type_high; + s32 rc; + + rc = ixgbe_aci_get_link_info(hw, false, NULL); + if (rc) { + goto err; + } + + /* If media is not available get default config */ + if (!(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE)) + rmode = IXGBE_ACI_REPORT_DFLT_CFG; + + rc = ixgbe_aci_get_phy_caps(hw, false, rmode, &pcaps); + if (rc) { + goto err; + } + + sup_phy_type_low = pcaps.phy_type_low; + sup_phy_type_high = pcaps.phy_type_high; + + /* Get Active configuration to avoid unintended changes */ + rc = ixgbe_aci_get_phy_caps(hw, false, IXGBE_ACI_REPORT_ACTIVE_CFG, + &pcaps); + if (rc) { + goto err; + } + ixgbe_copy_phy_caps_to_cfg(&pcaps, &pcfg); + + /* Set default PHY types for a given speed */ + pcfg.phy_type_low = 0; + pcfg.phy_type_high = 0; + + if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_10_FULL) { + pcfg.phy_type_high |= IXGBE_PHY_TYPE_HIGH_10BASE_T; + pcfg.phy_type_high |= IXGBE_PHY_TYPE_HIGH_10M_SGMII; + } + if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_100_FULL) { + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_100BASE_TX; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_100M_SGMII; + pcfg.phy_type_high |= IXGBE_PHY_TYPE_HIGH_100M_USXGMII; + } + if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_1GB_FULL) { + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_1000BASE_T; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_1000BASE_SX; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_1000BASE_LX; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_1000BASE_KX; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_1G_SGMII; + pcfg.phy_type_high |= IXGBE_PHY_TYPE_HIGH_1G_USXGMII; + } + if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_2_5GB_FULL) { + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_2500BASE_T; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_2500BASE_X; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_2500BASE_KX; + pcfg.phy_type_high |= IXGBE_PHY_TYPE_HIGH_2500M_SGMII; + pcfg.phy_type_high |= IXGBE_PHY_TYPE_HIGH_2500M_USXGMII; + } + if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_5GB_FULL) { + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_5GBASE_T; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_5GBASE_KR; + pcfg.phy_type_high |= IXGBE_PHY_TYPE_HIGH_5G_USXGMII; + } + if (hw->phy.autoneg_advertised & IXGBE_LINK_SPEED_10GB_FULL) { + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_10GBASE_T; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_10G_SFI_DA; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_10GBASE_SR; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_10GBASE_LR; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_10GBASE_KR_CR1; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_10G_SFI_AOC_ACC; + pcfg.phy_type_low |= IXGBE_PHY_TYPE_LOW_10G_SFI_C2C; + pcfg.phy_type_high |= IXGBE_PHY_TYPE_HIGH_10G_USXGMII; + } + + /* Mask the set values to avoid requesting unsupported link types */ + pcfg.phy_type_low &= sup_phy_type_low; + pcfg.phy_type_high &= sup_phy_type_high; + + if (pcfg.phy_type_high != pcaps.phy_type_high || + pcfg.phy_type_low != pcaps.phy_type_low || + pcfg.caps != pcaps.caps) { + pcfg.caps |= IXGBE_ACI_PHY_ENA_LINK; + pcfg.caps |= IXGBE_ACI_PHY_ENA_AUTO_LINK_UPDT; + + rc = ixgbe_aci_set_phy_cfg(hw, &pcfg); + } + +err: + return rc; +} + +/** + * ixgbe_get_phy_firmware_version_E610 - Gets the PHY Firmware Version + * @hw: pointer to hardware structure + * @firmware_version: pointer to the PHY Firmware Version + * + * Determines PHY FW version based on response to Get PHY Capabilities + * admin command (0x0600). + * + * Return: the exit code of the operation. + */ +s32 ixgbe_get_phy_firmware_version_E610(struct ixgbe_hw *hw, + u16 *firmware_version) +{ + struct ixgbe_aci_cmd_get_phy_caps_data pcaps; + s32 status; + + if (!firmware_version) + return IXGBE_ERR_PARAM; + + status = ixgbe_aci_get_phy_caps(hw, false, + IXGBE_ACI_REPORT_ACTIVE_CFG, + &pcaps); + if (status) + return status; + + /* TODO: determine which bytes of the 8-byte phy_fw_ver + * field should be written to the 2-byte firmware_version + * output argument. */ + memcpy(firmware_version, pcaps.phy_fw_ver, sizeof(u16)); + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_read_i2c_sff8472_E610 - Reads 8 bit word over I2C interface + * @hw: pointer to hardware structure + * @byte_offset: byte offset at address 0xA2 + * @sff8472_data: value read + * + * Performs byte read operation from SFP module's SFF-8472 data over I2C. + * + * Return: the exit code of the operation. + **/ +s32 ixgbe_read_i2c_sff8472_E610(struct ixgbe_hw *hw, u8 byte_offset, + u8 *sff8472_data) +{ + return ixgbe_aci_sff_eeprom(hw, 0, IXGBE_I2C_EEPROM_DEV_ADDR2, + byte_offset, 0, + IXGBE_ACI_SFF_NO_PAGE_BANK_UPDATE, + sff8472_data, 1, false); +} + +/** + * ixgbe_read_i2c_eeprom_E610 - Reads 8 bit EEPROM word over I2C interface + * @hw: pointer to hardware structure + * @byte_offset: EEPROM byte offset to read + * @eeprom_data: value read + * + * Performs byte read operation from SFP module's EEPROM over I2C interface. + * + * Return: the exit code of the operation. + **/ +s32 ixgbe_read_i2c_eeprom_E610(struct ixgbe_hw *hw, u8 byte_offset, + u8 *eeprom_data) +{ + return ixgbe_aci_sff_eeprom(hw, 0, IXGBE_I2C_EEPROM_DEV_ADDR, + byte_offset, 0, + IXGBE_ACI_SFF_NO_PAGE_BANK_UPDATE, + eeprom_data, 1, false); +} + +/** + * ixgbe_write_i2c_eeprom_E610 - Writes 8 bit EEPROM word over I2C interface + * @hw: pointer to hardware structure + * @byte_offset: EEPROM byte offset to write + * @eeprom_data: value to write + * + * Performs byte write operation to SFP module's EEPROM over I2C interface. + * + * Return: the exit code of the operation. + **/ +s32 ixgbe_write_i2c_eeprom_E610(struct ixgbe_hw *hw, u8 byte_offset, + u8 eeprom_data) +{ + return ixgbe_aci_sff_eeprom(hw, 0, IXGBE_I2C_EEPROM_DEV_ADDR, + byte_offset, 0, + IXGBE_ACI_SFF_NO_PAGE_BANK_UPDATE, + &eeprom_data, 1, true); +} + +/** + * ixgbe_check_overtemp_E610 - Check firmware-controlled PHYs for overtemp + * @hw: pointer to hardware structure + * + * Get the link status and check if the PHY temperature alarm detected. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_check_overtemp_E610(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_get_link_status_data link_data = { 0 }; + struct ixgbe_aci_cmd_get_link_status *resp; + struct ixgbe_aci_desc desc; + s32 status = IXGBE_SUCCESS; + + if (!hw) + return IXGBE_ERR_PARAM; + + ixgbe_fill_dflt_direct_cmd_desc(&desc, ixgbe_aci_opc_get_link_status); + resp = &desc.params.get_link_status; + resp->cmd_flags = IXGBE_CPU_TO_LE16(IXGBE_ACI_LSE_NOP); + + status = ixgbe_aci_send_cmd(hw, &desc, &link_data, sizeof(link_data)); + if (status != IXGBE_SUCCESS) + return status; + + if (link_data.ext_info & IXGBE_ACI_LINK_PHY_TEMP_ALARM) { + ERROR_REPORT1(IXGBE_ERROR_CAUTION, + "PHY Temperature Alarm detected"); + status = IXGBE_ERR_OVERTEMP; + } + + return status; +} + +/** + * ixgbe_set_phy_power_E610 - Control power for copper PHY + * @hw: pointer to hardware structure + * @on: true for on, false for off + * + * Set the power on/off of the PHY + * by getting its capabilities and setting the appropriate + * configuration parameters. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_set_phy_power_E610(struct ixgbe_hw *hw, bool on) +{ + struct ixgbe_aci_cmd_get_phy_caps_data phy_caps = { 0 }; + struct ixgbe_aci_cmd_set_phy_cfg_data phy_cfg = { 0 }; + s32 status; + + status = ixgbe_aci_get_phy_caps(hw, false, + IXGBE_ACI_REPORT_ACTIVE_CFG, &phy_caps); + if (status != IXGBE_SUCCESS) + return status; + + ixgbe_copy_phy_caps_to_cfg(&phy_caps, &phy_cfg); + + if (on) { + phy_cfg.caps &= ~IXGBE_ACI_PHY_ENA_LOW_POWER; + } else { + phy_cfg.caps |= IXGBE_ACI_PHY_ENA_LOW_POWER; + } + + /* PHY is already in requested power mode */ + if (phy_caps.caps == phy_cfg.caps) + return IXGBE_SUCCESS; + + phy_cfg.caps |= IXGBE_ACI_PHY_ENA_LINK; + phy_cfg.caps |= IXGBE_ACI_PHY_ENA_AUTO_LINK_UPDT; + + status = ixgbe_aci_set_phy_cfg(hw, &phy_cfg); + + return status; +} + +/** + * ixgbe_enter_lplu_E610 - Transition to low power states + * @hw: pointer to hardware structure + * + * Configures Low Power Link Up on transition to low power states + * (from D0 to non-D0). Link is required to enter LPLU so avoid resetting the + * X557 PHY immediately prior to entering LPLU. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_enter_lplu_E610(struct ixgbe_hw *hw) +{ + struct ixgbe_aci_cmd_get_phy_caps_data phy_caps = { 0 }; + struct ixgbe_aci_cmd_set_phy_cfg_data phy_cfg = { 0 }; + s32 status; + + status = ixgbe_aci_get_phy_caps(hw, false, + IXGBE_ACI_REPORT_ACTIVE_CFG, &phy_caps); + if (status != IXGBE_SUCCESS) + return status; + + ixgbe_copy_phy_caps_to_cfg(&phy_caps, &phy_cfg); + + phy_cfg.low_power_ctrl_an |= IXGBE_ACI_PHY_EN_D3COLD_LOW_POWER_AUTONEG; + + status = ixgbe_aci_set_phy_cfg(hw, &phy_cfg); + + return status; +} + +/** + * ixgbe_init_eeprom_params_E610 - Initialize EEPROM params + * @hw: pointer to hardware structure + * + * Initializes the EEPROM parameters ixgbe_eeprom_info within the + * ixgbe_hw struct in order to set up EEPROM access. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_init_eeprom_params_E610(struct ixgbe_hw *hw) +{ + struct ixgbe_eeprom_info *eeprom = &hw->eeprom; + u32 gens_stat; + u8 sr_size; + + if (eeprom->type == ixgbe_eeprom_uninitialized) { + eeprom->type = ixgbe_flash; + + gens_stat = IXGBE_READ_REG(hw, GLNVM_GENS); + sr_size = (gens_stat & GLNVM_GENS_SR_SIZE_M) >> + GLNVM_GENS_SR_SIZE_S; + + /* Switching to words (sr_size contains power of 2) */ + eeprom->word_size = BIT(sr_size) * IXGBE_SR_WORDS_IN_1KB; + + DEBUGOUT2("Eeprom params: type = %d, size = %d\n", + eeprom->type, eeprom->word_size); + } + + return IXGBE_SUCCESS; +} + +/** + * ixgbe_read_ee_aci_E610 - Read EEPROM word using the admin command. + * @hw: pointer to hardware structure + * @offset: offset of word in the EEPROM to read + * @data: word read from the EEPROM + * + * Reads a 16 bit word from the EEPROM using the ACI. + * If the EEPROM params are not initialized, the function + * initialize them before proceeding with reading. + * The function acquires and then releases the NVM ownership. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_read_ee_aci_E610(struct ixgbe_hw *hw, u16 offset, u16 *data) +{ + s32 status; + + if (hw->eeprom.type == ixgbe_eeprom_uninitialized) { + status = ixgbe_init_eeprom_params(hw); + if (status) + return status; + } + + status = ixgbe_acquire_nvm(hw, IXGBE_RES_READ); + if (status) + return status; + + status = ixgbe_read_sr_word_aci(hw, offset, data); + ixgbe_release_nvm(hw); + + return status; +} + +/** + * ixgbe_read_ee_aci_buffer_E610- Read EEPROM word(s) using admin commands. + * @hw: pointer to hardware structure + * @offset: offset of word in the EEPROM to read + * @words: number of words + * @data: word(s) read from the EEPROM + * + * Reads a 16 bit word(s) from the EEPROM using the ACI. + * If the EEPROM params are not initialized, the function + * initialize them before proceeding with reading. + * The function acquires and then releases the NVM ownership. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_read_ee_aci_buffer_E610(struct ixgbe_hw *hw, u16 offset, + u16 words, u16 *data) +{ + s32 status; + + if (hw->eeprom.type == ixgbe_eeprom_uninitialized) { + status = ixgbe_init_eeprom_params(hw); + if (status) + return status; + } + + status = ixgbe_acquire_nvm(hw, IXGBE_RES_READ); + if (status) + return status; + + status = ixgbe_read_sr_buf_aci(hw, offset, &words, data); + ixgbe_release_nvm(hw); + + return status; +} + +/** + * ixgbe_write_ee_aci_E610 - Write EEPROM word using the admin command. + * @hw: pointer to hardware structure + * @offset: offset of word in the EEPROM to write + * @data: word write to the EEPROM + * + * Write a 16 bit word to the EEPROM using the ACI. + * If the EEPROM params are not initialized, the function + * initialize them before proceeding with writing. + * The function acquires and then releases the NVM ownership. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_write_ee_aci_E610(struct ixgbe_hw *hw, u16 offset, u16 data) +{ + s32 status; + + if (hw->eeprom.type == ixgbe_eeprom_uninitialized) { + status = ixgbe_init_eeprom_params(hw); + if (status) + return status; + } + + status = ixgbe_acquire_nvm(hw, IXGBE_RES_WRITE); + if (status) + return status; + + status = ixgbe_write_sr_word_aci(hw, (u32)offset, &data); + ixgbe_release_nvm(hw); + + return status; +} + +/** + * ixgbe_write_ee_aci_buffer_E610 - Write EEPROM word(s) using admin commands. + * @hw: pointer to hardware structure + * @offset: offset of word in the EEPROM to write + * @words: number of words + * @data: word(s) write to the EEPROM + * + * Write a 16 bit word(s) to the EEPROM using the ACI. + * If the EEPROM params are not initialized, the function + * initialize them before proceeding with writing. + * The function acquires and then releases the NVM ownership. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_write_ee_aci_buffer_E610(struct ixgbe_hw *hw, u16 offset, + u16 words, u16 *data) +{ + s32 status; + + if (hw->eeprom.type == ixgbe_eeprom_uninitialized) { + status = ixgbe_init_eeprom_params(hw); + if (status) + return status; + } + + status = ixgbe_acquire_nvm(hw, IXGBE_RES_WRITE); + if (status) + return status; + + status = ixgbe_write_sr_buf_aci(hw, (u32)offset, words, data); + ixgbe_release_nvm(hw); + + return status; +} + +/** + * ixgbe_calc_eeprom_checksum_E610 - Calculates and returns the checksum + * @hw: pointer to hardware structure + * + * Calculate SW Checksum that covers the whole 64kB shadow RAM + * except the VPD and PCIe ALT Auto-load modules. The structure and size of VPD + * is customer specific and unknown. Therefore, this function skips all maximum + * possible size of VPD (1kB). + * If the EEPROM params are not initialized, the function + * initializes them before proceeding. + * The function acquires and then releases the NVM ownership. + * + * Return: the negative error code on error, or the 16-bit checksum + */ +s32 ixgbe_calc_eeprom_checksum_E610(struct ixgbe_hw *hw) +{ + bool nvm_acquired = false; + u16 pcie_alt_module = 0; + u16 checksum_local = 0; + u16 checksum = 0; + u16 vpd_module; + void *vmem; + s32 status; + u16 *data; + u16 i; + + if (hw->eeprom.type == ixgbe_eeprom_uninitialized) { + status = ixgbe_init_eeprom_params(hw); + if (status) + return status; + } + + vmem = ixgbe_calloc(hw, IXGBE_SR_SECTOR_SIZE_IN_WORDS, sizeof(u16)); + if (!vmem) + return IXGBE_ERR_OUT_OF_MEM; + data = (u16 *)vmem; + status = ixgbe_acquire_nvm(hw, IXGBE_RES_READ); + if (status) + goto ixgbe_calc_sr_checksum_exit; + nvm_acquired = true; + + /* read pointer to VPD area */ + status = ixgbe_read_sr_word_aci(hw, E610_SR_VPD_PTR, &vpd_module); + if (status) + goto ixgbe_calc_sr_checksum_exit; + + /* read pointer to PCIe Alt Auto-load module */ + status = ixgbe_read_sr_word_aci(hw, E610_SR_PCIE_ALT_AUTO_LOAD_PTR, + &pcie_alt_module); + if (status) + goto ixgbe_calc_sr_checksum_exit; + + /* Calculate SW checksum that covers the whole 64kB shadow RAM + * except the VPD and PCIe ALT Auto-load modules + */ + for (i = 0; i < hw->eeprom.word_size; i++) { + /* Read SR page */ + if ((i % IXGBE_SR_SECTOR_SIZE_IN_WORDS) == 0) { + u16 words = IXGBE_SR_SECTOR_SIZE_IN_WORDS; + + status = ixgbe_read_sr_buf_aci(hw, i, &words, data); + if (status != IXGBE_SUCCESS) + goto ixgbe_calc_sr_checksum_exit; + } + + /* Skip Checksum word */ + if (i == E610_SR_SW_CHECKSUM_WORD) + continue; + /* Skip VPD module (convert byte size to word count) */ + if (i >= (u32)vpd_module && + i < ((u32)vpd_module + E610_SR_VPD_SIZE_WORDS)) + continue; + /* Skip PCIe ALT module (convert byte size to word count) */ + if (i >= (u32)pcie_alt_module && + i < ((u32)pcie_alt_module + E610_SR_PCIE_ALT_SIZE_WORDS)) + continue; + + checksum_local += data[i % IXGBE_SR_SECTOR_SIZE_IN_WORDS]; + } + + checksum = (u16)IXGBE_SR_SW_CHECKSUM_BASE - checksum_local; + +ixgbe_calc_sr_checksum_exit: + if(nvm_acquired) + ixgbe_release_nvm(hw); + ixgbe_free(hw, vmem); + + if(!status) + return (s32)checksum; + else + return status; +} + +/** + * ixgbe_update_eeprom_checksum_E610 - Updates the EEPROM checksum and flash + * @hw: pointer to hardware structure + * + * After writing EEPROM to Shadow RAM, software sends the admin command + * to recalculate and update EEPROM checksum and instructs the hardware + * to update the flash. + * If the EEPROM params are not initialized, the function + * initialize them before proceeding. + * The function acquires and then releases the NVM ownership. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_update_eeprom_checksum_E610(struct ixgbe_hw *hw) +{ + s32 status; + + if (hw->eeprom.type == ixgbe_eeprom_uninitialized) { + status = ixgbe_init_eeprom_params(hw); + if (status) + return status; + } + + status = ixgbe_nvm_recalculate_checksum(hw); + if (status) + return status; + status = ixgbe_acquire_nvm(hw, IXGBE_RES_WRITE); + if (status) + return status; + + status = ixgbe_nvm_write_activate(hw, IXGBE_ACI_NVM_ACTIV_REQ_EMPR, + NULL); + ixgbe_release_nvm(hw); + + return status; +} + +/** + * ixgbe_validate_eeprom_checksum_E610 - Validate EEPROM checksum + * @hw: pointer to hardware structure + * @checksum_val: calculated checksum + * + * Performs checksum calculation and validates the EEPROM checksum. If the + * caller does not need checksum_val, the value can be NULL. + * If the EEPROM params are not initialized, the function + * initialize them before proceeding. + * The function acquires and then releases the NVM ownership. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_validate_eeprom_checksum_E610(struct ixgbe_hw *hw, u16 *checksum_val) +{ + u32 status; + + if (hw->eeprom.type == ixgbe_eeprom_uninitialized) { + status = ixgbe_init_eeprom_params(hw); + if (status) + return status; + } + + status = ixgbe_nvm_validate_checksum(hw); + + if (status) + return status; + + if (checksum_val) { + u16 tmp_checksum; + status = ixgbe_acquire_nvm(hw, IXGBE_RES_READ); + if (status) + return status; + + status = ixgbe_read_sr_word_aci(hw, E610_SR_SW_CHECKSUM_WORD, + &tmp_checksum); + ixgbe_release_nvm(hw); + + if (!status) + *checksum_val = tmp_checksum; + } + + return status; +} + +/** + * ixgbe_get_pfa_module_tlv - Reads sub module TLV from NVM PFA + * @hw: pointer to hardware structure + * @module_tlv: pointer to module TLV to return + * @module_tlv_len: pointer to module TLV length to return + * @module_type: module type requested + * + * Finds the requested sub module TLV type from the Preserved Field + * Area (PFA) and returns the TLV pointer and length. The caller can + * use these to read the variable length TLV value. + * + * Return: the exit code of the operation. + */ +static s32 ixgbe_get_pfa_module_tlv(struct ixgbe_hw *hw, u16 *module_tlv, + u16 *module_tlv_len, u16 module_type) +{ + u16 pfa_len, pfa_ptr, pfa_end_ptr; + u16 next_tlv; + s32 status; + + status = ixgbe_read_ee_aci_E610(hw, E610_SR_PFA_PTR, &pfa_ptr); + if (status != IXGBE_SUCCESS) { + return status; + } + status = ixgbe_read_ee_aci_E610(hw, pfa_ptr, &pfa_len); + if (status != IXGBE_SUCCESS) { + return status; + } + /* Starting with first TLV after PFA length, iterate through the list + * of TLVs to find the requested one. + */ + next_tlv = pfa_ptr + 1; + pfa_end_ptr = pfa_ptr + pfa_len; + while (next_tlv < pfa_end_ptr) { + u16 tlv_sub_module_type, tlv_len; + + /* Read TLV type */ + status = ixgbe_read_ee_aci_E610(hw, next_tlv, + &tlv_sub_module_type); + if (status != IXGBE_SUCCESS) { + break; + } + /* Read TLV length */ + status = ixgbe_read_ee_aci_E610(hw, next_tlv + 1, &tlv_len); + if (status != IXGBE_SUCCESS) { + break; + } + if (tlv_sub_module_type == module_type) { + if (tlv_len) { + *module_tlv = next_tlv; + *module_tlv_len = tlv_len; + return IXGBE_SUCCESS; + } + return IXGBE_ERR_INVAL_SIZE; + } + /* Check next TLV, i.e. current TLV pointer + length + 2 words + * (for current TLV's type and length) + */ + next_tlv = next_tlv + tlv_len + 2; + } + /* Module does not exist */ + return IXGBE_ERR_DOES_NOT_EXIST; +} + +/** + * ixgbe_read_pba_string_E610 - Reads part number string from NVM + * @hw: pointer to hardware structure + * @pba_num: stores the part number string from the NVM + * @pba_num_size: part number string buffer length + * + * Reads the part number string from the NVM. + * + * Return: the exit code of the operation. + */ +s32 ixgbe_read_pba_string_E610(struct ixgbe_hw *hw, u8 *pba_num, + u32 pba_num_size) +{ + u16 pba_tlv, pba_tlv_len; + u16 pba_word, pba_size; + s32 status; + u16 i; + + status = ixgbe_get_pfa_module_tlv(hw, &pba_tlv, &pba_tlv_len, + E610_SR_PBA_BLOCK_PTR); + if (status != IXGBE_SUCCESS) { + return status; + } + + /* pba_size is the next word */ + status = ixgbe_read_ee_aci_E610(hw, (pba_tlv + 2), &pba_size); + if (status != IXGBE_SUCCESS) { + return status; + } + + if (pba_tlv_len < pba_size) { + return IXGBE_ERR_INVAL_SIZE; + } + + /* Subtract one to get PBA word count (PBA Size word is included in + * total size) + */ + pba_size--; + if (pba_num_size < (((u32)pba_size * 2) + 1)) { + return IXGBE_ERR_PARAM; + } + + for (i = 0; i < pba_size; i++) { + status = ixgbe_read_ee_aci_E610(hw, (pba_tlv + 2 + 1) + i, + &pba_word); + if (status != IXGBE_SUCCESS) { + return status; + } + + pba_num[(i * 2)] = (pba_word >> 8) & 0xFF; + pba_num[(i * 2) + 1] = pba_word & 0xFF; + } + pba_num[(pba_size * 2)] = '\0'; + + return status; +} diff --git a/sys/dev/ixgbe/ixgbe_e610.h b/sys/dev/ixgbe/ixgbe_e610.h new file mode 100644 index 000000000000..94e600139499 --- /dev/null +++ b/sys/dev/ixgbe/ixgbe_e610.h @@ -0,0 +1,224 @@ +/****************************************************************************** + SPDX-License-Identifier: BSD-3-Clause + + Copyright (c) 2025, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ + +#ifndef _IXGBE_E610_H_ +#define _IXGBE_E610_H_ + +#include "ixgbe_type.h" + +void ixgbe_init_aci(struct ixgbe_hw *hw); +void ixgbe_shutdown_aci(struct ixgbe_hw *hw); +s32 ixgbe_aci_send_cmd(struct ixgbe_hw *hw, struct ixgbe_aci_desc *desc, + void *buf, u16 buf_size); +bool ixgbe_aci_check_event_pending(struct ixgbe_hw *hw); +s32 ixgbe_aci_get_event(struct ixgbe_hw *hw, struct ixgbe_aci_event *e, + bool *pending); + +void ixgbe_fill_dflt_direct_cmd_desc(struct ixgbe_aci_desc *desc, u16 opcode); + +s32 ixgbe_aci_get_fw_ver(struct ixgbe_hw *hw); +s32 ixgbe_aci_send_driver_ver(struct ixgbe_hw *hw, struct ixgbe_driver_ver *dv); +s32 ixgbe_aci_set_pf_context(struct ixgbe_hw *hw, u8 pf_id); + +s32 ixgbe_acquire_res(struct ixgbe_hw *hw, enum ixgbe_aci_res_ids res, + enum ixgbe_aci_res_access_type access, u32 timeout); +void ixgbe_release_res(struct ixgbe_hw *hw, enum ixgbe_aci_res_ids res); +s32 ixgbe_aci_list_caps(struct ixgbe_hw *hw, void *buf, u16 buf_size, + u32 *cap_count, enum ixgbe_aci_opc opc); +s32 ixgbe_discover_dev_caps(struct ixgbe_hw *hw, + struct ixgbe_hw_dev_caps *dev_caps); +s32 ixgbe_discover_func_caps(struct ixgbe_hw* hw, + struct ixgbe_hw_func_caps* func_caps); +s32 ixgbe_get_caps(struct ixgbe_hw *hw); +s32 ixgbe_aci_disable_rxen(struct ixgbe_hw *hw); +s32 ixgbe_aci_get_phy_caps(struct ixgbe_hw *hw, bool qual_mods, u8 report_mode, + struct ixgbe_aci_cmd_get_phy_caps_data *pcaps); +bool ixgbe_phy_caps_equals_cfg(struct ixgbe_aci_cmd_get_phy_caps_data *caps, + struct ixgbe_aci_cmd_set_phy_cfg_data *cfg); +void ixgbe_copy_phy_caps_to_cfg(struct ixgbe_aci_cmd_get_phy_caps_data *caps, + struct ixgbe_aci_cmd_set_phy_cfg_data *cfg); +s32 ixgbe_aci_set_phy_cfg(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_set_phy_cfg_data *cfg); +s32 ixgbe_aci_set_link_restart_an(struct ixgbe_hw *hw, bool ena_link); +s32 ixgbe_update_link_info(struct ixgbe_hw *hw); +s32 ixgbe_get_link_status(struct ixgbe_hw *hw, bool *link_up); +s32 ixgbe_aci_get_link_info(struct ixgbe_hw *hw, bool ena_lse, + struct ixgbe_link_status *link); +s32 ixgbe_aci_set_event_mask(struct ixgbe_hw *hw, u8 port_num, u16 mask); +s32 ixgbe_configure_lse(struct ixgbe_hw *hw, bool activate, u16 mask); + +s32 ixgbe_aci_get_netlist_node(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_get_link_topo *cmd, + u8 *node_part_number, u16 *node_handle); +s32 ixgbe_find_netlist_node(struct ixgbe_hw *hw, u8 node_type_ctx, + u8 node_part_number, u16 *node_handle); +s32 ixgbe_aci_read_i2c(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_link_topo_addr topo_addr, + u16 bus_addr, __le16 addr, u8 params, u8 *data); +s32 ixgbe_aci_write_i2c(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_link_topo_addr topo_addr, + u16 bus_addr, __le16 addr, u8 params, u8 *data); + +s32 ixgbe_aci_set_port_id_led(struct ixgbe_hw *hw, bool orig_mode); +s32 ixgbe_aci_set_gpio(struct ixgbe_hw *hw, u16 gpio_ctrl_handle, u8 pin_idx, + bool value); +s32 ixgbe_aci_get_gpio(struct ixgbe_hw *hw, u16 gpio_ctrl_handle, u8 pin_idx, + bool *value); +s32 ixgbe_aci_sff_eeprom(struct ixgbe_hw *hw, u16 lport, u8 bus_addr, + u16 mem_addr, u8 page, u8 page_bank_ctrl, u8 *data, + u8 length, bool write); +s32 ixgbe_aci_prog_topo_dev_nvm(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_link_topo_params *topo_params); +s32 ixgbe_aci_read_topo_dev_nvm(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_link_topo_params *topo_params, + u32 start_address, u8 *data, u8 data_size); + +s32 ixgbe_acquire_nvm(struct ixgbe_hw *hw, + enum ixgbe_aci_res_access_type access); +void ixgbe_release_nvm(struct ixgbe_hw *hw); + +s32 ixgbe_aci_read_nvm(struct ixgbe_hw *hw, u16 module_typeid, u32 offset, + u16 length, void *data, bool last_command, + bool read_shadow_ram); + +s32 ixgbe_aci_erase_nvm(struct ixgbe_hw *hw, u16 module_typeid); +s32 ixgbe_aci_update_nvm(struct ixgbe_hw *hw, u16 module_typeid, + u32 offset, u16 length, void *data, + bool last_command, u8 command_flags); + +s32 ixgbe_aci_read_nvm_cfg(struct ixgbe_hw *hw, u8 cmd_flags, + u16 field_id, void *data, u16 buf_size, + u16 *elem_count); +s32 ixgbe_aci_write_nvm_cfg(struct ixgbe_hw *hw, u8 cmd_flags, + void *data, u16 buf_size, u16 elem_count); + +s32 ixgbe_nvm_validate_checksum(struct ixgbe_hw *hw); +s32 ixgbe_nvm_recalculate_checksum(struct ixgbe_hw *hw); + +s32 ixgbe_nvm_write_activate(struct ixgbe_hw *hw, u16 cmd_flags, + u8 *response_flags); + +s32 ixgbe_get_nvm_minsrevs(struct ixgbe_hw *hw, struct ixgbe_minsrev_info *minsrevs); +s32 ixgbe_update_nvm_minsrevs(struct ixgbe_hw *hw, struct ixgbe_minsrev_info *minsrevs); + +s32 ixgbe_get_inactive_nvm_ver(struct ixgbe_hw *hw, struct ixgbe_nvm_info *nvm); +s32 ixgbe_get_active_nvm_ver(struct ixgbe_hw *hw, struct ixgbe_nvm_info *nvm); + +s32 ixgbe_get_inactive_netlist_ver(struct ixgbe_hw *hw, struct ixgbe_netlist_info *netlist); +s32 ixgbe_init_nvm(struct ixgbe_hw *hw); + +s32 ixgbe_sanitize_operate(struct ixgbe_hw *hw); +s32 ixgbe_sanitize_nvm(struct ixgbe_hw *hw, u8 cmd_flags, u8 *values); + +s32 ixgbe_read_sr_word_aci(struct ixgbe_hw *hw, u16 offset, u16 *data); +s32 ixgbe_read_sr_buf_aci(struct ixgbe_hw *hw, u16 offset, u16 *words, u16 *data); +s32 ixgbe_read_flat_nvm(struct ixgbe_hw *hw, u32 offset, u32 *length, + u8 *data, bool read_shadow_ram); + +s32 ixgbe_write_sr_word_aci(struct ixgbe_hw *hw, u32 offset, const u16 *data); +s32 ixgbe_write_sr_buf_aci(struct ixgbe_hw *hw, u32 offset, u16 words, const u16 *data); + +s32 ixgbe_aci_alternate_write(struct ixgbe_hw *hw, u32 reg_addr0, + u32 reg_val0, u32 reg_addr1, u32 reg_val1); +s32 ixgbe_aci_alternate_read(struct ixgbe_hw *hw, u32 reg_addr0, + u32 *reg_val0, u32 reg_addr1, u32 *reg_val1); +s32 ixgbe_aci_alternate_write_done(struct ixgbe_hw *hw, u8 bios_mode, + bool *reset_needed); +s32 ixgbe_aci_alternate_clear(struct ixgbe_hw *hw); + +s32 ixgbe_aci_get_internal_data(struct ixgbe_hw *hw, u16 cluster_id, + u16 table_id, u32 start, void *buf, + u16 buf_size, u16 *ret_buf_size, + u16 *ret_next_cluster, u16 *ret_next_table, + u32 *ret_next_index); + +s32 ixgbe_handle_nvm_access(struct ixgbe_hw *hw, + struct ixgbe_nvm_access_cmd *cmd, + struct ixgbe_nvm_access_data *data); + +s32 ixgbe_aci_set_health_status_config(struct ixgbe_hw *hw, u8 event_source); + +/* E610 operations */ +s32 ixgbe_init_ops_E610(struct ixgbe_hw *hw); +s32 ixgbe_reset_hw_E610(struct ixgbe_hw *hw); +s32 ixgbe_start_hw_E610(struct ixgbe_hw *hw); +enum ixgbe_media_type ixgbe_get_media_type_E610(struct ixgbe_hw *hw); +u64 ixgbe_get_supported_physical_layer_E610(struct ixgbe_hw *hw); +s32 ixgbe_setup_link_E610(struct ixgbe_hw *hw, ixgbe_link_speed speed, + bool autoneg_wait); +s32 ixgbe_check_link_E610(struct ixgbe_hw *hw, ixgbe_link_speed *speed, + bool *link_up, bool link_up_wait_to_complete); +s32 ixgbe_get_link_capabilities_E610(struct ixgbe_hw *hw, + ixgbe_link_speed *speed, + bool *autoneg); +s32 ixgbe_cfg_phy_fc(struct ixgbe_hw *hw, + struct ixgbe_aci_cmd_set_phy_cfg_data *cfg, + enum ixgbe_fc_mode req_mode); +s32 ixgbe_setup_fc_E610(struct ixgbe_hw *hw); +void ixgbe_fc_autoneg_E610(struct ixgbe_hw *hw); +s32 ixgbe_set_fw_drv_ver_E610(struct ixgbe_hw *hw, u8 maj, u8 min, u8 build, + u8 sub, u16 len, const char *driver_ver); +void ixgbe_disable_rx_E610(struct ixgbe_hw *hw); +s32 ixgbe_setup_eee_E610(struct ixgbe_hw *hw, bool enable_eee); +bool ixgbe_fw_recovery_mode_E610(struct ixgbe_hw *hw); +bool ixgbe_fw_rollback_mode_E610(struct ixgbe_hw *hw); +bool ixgbe_get_fw_tsam_mode_E610(struct ixgbe_hw *hw); +s32 ixgbe_init_phy_ops_E610(struct ixgbe_hw *hw); +s32 ixgbe_identify_phy_E610(struct ixgbe_hw *hw); +s32 ixgbe_identify_module_E610(struct ixgbe_hw *hw); +s32 ixgbe_setup_phy_link_E610(struct ixgbe_hw *hw); +s32 ixgbe_get_phy_firmware_version_E610(struct ixgbe_hw *hw, + u16 *firmware_version); +s32 ixgbe_read_i2c_sff8472_E610(struct ixgbe_hw *hw, u8 byte_offset, + u8 *sff8472_data); +s32 ixgbe_read_i2c_eeprom_E610(struct ixgbe_hw *hw, u8 byte_offset, + u8 *eeprom_data); +s32 ixgbe_write_i2c_eeprom_E610(struct ixgbe_hw *hw, u8 byte_offset, + u8 eeprom_data); +s32 ixgbe_check_overtemp_E610(struct ixgbe_hw *hw); +s32 ixgbe_set_phy_power_E610(struct ixgbe_hw *hw, bool on); +s32 ixgbe_enter_lplu_E610(struct ixgbe_hw *hw); +s32 ixgbe_init_eeprom_params_E610(struct ixgbe_hw *hw); +s32 ixgbe_read_ee_aci_E610(struct ixgbe_hw *hw, u16 offset, u16 *data); +s32 ixgbe_read_ee_aci_buffer_E610(struct ixgbe_hw *hw, u16 offset, + u16 words, u16 *data); +s32 ixgbe_write_ee_aci_E610(struct ixgbe_hw *hw, u16 offset, u16 data); +s32 ixgbe_write_ee_aci_buffer_E610(struct ixgbe_hw *hw, u16 offset, + u16 words, u16 *data); +s32 ixgbe_calc_eeprom_checksum_E610(struct ixgbe_hw *hw); +s32 ixgbe_update_eeprom_checksum_E610(struct ixgbe_hw *hw); +s32 ixgbe_validate_eeprom_checksum_E610(struct ixgbe_hw *hw, u16 *checksum_val); +s32 ixgbe_read_pba_string_E610(struct ixgbe_hw *hw, u8 *pba_num, u32 pba_num_size); + +#endif /* _IXGBE_E610_H_ */ diff --git a/sys/dev/ixgbe/ixgbe_osdep.c b/sys/dev/ixgbe/ixgbe_osdep.c index 892924712c38..9bd9ce63b786 100644 --- a/sys/dev/ixgbe/ixgbe_osdep.c +++ b/sys/dev/ixgbe/ixgbe_osdep.c @@ -114,3 +114,29 @@ ixgbe_link_speed_to_baudrate(ixgbe_link_speed speed) return baudrate; } + +void +ixgbe_init_lock(struct ixgbe_lock *lock) +{ + mtx_init(&lock->mutex, "mutex", + "ixgbe ACI lock", MTX_DEF | MTX_DUPOK); +} + +void +ixgbe_acquire_lock(struct ixgbe_lock *lock) +{ + mtx_lock(&lock->mutex); +} + +void +ixgbe_release_lock(struct ixgbe_lock *lock) +{ + mtx_unlock(&lock->mutex); +} + +void +ixgbe_destroy_lock(struct ixgbe_lock *lock) +{ + if (mtx_initialized(&lock->mutex)) + mtx_destroy(&lock->mutex); +} diff --git a/sys/dev/ixgbe/ixgbe_osdep.h b/sys/dev/ixgbe/ixgbe_osdep.h index cf7c578fd684..8cf1d13736ce 100644 --- a/sys/dev/ixgbe/ixgbe_osdep.h +++ b/sys/dev/ixgbe/ixgbe_osdep.h @@ -133,7 +133,9 @@ enum { /* XXX these need to be revisited */ #define IXGBE_CPU_TO_LE16 htole16 #define IXGBE_CPU_TO_LE32 htole32 +#define IXGBE_LE16_TO_CPU le16toh #define IXGBE_LE32_TO_CPU le32toh +#define IXGBE_LE64_TO_CPU le64toh #define IXGBE_LE32_TO_CPUS(x) *(x) = le32dec(x) #define IXGBE_CPU_TO_BE16 htobe16 #define IXGBE_CPU_TO_BE32 htobe32 @@ -146,6 +148,7 @@ typedef int16_t s16; typedef uint32_t u32; typedef int32_t s32; typedef uint64_t u64; +typedef int64_t s64; #ifndef __bool_true_false_are_defined typedef boolean_t bool; #endif @@ -195,6 +198,11 @@ struct ixgbe_osdep bus_space_handle_t mem_bus_space_handle; }; +struct ixgbe_lock +{ + struct mtx mutex; +}; + /* These routines need struct ixgbe_hw declared */ struct ixgbe_hw; device_t ixgbe_dev_from_hw(struct ixgbe_hw *hw); @@ -222,4 +230,27 @@ extern void ixgbe_write_reg_array(struct ixgbe_hw *, u32, u32, u32); #define IXGBE_WRITE_REG_ARRAY(a, reg, offset, val) \ ixgbe_write_reg_array(a, reg, offset, val) +void ixgbe_init_lock(struct ixgbe_lock *); +void ixgbe_destroy_lock(struct ixgbe_lock *); +void ixgbe_acquire_lock(struct ixgbe_lock *); +void ixgbe_release_lock(struct ixgbe_lock *); + +static inline void * +ixgbe_calloc(struct ixgbe_hw __unused *hw, size_t count, size_t size) +{ + return (malloc(count * size, M_DEVBUF, M_ZERO | M_NOWAIT)); +} + +static inline void * +ixgbe_malloc(struct ixgbe_hw __unused *hw, size_t size) +{ + return (malloc(size, M_DEVBUF, M_ZERO | M_NOWAIT)); +} + +static inline void +ixgbe_free(struct ixgbe_hw __unused *hw, void *addr) +{ + free(addr, M_DEVBUF); +} + #endif /* _IXGBE_OSDEP_H_ */ diff --git a/sys/dev/ixgbe/ixgbe_type.h b/sys/dev/ixgbe/ixgbe_type.h index 91b46da72c75..0bbe7806d41d 100644 --- a/sys/dev/ixgbe/ixgbe_type.h +++ b/sys/dev/ixgbe/ixgbe_type.h @@ -74,6 +74,7 @@ */ #include "ixgbe_osdep.h" +#include "ixgbe_type_e610.h" /* Override this by setting IOMEM in your ixgbe_osdep.h header */ #define IOMEM @@ -150,12 +151,19 @@ #define IXGBE_DEV_ID_X550EM_X_10G_T 0x15AD #define IXGBE_DEV_ID_X550EM_X_1G_T 0x15AE #define IXGBE_DEV_ID_X550EM_X_XFI 0x15B0 +#define IXGBE_DEV_ID_E610_BACKPLANE 0x57AE +#define IXGBE_DEV_ID_E610_SFP 0x57AF +#define IXGBE_DEV_ID_E610_10G_T 0x57B0 +#define IXGBE_DEV_ID_E610_2_5G_T 0x57B1 +#define IXGBE_DEV_ID_E610_SGMII 0x57B2 #define IXGBE_DEV_ID_X550_VF_HV 0x1564 #define IXGBE_DEV_ID_X550_VF 0x1565 #define IXGBE_DEV_ID_X550EM_A_VF 0x15C5 #define IXGBE_DEV_ID_X550EM_A_VF_HV 0x15B4 #define IXGBE_DEV_ID_X550EM_X_VF 0x15A8 #define IXGBE_DEV_ID_X550EM_X_VF_HV 0x15A9 +#define IXGBE_DEV_ID_E610_VF 0x57AD +#define IXGBE_SUBDEV_ID_E610_VF_HV 0x0001 #define IXGBE_CAT(r, m) IXGBE_##r##m @@ -1969,6 +1977,7 @@ enum { #define IXGBE_EICR_MAILBOX 0x00080000 /* VF to PF Mailbox Interrupt */ #define IXGBE_EICR_LSC 0x00100000 /* Link Status Change */ #define IXGBE_EICR_LINKSEC 0x00200000 /* PN Threshold */ +#define IXGBE_EICR_FW_EVENT 0x00200000 /* Async FW event */ #define IXGBE_EICR_MNG 0x00400000 /* Manageability Event Interrupt */ #define IXGBE_EICR_TS 0x00800000 /* Thermal Sensor Event */ #define IXGBE_EICR_TIMESYNC 0x01000000 /* Timesync Event */ @@ -2004,6 +2013,7 @@ enum { #define IXGBE_EICS_PCI IXGBE_EICR_PCI /* PCI Exception */ #define IXGBE_EICS_MAILBOX IXGBE_EICR_MAILBOX /* VF to PF Mailbox Int */ #define IXGBE_EICS_LSC IXGBE_EICR_LSC /* Link Status Change */ +#define IXGBE_EICS_FW_EVENT IXGBE_EICR_FW_EVENT /* Async FW event */ #define IXGBE_EICS_MNG IXGBE_EICR_MNG /* MNG Event Interrupt */ #define IXGBE_EICS_TIMESYNC IXGBE_EICR_TIMESYNC /* Timesync Event */ #define IXGBE_EICS_GPI_SDP0 IXGBE_EICR_GPI_SDP0 /* SDP0 Gen Purpose Int */ @@ -2025,6 +2035,7 @@ enum { #define IXGBE_EIMS_PCI IXGBE_EICR_PCI /* PCI Exception */ #define IXGBE_EIMS_MAILBOX IXGBE_EICR_MAILBOX /* VF to PF Mailbox Int */ #define IXGBE_EIMS_LSC IXGBE_EICR_LSC /* Link Status Change */ +#define IXGBE_EIMS_FW_EVENT IXGBE_EICR_FW_EVENT /* Async FW event */ #define IXGBE_EIMS_MNG IXGBE_EICR_MNG /* MNG Event Interrupt */ #define IXGBE_EIMS_TS IXGBE_EICR_TS /* Thermal Sensor Event */ #define IXGBE_EIMS_TIMESYNC IXGBE_EICR_TIMESYNC /* Timesync Event */ @@ -2047,6 +2058,7 @@ enum { #define IXGBE_EIMC_PCI IXGBE_EICR_PCI /* PCI Exception */ #define IXGBE_EIMC_MAILBOX IXGBE_EICR_MAILBOX /* VF to PF Mailbox Int */ #define IXGBE_EIMC_LSC IXGBE_EICR_LSC /* Link Status Change */ +#define IXGBE_EIMC_FW_EVENT IXGBE_EICR_FW_EVENT /* Async FW event */ #define IXGBE_EIMC_MNG IXGBE_EICR_MNG /* MNG Event Interrupt */ #define IXGBE_EIMC_TIMESYNC IXGBE_EICR_TIMESYNC /* Timesync Event */ #define IXGBE_EIMC_GPI_SDP0 IXGBE_EICR_GPI_SDP0 /* SDP0 Gen Purpose Int */ @@ -2454,6 +2466,7 @@ enum { #define IXGBE_82599_SERIAL_NUMBER_MAC_ADDR 0x11 #define IXGBE_X550_SERIAL_NUMBER_MAC_ADDR 0x04 +#define IXGBE_PCIE_MSIX_E610_CAPS 0xB2 #define IXGBE_PCIE_MSIX_82599_CAPS 0x72 #define IXGBE_MAX_MSIX_VECTORS_82599 0x40 #define IXGBE_PCIE_MSIX_82598_CAPS 0x62 @@ -2571,6 +2584,7 @@ enum { #define IXGBE_PCI_DEVICE_STATUS 0xAA #define IXGBE_PCI_DEVICE_STATUS_TRANSACTION_PENDING 0x0020 #define IXGBE_PCI_LINK_STATUS 0xB2 +#define IXGBE_PCI_LINK_STATUS_E610 0x82 #define IXGBE_PCI_DEVICE_CONTROL2 0xC8 #define IXGBE_PCI_LINK_WIDTH 0x3F0 #define IXGBE_PCI_LINK_WIDTH_1 0x10 @@ -2581,6 +2595,7 @@ enum { #define IXGBE_PCI_LINK_SPEED_2500 0x1 #define IXGBE_PCI_LINK_SPEED_5000 0x2 #define IXGBE_PCI_LINK_SPEED_8000 0x3 +#define IXGBE_PCI_LINK_SPEED_16000 0x4 #define IXGBE_PCI_HEADER_TYPE_REGISTER 0x0E #define IXGBE_PCI_HEADER_TYPE_MULTIFUNC 0x80 #define IXGBE_PCI_DEVICE_CONTROL2_16ms 0x0005 @@ -3743,6 +3758,8 @@ enum ixgbe_mac_type { ixgbe_mac_X550_vf, ixgbe_mac_X550EM_x_vf, ixgbe_mac_X550EM_a_vf, + ixgbe_mac_E610, + ixgbe_mac_E610_vf, ixgbe_num_macs }; @@ -3822,7 +3839,9 @@ enum ixgbe_media_type { ixgbe_media_type_copper, ixgbe_media_type_backplane, ixgbe_media_type_cx4, - ixgbe_media_type_virtual + ixgbe_media_type_virtual, + ixgbe_media_type_da, + ixgbe_media_type_aui }; /* Flow Control Settings */ @@ -3831,6 +3850,7 @@ enum ixgbe_fc_mode { ixgbe_fc_rx_pause, ixgbe_fc_tx_pause, ixgbe_fc_full, + ixgbe_fc_auto, ixgbe_fc_default }; @@ -3863,6 +3883,7 @@ enum ixgbe_bus_speed { ixgbe_bus_speed_2500 = 2500, ixgbe_bus_speed_5000 = 5000, ixgbe_bus_speed_8000 = 8000, + ixgbe_bus_speed_16000 = 16000, ixgbe_bus_speed_reserved }; @@ -4007,6 +4028,7 @@ struct ixgbe_eeprom_operations { s32 (*validate_checksum)(struct ixgbe_hw *, u16 *); s32 (*update_checksum)(struct ixgbe_hw *); s32 (*calc_checksum)(struct ixgbe_hw *); + s32 (*read_pba_string)(struct ixgbe_hw *, u8 *, u32); }; struct ixgbe_mac_operations { @@ -4118,6 +4140,10 @@ struct ixgbe_mac_operations { void (*mdd_event)(struct ixgbe_hw *hw, u32 *vf_bitmap); void (*restore_mdd_vf)(struct ixgbe_hw *hw, u32 vf); bool (*fw_recovery_mode)(struct ixgbe_hw *hw); + bool (*fw_rollback_mode)(struct ixgbe_hw *hw); + bool (*get_fw_tsam_mode)(struct ixgbe_hw *hw); + s32 (*get_fw_version)(struct ixgbe_hw *hw); + s32 (*get_nvm_version)(struct ixgbe_hw *hw, struct ixgbe_nvm_info *nvm); }; struct ixgbe_phy_operations { @@ -4162,6 +4188,9 @@ struct ixgbe_link_operations { struct ixgbe_link_info { struct ixgbe_link_operations ops; u8 addr; + struct ixgbe_link_status link_info; + struct ixgbe_link_status link_info_old; + u8 get_link_info; }; struct ixgbe_eeprom_info { @@ -4233,6 +4262,9 @@ struct ixgbe_phy_info { bool reset_if_overtemp; bool qsfp_shared_i2c_bus; u32 nw_mng_if_sel; + u64 phy_type_low; + u64 phy_type_high; + struct ixgbe_aci_cmd_set_phy_cfg_data curr_user_phy_cfg; }; #include "ixgbe_mbx.h" @@ -4261,6 +4293,22 @@ struct ixgbe_hw { bool wol_enabled; bool need_crosstalk_fix; u32 fw_rst_cnt; + u8 api_branch; + u8 api_maj_ver; + u8 api_min_ver; + u8 api_patch; + u8 fw_branch; + u8 fw_maj_ver; + u8 fw_min_ver; + u8 fw_patch; + u32 fw_build; + struct ixgbe_aci_info aci; + struct ixgbe_flash_info flash; + struct ixgbe_hw_dev_caps dev_caps; + struct ixgbe_hw_func_caps func_caps; + struct ixgbe_fwlog_cfg fwlog_cfg; + bool fwlog_support_ena; + struct ixgbe_fwlog_ring fwlog_ring; }; #define ixgbe_call_func(hw, func, params, error) \ @@ -4312,6 +4360,24 @@ struct ixgbe_hw { #define IXGBE_ERR_MBX_NOMSG -42 #define IXGBE_ERR_TIMEOUT -43 +#define IXGBE_ERR_NOT_SUPPORTED -45 +#define IXGBE_ERR_OUT_OF_RANGE -46 + +#define IXGBE_ERR_NVM -50 +#define IXGBE_ERR_NVM_CHECKSUM -51 +#define IXGBE_ERR_BUF_TOO_SHORT -52 +#define IXGBE_ERR_NVM_BLANK_MODE -53 +#define IXGBE_ERR_INVAL_SIZE -54 +#define IXGBE_ERR_DOES_NOT_EXIST -55 + +#define IXGBE_ERR_ACI_ERROR -100 +#define IXGBE_ERR_ACI_DISABLED -101 +#define IXGBE_ERR_ACI_TIMEOUT -102 +#define IXGBE_ERR_ACI_BUSY -103 +#define IXGBE_ERR_ACI_NO_WORK -104 +#define IXGBE_ERR_ACI_NO_EVENTS -105 +#define IXGBE_ERR_FW_API_VER -106 + #define IXGBE_NOT_IMPLEMENTED 0x7FFFFFFF @@ -4540,5 +4606,6 @@ struct ixgbe_bypass_eeprom { #define IXGBE_REQUEST_TASK_FDIR 0x08 #define IXGBE_REQUEST_TASK_PHY 0x10 #define IXGBE_REQUEST_TASK_LSC 0x20 +#define IXGBE_REQUEST_TASK_FWEVENT 0x40 #endif /* _IXGBE_TYPE_H_ */ diff --git a/sys/dev/ixgbe/ixgbe_type_e610.h b/sys/dev/ixgbe/ixgbe_type_e610.h new file mode 100644 index 000000000000..e300030c3ba4 --- /dev/null +++ b/sys/dev/ixgbe/ixgbe_type_e610.h @@ -0,0 +1,2278 @@ +/****************************************************************************** + SPDX-License-Identifier: BSD-3-Clause + + Copyright (c) 2025, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ + +#ifndef _IXGBE_TYPE_E610_H_ +#define _IXGBE_TYPE_E610_H_ + + +/* Generic defines */ +#ifndef BIT +#define BIT(a) (1UL << (a)) +#endif /* !BIT */ +#ifndef BIT_ULL +#define BIT_ULL(a) (1ULL << (a)) +#endif /* !BIT_ULL */ +#ifndef BITS_PER_BYTE +#define BITS_PER_BYTE 8 +#endif /* !BITS_PER_BYTE */ +#ifndef DIVIDE_AND_ROUND_UP +#define DIVIDE_AND_ROUND_UP(a, b) (((a) + (b) - 1) / (b)) +#endif /* !DIVIDE_AND_ROUND_UP */ + +#ifndef ROUND_UP +/** + * ROUND_UP - round up to next arbitrary multiple (not a power of 2) + * @a: value to round up + * @b: arbitrary multiple + * + * Round up to the next multiple of the arbitrary b. + */ +#define ROUND_UP(a, b) ((b) * DIVIDE_AND_ROUND_UP((a), (b))) +#endif /* !ROUND_UP */ + +#define MAKEMASK(mask, shift) (mask << shift) + +#define BYTES_PER_WORD 2 +#define BYTES_PER_DWORD 4 + +#ifndef BITS_PER_LONG +#define BITS_PER_LONG 64 +#endif /* !BITS_PER_LONG */ +#ifndef BITS_PER_LONG_LONG +#define BITS_PER_LONG_LONG 64 +#endif /* !BITS_PER_LONG_LONG */ +#undef GENMASK +#define GENMASK(h, l) \ + (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) +#undef GENMASK_ULL +#define GENMASK_ULL(h, l) \ + (((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h)))) + +/* Data type manipulation macros. */ +#define HI_DWORD(x) ((u32)((((x) >> 16) >> 16) & 0xFFFFFFFF)) +#define LO_DWORD(x) ((u32)((x) & 0xFFFFFFFF)) +#define HI_WORD(x) ((u16)(((x) >> 16) & 0xFFFF)) +#define LO_WORD(x) ((u16)((x) & 0xFFFF)) +#define HI_BYTE(x) ((u8)(((x) >> 8) & 0xFF)) +#define LO_BYTE(x) ((u8)((x) & 0xFF)) + +#ifndef MIN_T +#define MIN_T(_t, _a, _b) min((_t)(_a), (_t)(_b)) +#endif + +#define IS_ASCII(_ch) ((_ch) < 0x80) + +/** + * ixgbe_struct_size - size of struct with C99 flexible array member + * @ptr: pointer to structure + * @field: flexible array member (last member of the structure) + * @num: number of elements of that flexible array member + */ +#define ixgbe_struct_size(ptr, field, num) \ + (sizeof(*(ptr)) + sizeof(*(ptr)->field) * (num)) + +/* General E610 defines */ +#define IXGBE_MAX_VSI 768 + +/* Auxiliary field, mask and shift definition for Shadow RAM and NVM Flash */ +#define E610_SR_VPD_SIZE_WORDS 512 +#define E610_SR_PCIE_ALT_SIZE_WORDS 512 + +/* Checksum and Shadow RAM pointers */ +#define E610_SR_NVM_DEV_STARTER_VER 0x18 +#define E610_NVM_VER_LO_SHIFT 0 +#define E610_NVM_VER_LO_MASK (0xff << E610_NVM_VER_LO_SHIFT) +#define E610_NVM_VER_HI_SHIFT 12 +#define E610_NVM_VER_HI_MASK (0xf << E610_NVM_VER_HI_SHIFT) +#define E610_SR_NVM_MAP_VER 0x29 +#define E610_SR_NVM_EETRACK_LO 0x2D +#define E610_SR_NVM_EETRACK_HI 0x2E +#define E610_SR_VPD_PTR 0x2F +#define E610_SR_PCIE_ALT_AUTO_LOAD_PTR 0x3E +#define E610_SR_SW_CHECKSUM_WORD 0x3F +#define E610_SR_PFA_PTR 0x40 +#define E610_SR_1ST_NVM_BANK_PTR 0x42 +#define E610_SR_NVM_BANK_SIZE 0x43 +#define E610_SR_1ST_OROM_BANK_PTR 0x44 +#define E610_SR_OROM_BANK_SIZE 0x45 +#define E610_SR_NETLIST_BANK_PTR 0x46 +#define E610_SR_NETLIST_BANK_SIZE 0x47 +#define E610_SR_POINTER_TYPE_BIT BIT(15) +#define E610_SR_POINTER_MASK 0x7fff +#define E610_SR_HALF_4KB_SECTOR_UNITS 2048 +#define E610_GET_PFA_POINTER_IN_WORDS(offset) \ + ((offset & E610_SR_POINTER_TYPE_BIT) == E610_SR_POINTER_TYPE_BIT) ? \ + ((offset & E610_SR_POINTER_MASK) * E610_SR_HALF_4KB_SECTOR_UNITS) : \ + (offset & E610_SR_POINTER_MASK) + +/* Checksum and Shadow RAM pointers */ +#define E610_SR_NVM_CTRL_WORD 0x00 +#define E610_SR_PBA_BLOCK_PTR 0x16 + +/* The Orom version topology */ +#define IXGBE_OROM_VER_PATCH_SHIFT 0 +#define IXGBE_OROM_VER_PATCH_MASK (0xff << IXGBE_OROM_VER_PATCH_SHIFT) +#define IXGBE_OROM_VER_BUILD_SHIFT 8 +#define IXGBE_OROM_VER_BUILD_MASK (0xffff << IXGBE_OROM_VER_BUILD_SHIFT) +#define IXGBE_OROM_VER_SHIFT 24 +#define IXGBE_OROM_VER_MASK (0xff << IXGBE_OROM_VER_SHIFT) + +/* CSS Header words */ +#define IXGBE_NVM_CSS_HDR_LEN_L 0x02 +#define IXGBE_NVM_CSS_HDR_LEN_H 0x03 +#define IXGBE_NVM_CSS_SREV_L 0x14 +#define IXGBE_NVM_CSS_SREV_H 0x15 + +/* Length of Authentication header section in words */ +#define IXGBE_NVM_AUTH_HEADER_LEN 0x08 + +/* The Netlist ID Block is located after all of the Link Topology nodes. */ +#define IXGBE_NETLIST_ID_BLK_SIZE 0x30 +#define IXGBE_NETLIST_ID_BLK_OFFSET(n) IXGBE_NETLIST_LINK_TOPO_OFFSET(0x0004 + 2 * (n)) + +/* netlist ID block field offsets (word offsets) */ +#define IXGBE_NETLIST_ID_BLK_MAJOR_VER_LOW 0x02 +#define IXGBE_NETLIST_ID_BLK_MAJOR_VER_HIGH 0x03 +#define IXGBE_NETLIST_ID_BLK_MINOR_VER_LOW 0x04 +#define IXGBE_NETLIST_ID_BLK_MINOR_VER_HIGH 0x05 +#define IXGBE_NETLIST_ID_BLK_TYPE_LOW 0x06 +#define IXGBE_NETLIST_ID_BLK_TYPE_HIGH 0x07 +#define IXGBE_NETLIST_ID_BLK_REV_LOW 0x08 +#define IXGBE_NETLIST_ID_BLK_REV_HIGH 0x09 +#define IXGBE_NETLIST_ID_BLK_SHA_HASH_WORD(n) (0x0A + (n)) +#define IXGBE_NETLIST_ID_BLK_CUST_VER 0x2F + +/* The Link Topology Netlist section is stored as a series of words. It is + * stored in the NVM as a TLV, with the first two words containing the type + * and length. + */ +#define IXGBE_NETLIST_LINK_TOPO_MOD_ID 0x011B +#define IXGBE_NETLIST_TYPE_OFFSET 0x0000 +#define IXGBE_NETLIST_LEN_OFFSET 0x0001 + +/* The Link Topology section follows the TLV header. When reading the netlist + * using ixgbe_read_netlist_module, we need to account for the 2-word TLV + * header. + */ +#define IXGBE_NETLIST_LINK_TOPO_OFFSET(n) ((n) + 2) +#define IXGBE_LINK_TOPO_MODULE_LEN IXGBE_NETLIST_LINK_TOPO_OFFSET(0x0000) +#define IXGBE_LINK_TOPO_NODE_COUNT IXGBE_NETLIST_LINK_TOPO_OFFSET(0x0001) +#define IXGBE_LINK_TOPO_NODE_COUNT_M MAKEMASK(0x3FF, 0) + +/* Auxiliary field, mask and shift definition for Shadow RAM and NVM Flash */ +#define IXGBE_SR_CTRL_WORD_1_S 0x06 +#define IXGBE_SR_CTRL_WORD_1_M (0x03 << IXGBE_SR_CTRL_WORD_1_S) +#define IXGBE_SR_CTRL_WORD_VALID 0x1 +#define IXGBE_SR_CTRL_WORD_OROM_BANK BIT(3) +#define IXGBE_SR_CTRL_WORD_NETLIST_BANK BIT(4) +#define IXGBE_SR_CTRL_WORD_NVM_BANK BIT(5) +#define IXGBE_SR_NVM_PTR_4KB_UNITS BIT(15) + +/* These macros strip from NVM Image Revision the particular part of NVM ver: + major ver, minor ver and image id */ +#define E610_NVM_MAJOR_VER(x) ((x & 0xF000) >> 12) +#define E610_NVM_MINOR_VER(x) (x & 0x00FF) + +/* Shadow RAM related */ +#define IXGBE_SR_SECTOR_SIZE_IN_WORDS 0x800 +#define IXGBE_SR_WORDS_IN_1KB 512 +/* Checksum should be calculated such that after adding all the words, + * including the checksum word itself, the sum should be 0xBABA. + */ +#define IXGBE_SR_SW_CHECKSUM_BASE 0xBABA + +/* Netlist */ +#define IXGBE_MAX_NETLIST_SIZE 10 + +/* General registers */ + +/* Firmware Status Register (GL_FWSTS) */ +#define GL_FWSTS 0x00083048 /* Reset Source: POR */ +#define GL_FWSTS_FWS0B_S 0 +#define GL_FWSTS_FWS0B_M MAKEMASK(0xFF, 0) +#define GL_FWSTS_FWROWD_S 8 +#define GL_FWSTS_FWROWD_M BIT(8) +#define GL_FWSTS_FWRI_S 9 +#define GL_FWSTS_FWRI_M BIT(9) +#define GL_FWSTS_FWS1B_S 16 +#define GL_FWSTS_FWS1B_M MAKEMASK(0xFF, 16) +#define GL_FWSTS_EP_PF0 BIT(24) +#define GL_FWSTS_EP_PF1 BIT(25) + +/* Recovery mode values of Firmware Status 1 Byte (FWS1B) bitfield */ +#define GL_FWSTS_FWS1B_RECOVERY_MODE_CORER_LEGACY 0x0B +#define GL_FWSTS_FWS1B_RECOVERY_MODE_GLOBR_LEGACY 0x0C +#define GL_FWSTS_FWS1B_RECOVERY_MODE_CORER 0x30 +#define GL_FWSTS_FWS1B_RECOVERY_MODE_GLOBR 0x31 +#define GL_FWSTS_FWS1B_RECOVERY_MODE_TRANSITION 0x32 +#define GL_FWSTS_FWS1B_RECOVERY_MODE_NVM 0x33 + +/* Firmware Status (GL_MNG_FWSM) */ +#define GL_MNG_FWSM 0x000B6134 /* Reset Source: POR */ +#define GL_MNG_FWSM_FW_MODES_S 0 +#define GL_MNG_FWSM_FW_MODES_M MAKEMASK(0x7, 0) +#define GL_MNG_FWSM_RSV0_S 2 +#define GL_MNG_FWSM_RSV0_M MAKEMASK(0xFF, 2) +#define GL_MNG_FWSM_EEP_RELOAD_IND_S 10 +#define GL_MNG_FWSM_EEP_RELOAD_IND_M BIT(10) +#define GL_MNG_FWSM_RSV1_S 11 +#define GL_MNG_FWSM_RSV1_M MAKEMASK(0xF, 11) +#define GL_MNG_FWSM_RSV2_S 15 +#define GL_MNG_FWSM_RSV2_M BIT(15) +#define GL_MNG_FWSM_PCIR_AL_FAILURE_S 16 +#define GL_MNG_FWSM_PCIR_AL_FAILURE_M BIT(16) +#define GL_MNG_FWSM_POR_AL_FAILURE_S 17 +#define GL_MNG_FWSM_POR_AL_FAILURE_M BIT(17) +#define GL_MNG_FWSM_RSV3_S 18 +#define GL_MNG_FWSM_RSV3_M BIT(18) +#define GL_MNG_FWSM_EXT_ERR_IND_S 19 +#define GL_MNG_FWSM_EXT_ERR_IND_M MAKEMASK(0x3F, 19) +#define GL_MNG_FWSM_RSV4_S 25 +#define GL_MNG_FWSM_RSV4_M BIT(25) +#define GL_MNG_FWSM_RESERVED_11_S 26 +#define GL_MNG_FWSM_RESERVED_11_M MAKEMASK(0xF, 26) +#define GL_MNG_FWSM_RSV5_S 30 +#define GL_MNG_FWSM_RSV5_M MAKEMASK(0x3, 30) + +/* FW mode indications */ +#define GL_MNG_FWSM_FW_MODES_DEBUG_M BIT(0) +#define GL_MNG_FWSM_FW_MODES_RECOVERY_M BIT(1) +#define GL_MNG_FWSM_FW_MODES_ROLLBACK_M BIT(2) + +/* Global NVM General Status Register */ +#define GLNVM_GENS 0x000B6100 /* Reset Source: POR */ +#define GLNVM_GENS_NVM_PRES_S 0 +#define GLNVM_GENS_NVM_PRES_M BIT(0) +#define GLNVM_GENS_SR_SIZE_S 5 +#define GLNVM_GENS_SR_SIZE_M MAKEMASK(0x7, 5) +#define GLNVM_GENS_BANK1VAL_S 8 +#define GLNVM_GENS_BANK1VAL_M BIT(8) +#define GLNVM_GENS_ALT_PRST_S 23 +#define GLNVM_GENS_ALT_PRST_M BIT(23) +#define GLNVM_GENS_FL_AUTO_RD_S 25 +#define GLNVM_GENS_FL_AUTO_RD_M BIT(25) + +/* Flash Access Register */ +#define GLNVM_FLA 0x000B6108 /* Reset Source: POR */ +#define GLNVM_FLA_LOCKED_S 6 +#define GLNVM_FLA_LOCKED_M BIT(6) + +/* Bit Bang registers */ +#define RDASB_MSGCTL 0x000B6820 +#define RDASB_MSGCTL_HDR_DWS_S 0 +#define RDASB_MSGCTL_EXP_RDW_S 8 +#define RDASB_MSGCTL_CMDV_M BIT(31) +#define RDASB_RSPCTL 0x000B6824 +#define RDASB_RSPCTL_BAD_LENGTH_M BIT(30) +#define RDASB_RSPCTL_NOT_SUCCESS_M BIT(31) +#define RDASB_WHDR0 0x000B68F4 +#define RDASB_WHDR1 0x000B68F8 +#define RDASB_WHDR2 0x000B68FC +#define RDASB_WHDR3 0x000B6900 +#define RDASB_WHDR4 0x000B6904 +#define RDASB_RHDR0 0x000B6AFC +#define RDASB_RHDR0_RESPONSE_S 27 +#define RDASB_RHDR0_RESPONSE_M MAKEMASK(0x7, 27) +#define RDASB_RDATA0 0x000B6B00 +#define RDASB_RDATA1 0x000B6B04 + +/* SPI Registers */ +#define SPISB_MSGCTL 0x000B7020 +#define SPISB_MSGCTL_HDR_DWS_S 0 +#define SPISB_MSGCTL_EXP_RDW_S 8 +#define SPISB_MSGCTL_MSG_MODE_S 26 +#define SPISB_MSGCTL_TOKEN_MODE_S 28 +#define SPISB_MSGCTL_BARCLR_S 30 +#define SPISB_MSGCTL_CMDV_S 31 +#define SPISB_MSGCTL_CMDV_M BIT(31) +#define SPISB_RSPCTL 0x000B7024 +#define SPISB_RSPCTL_BAD_LENGTH_M BIT(30) +#define SPISB_RSPCTL_NOT_SUCCESS_M BIT(31) +#define SPISB_WHDR0 0x000B70F4 +#define SPISB_WHDR0_DEST_SEL_S 12 +#define SPISB_WHDR0_OPCODE_SEL_S 16 +#define SPISB_WHDR0_TAG_S 24 +#define SPISB_WHDR1 0x000B70F8 +#define SPISB_WHDR2 0x000B70FC +#define SPISB_RDATA 0x000B7300 +#define SPISB_WDATA 0x000B7100 + +/* Firmware Reset Count register */ +#define GL_FWRESETCNT 0x00083100 /* Reset Source: POR */ +#define GL_FWRESETCNT_FWRESETCNT_S 0 +#define GL_FWRESETCNT_FWRESETCNT_M MAKEMASK(0xFFFFFFFF, 0) + +/* Admin Command Interface (ACI) registers */ +#define PF_HIDA(_i) (0x00085000 + ((_i) * 4)) +#define PF_HIDA_2(_i) (0x00085020 + ((_i) * 4)) +#define PF_HIBA(_i) (0x00084000 + ((_i) * 4)) +#define PF_HICR 0x00082048 + +#define PF_HIDA_MAX_INDEX 15 +#define PF_HIBA_MAX_INDEX 1023 + +#define PF_HICR_EN BIT(0) +#define PF_HICR_C BIT(1) +#define PF_HICR_SV BIT(2) +#define PF_HICR_EV BIT(3) + +#define GL_HIDA(_i) (0x00082000 + ((_i) * 4)) +#define GL_HIDA_2(_i) (0x00082020 + ((_i) * 4)) +#define GL_HIBA(_i) (0x00081000 + ((_i) * 4)) +#define GL_HICR 0x00082040 + +#define GL_HIDA_MAX_INDEX 15 +#define GL_HIBA_MAX_INDEX 1023 + +#define GL_HICR_C BIT(1) +#define GL_HICR_SV BIT(2) +#define GL_HICR_EV BIT(3) + +#define GL_HICR_EN 0x00082044 + +#define GL_HICR_EN_CHECK BIT(0) + +/* Admin Command Interface (ACI) defines */ +/* Defines that help manage the driver vs FW API checks. + */ +#define IXGBE_FW_API_VER_BRANCH 0x00 +#define IXGBE_FW_API_VER_MAJOR 0x01 +#define IXGBE_FW_API_VER_MINOR 0x07 +#define IXGBE_FW_API_VER_DIFF_ALLOWED 0x02 + +#define IXGBE_ACI_DESC_SIZE 32 +#define IXGBE_ACI_DESC_SIZE_IN_DWORDS IXGBE_ACI_DESC_SIZE / BYTES_PER_DWORD + +#define IXGBE_ACI_MAX_BUFFER_SIZE 4096 /* Size in bytes */ +#define IXGBE_ACI_DESC_COOKIE_L_DWORD_OFFSET 3 +#define IXGBE_ACI_SEND_DELAY_TIME_MS 10 +#define IXGBE_ACI_SEND_MAX_EXECUTE 3 +/* [ms] timeout of waiting for sync response */ +#define IXGBE_ACI_SYNC_RESPONSE_TIMEOUT 100000 +/* [ms] timeout of waiting for async response */ +#define IXGBE_ACI_ASYNC_RESPONSE_TIMEOUT 150000 +/* [ms] timeout of waiting for resource release */ +#define IXGBE_ACI_RELEASE_RES_TIMEOUT 10000 + +/* Timestamp spacing for Tools ACI: queue is active if spacing is within the range [LO..HI] */ +#define IXGBE_TOOLS_ACI_ACTIVE_STAMP_SPACING_LO 0 +#define IXGBE_TOOLS_ACI_ACTIVE_STAMP_SPACING_HI 200 + +/* Timestamp spacing for Tools ACI: queue is expired if spacing is outside the range [LO..HI] */ +#define IXGBE_TOOLS_ACI_EXPIRED_STAMP_SPACING_LO -5 +#define IXGBE_TOOLS_ACI_EXPIRED_STAMP_SPACING_HI 205 + +/* FW defined boundary for a large buffer, 4k >= Large buffer > 512 bytes */ +#define IXGBE_ACI_LG_BUF 512 + +/* Flags sub-structure + * |0 |1 |2 |3 |4 |5 |6 |7 |8 |9 |10 |11 |12 |13 |14 |15 | + * |DD |CMP|ERR|VFE| * * RESERVED * * |LB |RD |VFC|BUF|SI |EI |FE | + */ + +/* command flags and offsets */ +#define IXGBE_ACI_FLAG_DD_S 0 +#define IXGBE_ACI_FLAG_CMP_S 1 +#define IXGBE_ACI_FLAG_ERR_S 2 +#define IXGBE_ACI_FLAG_VFE_S 3 +#define IXGBE_ACI_FLAG_LB_S 9 +#define IXGBE_ACI_FLAG_RD_S 10 +#define IXGBE_ACI_FLAG_VFC_S 11 +#define IXGBE_ACI_FLAG_BUF_S 12 +#define IXGBE_ACI_FLAG_SI_S 13 +#define IXGBE_ACI_FLAG_EI_S 14 +#define IXGBE_ACI_FLAG_FE_S 15 + +#define IXGBE_ACI_FLAG_DD BIT(IXGBE_ACI_FLAG_DD_S) /* 0x1 */ +#define IXGBE_ACI_FLAG_CMP BIT(IXGBE_ACI_FLAG_CMP_S) /* 0x2 */ +#define IXGBE_ACI_FLAG_ERR BIT(IXGBE_ACI_FLAG_ERR_S) /* 0x4 */ +#define IXGBE_ACI_FLAG_VFE BIT(IXGBE_ACI_FLAG_VFE_S) /* 0x8 */ +#define IXGBE_ACI_FLAG_LB BIT(IXGBE_ACI_FLAG_LB_S) /* 0x200 */ +#define IXGBE_ACI_FLAG_RD BIT(IXGBE_ACI_FLAG_RD_S) /* 0x400 */ +#define IXGBE_ACI_FLAG_VFC BIT(IXGBE_ACI_FLAG_VFC_S) /* 0x800 */ +#define IXGBE_ACI_FLAG_BUF BIT(IXGBE_ACI_FLAG_BUF_S) /* 0x1000 */ +#define IXGBE_ACI_FLAG_SI BIT(IXGBE_ACI_FLAG_SI_S) /* 0x2000 */ +#define IXGBE_ACI_FLAG_EI BIT(IXGBE_ACI_FLAG_EI_S) /* 0x4000 */ +#define IXGBE_ACI_FLAG_FE BIT(IXGBE_ACI_FLAG_FE_S) /* 0x8000 */ + +/* Admin Command Interface (ACI) error codes */ +enum ixgbe_aci_err { + IXGBE_ACI_RC_OK = 0, /* Success */ + IXGBE_ACI_RC_EPERM = 1, /* Operation not permitted */ + IXGBE_ACI_RC_ENOENT = 2, /* No such element */ + IXGBE_ACI_RC_ESRCH = 3, /* Bad opcode */ + IXGBE_ACI_RC_EINTR = 4, /* Operation interrupted */ + IXGBE_ACI_RC_EIO = 5, /* I/O error */ + IXGBE_ACI_RC_ENXIO = 6, /* No such resource */ + IXGBE_ACI_RC_E2BIG = 7, /* Arg too long */ + IXGBE_ACI_RC_EAGAIN = 8, /* Try again */ + IXGBE_ACI_RC_ENOMEM = 9, /* Out of memory */ + IXGBE_ACI_RC_EACCES = 10, /* Permission denied */ + IXGBE_ACI_RC_EFAULT = 11, /* Bad address */ + IXGBE_ACI_RC_EBUSY = 12, /* Device or resource busy */ + IXGBE_ACI_RC_EEXIST = 13, /* Object already exists */ + IXGBE_ACI_RC_EINVAL = 14, /* Invalid argument */ + IXGBE_ACI_RC_ENOTTY = 15, /* Not a typewriter */ + IXGBE_ACI_RC_ENOSPC = 16, /* No space left or allocation failure */ + IXGBE_ACI_RC_ENOSYS = 17, /* Function not implemented */ + IXGBE_ACI_RC_ERANGE = 18, /* Parameter out of range */ + IXGBE_ACI_RC_EFLUSHED = 19, /* Cmd flushed due to prev cmd error */ + IXGBE_ACI_RC_BAD_ADDR = 20, /* Descriptor contains a bad pointer */ + IXGBE_ACI_RC_EMODE = 21, /* Op not allowed in current dev mode */ + IXGBE_ACI_RC_EFBIG = 22, /* File too big */ + IXGBE_ACI_RC_ESBCOMP = 23, /* SB-IOSF completion unsuccessful */ + IXGBE_ACI_RC_ENOSEC = 24, /* Missing security manifest */ + IXGBE_ACI_RC_EBADSIG = 25, /* Bad RSA signature */ + IXGBE_ACI_RC_ESVN = 26, /* SVN number prohibits this package */ + IXGBE_ACI_RC_EBADMAN = 27, /* Manifest hash mismatch */ + IXGBE_ACI_RC_EBADBUF = 28, /* Buffer hash mismatches manifest */ + IXGBE_ACI_RC_EACCES_BMCU = 29, /* BMC Update in progress */ +}; + +/* Admin Command Interface (ACI) opcodes */ +enum ixgbe_aci_opc { + ixgbe_aci_opc_get_ver = 0x0001, + ixgbe_aci_opc_driver_ver = 0x0002, + ixgbe_aci_opc_get_exp_err = 0x0005, + + /* resource ownership */ + ixgbe_aci_opc_req_res = 0x0008, + ixgbe_aci_opc_release_res = 0x0009, + + /* device/function capabilities */ + ixgbe_aci_opc_list_func_caps = 0x000A, + ixgbe_aci_opc_list_dev_caps = 0x000B, + + /* safe disable of RXEN */ + ixgbe_aci_opc_disable_rxen = 0x000C, + + /* FW events */ + ixgbe_aci_opc_get_fw_event = 0x0014, + + /* PHY commands */ + ixgbe_aci_opc_get_phy_caps = 0x0600, + ixgbe_aci_opc_set_phy_cfg = 0x0601, + ixgbe_aci_opc_restart_an = 0x0605, + ixgbe_aci_opc_get_link_status = 0x0607, + ixgbe_aci_opc_set_event_mask = 0x0613, + ixgbe_aci_opc_get_link_topo = 0x06E0, + ixgbe_aci_opc_read_i2c = 0x06E2, + ixgbe_aci_opc_write_i2c = 0x06E3, + ixgbe_aci_opc_read_mdio = 0x06E4, + ixgbe_aci_opc_write_mdio = 0x06E5, + ixgbe_aci_opc_set_gpio_by_func = 0x06E6, + ixgbe_aci_opc_get_gpio_by_func = 0x06E7, + ixgbe_aci_opc_set_port_id_led = 0x06E9, + ixgbe_aci_opc_set_gpio = 0x06EC, + ixgbe_aci_opc_get_gpio = 0x06ED, + ixgbe_aci_opc_sff_eeprom = 0x06EE, + ixgbe_aci_opc_prog_topo_dev_nvm = 0x06F2, + ixgbe_aci_opc_read_topo_dev_nvm = 0x06F3, + + /* NVM commands */ + ixgbe_aci_opc_nvm_read = 0x0701, + ixgbe_aci_opc_nvm_erase = 0x0702, + ixgbe_aci_opc_nvm_write = 0x0703, + ixgbe_aci_opc_nvm_cfg_read = 0x0704, + ixgbe_aci_opc_nvm_cfg_write = 0x0705, + ixgbe_aci_opc_nvm_checksum = 0x0706, + ixgbe_aci_opc_nvm_write_activate = 0x0707, + ixgbe_aci_opc_nvm_sr_dump = 0x0707, + ixgbe_aci_opc_nvm_save_factory_settings = 0x0708, + ixgbe_aci_opc_nvm_update_empr = 0x0709, + ixgbe_aci_opc_nvm_pkg_data = 0x070A, + ixgbe_aci_opc_nvm_pass_component_tbl = 0x070B, + ixgbe_aci_opc_nvm_sanitization = 0x070C, + + /* Alternate Structure Commands */ + ixgbe_aci_opc_write_alt_direct = 0x0900, + ixgbe_aci_opc_write_alt_indirect = 0x0901, + ixgbe_aci_opc_read_alt_direct = 0x0902, + ixgbe_aci_opc_read_alt_indirect = 0x0903, + ixgbe_aci_opc_done_alt_write = 0x0904, + ixgbe_aci_opc_clear_port_alt_write = 0x0906, + + ixgbe_aci_opc_temp_tca_event = 0x0C94, + + /* debug commands */ + ixgbe_aci_opc_debug_dump_internals = 0xFF08, + + /* SystemDiagnostic commands */ + ixgbe_aci_opc_set_health_status_config = 0xFF20, + ixgbe_aci_opc_get_supported_health_status_codes = 0xFF21, + ixgbe_aci_opc_get_health_status = 0xFF22, + ixgbe_aci_opc_clear_health_status = 0xFF23, + + /* FW Logging Commands */ + ixgbe_aci_opc_fw_logs_config = 0xFF30, + ixgbe_aci_opc_fw_logs_register = 0xFF31, + ixgbe_aci_opc_fw_logs_query = 0xFF32, + ixgbe_aci_opc_fw_logs_event = 0xFF33, + ixgbe_aci_opc_fw_logs_get = 0xFF34, + ixgbe_aci_opc_fw_logs_clear = 0xFF35 +}; + +/* This macro is used to generate a compilation error if a structure + * is not exactly the correct length. It gives a divide by zero error if the + * structure is not of the correct size, otherwise it creates an enum that is + * never used. + */ +#define IXGBE_CHECK_STRUCT_LEN(n, X) enum ixgbe_static_assert_enum_##X \ + { ixgbe_static_assert_##X = (n) / ((sizeof(struct X) == (n)) ? 1 : 0) } + +/* This macro is used to generate a compilation error if a variable-length + * structure is not exactly the correct length assuming a single element of + * the variable-length object as the last element of the structure. It gives + * a divide by zero error if the structure is not of the correct size, + * otherwise it creates an enum that is never used. + */ +#define IXGBE_CHECK_VAR_LEN_STRUCT_LEN(n, X, T) enum ixgbe_static_assert_enum_##X \ + { ixgbe_static_assert_##X = (n) / \ + (((sizeof(struct X) + sizeof(T)) == (n)) ? 1 : 0) } + +/* This macro is used to ensure that parameter structures (i.e. structures + * in the params union member of struct ixgbe_aci_desc) are 16 bytes in length. + * + * NOT intended to be used to check the size of an indirect command/response + * additional data buffer (e.g. struct foo) which should just happen to be 16 + * bytes (instead, use IXGBE_CHECK_STRUCT_LEN(16, foo) for that). + */ +#define IXGBE_CHECK_PARAM_LEN(X) IXGBE_CHECK_STRUCT_LEN(16, X) + +struct ixgbe_aci_cmd_generic { + __le32 param0; + __le32 param1; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_generic); + +/* Get version (direct 0x0001) */ +struct ixgbe_aci_cmd_get_ver { + __le32 rom_ver; + __le32 fw_build; + u8 fw_branch; + u8 fw_major; + u8 fw_minor; + u8 fw_patch; + u8 api_branch; + u8 api_major; + u8 api_minor; + u8 api_patch; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_ver); + +#define IXGBE_DRV_VER_STR_LEN_E610 32 + +struct ixgbe_driver_ver { + u8 major_ver; + u8 minor_ver; + u8 build_ver; + u8 subbuild_ver; + u8 driver_string[IXGBE_DRV_VER_STR_LEN_E610]; +}; + +/* Send driver version (indirect 0x0002) */ +struct ixgbe_aci_cmd_driver_ver { + u8 major_ver; + u8 minor_ver; + u8 build_ver; + u8 subbuild_ver; + u8 reserved[4]; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_driver_ver); + +/* Get Expanded Error Code (0x0005, direct) */ +struct ixgbe_aci_cmd_get_exp_err { + __le32 reason; +#define IXGBE_ACI_EXPANDED_ERROR_NOT_PROVIDED 0xFFFFFFFF + __le32 identifier; + u8 rsvd[8]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_exp_err); + +/* FW update timeout definitions are in milliseconds */ +#define IXGBE_NVM_TIMEOUT 180000 +#define IXGBE_CHANGE_LOCK_TIMEOUT 1000 +#define IXGBE_GLOBAL_CFG_LOCK_TIMEOUT 3000 + +enum ixgbe_aci_res_access_type { + IXGBE_RES_READ = 1, + IXGBE_RES_WRITE +}; + +enum ixgbe_aci_res_ids { + IXGBE_NVM_RES_ID = 1, + IXGBE_SPD_RES_ID, + IXGBE_CHANGE_LOCK_RES_ID, + IXGBE_GLOBAL_CFG_LOCK_RES_ID +}; + +/* Request resource ownership (direct 0x0008) + * Release resource ownership (direct 0x0009) + */ +struct ixgbe_aci_cmd_req_res { + __le16 res_id; +#define IXGBE_ACI_RES_ID_NVM 1 +#define IXGBE_ACI_RES_ID_SDP 2 +#define IXGBE_ACI_RES_ID_CHNG_LOCK 3 +#define IXGBE_ACI_RES_ID_GLBL_LOCK 4 + __le16 access_type; +#define IXGBE_ACI_RES_ACCESS_READ 1 +#define IXGBE_ACI_RES_ACCESS_WRITE 2 + + /* Upon successful completion, FW writes this value and driver is + * expected to release resource before timeout. This value is provided + * in milliseconds. + */ + __le32 timeout; +#define IXGBE_ACI_RES_NVM_READ_DFLT_TIMEOUT_MS 3000 +#define IXGBE_ACI_RES_NVM_WRITE_DFLT_TIMEOUT_MS 180000 +#define IXGBE_ACI_RES_CHNG_LOCK_DFLT_TIMEOUT_MS 1000 +#define IXGBE_ACI_RES_GLBL_LOCK_DFLT_TIMEOUT_MS 3000 + /* For SDP: pin ID of the SDP */ + __le32 res_number; + /* Status is only used for IXGBE_ACI_RES_ID_GLBL_LOCK */ + __le16 status; +#define IXGBE_ACI_RES_GLBL_SUCCESS 0 +#define IXGBE_ACI_RES_GLBL_IN_PROG 1 +#define IXGBE_ACI_RES_GLBL_DONE 2 + u8 reserved[2]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_req_res); + +/* Get function capabilities (indirect 0x000A) + * Get device capabilities (indirect 0x000B) + */ +struct ixgbe_aci_cmd_list_caps { + u8 cmd_flags; + u8 pf_index; + u8 reserved[2]; + __le32 count; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_list_caps); + +/* Device/Function buffer entry, repeated per reported capability */ +struct ixgbe_aci_cmd_list_caps_elem { + __le16 cap; +#define IXGBE_ACI_CAPS_VALID_FUNCTIONS 0x0005 +#define IXGBE_ACI_MAX_VALID_FUNCTIONS 0x8 +#define IXGBE_ACI_CAPS_SRIOV 0x0012 +#define IXGBE_ACI_CAPS_VF 0x0013 +#define IXGBE_ACI_CAPS_VMDQ 0x0014 +#define IXGBE_ACI_CAPS_VSI 0x0017 +#define IXGBE_ACI_CAPS_DCB 0x0018 +#define IXGBE_ACI_CAPS_RSS 0x0040 +#define IXGBE_ACI_CAPS_RXQS 0x0041 +#define IXGBE_ACI_CAPS_TXQS 0x0042 +#define IXGBE_ACI_CAPS_MSIX 0x0043 +#define IXGBE_ACI_CAPS_FD 0x0045 +#define IXGBE_ACI_CAPS_MAX_MTU 0x0047 +#define IXGBE_ACI_CAPS_NVM_VER 0x0048 +#define IXGBE_ACI_CAPS_INLINE_IPSEC 0x0070 +#define IXGBE_ACI_CAPS_NUM_ENABLED_PORTS 0x0072 +#define IXGBE_ACI_CAPS_PCIE_RESET_AVOIDANCE 0x0076 +#define IXGBE_ACI_CAPS_POST_UPDATE_RESET_RESTRICT 0x0077 +#define IXGBE_ACI_CAPS_NVM_MGMT 0x0080 +#define IXGBE_ACI_CAPS_EXT_TOPO_DEV_IMG0 0x0081 +#define IXGBE_ACI_CAPS_EXT_TOPO_DEV_IMG1 0x0082 +#define IXGBE_ACI_CAPS_EXT_TOPO_DEV_IMG2 0x0083 +#define IXGBE_ACI_CAPS_EXT_TOPO_DEV_IMG3 0x0084 +#define IXGBE_ACI_CAPS_OROM_RECOVERY_UPDATE 0x0090 +#define IXGBE_ACI_CAPS_NEXT_CLUSTER_ID 0x0096 + u8 major_ver; + u8 minor_ver; + /* Number of resources described by this capability */ + __le32 number; + /* Only meaningful for some types of resources */ + __le32 logical_id; + /* Only meaningful for some types of resources */ + __le32 phys_id; + __le64 rsvd1; + __le64 rsvd2; +}; + +IXGBE_CHECK_STRUCT_LEN(32, ixgbe_aci_cmd_list_caps_elem); + +/* Disable RXEN (direct 0x000C) */ +struct ixgbe_aci_cmd_disable_rxen { + u8 lport_num; + u8 reserved[15]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_disable_rxen); + +/* Get FW Event (indirect 0x0014) */ +struct ixgbe_aci_cmd_get_fw_event { + __le16 fw_buf_status; +#define IXGBE_ACI_GET_FW_EVENT_STATUS_OBTAINED BIT(0) +#define IXGBE_ACI_GET_FW_EVENT_STATUS_PENDING BIT(1) + u8 rsvd[14]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_fw_event); + +/* Get PHY capabilities (indirect 0x0600) */ +struct ixgbe_aci_cmd_get_phy_caps { + u8 lport_num; + u8 reserved; + __le16 param0; + /* 18.0 - Report qualified modules */ +#define IXGBE_ACI_GET_PHY_RQM BIT(0) + /* 18.1 - 18.3 : Report mode + * 000b - Report topology capabilities, without media + * 001b - Report topology capabilities, with media + * 010b - Report Active configuration + * 011b - Report PHY Type and FEC mode capabilities + * 100b - Report Default capabilities + */ +#define IXGBE_ACI_REPORT_MODE_S 1 +#define IXGBE_ACI_REPORT_MODE_M (7 << IXGBE_ACI_REPORT_MODE_S) +#define IXGBE_ACI_REPORT_TOPO_CAP_NO_MEDIA 0 +#define IXGBE_ACI_REPORT_TOPO_CAP_MEDIA BIT(1) +#define IXGBE_ACI_REPORT_ACTIVE_CFG BIT(2) +#define IXGBE_ACI_REPORT_DFLT_CFG BIT(3) + __le32 reserved1; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_phy_caps); + +/* This is #define of PHY type (Extended): + * The first set of defines is for phy_type_low. + */ +#define IXGBE_PHY_TYPE_LOW_100BASE_TX BIT_ULL(0) +#define IXGBE_PHY_TYPE_LOW_100M_SGMII BIT_ULL(1) +#define IXGBE_PHY_TYPE_LOW_1000BASE_T BIT_ULL(2) +#define IXGBE_PHY_TYPE_LOW_1000BASE_SX BIT_ULL(3) +#define IXGBE_PHY_TYPE_LOW_1000BASE_LX BIT_ULL(4) +#define IXGBE_PHY_TYPE_LOW_1000BASE_KX BIT_ULL(5) +#define IXGBE_PHY_TYPE_LOW_1G_SGMII BIT_ULL(6) +#define IXGBE_PHY_TYPE_LOW_2500BASE_T BIT_ULL(7) +#define IXGBE_PHY_TYPE_LOW_2500BASE_X BIT_ULL(8) +#define IXGBE_PHY_TYPE_LOW_2500BASE_KX BIT_ULL(9) +#define IXGBE_PHY_TYPE_LOW_5GBASE_T BIT_ULL(10) +#define IXGBE_PHY_TYPE_LOW_5GBASE_KR BIT_ULL(11) +#define IXGBE_PHY_TYPE_LOW_10GBASE_T BIT_ULL(12) +#define IXGBE_PHY_TYPE_LOW_10G_SFI_DA BIT_ULL(13) +#define IXGBE_PHY_TYPE_LOW_10GBASE_SR BIT_ULL(14) +#define IXGBE_PHY_TYPE_LOW_10GBASE_LR BIT_ULL(15) +#define IXGBE_PHY_TYPE_LOW_10GBASE_KR_CR1 BIT_ULL(16) +#define IXGBE_PHY_TYPE_LOW_10G_SFI_AOC_ACC BIT_ULL(17) +#define IXGBE_PHY_TYPE_LOW_10G_SFI_C2C BIT_ULL(18) +#define IXGBE_PHY_TYPE_LOW_MAX_INDEX 18 +/* The second set of defines is for phy_type_high. */ +#define IXGBE_PHY_TYPE_HIGH_10BASE_T BIT_ULL(1) +#define IXGBE_PHY_TYPE_HIGH_10M_SGMII BIT_ULL(2) +#define IXGBE_PHY_TYPE_HIGH_2500M_SGMII BIT_ULL(56) +#define IXGBE_PHY_TYPE_HIGH_100M_USXGMII BIT_ULL(57) +#define IXGBE_PHY_TYPE_HIGH_1G_USXGMII BIT_ULL(58) +#define IXGBE_PHY_TYPE_HIGH_2500M_USXGMII BIT_ULL(59) +#define IXGBE_PHY_TYPE_HIGH_5G_USXGMII BIT_ULL(60) +#define IXGBE_PHY_TYPE_HIGH_10G_USXGMII BIT_ULL(61) +#define IXGBE_PHY_TYPE_HIGH_MAX_INDEX 61 + +struct ixgbe_aci_cmd_get_phy_caps_data { + __le64 phy_type_low; /* Use values from IXGBE_PHY_TYPE_LOW_* */ + __le64 phy_type_high; /* Use values from IXGBE_PHY_TYPE_HIGH_* */ + u8 caps; +#define IXGBE_ACI_PHY_EN_TX_LINK_PAUSE BIT(0) +#define IXGBE_ACI_PHY_EN_RX_LINK_PAUSE BIT(1) +#define IXGBE_ACI_PHY_LOW_POWER_MODE BIT(2) +#define IXGBE_ACI_PHY_EN_LINK BIT(3) +#define IXGBE_ACI_PHY_AN_MODE BIT(4) +#define IXGBE_ACI_PHY_EN_MOD_QUAL BIT(5) +#define IXGBE_ACI_PHY_EN_LESM BIT(6) +#define IXGBE_ACI_PHY_EN_AUTO_FEC BIT(7) +#define IXGBE_ACI_PHY_CAPS_MASK MAKEMASK(0xff, 0) + u8 low_power_ctrl_an; +#define IXGBE_ACI_PHY_EN_D3COLD_LOW_POWER_AUTONEG BIT(0) +#define IXGBE_ACI_PHY_AN_EN_CLAUSE28 BIT(1) +#define IXGBE_ACI_PHY_AN_EN_CLAUSE73 BIT(2) +#define IXGBE_ACI_PHY_AN_EN_CLAUSE37 BIT(3) + __le16 eee_cap; +#define IXGBE_ACI_PHY_EEE_EN_100BASE_TX BIT(0) +#define IXGBE_ACI_PHY_EEE_EN_1000BASE_T BIT(1) +#define IXGBE_ACI_PHY_EEE_EN_10GBASE_T BIT(2) +#define IXGBE_ACI_PHY_EEE_EN_1000BASE_KX BIT(3) +#define IXGBE_ACI_PHY_EEE_EN_10GBASE_KR BIT(4) +#define IXGBE_ACI_PHY_EEE_EN_25GBASE_KR BIT(5) +#define IXGBE_ACI_PHY_EEE_EN_10BASE_T BIT(11) + __le16 eeer_value; + u8 phy_id_oui[4]; /* PHY/Module ID connected on the port */ + u8 phy_fw_ver[8]; + u8 link_fec_options; +#define IXGBE_ACI_PHY_FEC_10G_KR_40G_KR4_EN BIT(0) +#define IXGBE_ACI_PHY_FEC_10G_KR_40G_KR4_REQ BIT(1) +#define IXGBE_ACI_PHY_FEC_25G_RS_528_REQ BIT(2) +#define IXGBE_ACI_PHY_FEC_25G_KR_REQ BIT(3) +#define IXGBE_ACI_PHY_FEC_25G_RS_544_REQ BIT(4) +#define IXGBE_ACI_PHY_FEC_25G_RS_CLAUSE91_EN BIT(6) +#define IXGBE_ACI_PHY_FEC_25G_KR_CLAUSE74_EN BIT(7) +#define IXGBE_ACI_PHY_FEC_MASK MAKEMASK(0xdf, 0) + u8 module_compliance_enforcement; +#define IXGBE_ACI_MOD_ENFORCE_STRICT_MODE BIT(0) + u8 extended_compliance_code; +#define IXGBE_ACI_MODULE_TYPE_TOTAL_BYTE 3 + u8 module_type[IXGBE_ACI_MODULE_TYPE_TOTAL_BYTE]; +#define IXGBE_ACI_MOD_TYPE_BYTE0_SFP_PLUS 0xA0 +#define IXGBE_ACI_MOD_TYPE_BYTE0_QSFP_PLUS 0x80 +#define IXGBE_ACI_MOD_TYPE_IDENT 1 +#define IXGBE_ACI_MOD_TYPE_BYTE1_SFP_PLUS_CU_PASSIVE BIT(0) +#define IXGBE_ACI_MOD_TYPE_BYTE1_SFP_PLUS_CU_ACTIVE BIT(1) +#define IXGBE_ACI_MOD_TYPE_BYTE1_10G_BASE_SR BIT(4) +#define IXGBE_ACI_MOD_TYPE_BYTE1_10G_BASE_LR BIT(5) +#define IXGBE_ACI_MOD_TYPE_BYTE1_10G_BASE_LRM BIT(6) +#define IXGBE_ACI_MOD_TYPE_BYTE1_10G_BASE_ER BIT(7) +#define IXGBE_ACI_MOD_TYPE_BYTE2_SFP_PLUS 0xA0 +#define IXGBE_ACI_MOD_TYPE_BYTE2_QSFP_PLUS 0x86 + u8 qualified_module_count; + u8 rsvd2[7]; /* Bytes 47:41 reserved */ +#define IXGBE_ACI_QUAL_MOD_COUNT_MAX 16 + struct { + u8 v_oui[3]; + u8 rsvd3; + u8 v_part[16]; + __le32 v_rev; + __le64 rsvd4; + } qual_modules[IXGBE_ACI_QUAL_MOD_COUNT_MAX]; +}; + +IXGBE_CHECK_STRUCT_LEN(560, ixgbe_aci_cmd_get_phy_caps_data); + +/* Set PHY capabilities (direct 0x0601) + * NOTE: This command must be followed by setup link and restart auto-neg + */ +struct ixgbe_aci_cmd_set_phy_cfg { + u8 reserved[8]; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_set_phy_cfg); + +/* Set PHY config command data structure */ +struct ixgbe_aci_cmd_set_phy_cfg_data { + __le64 phy_type_low; /* Use values from IXGBE_PHY_TYPE_LOW_* */ + __le64 phy_type_high; /* Use values from IXGBE_PHY_TYPE_HIGH_* */ + u8 caps; +#define IXGBE_ACI_PHY_ENA_VALID_MASK MAKEMASK(0xef, 0) +#define IXGBE_ACI_PHY_ENA_TX_PAUSE_ABILITY BIT(0) +#define IXGBE_ACI_PHY_ENA_RX_PAUSE_ABILITY BIT(1) +#define IXGBE_ACI_PHY_ENA_LOW_POWER BIT(2) +#define IXGBE_ACI_PHY_ENA_LINK BIT(3) +#define IXGBE_ACI_PHY_ENA_AUTO_LINK_UPDT BIT(5) +#define IXGBE_ACI_PHY_ENA_LESM BIT(6) +#define IXGBE_ACI_PHY_ENA_AUTO_FEC BIT(7) + u8 low_power_ctrl_an; + __le16 eee_cap; /* Value from ixgbe_aci_get_phy_caps */ + __le16 eeer_value; /* Use defines from ixgbe_aci_get_phy_caps */ + u8 link_fec_opt; /* Use defines from ixgbe_aci_get_phy_caps */ + u8 module_compliance_enforcement; +}; + +IXGBE_CHECK_STRUCT_LEN(24, ixgbe_aci_cmd_set_phy_cfg_data); + +/* Restart AN command data structure (direct 0x0605) + * Also used for response, with only the lport_num field present. + */ +struct ixgbe_aci_cmd_restart_an { + u8 reserved[2]; + u8 cmd_flags; +#define IXGBE_ACI_RESTART_AN_LINK_RESTART BIT(1) +#define IXGBE_ACI_RESTART_AN_LINK_ENABLE BIT(2) + u8 reserved2[13]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_restart_an); + +#pragma pack(1) +/* Get link status (indirect 0x0607), also used for Link Status Event */ +struct ixgbe_aci_cmd_get_link_status { + u8 reserved[2]; + u8 cmd_flags; +#define IXGBE_ACI_LSE_M 0x3 +#define IXGBE_ACI_LSE_NOP 0x0 +#define IXGBE_ACI_LSE_DIS 0x2 +#define IXGBE_ACI_LSE_ENA 0x3 + /* only response uses this flag */ +#define IXGBE_ACI_LSE_IS_ENABLED 0x1 + u8 reserved2[5]; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_link_status); + +/* Get link status response data structure, also used for Link Status Event */ +struct ixgbe_aci_cmd_get_link_status_data { + u8 topo_media_conflict; +#define IXGBE_ACI_LINK_TOPO_CONFLICT BIT(0) +#define IXGBE_ACI_LINK_MEDIA_CONFLICT BIT(1) +#define IXGBE_ACI_LINK_TOPO_CORRUPT BIT(2) +#define IXGBE_ACI_LINK_TOPO_UNREACH_PRT BIT(4) +#define IXGBE_ACI_LINK_TOPO_UNDRUTIL_PRT BIT(5) +#define IXGBE_ACI_LINK_TOPO_UNDRUTIL_MEDIA BIT(6) +#define IXGBE_ACI_LINK_TOPO_UNSUPP_MEDIA BIT(7) + u8 link_cfg_err; +#define IXGBE_ACI_LINK_CFG_ERR BIT(0) +#define IXGBE_ACI_LINK_CFG_COMPLETED BIT(1) +#define IXGBE_ACI_LINK_ACT_PORT_OPT_INVAL BIT(2) +#define IXGBE_ACI_LINK_FEAT_ID_OR_CONFIG_ID_INVAL BIT(3) +#define IXGBE_ACI_LINK_TOPO_CRITICAL_SDP_ERR BIT(4) +#define IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED BIT(5) +#define IXGBE_ACI_LINK_EXTERNAL_PHY_LOAD_FAILURE BIT(6) +#define IXGBE_ACI_LINK_INVAL_MAX_POWER_LIMIT BIT(7) + u8 link_info; +#define IXGBE_ACI_LINK_UP BIT(0) /* Link Status */ +#define IXGBE_ACI_LINK_FAULT BIT(1) +#define IXGBE_ACI_LINK_FAULT_TX BIT(2) +#define IXGBE_ACI_LINK_FAULT_RX BIT(3) +#define IXGBE_ACI_LINK_FAULT_REMOTE BIT(4) +#define IXGBE_ACI_LINK_UP_PORT BIT(5) /* External Port Link Status */ +#define IXGBE_ACI_MEDIA_AVAILABLE BIT(6) +#define IXGBE_ACI_SIGNAL_DETECT BIT(7) + u8 an_info; +#define IXGBE_ACI_AN_COMPLETED BIT(0) +#define IXGBE_ACI_LP_AN_ABILITY BIT(1) +#define IXGBE_ACI_PD_FAULT BIT(2) /* Parallel Detection Fault */ +#define IXGBE_ACI_FEC_EN BIT(3) +#define IXGBE_ACI_PHY_LOW_POWER BIT(4) /* Low Power State */ +#define IXGBE_ACI_LINK_PAUSE_TX BIT(5) +#define IXGBE_ACI_LINK_PAUSE_RX BIT(6) +#define IXGBE_ACI_QUALIFIED_MODULE BIT(7) + u8 ext_info; +#define IXGBE_ACI_LINK_PHY_TEMP_ALARM BIT(0) +#define IXGBE_ACI_LINK_EXCESSIVE_ERRORS BIT(1) /* Excessive Link Errors */ + /* Port Tx Suspended */ +#define IXGBE_ACI_LINK_TX_S 2 +#define IXGBE_ACI_LINK_TX_M (0x03 << IXGBE_ACI_LINK_TX_S) +#define IXGBE_ACI_LINK_TX_ACTIVE 0 +#define IXGBE_ACI_LINK_TX_DRAINED 1 +#define IXGBE_ACI_LINK_TX_FLUSHED 3 + u8 lb_status; +#define IXGBE_ACI_LINK_LB_PHY_LCL BIT(0) +#define IXGBE_ACI_LINK_LB_PHY_RMT BIT(1) +#define IXGBE_ACI_LINK_LB_MAC_LCL BIT(2) +#define IXGBE_ACI_LINK_LB_PHY_IDX_S 3 +#define IXGBE_ACI_LINK_LB_PHY_IDX_M (0x7 << IXGBE_ACI_LB_PHY_IDX_S) + __le16 max_frame_size; + u8 cfg; +#define IXGBE_ACI_LINK_25G_KR_FEC_EN BIT(0) +#define IXGBE_ACI_LINK_25G_RS_528_FEC_EN BIT(1) +#define IXGBE_ACI_LINK_25G_RS_544_FEC_EN BIT(2) +#define IXGBE_ACI_FEC_MASK MAKEMASK(0x7, 0) + /* Pacing Config */ +#define IXGBE_ACI_CFG_PACING_S 3 +#define IXGBE_ACI_CFG_PACING_M (0xF << IXGBE_ACI_CFG_PACING_S) +#define IXGBE_ACI_CFG_PACING_TYPE_M BIT(7) +#define IXGBE_ACI_CFG_PACING_TYPE_AVG 0 +#define IXGBE_ACI_CFG_PACING_TYPE_FIXED IXGBE_ACI_CFG_PACING_TYPE_M + /* External Device Power Ability */ + u8 power_desc; +#define IXGBE_ACI_PWR_CLASS_M 0x3F +#define IXGBE_ACI_LINK_PWR_BASET_LOW_HIGH 0 +#define IXGBE_ACI_LINK_PWR_BASET_HIGH 1 +#define IXGBE_ACI_LINK_PWR_QSFP_CLASS_1 0 +#define IXGBE_ACI_LINK_PWR_QSFP_CLASS_2 1 +#define IXGBE_ACI_LINK_PWR_QSFP_CLASS_3 2 +#define IXGBE_ACI_LINK_PWR_QSFP_CLASS_4 3 + __le16 link_speed; +#define IXGBE_ACI_LINK_SPEED_M 0x7FF +#define IXGBE_ACI_LINK_SPEED_10MB BIT(0) +#define IXGBE_ACI_LINK_SPEED_100MB BIT(1) +#define IXGBE_ACI_LINK_SPEED_1000MB BIT(2) +#define IXGBE_ACI_LINK_SPEED_2500MB BIT(3) +#define IXGBE_ACI_LINK_SPEED_5GB BIT(4) +#define IXGBE_ACI_LINK_SPEED_10GB BIT(5) +#define IXGBE_ACI_LINK_SPEED_20GB BIT(6) +#define IXGBE_ACI_LINK_SPEED_25GB BIT(7) +#define IXGBE_ACI_LINK_SPEED_40GB BIT(8) +#define IXGBE_ACI_LINK_SPEED_50GB BIT(9) +#define IXGBE_ACI_LINK_SPEED_100GB BIT(10) +#define IXGBE_ACI_LINK_SPEED_200GB BIT(11) +#define IXGBE_ACI_LINK_SPEED_UNKNOWN BIT(15) + __le16 reserved3; /* Aligns next field to 8-byte boundary */ + u8 ext_fec_status; +#define IXGBE_ACI_LINK_RS_272_FEC_EN BIT(0) /* RS 272 FEC enabled */ + u8 reserved4; + __le64 phy_type_low; /* Use values from IXGBE_PHY_TYPE_LOW_* */ + __le64 phy_type_high; /* Use values from IXGBE_PHY_TYPE_HIGH_* */ + /* Get link status version 2 link partner data */ + __le64 lp_phy_type_low; /* Use values from IXGBE_PHY_TYPE_LOW_* */ + __le64 lp_phy_type_high; /* Use values from IXGBE_PHY_TYPE_HIGH_* */ + u8 lp_fec_adv; +#define IXGBE_ACI_LINK_LP_10G_KR_FEC_CAP BIT(0) +#define IXGBE_ACI_LINK_LP_25G_KR_FEC_CAP BIT(1) +#define IXGBE_ACI_LINK_LP_RS_528_FEC_CAP BIT(2) +#define IXGBE_ACI_LINK_LP_50G_KR_272_FEC_CAP BIT(3) +#define IXGBE_ACI_LINK_LP_100G_KR_272_FEC_CAP BIT(4) +#define IXGBE_ACI_LINK_LP_200G_KR_272_FEC_CAP BIT(5) + u8 lp_fec_req; +#define IXGBE_ACI_LINK_LP_10G_KR_FEC_REQ BIT(0) +#define IXGBE_ACI_LINK_LP_25G_KR_FEC_REQ BIT(1) +#define IXGBE_ACI_LINK_LP_RS_528_FEC_REQ BIT(2) +#define IXGBE_ACI_LINK_LP_KR_272_FEC_REQ BIT(3) + u8 lp_flowcontrol; +#define IXGBE_ACI_LINK_LP_PAUSE_ADV BIT(0) +#define IXGBE_ACI_LINK_LP_ASM_DIR_ADV BIT(1) + u8 reserved5[5]; +}; +#pragma pack() + +IXGBE_CHECK_STRUCT_LEN(56, ixgbe_aci_cmd_get_link_status_data); + +/* Set event mask command (direct 0x0613) */ +struct ixgbe_aci_cmd_set_event_mask { + u8 reserved[8]; + __le16 event_mask; +#define IXGBE_ACI_LINK_EVENT_UPDOWN BIT(1) +#define IXGBE_ACI_LINK_EVENT_MEDIA_NA BIT(2) +#define IXGBE_ACI_LINK_EVENT_LINK_FAULT BIT(3) +#define IXGBE_ACI_LINK_EVENT_PHY_TEMP_ALARM BIT(4) +#define IXGBE_ACI_LINK_EVENT_EXCESSIVE_ERRORS BIT(5) +#define IXGBE_ACI_LINK_EVENT_SIGNAL_DETECT BIT(6) +#define IXGBE_ACI_LINK_EVENT_AN_COMPLETED BIT(7) +#define IXGBE_ACI_LINK_EVENT_MODULE_QUAL_FAIL BIT(8) +#define IXGBE_ACI_LINK_EVENT_PORT_TX_SUSPENDED BIT(9) +#define IXGBE_ACI_LINK_EVENT_TOPO_CONFLICT BIT(10) +#define IXGBE_ACI_LINK_EVENT_MEDIA_CONFLICT BIT(11) +#define IXGBE_ACI_LINK_EVENT_PHY_FW_LOAD_FAIL BIT(12) + u8 reserved1[6]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_set_event_mask); + +struct ixgbe_aci_cmd_link_topo_params { + u8 lport_num; + u8 lport_num_valid; +#define IXGBE_ACI_LINK_TOPO_PORT_NUM_VALID BIT(0) + u8 node_type_ctx; +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_S 0 +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_M (0xF << IXGBE_ACI_LINK_TOPO_NODE_TYPE_S) +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_PHY 0 +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_GPIO_CTRL 1 +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_MUX_CTRL 2 +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_LED_CTRL 3 +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_LED 4 +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_THERMAL 5 +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_CAGE 6 +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_MEZZ 7 +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_ID_EEPROM 8 +#define IXGBE_ACI_LINK_TOPO_NODE_TYPE_GPS 11 +#define IXGBE_ACI_LINK_TOPO_NODE_CTX_S 4 +#define IXGBE_ACI_LINK_TOPO_NODE_CTX_M \ + (0xF << IXGBE_ACI_LINK_TOPO_NODE_CTX_S) +#define IXGBE_ACI_LINK_TOPO_NODE_CTX_GLOBAL 0 +#define IXGBE_ACI_LINK_TOPO_NODE_CTX_BOARD 1 +#define IXGBE_ACI_LINK_TOPO_NODE_CTX_PORT 2 +#define IXGBE_ACI_LINK_TOPO_NODE_CTX_NODE 3 +#define IXGBE_ACI_LINK_TOPO_NODE_CTX_NODE_HANDLE 4 +#define IXGBE_ACI_LINK_TOPO_NODE_CTX_DIRECT_BUS_ACCESS 5 +#define IXGBE_ACI_LINK_TOPO_NODE_CTX_NODE_HANDLE_BUS_ADDRESS 6 + u8 index; +}; + +IXGBE_CHECK_STRUCT_LEN(4, ixgbe_aci_cmd_link_topo_params); + +struct ixgbe_aci_cmd_link_topo_addr { + struct ixgbe_aci_cmd_link_topo_params topo_params; + __le16 handle; +#define IXGBE_ACI_LINK_TOPO_HANDLE_S 0 +#define IXGBE_ACI_LINK_TOPO_HANDLE_M (0x3FF << IXGBE_ACI_LINK_TOPO_HANDLE_S) +/* Used to decode the handle field */ +#define IXGBE_ACI_LINK_TOPO_HANDLE_BRD_TYPE_M BIT(9) +#define IXGBE_ACI_LINK_TOPO_HANDLE_BRD_TYPE_LOM BIT(9) +#define IXGBE_ACI_LINK_TOPO_HANDLE_BRD_TYPE_MEZZ 0 +#define IXGBE_ACI_LINK_TOPO_HANDLE_NODE_S 0 +/* In case of a Mezzanine type */ +#define IXGBE_ACI_LINK_TOPO_HANDLE_MEZZ_NODE_M \ + (0x3F << IXGBE_ACI_LINK_TOPO_HANDLE_NODE_S) +#define IXGBE_ACI_LINK_TOPO_HANDLE_MEZZ_S 6 +#define IXGBE_ACI_LINK_TOPO_HANDLE_MEZZ_M \ + (0x7 << IXGBE_ACI_LINK_TOPO_HANDLE_MEZZ_S) +/* In case of a LOM type */ +#define IXGBE_ACI_LINK_TOPO_HANDLE_LOM_NODE_M \ + (0x1FF << IXGBE_ACI_LINK_TOPO_HANDLE_NODE_S) +}; + +IXGBE_CHECK_STRUCT_LEN(6, ixgbe_aci_cmd_link_topo_addr); + +/* Get Link Topology Handle (direct, 0x06E0) */ +struct ixgbe_aci_cmd_get_link_topo { + struct ixgbe_aci_cmd_link_topo_addr addr; + u8 node_part_num; +#define IXGBE_ACI_GET_LINK_TOPO_NODE_NR_PCA9575 0x21 +#define IXGBE_ACI_GET_LINK_TOPO_NODE_NR_GEN_GPS 0x48 +#define IXGBE_ACI_GET_LINK_TOPO_NODE_NR_E610_PTC 0x49 + u8 rsvd[9]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_link_topo); + +/* Read/Write I2C (direct, 0x06E2/0x06E3) */ +struct ixgbe_aci_cmd_i2c { + struct ixgbe_aci_cmd_link_topo_addr topo_addr; + __le16 i2c_addr; + u8 i2c_params; +#define IXGBE_ACI_I2C_DATA_SIZE_S 0 +#define IXGBE_ACI_I2C_DATA_SIZE_M (0xF << IXGBE_ACI_I2C_DATA_SIZE_S) +#define IXGBE_ACI_I2C_ADDR_TYPE_M BIT(4) +#define IXGBE_ACI_I2C_ADDR_TYPE_7BIT 0 +#define IXGBE_ACI_I2C_ADDR_TYPE_10BIT IXGBE_ACI_I2C_ADDR_TYPE_M +#define IXGBE_ACI_I2C_DATA_OFFSET_S 5 +#define IXGBE_ACI_I2C_DATA_OFFSET_M (0x3 << IXGBE_ACI_I2C_DATA_OFFSET_S) +#define IXGBE_ACI_I2C_USE_REPEATED_START BIT(7) + u8 rsvd; + __le16 i2c_bus_addr; +#define IXGBE_ACI_I2C_ADDR_7BIT_MASK 0x7F +#define IXGBE_ACI_I2C_ADDR_10BIT_MASK 0x3FF + u8 i2c_data[4]; /* Used only by write command, reserved in read. */ +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_i2c); + +/* Read I2C Response (direct, 0x06E2) */ +struct ixgbe_aci_cmd_read_i2c_resp { + u8 i2c_data[16]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_read_i2c_resp); + +/* Read/Write MDIO (direct, 0x06E4/0x06E5) */ +struct ixgbe_aci_cmd_mdio { + struct ixgbe_aci_cmd_link_topo_addr topo_addr; + u8 mdio_device_addr; +#define IXGBE_ACI_MDIO_DEV_S 0 +#define IXGBE_ACI_MDIO_DEV_M (0x1F << IXGBE_ACI_MDIO_DEV_S) +#define IXGBE_ACI_MDIO_CLAUSE_22 BIT(5) +#define IXGBE_ACI_MDIO_CLAUSE_45 BIT(6) + u8 mdio_bus_address; +#define IXGBE_ACI_MDIO_BUS_ADDR_S 0 +#define IXGBE_ACI_MDIO_BUS_ADDR_M (0x1F << IXGBE_ACI_MDIO_BUS_ADDR_S) + __le16 offset; + __le16 data; /* Input in write cmd, output in read cmd. */ + u8 rsvd1[4]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_mdio); + +/* Set/Get GPIO By Function (direct, 0x06E6/0x06E7) */ +struct ixgbe_aci_cmd_gpio_by_func { + struct ixgbe_aci_cmd_link_topo_addr topo_addr; + u8 io_func_num; +#define IXGBE_ACI_GPIO_FUNC_S 0 +#define IXGBE_ACI_GPIO_FUNC_M (0x1F << IXGBE_ACI_GPIO_IO_FUNC_NUM_S) + u8 io_value; /* Input in write cmd, output in read cmd. */ +#define IXGBE_ACI_GPIO_ON BIT(0) +#define IXGBE_ACI_GPIO_OFF 0 + u8 rsvd[8]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_gpio_by_func); + +/* Set Port Identification LED (direct, 0x06E9) */ +struct ixgbe_aci_cmd_set_port_id_led { + u8 lport_num; + u8 lport_num_valid; +#define IXGBE_ACI_PORT_ID_PORT_NUM_VALID BIT(0) + u8 ident_mode; +#define IXGBE_ACI_PORT_IDENT_LED_BLINK BIT(0) +#define IXGBE_ACI_PORT_IDENT_LED_ORIG 0 + u8 rsvd[13]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_set_port_id_led); + +/* Set/Get GPIO (direct, 0x06EC/0x06ED) */ +struct ixgbe_aci_cmd_gpio { + __le16 gpio_ctrl_handle; +#define IXGBE_ACI_GPIO_HANDLE_S 0 +#define IXGBE_ACI_GPIO_HANDLE_M (0x3FF << IXGBE_ACI_GPIO_HANDLE_S) + u8 gpio_num; + u8 gpio_val; + u8 rsvd[12]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_gpio); + +/* Read/Write SFF EEPROM command (indirect 0x06EE) */ +struct ixgbe_aci_cmd_sff_eeprom { + u8 lport_num; + u8 lport_num_valid; +#define IXGBE_ACI_SFF_PORT_NUM_VALID BIT(0) + __le16 i2c_bus_addr; +#define IXGBE_ACI_SFF_I2CBUS_7BIT_M 0x7F +#define IXGBE_ACI_SFF_I2CBUS_10BIT_M 0x3FF +#define IXGBE_ACI_SFF_I2CBUS_TYPE_M BIT(10) +#define IXGBE_ACI_SFF_I2CBUS_TYPE_7BIT 0 +#define IXGBE_ACI_SFF_I2CBUS_TYPE_10BIT IXGBE_ACI_SFF_I2CBUS_TYPE_M +#define IXGBE_ACI_SFF_PAGE_BANK_CTRL_S 11 +#define IXGBE_ACI_SFF_PAGE_BANK_CTRL_M (0x3 << IXGBE_ACI_SFF_PAGE_BANK_CTRL_S) +#define IXGBE_ACI_SFF_NO_PAGE_BANK_UPDATE 0 +#define IXGBE_ACI_SFF_UPDATE_PAGE 1 +#define IXGBE_ACI_SFF_UPDATE_BANK 2 +#define IXGBE_ACI_SFF_UPDATE_PAGE_BANK 3 +#define IXGBE_ACI_SFF_IS_WRITE BIT(15) + __le16 i2c_offset; + u8 module_bank; + u8 module_page; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_sff_eeprom); + +/* Program Topology Device NVM (direct, 0x06F2) */ +struct ixgbe_aci_cmd_prog_topo_dev_nvm { + struct ixgbe_aci_cmd_link_topo_params topo_params; + u8 rsvd[12]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_prog_topo_dev_nvm); + +/* Read Topology Device NVM (direct, 0x06F3) */ +struct ixgbe_aci_cmd_read_topo_dev_nvm { + struct ixgbe_aci_cmd_link_topo_params topo_params; + __le32 start_address; +#define IXGBE_ACI_READ_TOPO_DEV_NVM_DATA_READ_SIZE 8 + u8 data_read[IXGBE_ACI_READ_TOPO_DEV_NVM_DATA_READ_SIZE]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_read_topo_dev_nvm); + +/* NVM Read command (indirect 0x0701) + * NVM Erase commands (direct 0x0702) + * NVM Write commands (indirect 0x0703) + * NVM Write Activate commands (direct 0x0707) + * NVM Shadow RAM Dump commands (direct 0x0707) + */ +struct ixgbe_aci_cmd_nvm { +#define IXGBE_ACI_NVM_MAX_OFFSET 0xFFFFFF + __le16 offset_low; + u8 offset_high; /* For Write Activate offset_high is used as flags2 */ + u8 cmd_flags; +#define IXGBE_ACI_NVM_LAST_CMD BIT(0) +#define IXGBE_ACI_NVM_PCIR_REQ BIT(0) /* Used by NVM Write reply */ +#define IXGBE_ACI_NVM_PRESERVATION_S 1 /* Used by NVM Write Activate only */ +#define IXGBE_ACI_NVM_PRESERVATION_M (3 << IXGBE_ACI_NVM_PRESERVATION_S) +#define IXGBE_ACI_NVM_NO_PRESERVATION (0 << IXGBE_ACI_NVM_PRESERVATION_S) +#define IXGBE_ACI_NVM_PRESERVE_ALL BIT(1) +#define IXGBE_ACI_NVM_FACTORY_DEFAULT (2 << IXGBE_ACI_NVM_PRESERVATION_S) +#define IXGBE_ACI_NVM_PRESERVE_SELECTED (3 << IXGBE_ACI_NVM_PRESERVATION_S) +#define IXGBE_ACI_NVM_ACTIV_SEL_NVM BIT(3) /* Write Activate/SR Dump only */ +#define IXGBE_ACI_NVM_ACTIV_SEL_OROM BIT(4) +#define IXGBE_ACI_NVM_ACTIV_SEL_NETLIST BIT(5) +#define IXGBE_ACI_NVM_SPECIAL_UPDATE BIT(6) +#define IXGBE_ACI_NVM_REVERT_LAST_ACTIV BIT(6) /* Write Activate only */ +#define IXGBE_ACI_NVM_ACTIV_SEL_MASK MAKEMASK(0x7, 3) +#define IXGBE_ACI_NVM_FLASH_ONLY BIT(7) +#define IXGBE_ACI_NVM_RESET_LVL_M MAKEMASK(0x3, 0) /* Write reply only */ +#define IXGBE_ACI_NVM_POR_FLAG 0 +#define IXGBE_ACI_NVM_PERST_FLAG 1 +#define IXGBE_ACI_NVM_EMPR_FLAG 2 +#define IXGBE_ACI_NVM_EMPR_ENA BIT(0) /* Write Activate reply only */ + /* For Write Activate, several flags are sent as part of a separate + * flags2 field using a separate byte. For simplicity of the software + * interface, we pass the flags as a 16 bit value so these flags are + * all offset by 8 bits + */ +#define IXGBE_ACI_NVM_ACTIV_REQ_EMPR BIT(8) /* NVM Write Activate only */ + __le16 module_typeid; + __le16 length; +#define IXGBE_ACI_NVM_ERASE_LEN 0xFFFF + __le32 addr_high; + __le32 addr_low; +}; + +/* NVM Module_Type ID, needed offset and read_len for struct ixgbe_aci_cmd_nvm. */ +#define IXGBE_ACI_NVM_SECTOR_UNIT 4096 /* In Bytes */ +#define IXGBE_ACI_NVM_WORD_UNIT 2 /* In Bytes */ + +#define IXGBE_ACI_NVM_START_POINT 0 +#define IXGBE_ACI_NVM_EMP_SR_PTR_OFFSET 0x90 +#define IXGBE_ACI_NVM_EMP_SR_PTR_RD_LEN 2 /* In Bytes */ +#define IXGBE_ACI_NVM_EMP_SR_PTR_M MAKEMASK(0x7FFF, 0) +#define IXGBE_ACI_NVM_EMP_SR_PTR_TYPE_S 15 +#define IXGBE_ACI_NVM_EMP_SR_PTR_TYPE_M BIT(15) +#define IXGBE_ACI_NVM_EMP_SR_PTR_TYPE_SECTOR 1 + +#define IXGBE_ACI_NVM_LLDP_CFG_PTR_OFFSET 0x46 +#define IXGBE_ACI_NVM_LLDP_CFG_HEADER_LEN 2 /* In Bytes */ +#define IXGBE_ACI_NVM_LLDP_CFG_PTR_RD_LEN 2 /* In Bytes */ + +#define IXGBE_ACI_NVM_LLDP_PRESERVED_MOD_ID 0x129 +#define IXGBE_ACI_NVM_CUR_LLDP_PERSIST_RD_OFFSET 2 /* In Bytes */ +#define IXGBE_ACI_NVM_LLDP_STATUS_M MAKEMASK(0xF, 0) +#define IXGBE_ACI_NVM_LLDP_STATUS_M_LEN 4 /* In Bits */ +#define IXGBE_ACI_NVM_LLDP_STATUS_RD_LEN 4 /* In Bytes */ + +#define IXGBE_ACI_NVM_MINSREV_MOD_ID 0x130 + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_nvm); + +/* Used for reading and writing MinSRev using 0x0701 and 0x0703. Note that the + * type field is excluded from the section when reading and writing from + * a module using the module_typeid field with these AQ commands. + */ +struct ixgbe_aci_cmd_nvm_minsrev { + __le16 length; + __le16 validity; +#define IXGBE_ACI_NVM_MINSREV_NVM_VALID BIT(0) +#define IXGBE_ACI_NVM_MINSREV_OROM_VALID BIT(1) + __le16 nvm_minsrev_l; + __le16 nvm_minsrev_h; + __le16 orom_minsrev_l; + __le16 orom_minsrev_h; +}; + +IXGBE_CHECK_STRUCT_LEN(12, ixgbe_aci_cmd_nvm_minsrev); + +/* Used for 0x0704 as well as for 0x0705 commands */ +struct ixgbe_aci_cmd_nvm_cfg { + u8 cmd_flags; +#define IXGBE_ACI_ANVM_MULTIPLE_ELEMS BIT(0) +#define IXGBE_ACI_ANVM_IMMEDIATE_FIELD BIT(1) +#define IXGBE_ACI_ANVM_NEW_CFG BIT(2) + u8 reserved; + __le16 count; + __le16 id; + u8 reserved1[2]; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_nvm_cfg); + +struct ixgbe_aci_cmd_nvm_cfg_data { + __le16 field_id; + __le16 field_options; + __le16 field_value; +}; + +IXGBE_CHECK_STRUCT_LEN(6, ixgbe_aci_cmd_nvm_cfg_data); + +/* NVM Checksum Command (direct, 0x0706) */ +struct ixgbe_aci_cmd_nvm_checksum { + u8 flags; +#define IXGBE_ACI_NVM_CHECKSUM_VERIFY BIT(0) +#define IXGBE_ACI_NVM_CHECKSUM_RECALC BIT(1) + u8 rsvd; + __le16 checksum; /* Used only by response */ +#define IXGBE_ACI_NVM_CHECKSUM_CORRECT 0xBABA + u8 rsvd2[12]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_nvm_checksum); + +/* Used for NVM Sanitization command - 0x070C */ +struct ixgbe_aci_cmd_nvm_sanitization { + u8 cmd_flags; +#define IXGBE_ACI_SANITIZE_REQ_READ 0 +#define IXGBE_ACI_SANITIZE_REQ_OPERATE BIT(0) + +#define IXGBE_ACI_SANITIZE_READ_SUBJECT_NVM_BITS 0 +#define IXGBE_ACI_SANITIZE_READ_SUBJECT_NVM_STATE BIT(1) +#define IXGBE_ACI_SANITIZE_OPERATE_SUBJECT_CLEAR 0 + u8 values; +#define IXGBE_ACI_SANITIZE_NVM_BITS_HOST_CLEAN_SUPPORT BIT(0) +#define IXGBE_ACI_SANITIZE_NVM_BITS_BMC_CLEAN_SUPPORT BIT(2) +#define IXGBE_ACI_SANITIZE_NVM_STATE_HOST_CLEAN_DONE BIT(0) +#define IXGBE_ACI_SANITIZE_NVM_STATE_HOST_CLEAN_SUCCESS BIT(1) +#define IXGBE_ACI_SANITIZE_NVM_STATE_BMC_CLEAN_DONE BIT(2) +#define IXGBE_ACI_SANITIZE_NVM_STATE_BMC_CLEAN_SUCCESS BIT(3) +#define IXGBE_ACI_SANITIZE_OPERATE_HOST_CLEAN_DONE BIT(0) +#define IXGBE_ACI_SANITIZE_OPERATE_HOST_CLEAN_SUCCESS BIT(1) +#define IXGBE_ACI_SANITIZE_OPERATE_BMC_CLEAN_DONE BIT(2) +#define IXGBE_ACI_SANITIZE_OPERATE_BMC_CLEAN_SUCCESS BIT(3) + u8 reserved[14]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_nvm_sanitization); + +/* Write/Read Alternate - Direct (direct 0x0900/0x0902) */ +struct ixgbe_aci_cmd_read_write_alt_direct { + __le32 dword0_addr; + __le32 dword0_value; + __le32 dword1_addr; + __le32 dword1_value; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_read_write_alt_direct); + +/* Write/Read Alternate - Indirect (indirect 0x0901/0x0903) */ +struct ixgbe_aci_cmd_read_write_alt_indirect { + __le32 base_dword_addr; + __le32 num_dwords; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_read_write_alt_indirect); + +/* Done Alternate Write (direct 0x0904) */ +struct ixgbe_aci_cmd_done_alt_write { + u8 flags; +#define IXGBE_ACI_CMD_UEFI_BIOS_MODE BIT(0) +#define IXGBE_ACI_RESP_RESET_NEEDED BIT(1) + u8 reserved[15]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_done_alt_write); + +/* Clear Port Alternate Write (direct 0x0906) */ +struct ixgbe_aci_cmd_clear_port_alt_write { + u8 reserved[16]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_clear_port_alt_write); + +/* Get CGU abilities command response data structure (indirect 0x0C61) */ +struct ixgbe_aci_cmd_get_cgu_abilities { + u8 num_inputs; + u8 num_outputs; + u8 pps_dpll_idx; + u8 synce_dpll_idx; + __le32 max_in_freq; + __le32 max_in_phase_adj; + __le32 max_out_freq; + __le32 max_out_phase_adj; + u8 cgu_part_num; + u8 rsvd[3]; +}; + +IXGBE_CHECK_STRUCT_LEN(24, ixgbe_aci_cmd_get_cgu_abilities); + +#define IXGBE_ACI_NODE_HANDLE_VALID BIT(10) +#define IXGBE_ACI_NODE_HANDLE MAKEMASK(0x3FF, 0) +#define IXGBE_ACI_DRIVING_CLK_NUM_SHIFT 10 +#define IXGBE_ACI_DRIVING_CLK_NUM MAKEMASK(0x3F, IXGBE_ACI_DRIVING_CLK_NUM_SHIFT) + +/* Set CGU input config (direct 0x0C62) */ +struct ixgbe_aci_cmd_set_cgu_input_config { + u8 input_idx; + u8 flags1; +#define IXGBE_ACI_SET_CGU_IN_CFG_FLG1_UPDATE_FREQ BIT(6) +#define IXGBE_ACI_SET_CGU_IN_CFG_FLG1_UPDATE_DELAY BIT(7) + u8 flags2; +#define IXGBE_ACI_SET_CGU_IN_CFG_FLG2_INPUT_EN BIT(5) +#define IXGBE_ACI_SET_CGU_IN_CFG_FLG2_ESYNC_EN BIT(6) + u8 rsvd; + __le32 freq; + __le32 phase_delay; + u8 rsvd2[2]; + __le16 node_handle; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_set_cgu_input_config); + +/* Get CGU input config response descriptor structure (direct 0x0C63) */ +struct ixgbe_aci_cmd_get_cgu_input_config { + u8 input_idx; + u8 status; +#define IXGBE_ACI_GET_CGU_IN_CFG_STATUS_LOS BIT(0) +#define IXGBE_ACI_GET_CGU_IN_CFG_STATUS_SCM_FAIL BIT(1) +#define IXGBE_ACI_GET_CGU_IN_CFG_STATUS_CFM_FAIL BIT(2) +#define IXGBE_ACI_GET_CGU_IN_CFG_STATUS_GST_FAIL BIT(3) +#define IXGBE_ACI_GET_CGU_IN_CFG_STATUS_PFM_FAIL BIT(4) +#define IXGBE_ACI_GET_CGU_IN_CFG_STATUS_ESYNC_FAIL BIT(6) +#define IXGBE_ACI_GET_CGU_IN_CFG_STATUS_ESYNC_CAP BIT(7) + u8 type; +#define IXGBE_ACI_GET_CGU_IN_CFG_TYPE_READ_ONLY BIT(0) +#define IXGBE_ACI_GET_CGU_IN_CFG_TYPE_GPS BIT(4) +#define IXGBE_ACI_GET_CGU_IN_CFG_TYPE_EXTERNAL BIT(5) +#define IXGBE_ACI_GET_CGU_IN_CFG_TYPE_PHY BIT(6) + u8 flags1; +#define IXGBE_ACI_GET_CGU_IN_CFG_FLG1_PHASE_DELAY_SUPP BIT(0) +#define IXGBE_ACI_GET_CGU_IN_CFG_FLG1_1PPS_SUPP BIT(2) +#define IXGBE_ACI_GET_CGU_IN_CFG_FLG1_10MHZ_SUPP BIT(3) +#define IXGBE_ACI_GET_CGU_IN_CFG_FLG1_ANYFREQ BIT(7) + __le32 freq; + __le32 phase_delay; + u8 flags2; +#define IXGBE_ACI_GET_CGU_IN_CFG_FLG2_INPUT_EN BIT(5) +#define IXGBE_ACI_GET_CGU_IN_CFG_FLG2_ESYNC_EN BIT(6) + u8 rsvd[1]; + __le16 node_handle; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_cgu_input_config); + +/* Set CGU output config (direct 0x0C64) */ +struct ixgbe_aci_cmd_set_cgu_output_config { + u8 output_idx; + u8 flags; +#define IXGBE_ACI_SET_CGU_OUT_CFG_OUT_EN BIT(0) +#define IXGBE_ACI_SET_CGU_OUT_CFG_ESYNC_EN BIT(1) +#define IXGBE_ACI_SET_CGU_OUT_CFG_UPDATE_FREQ BIT(2) +#define IXGBE_ACI_SET_CGU_OUT_CFG_UPDATE_PHASE BIT(3) +#define IXGBE_ACI_SET_CGU_OUT_CFG_UPDATE_SRC_SEL BIT(4) + u8 src_sel; +#define IXGBE_ACI_SET_CGU_OUT_CFG_DPLL_SRC_SEL MAKEMASK(0x1F, 0) + u8 rsvd; + __le32 freq; + __le32 phase_delay; + u8 rsvd2[2]; + __le16 node_handle; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_set_cgu_output_config); + +/* Get CGU output config (direct 0x0C65) */ +struct ixgbe_aci_cmd_get_cgu_output_config { + u8 output_idx; + u8 flags; +#define IXGBE_ACI_GET_CGU_OUT_CFG_OUT_EN BIT(0) +#define IXGBE_ACI_GET_CGU_OUT_CFG_ESYNC_EN BIT(1) +#define IXGBE_ACI_GET_CGU_OUT_CFG_ESYNC_ABILITY BIT(2) + u8 src_sel; +#define IXGBE_ACI_GET_CGU_OUT_CFG_DPLL_SRC_SEL_SHIFT 0 +#define IXGBE_ACI_GET_CGU_OUT_CFG_DPLL_SRC_SEL \ + MAKEMASK(0x1F, IXGBE_ACI_GET_CGU_OUT_CFG_DPLL_SRC_SEL_SHIFT) +#define IXGBE_ACI_GET_CGU_OUT_CFG_DPLL_MODE_SHIFT 5 +#define IXGBE_ACI_GET_CGU_OUT_CFG_DPLL_MODE \ + MAKEMASK(0x7, IXGBE_ACI_GET_CGU_OUT_CFG_DPLL_MODE_SHIFT) + u8 rsvd; + __le32 freq; + __le32 src_freq; + u8 rsvd2[2]; + __le16 node_handle; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_cgu_output_config); + +/* Get CGU DPLL status (direct 0x0C66) */ +struct ixgbe_aci_cmd_get_cgu_dpll_status { + u8 dpll_num; + u8 ref_state; +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_REF_SW_LOS BIT(0) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_REF_SW_SCM BIT(1) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_REF_SW_CFM BIT(2) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_REF_SW_GST BIT(3) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_REF_SW_PFM BIT(4) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_FAST_LOCK_EN BIT(5) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_REF_SW_ESYNC BIT(6) + __le16 dpll_state; +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_LOCK BIT(0) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_HO BIT(1) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_HO_READY BIT(2) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_FLHIT BIT(5) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_PSLHIT BIT(7) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_CLK_REF_SHIFT 8 +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_CLK_REF_SEL \ + MAKEMASK(0x1F, IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_CLK_REF_SHIFT) +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_MODE_SHIFT 13 +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_MODE \ + MAKEMASK(0x7, IXGBE_ACI_GET_CGU_DPLL_STATUS_STATE_MODE_SHIFT) + __le32 phase_offset_h; + __le32 phase_offset_l; + u8 eec_mode; +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_EEC_MODE_1 0xA +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_EEC_MODE_2 0xB +#define IXGBE_ACI_GET_CGU_DPLL_STATUS_EEC_MODE_UNKNOWN 0xF + u8 rsvd[1]; + __le16 node_handle; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_cgu_dpll_status); + +/* Set CGU DPLL config (direct 0x0C67) */ +struct ixgbe_aci_cmd_set_cgu_dpll_config { + u8 dpll_num; + u8 ref_state; +#define IXGBE_ACI_SET_CGU_DPLL_CONFIG_REF_SW_LOS BIT(0) +#define IXGBE_ACI_SET_CGU_DPLL_CONFIG_REF_SW_SCM BIT(1) +#define IXGBE_ACI_SET_CGU_DPLL_CONFIG_REF_SW_CFM BIT(2) +#define IXGBE_ACI_SET_CGU_DPLL_CONFIG_REF_SW_GST BIT(3) +#define IXGBE_ACI_SET_CGU_DPLL_CONFIG_REF_SW_PFM BIT(4) +#define IXGBE_ACI_SET_CGU_DPLL_CONFIG_REF_FLOCK_EN BIT(5) +#define IXGBE_ACI_SET_CGU_DPLL_CONFIG_REF_SW_ESYNC BIT(6) + u8 rsvd; + u8 config; +#define IXGBE_ACI_SET_CGU_DPLL_CONFIG_CLK_REF_SEL MAKEMASK(0x1F, 0) +#define IXGBE_ACI_SET_CGU_DPLL_CONFIG_MODE MAKEMASK(0x7, 5) + u8 rsvd2[8]; + u8 eec_mode; + u8 rsvd3[1]; + __le16 node_handle; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_set_cgu_dpll_config); + +/* Set CGU reference priority (direct 0x0C68) */ +struct ixgbe_aci_cmd_set_cgu_ref_prio { + u8 dpll_num; + u8 ref_idx; + u8 ref_priority; + u8 rsvd[11]; + __le16 node_handle; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_set_cgu_ref_prio); + +/* Get CGU reference priority (direct 0x0C69) */ +struct ixgbe_aci_cmd_get_cgu_ref_prio { + u8 dpll_num; + u8 ref_idx; + u8 ref_priority; /* Valid only in response */ + u8 rsvd[13]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_cgu_ref_prio); + +/* Get CGU info (direct 0x0C6A) */ +struct ixgbe_aci_cmd_get_cgu_info { + __le32 cgu_id; + __le32 cgu_cfg_ver; + __le32 cgu_fw_ver; + u8 node_part_num; + u8 dev_rev; + __le16 node_handle; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_cgu_info); + +struct ixgbe_aci_cmd_temp_tca_event { + u8 event_desc; +#define IXGBE_TEMP_TCA_EVENT_DESC_SUBJ_SHIFT 0 +#define IXGBE_TEMP_TCA_EVENT_DESC_SUBJ_NVM 0 +#define IXGBE_TEMP_TCA_EVENT_DESC_SUBJ_EVENT_STATE 1 +#define IXGBE_TEMP_TCA_EVENT_DESC_SUBJ_ALL 2 + +#define IXGBE_TEMP_TCA_EVENT_DESC_ALARM_SHIFT 2 +#define IXGBE_TEMP_TCA_EVENT_DESC_WARNING_CLEARED 0 +#define IXGBE_TEMP_TCA_EVENT_DESC_ALARM_CLEARED 1 +#define IXGBE_TEMP_TCA_EVENT_DESC_WARNING_RAISED 2 +#define IXGBE_TEMP_TCA_EVENT_DESC_ALARM_RAISED 3 + + u8 reserved; + __le16 temperature; + __le16 thermal_sensor_max_value; + __le16 thermal_sensor_min_value; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_temp_tca_event); + +/* Debug Dump Internal Data (indirect 0xFF08) */ +struct ixgbe_aci_cmd_debug_dump_internals { + __le16 cluster_id; /* Expresses next cluster ID in response */ +#define IXGBE_ACI_DBG_DUMP_CLUSTER_ID_LINK 0 +#define IXGBE_ACI_DBG_DUMP_CLUSTER_ID_FULL_CSR_SPACE 1 + __le16 table_id; /* Used only for non-memory clusters */ + __le32 idx; /* In table entries for tables, in bytes for memory */ + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_debug_dump_internals); + +/* Set Health Status (direct 0xFF20) */ +struct ixgbe_aci_cmd_set_health_status_config { + u8 event_source; +#define IXGBE_ACI_HEALTH_STATUS_SET_PF_SPECIFIC_MASK BIT(0) +#define IXGBE_ACI_HEALTH_STATUS_SET_ALL_PF_MASK BIT(1) +#define IXGBE_ACI_HEALTH_STATUS_SET_GLOBAL_MASK BIT(2) + u8 reserved[15]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_set_health_status_config); + +#define IXGBE_ACI_HEALTH_STATUS_ERR_UNKNOWN_MOD_STRICT 0x101 +#define IXGBE_ACI_HEALTH_STATUS_ERR_MOD_TYPE 0x102 +#define IXGBE_ACI_HEALTH_STATUS_ERR_MOD_QUAL 0x103 +#define IXGBE_ACI_HEALTH_STATUS_ERR_MOD_COMM 0x104 +#define IXGBE_ACI_HEALTH_STATUS_ERR_MOD_CONFLICT 0x105 +#define IXGBE_ACI_HEALTH_STATUS_ERR_MOD_NOT_PRESENT 0x106 +#define IXGBE_ACI_HEALTH_STATUS_INFO_MOD_UNDERUTILIZED 0x107 +#define IXGBE_ACI_HEALTH_STATUS_ERR_UNKNOWN_MOD_LENIENT 0x108 +#define IXGBE_ACI_HEALTH_STATUS_ERR_MOD_DIAGNOSTIC_FEATURE 0x109 +#define IXGBE_ACI_HEALTH_STATUS_ERR_INVALID_LINK_CFG 0x10B +#define IXGBE_ACI_HEALTH_STATUS_ERR_PORT_ACCESS 0x10C +#define IXGBE_ACI_HEALTH_STATUS_ERR_PORT_UNREACHABLE 0x10D +#define IXGBE_ACI_HEALTH_STATUS_INFO_PORT_SPEED_MOD_LIMITED 0x10F +#define IXGBE_ACI_HEALTH_STATUS_ERR_PARALLEL_FAULT 0x110 +#define IXGBE_ACI_HEALTH_STATUS_INFO_PORT_SPEED_PHY_LIMITED 0x111 +#define IXGBE_ACI_HEALTH_STATUS_ERR_NETLIST_TOPO 0x112 +#define IXGBE_ACI_HEALTH_STATUS_ERR_NETLIST 0x113 +#define IXGBE_ACI_HEALTH_STATUS_ERR_TOPO_CONFLICT 0x114 +#define IXGBE_ACI_HEALTH_STATUS_ERR_LINK_HW_ACCESS 0x115 +#define IXGBE_ACI_HEALTH_STATUS_ERR_LINK_RUNTIME 0x116 +#define IXGBE_ACI_HEALTH_STATUS_ERR_DNL_INIT 0x117 +#define IXGBE_ACI_HEALTH_STATUS_ERR_PHY_NVM_PROG 0x120 +#define IXGBE_ACI_HEALTH_STATUS_ERR_PHY_FW_LOAD 0x121 +#define IXGBE_ACI_HEALTH_STATUS_INFO_RECOVERY 0x500 +#define IXGBE_ACI_HEALTH_STATUS_ERR_FLASH_ACCESS 0x501 +#define IXGBE_ACI_HEALTH_STATUS_ERR_NVM_AUTH 0x502 +#define IXGBE_ACI_HEALTH_STATUS_ERR_OROM_AUTH 0x503 +#define IXGBE_ACI_HEALTH_STATUS_ERR_DDP_AUTH 0x504 +#define IXGBE_ACI_HEALTH_STATUS_ERR_NVM_COMPAT 0x505 +#define IXGBE_ACI_HEALTH_STATUS_ERR_OROM_COMPAT 0x506 +#define IXGBE_ACI_HEALTH_STATUS_ERR_NVM_SEC_VIOLATION 0x507 +#define IXGBE_ACI_HEALTH_STATUS_ERR_OROM_SEC_VIOLATION 0x508 +#define IXGBE_ACI_HEALTH_STATUS_ERR_DCB_MIB 0x509 +#define IXGBE_ACI_HEALTH_STATUS_ERR_MNG_TIMEOUT 0x50A +#define IXGBE_ACI_HEALTH_STATUS_ERR_BMC_RESET 0x50B +#define IXGBE_ACI_HEALTH_STATUS_ERR_LAST_MNG_FAIL 0x50C +#define IXGBE_ACI_HEALTH_STATUS_ERR_RESOURCE_ALLOC_FAIL 0x50D +#define IXGBE_ACI_HEALTH_STATUS_ERR_FW_LOOP 0x1000 +#define IXGBE_ACI_HEALTH_STATUS_ERR_FW_PFR_FAIL 0x1001 +#define IXGBE_ACI_HEALTH_STATUS_ERR_LAST_FAIL_AQ 0x1002 + +/* Get Health Status codes (indirect 0xFF21) */ +struct ixgbe_aci_cmd_get_supported_health_status_codes { + __le16 health_code_count; + u8 reserved[6]; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_supported_health_status_codes); + +/* Get Health Status (indirect 0xFF22) */ +struct ixgbe_aci_cmd_get_health_status { + __le16 health_status_count; + u8 reserved[6]; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_get_health_status); + +/* Get Health Status event buffer entry, (0xFF22) + * repeated per reported health status + */ +struct ixgbe_aci_cmd_health_status_elem { + __le16 health_status_code; + __le16 event_source; +#define IXGBE_ACI_HEALTH_STATUS_PF (0x1) +#define IXGBE_ACI_HEALTH_STATUS_PORT (0x2) +#define IXGBE_ACI_HEALTH_STATUS_GLOBAL (0x3) + __le32 internal_data1; +#define IXGBE_ACI_HEALTH_STATUS_UNDEFINED_DATA (0xDEADBEEF) + __le32 internal_data2; +}; + +IXGBE_CHECK_STRUCT_LEN(12, ixgbe_aci_cmd_health_status_elem); + +/* Clear Health Status (direct 0xFF23) */ +struct ixgbe_aci_cmd_clear_health_status { + __le32 reserved[4]; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_clear_health_status); + +enum ixgbe_aci_fw_logging_mod { + IXGBE_ACI_FW_LOG_ID_GENERAL = 0, + IXGBE_ACI_FW_LOG_ID_CTRL = 1, + IXGBE_ACI_FW_LOG_ID_LINK = 2, + IXGBE_ACI_FW_LOG_ID_LINK_TOPO = 3, + IXGBE_ACI_FW_LOG_ID_DNL = 4, + IXGBE_ACI_FW_LOG_ID_I2C = 5, + IXGBE_ACI_FW_LOG_ID_SDP = 6, + IXGBE_ACI_FW_LOG_ID_MDIO = 7, + IXGBE_ACI_FW_LOG_ID_ADMINQ = 8, + IXGBE_ACI_FW_LOG_ID_HDMA = 9, + IXGBE_ACI_FW_LOG_ID_LLDP = 10, + IXGBE_ACI_FW_LOG_ID_DCBX = 11, + IXGBE_ACI_FW_LOG_ID_DCB = 12, + IXGBE_ACI_FW_LOG_ID_XLR = 13, + IXGBE_ACI_FW_LOG_ID_NVM = 14, + IXGBE_ACI_FW_LOG_ID_AUTH = 15, + IXGBE_ACI_FW_LOG_ID_VPD = 16, + IXGBE_ACI_FW_LOG_ID_IOSF = 17, + IXGBE_ACI_FW_LOG_ID_PARSER = 18, + IXGBE_ACI_FW_LOG_ID_SW = 19, + IXGBE_ACI_FW_LOG_ID_SCHEDULER = 20, + IXGBE_ACI_FW_LOG_ID_TXQ = 21, + IXGBE_ACI_FW_LOG_ID_ACL = 22, + IXGBE_ACI_FW_LOG_ID_POST = 23, + IXGBE_ACI_FW_LOG_ID_WATCHDOG = 24, + IXGBE_ACI_FW_LOG_ID_TASK_DISPATCH = 25, + IXGBE_ACI_FW_LOG_ID_MNG = 26, + IXGBE_ACI_FW_LOG_ID_SYNCE = 27, + IXGBE_ACI_FW_LOG_ID_HEALTH = 28, + IXGBE_ACI_FW_LOG_ID_TSDRV = 29, + IXGBE_ACI_FW_LOG_ID_PFREG = 30, + IXGBE_ACI_FW_LOG_ID_MDLVER = 31, + IXGBE_ACI_FW_LOG_ID_MAX = 32, +}; + +/* Only a single log level should be set and all log levels under the set value + * are enabled, e.g. if log level is set to IXGBE_FWLOG_LEVEL_VERBOSE, then all + * other log levels are included (except IXGBE_FWLOG_LEVEL_NONE) + */ +enum ixgbe_fwlog_level { + IXGBE_FWLOG_LEVEL_NONE = 0, + IXGBE_FWLOG_LEVEL_ERROR = 1, + IXGBE_FWLOG_LEVEL_WARNING = 2, + IXGBE_FWLOG_LEVEL_NORMAL = 3, + IXGBE_FWLOG_LEVEL_VERBOSE = 4, + IXGBE_FWLOG_LEVEL_INVALID, /* all values >= this entry are invalid */ +}; + +struct ixgbe_fwlog_module_entry { + /* module ID for the corresponding firmware logging event */ + u16 module_id; + /* verbosity level for the module_id */ + u8 log_level; +}; + +struct ixgbe_fwlog_cfg { + /* list of modules for configuring log level */ + struct ixgbe_fwlog_module_entry module_entries[IXGBE_ACI_FW_LOG_ID_MAX]; +#define IXGBE_FWLOG_OPTION_ARQ_ENA BIT(0) +#define IXGBE_FWLOG_OPTION_UART_ENA BIT(1) + /* set before calling ixgbe_fwlog_init() so the PF registers for firmware + * logging on initialization + */ +#define IXGBE_FWLOG_OPTION_REGISTER_ON_INIT BIT(2) + /* set in the ixgbe_fwlog_get() response if the PF is registered for FW + * logging events over ARQ + */ +#define IXGBE_FWLOG_OPTION_IS_REGISTERED BIT(3) + /* options used to configure firmware logging */ + u16 options; + /* minimum number of log events sent per Admin Receive Queue event */ + u8 log_resolution; +}; + +struct ixgbe_fwlog_data { + u16 data_size; + u8 *data; +}; + +struct ixgbe_fwlog_ring { + struct ixgbe_fwlog_data *rings; + u16 size; + u16 head; + u16 tail; +}; + +#define IXGBE_FWLOG_RING_SIZE_DFLT 256 +#define IXGBE_FWLOG_RING_SIZE_MAX 512 + +/* Set FW Logging configuration (indirect 0xFF30) + * Register for FW Logging (indirect 0xFF31) + * Query FW Logging (indirect 0xFF32) + * FW Log Event (indirect 0xFF33) + * Get FW Log (indirect 0xFF34) + * Clear FW Log (indirect 0xFF35) + */ +struct ixgbe_aci_cmd_fw_log { + u8 cmd_flags; +#define IXGBE_ACI_FW_LOG_CONF_UART_EN BIT(0) +#define IXGBE_ACI_FW_LOG_CONF_AQ_EN BIT(1) +#define IXGBE_ACI_FW_LOG_QUERY_REGISTERED BIT(2) +#define IXGBE_ACI_FW_LOG_CONF_SET_VALID BIT(3) +#define IXGBE_ACI_FW_LOG_AQ_REGISTER BIT(0) +#define IXGBE_ACI_FW_LOG_AQ_QUERY BIT(2) +#define IXGBE_ACI_FW_LOG_PERSISTENT BIT(0) + u8 rsp_flag; +#define IXGBE_ACI_FW_LOG_MORE_DATA BIT(1) + __le16 fw_rt_msb; + union { + struct { + __le32 fw_rt_lsb; + } sync; + struct { + __le16 log_resolution; +#define IXGBE_ACI_FW_LOG_MIN_RESOLUTION (1) +#define IXGBE_ACI_FW_LOG_MAX_RESOLUTION (128) + __le16 mdl_cnt; + } cfg; + } ops; + __le32 addr_high; + __le32 addr_low; +}; + +IXGBE_CHECK_PARAM_LEN(ixgbe_aci_cmd_fw_log); + +/* Response Buffer for: + * Set Firmware Logging Configuration (0xFF30) + * Query FW Logging (0xFF32) + */ +struct ixgbe_aci_cmd_fw_log_cfg_resp { + __le16 module_identifier; + u8 log_level; + u8 rsvd0; +}; + +IXGBE_CHECK_STRUCT_LEN(4, ixgbe_aci_cmd_fw_log_cfg_resp); + +/** + * struct ixgbe_aq_desc - Admin Command (AC) descriptor + * @flags: IXGBE_ACI_FLAG_* flags + * @opcode: Admin command opcode + * @datalen: length in bytes of indirect/external data buffer + * @retval: return value from firmware + * @cookie_high: opaque data high-half + * @cookie_low: opaque data low-half + * @params: command-specific parameters + * + * Descriptor format for commands the driver posts via the Admin Command Interface + * (ACI). The firmware writes back onto the command descriptor and returns + * the result of the command. Asynchronous events that are not an immediate + * result of the command are written to the Admin Command Interface (ACI) using + * the same descriptor format. Descriptors are in little-endian notation with + * 32-bit words. + */ +struct ixgbe_aci_desc { + __le16 flags; + __le16 opcode; + __le16 datalen; + __le16 retval; + __le32 cookie_high; + __le32 cookie_low; + union { + u8 raw[16]; + struct ixgbe_aci_cmd_generic generic; + struct ixgbe_aci_cmd_get_ver get_ver; + struct ixgbe_aci_cmd_driver_ver driver_ver; + struct ixgbe_aci_cmd_get_exp_err exp_err; + struct ixgbe_aci_cmd_req_res res_owner; + struct ixgbe_aci_cmd_list_caps get_cap; + struct ixgbe_aci_cmd_disable_rxen disable_rxen; + struct ixgbe_aci_cmd_get_fw_event get_fw_event; + struct ixgbe_aci_cmd_get_phy_caps get_phy; + struct ixgbe_aci_cmd_set_phy_cfg set_phy; + struct ixgbe_aci_cmd_restart_an restart_an; + struct ixgbe_aci_cmd_get_link_status get_link_status; + struct ixgbe_aci_cmd_set_event_mask set_event_mask; + struct ixgbe_aci_cmd_get_link_topo get_link_topo; + struct ixgbe_aci_cmd_i2c read_write_i2c; + struct ixgbe_aci_cmd_read_i2c_resp read_i2c_resp; + struct ixgbe_aci_cmd_mdio read_write_mdio; + struct ixgbe_aci_cmd_mdio read_mdio; + struct ixgbe_aci_cmd_mdio write_mdio; + struct ixgbe_aci_cmd_set_port_id_led set_port_id_led; + struct ixgbe_aci_cmd_gpio_by_func read_write_gpio_by_func; + struct ixgbe_aci_cmd_gpio read_write_gpio; + struct ixgbe_aci_cmd_sff_eeprom read_write_sff_param; + struct ixgbe_aci_cmd_prog_topo_dev_nvm prog_topo_dev_nvm; + struct ixgbe_aci_cmd_read_topo_dev_nvm read_topo_dev_nvm; + struct ixgbe_aci_cmd_nvm nvm; + struct ixgbe_aci_cmd_nvm_cfg nvm_cfg; + struct ixgbe_aci_cmd_nvm_checksum nvm_checksum; + struct ixgbe_aci_cmd_read_write_alt_direct read_write_alt_direct; + struct ixgbe_aci_cmd_read_write_alt_indirect read_write_alt_indirect; + struct ixgbe_aci_cmd_done_alt_write done_alt_write; + struct ixgbe_aci_cmd_clear_port_alt_write clear_port_alt_write; + struct ixgbe_aci_cmd_debug_dump_internals debug_dump; + struct ixgbe_aci_cmd_set_health_status_config + set_health_status_config; + struct ixgbe_aci_cmd_get_supported_health_status_codes + get_supported_health_status_codes; + struct ixgbe_aci_cmd_get_health_status get_health_status; + struct ixgbe_aci_cmd_clear_health_status clear_health_status; + struct ixgbe_aci_cmd_fw_log fw_log; + struct ixgbe_aci_cmd_nvm_sanitization nvm_sanitization; + } params; +}; + +/* E610-specific adapter context structures */ + +struct ixgbe_link_status { + /* Refer to ixgbe_aci_phy_type for bits definition */ + u64 phy_type_low; + u64 phy_type_high; + u8 topo_media_conflict; + u16 max_frame_size; + u16 link_speed; + u16 req_speeds; + u8 link_cfg_err; + u8 lse_ena; /* Link Status Event notification */ + u8 link_info; + u8 an_info; + u8 ext_info; + u8 fec_info; + u8 pacing; + /* Refer to #define from module_type[IXGBE_ACI_MODULE_TYPE_TOTAL_BYTE] of + * ixgbe_aci_get_phy_caps structure + */ + u8 module_type[IXGBE_ACI_MODULE_TYPE_TOTAL_BYTE]; +}; + +/* Common HW capabilities for SW use */ +struct ixgbe_hw_common_caps { + /* Write CSR protection */ + u64 wr_csr_prot; + u32 switching_mode; + /* switching mode supported - EVB switching (including cloud) */ +#define IXGBE_NVM_IMAGE_TYPE_EVB 0x0 + + /* Manageability mode & supported protocols over MCTP */ + u32 mgmt_mode; +#define IXGBE_MGMT_MODE_PASS_THRU_MODE_M 0xF +#define IXGBE_MGMT_MODE_CTL_INTERFACE_M 0xF0 +#define IXGBE_MGMT_MODE_REDIR_SB_INTERFACE_M 0xF00 + + u32 mgmt_protocols_mctp; +#define IXGBE_MGMT_MODE_PROTO_RSVD BIT(0) +#define IXGBE_MGMT_MODE_PROTO_PLDM BIT(1) +#define IXGBE_MGMT_MODE_PROTO_OEM BIT(2) +#define IXGBE_MGMT_MODE_PROTO_NC_SI BIT(3) + + u32 os2bmc; + u32 valid_functions; + /* DCB capabilities */ + u32 active_tc_bitmap; + u32 maxtc; + + /* RSS related capabilities */ + u32 rss_table_size; /* 512 for PFs and 64 for VFs */ + u32 rss_table_entry_width; /* RSS Entry width in bits */ + + /* Tx/Rx queues */ + u32 num_rxq; /* Number/Total Rx queues */ + u32 rxq_first_id; /* First queue ID for Rx queues */ + u32 num_txq; /* Number/Total Tx queues */ + u32 txq_first_id; /* First queue ID for Tx queues */ + + /* MSI-X vectors */ + u32 num_msix_vectors; + u32 msix_vector_first_id; + + /* Max MTU for function or device */ + u32 max_mtu; + + /* WOL related */ + u32 num_wol_proxy_fltr; + u32 wol_proxy_vsi_seid; + + /* LED/SDP pin count */ + u32 led_pin_num; + u32 sdp_pin_num; + + /* LED/SDP - Supports up to 12 LED pins and 8 SDP signals */ +#define IXGBE_MAX_SUPPORTED_GPIO_LED 12 +#define IXGBE_MAX_SUPPORTED_GPIO_SDP 8 + u8 led[IXGBE_MAX_SUPPORTED_GPIO_LED]; + u8 sdp[IXGBE_MAX_SUPPORTED_GPIO_SDP]; + /* SR-IOV virtualization */ + u8 sr_iov_1_1; /* SR-IOV enabled */ + /* VMDQ */ + u8 vmdq; /* VMDQ supported */ + + /* EVB capabilities */ + u8 evb_802_1_qbg; /* Edge Virtual Bridging */ + u8 evb_802_1_qbh; /* Bridge Port Extension */ + + u8 dcb; + u8 iscsi; + u8 mgmt_cem; + + /* WoL and APM support */ +#define IXGBE_WOL_SUPPORT_M BIT(0) +#define IXGBE_ACPI_PROG_MTHD_M BIT(1) +#define IXGBE_PROXY_SUPPORT_M BIT(2) + u8 apm_wol_support; + u8 acpi_prog_mthd; + u8 proxy_support; + bool sec_rev_disabled; + bool update_disabled; + bool nvm_unified_update; + bool netlist_auth; +#define IXGBE_NVM_MGMT_SEC_REV_DISABLED BIT(0) +#define IXGBE_NVM_MGMT_UPDATE_DISABLED BIT(1) +#define IXGBE_NVM_MGMT_UNIFIED_UPD_SUPPORT BIT(3) +#define IXGBE_NVM_MGMT_NETLIST_AUTH_SUPPORT BIT(5) + bool no_drop_policy_support; + /* PCIe reset avoidance */ + bool pcie_reset_avoidance; /* false: not supported, true: supported */ + /* Post update reset restriction */ + bool reset_restrict_support; /* false: not supported, true: supported */ + + /* External topology device images within the NVM */ +#define IXGBE_EXT_TOPO_DEV_IMG_COUNT 4 + u32 ext_topo_dev_img_ver_high[IXGBE_EXT_TOPO_DEV_IMG_COUNT]; + u32 ext_topo_dev_img_ver_low[IXGBE_EXT_TOPO_DEV_IMG_COUNT]; + u8 ext_topo_dev_img_part_num[IXGBE_EXT_TOPO_DEV_IMG_COUNT]; +#define IXGBE_EXT_TOPO_DEV_IMG_PART_NUM_S 8 +#define IXGBE_EXT_TOPO_DEV_IMG_PART_NUM_M \ + MAKEMASK(0xFF, IXGBE_EXT_TOPO_DEV_IMG_PART_NUM_S) + bool ext_topo_dev_img_load_en[IXGBE_EXT_TOPO_DEV_IMG_COUNT]; +#define IXGBE_EXT_TOPO_DEV_IMG_LOAD_EN BIT(0) + bool ext_topo_dev_img_prog_en[IXGBE_EXT_TOPO_DEV_IMG_COUNT]; +#define IXGBE_EXT_TOPO_DEV_IMG_PROG_EN BIT(1) + /* Support for OROM update in Recovery Mode. */ + bool orom_recovery_update; + bool next_cluster_id_support; +}; + +#pragma pack(1) +struct ixgbe_orom_civd_info { + u8 signature[4]; /* Must match ASCII '$CIV' characters */ + u8 checksum; /* Simple modulo 256 sum of all structure bytes must equal 0 */ + __le32 combo_ver; /* Combo Image Version number */ + u8 combo_name_len; /* Length of the unicode combo image version string, max of 32 */ + __le16 combo_name[32]; /* Unicode string representing the Combo Image version */ +}; +#pragma pack() + +/* Function specific capabilities */ +struct ixgbe_hw_func_caps { + struct ixgbe_hw_common_caps common_cap; + u32 num_allocd_vfs; /* Number of allocated VFs */ + u32 vf_base_id; /* Logical ID of the first VF */ + u32 guar_num_vsi; + bool no_drop_policy_ena; +}; + +/* Device wide capabilities */ +struct ixgbe_hw_dev_caps { + struct ixgbe_hw_common_caps common_cap; + u32 num_vfs_exposed; /* Total number of VFs exposed */ + u32 num_vsi_allocd_to_host; /* Excluding EMP VSI */ + u32 num_flow_director_fltr; /* Number of FD filters available */ + u32 num_funcs; +}; + +/* ACI event information */ +struct ixgbe_aci_event { + struct ixgbe_aci_desc desc; + u16 msg_len; + u16 buf_len; + u8 *msg_buf; +}; + +struct ixgbe_aci_info { + enum ixgbe_aci_err last_status; /* last status of sent admin command */ + struct ixgbe_lock lock; /* admin command interface lock */ +}; + +/* Minimum Security Revision information */ +struct ixgbe_minsrev_info { + u32 nvm; + u32 orom; + u8 nvm_valid : 1; + u8 orom_valid : 1; +}; + +/* Enumeration of which flash bank is desired to read from, either the active + * bank or the inactive bank. Used to abstract 1st and 2nd bank notion from + * code which just wants to read the active or inactive flash bank. + */ +enum ixgbe_bank_select { + IXGBE_ACTIVE_FLASH_BANK, + IXGBE_INACTIVE_FLASH_BANK, +}; + +/* Option ROM version information */ +struct ixgbe_orom_info { + u8 major; /* Major version of OROM */ + u8 patch; /* Patch version of OROM */ + u16 build; /* Build version of OROM */ + u32 srev; /* Security revision */ +}; + +/* NVM version information */ +struct ixgbe_nvm_info { + u32 eetrack; + u32 srev; + u8 major; + u8 minor; +}; + +/* netlist version information */ +struct ixgbe_netlist_info { + u32 major; /* major high/low */ + u32 minor; /* minor high/low */ + u32 type; /* type high/low */ + u32 rev; /* revision high/low */ + u32 hash; /* SHA-1 hash word */ + u16 cust_ver; /* customer version */ +}; + +/* Enumeration of possible flash banks for the NVM, OROM, and Netlist modules + * of the flash image. + */ +enum ixgbe_flash_bank { + IXGBE_INVALID_FLASH_BANK, + IXGBE_1ST_FLASH_BANK, + IXGBE_2ND_FLASH_BANK, +}; + +/* information for accessing NVM, OROM, and Netlist flash banks */ +struct ixgbe_bank_info { + u32 nvm_ptr; /* Pointer to 1st NVM bank */ + u32 nvm_size; /* Size of NVM bank */ + u32 orom_ptr; /* Pointer to 1st OROM bank */ + u32 orom_size; /* Size of OROM bank */ + u32 netlist_ptr; /* Pointer to 1st Netlist bank */ + u32 netlist_size; /* Size of Netlist bank */ + enum ixgbe_flash_bank nvm_bank; /* Active NVM bank */ + enum ixgbe_flash_bank orom_bank; /* Active OROM bank */ + enum ixgbe_flash_bank netlist_bank; /* Active Netlist bank */ +}; + +/* Flash Chip Information */ +struct ixgbe_flash_info { + struct ixgbe_orom_info orom; /* Option ROM version info */ + struct ixgbe_nvm_info nvm; /* NVM version information */ + struct ixgbe_netlist_info netlist; /* Netlist version info */ + struct ixgbe_bank_info banks; /* Flash Bank information */ + u16 sr_words; /* Shadow RAM size in words */ + u32 flash_size; /* Size of available flash in bytes */ + u8 blank_nvm_mode; /* is NVM empty (no FW present) */ +}; + +#define IXGBE_NVM_CMD_READ 0x0000000B +#define IXGBE_NVM_CMD_WRITE 0x0000000C + +/* NVM Access command */ +struct ixgbe_nvm_access_cmd { + u32 command; /* NVM command: READ or WRITE */ + u32 offset; /* Offset to read/write, in bytes */ + u32 data_size; /* Size of data field, in bytes */ +}; + +/* NVM Access data */ +struct ixgbe_nvm_access_data { + u32 regval; /* Storage for register value */ +}; + +#endif /* _IXGBE_TYPE_E610_H_ */ diff --git a/sys/dev/ixgbe/ixgbe_vf.c b/sys/dev/ixgbe/ixgbe_vf.c index cac3c6b5e5e7..4e48f7f33c9d 100644 --- a/sys/dev/ixgbe/ixgbe_vf.c +++ b/sys/dev/ixgbe/ixgbe_vf.c @@ -656,7 +656,8 @@ s32 ixgbe_check_mac_link_vf(struct ixgbe_hw *hw, ixgbe_link_speed *speed, break; case IXGBE_LINKS_SPEED_100_82599: *speed = IXGBE_LINK_SPEED_100_FULL; - if (hw->mac.type == ixgbe_mac_X550_vf) { + if (hw->mac.type == ixgbe_mac_X550_vf || + hw->mac.type == ixgbe_mac_E610_vf) { if (links_reg & IXGBE_LINKS_SPEED_NON_STD) *speed = IXGBE_LINK_SPEED_5GB_FULL; } diff --git a/sys/dev/ixl/if_ixl.c b/sys/dev/ixl/if_ixl.c index 60e66aeaf579..43c3af056b67 100644 --- a/sys/dev/ixl/if_ixl.c +++ b/sys/dev/ixl/if_ixl.c @@ -1151,13 +1151,20 @@ ixl_if_enable_intr(if_ctx_t ctx) struct ixl_pf *pf = iflib_get_softc(ctx); struct ixl_vsi *vsi = &pf->vsi; struct i40e_hw *hw = vsi->hw; - struct ixl_rx_queue *que = vsi->rx_queues; + struct ixl_rx_queue *rx_que = vsi->rx_queues; ixl_enable_intr0(hw); /* Enable queue interrupts */ - for (int i = 0; i < vsi->num_rx_queues; i++, que++) - /* TODO: Queue index parameter is probably wrong */ - ixl_enable_queue(hw, que->rxr.me); + if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { + for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) + ixl_enable_queue(hw, rx_que->rxr.me); + } else { + /* + * Set PFINT_LNKLST0 FIRSTQ_INDX to 0x0 to enable + * triggering interrupts by queues. + */ + wr32(hw, I40E_PFINT_LNKLST0, 0x0); + } } /* @@ -1175,11 +1182,13 @@ ixl_if_disable_intr(if_ctx_t ctx) if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) - ixl_disable_queue(hw, rx_que->msix - 1); + ixl_disable_queue(hw, rx_que->rxr.me); } else { - // Set PFINT_LNKLST0 FIRSTQ_INDX to 0x7FF - // stops queues from triggering interrupts - wr32(hw, I40E_PFINT_LNKLST0, 0x7FF); + /* + * Set PFINT_LNKLST0 FIRSTQ_INDX to End of List (0x7FF) + * to stop queues from triggering interrupts. + */ + wr32(hw, I40E_PFINT_LNKLST0, IXL_QUEUE_EOL); } } diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c index 978e5f25ceaf..cc0bc1f3fcd2 100644 --- a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c +++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c @@ -120,7 +120,7 @@ static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn, switch (attrs->dir) { case IPSEC_DIR_OUTBOUND: - if (attrs->replay_esn.replay_window != 0) + if (attrs->replay_esn.trigger) MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN); else MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_MODE); diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c index 6e24395b5577..c45f02cdaf42 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c @@ -1783,8 +1783,8 @@ mlx5e_add_vxlan_rule(struct mlx5e_priv *priv, sa_family_t family, u_int port) el->refcount++; if (el->installed) return (0); - } - el = mlx5e_vxlan_alloc_db_el(priv, proto, port); + } else + el = mlx5e_vxlan_alloc_db_el(priv, proto, port); if ((if_getcapenable(priv->ifp) & IFCAP_VXLAN_HWCSUM) != 0) { err = mlx5e_add_vxlan_rule_from_db(priv, el); diff --git a/sys/dev/netmap/if_ptnet.c b/sys/dev/netmap/if_ptnet.c index bf14bfdb73ea..9c06f7fec530 100644 --- a/sys/dev/netmap/if_ptnet.c +++ b/sys/dev/netmap/if_ptnet.c @@ -27,8 +27,9 @@ /* Driver for ptnet paravirtualized network device. */ #include <sys/cdefs.h> +#include "opt_inet.h" +#include "opt_inet6.h" -#include <sys/types.h> #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> @@ -75,9 +76,6 @@ #include <dev/pci/pcivar.h> #include <dev/pci/pcireg.h> -#include "opt_inet.h" -#include "opt_inet6.h" - #include <sys/selinfo.h> #include <net/netmap.h> #include <dev/netmap/netmap_kern.h> diff --git a/sys/dev/psci/smccc_trng.c b/sys/dev/psci/smccc_trng.c index ab98837d3841..8a2e5508ef48 100644 --- a/sys/dev/psci/smccc_trng.c +++ b/sys/dev/psci/smccc_trng.c @@ -58,7 +58,7 @@ static device_attach_t trng_attach; static unsigned trng_read(void *, unsigned); -static struct random_source random_trng = { +static const struct random_source random_trng = { .rs_ident = "Arm SMCCC TRNG", .rs_source = RANDOM_PURE_ARM_TRNG, .rs_read = trng_read, diff --git a/sys/dev/qcom_rnd/qcom_rnd.c b/sys/dev/qcom_rnd/qcom_rnd.c index fdd0b553523e..a5ece7e00f28 100644 --- a/sys/dev/qcom_rnd/qcom_rnd.c +++ b/sys/dev/qcom_rnd/qcom_rnd.c @@ -63,7 +63,7 @@ static int qcom_rnd_detach(device_t); static int qcom_rnd_harvest(struct qcom_rnd_softc *, void *, size_t *); static unsigned qcom_rnd_read(void *, unsigned); -static struct random_source random_qcom_rnd = { +static const struct random_source random_qcom_rnd = { .rs_ident = "Qualcomm Entropy Adapter", .rs_source = RANDOM_PURE_QUALCOMM, .rs_read = qcom_rnd_read, diff --git a/sys/dev/random/armv8rng.c b/sys/dev/random/armv8rng.c index 61698bfff820..524d80317681 100644 --- a/sys/dev/random/armv8rng.c +++ b/sys/dev/random/armv8rng.c @@ -44,7 +44,7 @@ static u_int random_rndr_read(void *, u_int); static bool has_rndr; -static struct random_source random_armv8_rndr = { +static const struct random_source random_armv8_rndr = { .rs_ident = "Armv8 rndr RNG", .rs_source = RANDOM_PURE_ARMV8, .rs_read = random_rndr_read, diff --git a/sys/dev/random/darn.c b/sys/dev/random/darn.c index a66754e095fb..9bb4991df82f 100644 --- a/sys/dev/random/darn.c +++ b/sys/dev/random/darn.c @@ -56,7 +56,7 @@ static u_int random_darn_read(void *, u_int); -static struct random_source random_darn = { +static const struct random_source random_darn = { .rs_ident = "PowerISA DARN random number generator", .rs_source = RANDOM_PURE_DARN, .rs_read = random_darn_read diff --git a/sys/dev/random/ivy.c b/sys/dev/random/ivy.c index 05474d977276..fa1e4831f1b9 100644 --- a/sys/dev/random/ivy.c +++ b/sys/dev/random/ivy.c @@ -51,7 +51,7 @@ static bool has_rdrand, has_rdseed; static u_int random_ivy_read(void *, u_int); -static struct random_source random_ivy = { +static const struct random_source random_ivy = { .rs_ident = "Intel Secure Key RNG", .rs_source = RANDOM_PURE_RDRAND, .rs_read = random_ivy_read diff --git a/sys/dev/random/nehemiah.c b/sys/dev/random/nehemiah.c index f76071290b8f..56f144169dae 100644 --- a/sys/dev/random/nehemiah.c +++ b/sys/dev/random/nehemiah.c @@ -44,7 +44,7 @@ static u_int random_nehemiah_read(void *, u_int); -static struct random_source random_nehemiah = { +static const struct random_source random_nehemiah = { .rs_ident = "VIA Nehemiah Padlock RNG", .rs_source = RANDOM_PURE_NEHEMIAH, .rs_read = random_nehemiah_read diff --git a/sys/dev/random/random_harvestq.c b/sys/dev/random/random_harvestq.c index c7762967c4fb..84ec174bd08e 100644 --- a/sys/dev/random/random_harvestq.c +++ b/sys/dev/random/random_harvestq.c @@ -110,7 +110,7 @@ __read_frequently u_int hc_source_mask; struct random_sources { CK_LIST_ENTRY(random_sources) rrs_entries; - struct random_source *rrs_source; + const struct random_source *rrs_source; }; static CK_LIST_HEAD(sources_head, random_sources) source_list = @@ -493,9 +493,9 @@ random_healthtest_init(enum random_entropy_source source) * The RCT limit comes from the formula in section 4.4.1. * * The APT cutoff is calculated using the formula in section 4.4.2 - * footnote 10 with the window size changed from 512 to 511, since the - * test as written counts the number of samples equal to the first - * sample in the window, and thus tests W-1 samples. + * footnote 10 with the number of Bernoulli trials changed from W to + * W-1, since the test as written counts the number of samples equal to + * the first sample in the window, and thus tests W-1 samples. */ ht->ht_rct_limit = 35; ht->ht_apt_cutoff = 330; @@ -849,7 +849,7 @@ random_harvest_deregister_source(enum random_entropy_source source) } void -random_source_register(struct random_source *rsource) +random_source_register(const struct random_source *rsource) { struct random_sources *rrs; @@ -868,7 +868,7 @@ random_source_register(struct random_source *rsource) } void -random_source_deregister(struct random_source *rsource) +random_source_deregister(const struct random_source *rsource) { struct random_sources *rrs = NULL; diff --git a/sys/dev/random/randomdev.h b/sys/dev/random/randomdev.h index e1c9ac7b680d..6d742447ea8b 100644 --- a/sys/dev/random/randomdev.h +++ b/sys/dev/random/randomdev.h @@ -103,8 +103,8 @@ struct random_source { random_source_read_t *rs_read; }; -void random_source_register(struct random_source *); -void random_source_deregister(struct random_source *); +void random_source_register(const struct random_source *); +void random_source_deregister(const struct random_source *); #endif /* _KERNEL */ diff --git a/sys/dev/ufshci/ufshci.h b/sys/dev/ufshci/ufshci.h index 9f0faaadeb57..b96d82ff836e 100644 --- a/sys/dev/ufshci/ufshci.h +++ b/sys/dev/ufshci/ufshci.h @@ -160,19 +160,19 @@ enum ufshci_data_direction { UFSHCI_DATA_DIRECTION_RESERVED = 0b11, }; -enum ufshci_overall_command_status { - UFSHCI_OCS_SUCCESS = 0x0, - UFSHCI_OCS_INVALID_COMMAND_TABLE_ATTRIBUTES = 0x01, - UFSHCI_OCS_INVALID_PRDT_ATTRIBUTES = 0x02, - UFSHCI_OCS_MISMATCH_DATA_BUFFER_SIZE = 0x03, - UFSHCI_OCS_MISMATCH_RESPONSE_UPIU_SIZE = 0x04, - UFSHCI_OCS_COMMUNICATION_FAILURE_WITHIN_UIC_LAYERS = 0x05, - UFSHCI_OCS_ABORTED = 0x06, - UFSHCI_OCS_HOST_CONTROLLER_FATAL_ERROR = 0x07, - UFSHCI_OCS_DEVICE_FATAL_ERROR = 0x08, - UFSHCI_OCS_INVALID_CRYPTO_CONFIGURATION = 0x09, - UFSHCI_OCS_GENERAL_CRYPTO_ERROR = 0x0A, - UFSHCI_OCS_INVALID = 0xF, +enum ufshci_utr_overall_command_status { + UFSHCI_UTR_OCS_SUCCESS = 0x0, + UFSHCI_UTR_OCS_INVALID_COMMAND_TABLE_ATTRIBUTES = 0x01, + UFSHCI_UTR_OCS_INVALID_PRDT_ATTRIBUTES = 0x02, + UFSHCI_UTR_OCS_MISMATCH_DATA_BUFFER_SIZE = 0x03, + UFSHCI_UTR_OCS_MISMATCH_RESPONSE_UPIU_SIZE = 0x04, + UFSHCI_UTR_OCS_COMMUNICATION_FAILURE_WITHIN_UIC_LAYERS = 0x05, + UFSHCI_UTR_OCS_ABORTED = 0x06, + UFSHCI_UTR_OCS_HOST_CONTROLLER_FATAL_ERROR = 0x07, + UFSHCI_UTR_OCS_DEVICE_FATAL_ERROR = 0x08, + UFSHCI_UTR_OCS_INVALID_CRYPTO_CONFIGURATION = 0x09, + UFSHCI_UTR_OCS_GENERAL_CRYPTO_ERROR = 0x0A, + UFSHCI_UTR_OCS_INVALID = 0xF, }; struct ufshci_utp_xfer_req_desc { @@ -271,6 +271,18 @@ _Static_assert(sizeof(struct ufshci_utp_cmd_desc) == #define UFSHCI_UTP_TASK_MGMT_REQ_SIZE 32 #define UFSHCI_UTP_TASK_MGMT_RESP_SIZE 32 +enum ufshci_utmr_overall_command_status { + UFSHCI_UTMR_OCS_SUCCESS = 0x0, + UFSHCI_UTMR_OCS_INVALID_TASK_MANAGEMENT_FUNCTION_ATTRIBUTES = 0x01, + UFSHCI_UTMR_OCS_MISMATCH_TASK_MANAGEMENT_REQUEST_SIZE = 0x02, + UFSHCI_UTMR_OCS_MISMATCH_TASK_MANAGEMENT_RESPONSE_SIZE = 0x03, + UFSHCI_UTMR_OCS_PEER_COMMUNICATION_FAILURE = 0x04, + UFSHCI_UTMR_OCS_ABORTED = 0x05, + UFSHCI_UTMR_OCS_FATAL_ERROR = 0x06, + UFSHCI_UTMR_OCS_DEVICE_FATAL_ERROR = 0x07, + UFSHCI_UTMR_OCS_INVALID = 0xF, +}; + /* UFSHCI spec 4.1, section 6.3.1 "UTP Task Management Request Descriptor" */ struct ufshci_utp_task_mgmt_req_desc { /* dword 0 */ @@ -356,6 +368,7 @@ struct ufshci_upiu { _Static_assert(sizeof(struct ufshci_upiu) == 512, "ufshci_upiu must be 512 bytes"); +/* UFS Spec 4.1, section 10.7.1 "COMMAND UPIU" */ struct ufshci_cmd_command_upiu { /* dword 0-2 */ struct ufshci_upiu_header header; @@ -376,6 +389,7 @@ _Static_assert(sizeof(struct ufshci_cmd_command_upiu) % UFSHCI_UPIU_ALIGNMENT == 0, "UPIU requires 64-bit alignment"); +/* UFS Spec 4.1, section 10.7.2 "RESPONSE UPIU" */ struct ufshci_cmd_response_upiu { /* dword 0-2 */ struct ufshci_upiu_header header; @@ -403,6 +417,69 @@ _Static_assert(sizeof(struct ufshci_cmd_response_upiu) % 0, "UPIU requires 64-bit alignment"); +enum task_management_function { + UFSHCI_TASK_MGMT_FUNCTION_ABORT_TASK = 0x01, + UFSHCI_TASK_MGMT_FUNCTION_ABORT_TASK_SET = 0x02, + UFSHCI_TASK_MGMT_FUNCTION_CLEAR_TASK_SET = 0x04, + UFSHCI_TASK_MGMT_FUNCTION_LOGICAL_UNIT_RESET = 0x08, + UFSHCI_TASK_MGMT_FUNCTION_QUERY_TASK = 0x80, + UFSHCI_TASK_MGMT_FUNCTION_QUERY_TASKSET = 0x81, +}; + +/* UFS Spec 4.1, section 10.7.6 "TASK MANAGEMENT REQUEST UPIU" */ +struct ufshci_task_mgmt_request_upiu { + /* dword 0-2 */ + struct ufshci_upiu_header header; + /* dword 3 */ + uint32_t input_param1; /* (Big-endian) */ + /* dword 4 */ + uint32_t input_param2; /* (Big-endian) */ + /* dword 5 */ + uint32_t input_param3; /* (Big-endian) */ + /* dword 6-7 */ + uint8_t reserved[8]; +} __packed __aligned(4); + +_Static_assert(sizeof(struct ufshci_task_mgmt_request_upiu) == 32, + "bad size for ufshci_task_mgmt_request_upiu"); +_Static_assert(sizeof(struct ufshci_task_mgmt_request_upiu) <= + UFSHCI_UTP_XFER_RESP_SIZE, + "bad size for ufshci_task_mgmt_request_upiu"); +_Static_assert(sizeof(struct ufshci_task_mgmt_request_upiu) % + UFSHCI_UPIU_ALIGNMENT == + 0, + "UPIU requires 64-bit alignment"); + +enum task_management_service_response { + UFSHCI_TASK_MGMT_SERVICE_RESPONSE_FUNCTION_COMPLETE = 0x00, + UFSHCI_TASK_MGMT_SERVICE_RESPONSE_FUNCTION_NOT_SUPPORTED = 0x04, + UFSHCI_TASK_MGMT_SERVICE_RESPONSE_FUNCTION_FAILED = 0x05, + UFSHCI_TASK_MGMT_SERVICE_RESPONSE_FUNCTION_SUCCEEDED = 0x08, + UFSHCI_TASK_MGMT_SERVICE_RESPONSE_INCORRECT_LUN = 0x09, +}; + +/* UFS Spec 4.1, section 10.7.7 "TASK MANAGEMENT RESPONSE UPIU" */ +struct ufshci_task_mgmt_response_upiu { + /* dword 0-2 */ + struct ufshci_upiu_header header; + /* dword 3 */ + uint32_t output_param1; /* (Big-endian) */ + /* dword 4 */ + uint32_t output_param2; /* (Big-endian) */ + /* dword 5-7 */ + uint8_t reserved[12]; +} __packed __aligned(4); + +_Static_assert(sizeof(struct ufshci_task_mgmt_response_upiu) == 32, + "bad size for ufshci_task_mgmt_response_upiu"); +_Static_assert(sizeof(struct ufshci_task_mgmt_response_upiu) <= + UFSHCI_UTP_XFER_RESP_SIZE, + "bad size for ufshci_task_mgmt_response_upiu"); +_Static_assert(sizeof(struct ufshci_task_mgmt_response_upiu) % + UFSHCI_UPIU_ALIGNMENT == + 0, + "UPIU requires 64-bit alignment"); + /* UFS Spec 4.1, section 10.7.8 "QUERY REQUEST UPIU" */ enum ufshci_query_function { UFSHCI_QUERY_FUNC_STANDARD_READ_REQUEST = 0x01, @@ -554,6 +631,7 @@ union ufshci_reponse_upiu { struct ufshci_upiu_header header; struct ufshci_cmd_response_upiu cmd_response_upiu; struct ufshci_query_response_upiu query_response_upiu; + struct ufshci_task_mgmt_response_upiu task_mgmt_response_upiu; struct ufshci_nop_in_upiu nop_in_upiu; }; diff --git a/sys/dev/ufshci/ufshci_ctrlr.c b/sys/dev/ufshci/ufshci_ctrlr.c index 55d8363d3287..37bd32665b2b 100644 --- a/sys/dev/ufshci/ufshci_ctrlr.c +++ b/sys/dev/ufshci/ufshci_ctrlr.c @@ -154,12 +154,12 @@ ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev) /* TODO: Initialize interrupt Aggregation Control Register (UTRIACR) */ /* Allocate and initialize UTP Task Management Request List. */ - error = ufshci_utm_req_queue_construct(ctrlr); + error = ufshci_utmr_req_queue_construct(ctrlr); if (error) return (error); /* Allocate and initialize UTP Transfer Request List or SQ/CQ. */ - error = ufshci_ut_req_queue_construct(ctrlr); + error = ufshci_utr_req_queue_construct(ctrlr); if (error) return (error); @@ -179,8 +179,8 @@ ufshci_ctrlr_destruct(struct ufshci_controller *ctrlr, device_t dev) /* TODO: Flush In-flight IOs */ /* Release resources */ - ufshci_utm_req_queue_destroy(ctrlr); - ufshci_ut_req_queue_destroy(ctrlr); + ufshci_utmr_req_queue_destroy(ctrlr); + ufshci_utr_req_queue_destroy(ctrlr); if (ctrlr->tag) bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag); @@ -215,8 +215,8 @@ ufshci_ctrlr_reset(struct ufshci_controller *ctrlr) ufshci_mmio_write_4(ctrlr, ie, 0); /* Release resources */ - ufshci_utm_req_queue_destroy(ctrlr); - ufshci_ut_req_queue_destroy(ctrlr); + ufshci_utmr_req_queue_destroy(ctrlr); + ufshci_utr_req_queue_destroy(ctrlr); /* Reset Host Controller */ error = ufshci_ctrlr_enable_host_ctrlr(ctrlr); @@ -232,12 +232,12 @@ ufshci_ctrlr_reset(struct ufshci_controller *ctrlr) ufshci_mmio_write_4(ctrlr, ie, ie); /* Allocate and initialize UTP Task Management Request List. */ - error = ufshci_utm_req_queue_construct(ctrlr); + error = ufshci_utmr_req_queue_construct(ctrlr); if (error) return (error); /* Allocate and initialize UTP Transfer Request List or SQ/CQ. */ - error = ufshci_ut_req_queue_construct(ctrlr); + error = ufshci_utr_req_queue_construct(ctrlr); if (error) return (error); @@ -245,6 +245,15 @@ ufshci_ctrlr_reset(struct ufshci_controller *ctrlr) } int +ufshci_ctrlr_submit_task_mgmt_request(struct ufshci_controller *ctrlr, + struct ufshci_request *req) +{ + return ( + ufshci_req_queue_submit_request(&ctrlr->task_mgmt_req_queue, req, + /*is_admin*/ false)); +} + +int ufshci_ctrlr_submit_admin_request(struct ufshci_controller *ctrlr, struct ufshci_request *req) { @@ -360,8 +369,8 @@ ufshci_ctrlr_start_config_hook(void *arg) TSENTER(); - if (ufshci_utm_req_queue_enable(ctrlr) == 0 && - ufshci_ut_req_queue_enable(ctrlr) == 0) + if (ufshci_utmr_req_queue_enable(ctrlr) == 0 && + ufshci_utr_req_queue_enable(ctrlr) == 0) ufshci_ctrlr_start(ctrlr); else ufshci_ctrlr_fail(ctrlr, false); @@ -445,9 +454,9 @@ ufshci_ctrlr_poll(struct ufshci_controller *ctrlr) } /* UTP Task Management Request Completion Status */ if (is & UFSHCIM(UFSHCI_IS_REG_UTMRCS)) { - ufshci_printf(ctrlr, "TODO: Implement UTMR completion\n"); ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UTMRCS)); - /* TODO: Implement UTMR completion */ + ufshci_req_queue_process_completions( + &ctrlr->task_mgmt_req_queue); } /* UTP Transfer Request Completion Status */ if (is & UFSHCIM(UFSHCI_IS_REG_UTRCS)) { diff --git a/sys/dev/ufshci/ufshci_ctrlr_cmd.c b/sys/dev/ufshci/ufshci_ctrlr_cmd.c index ddf28c58fa88..71d163d998af 100644 --- a/sys/dev/ufshci/ufshci_ctrlr_cmd.c +++ b/sys/dev/ufshci/ufshci_ctrlr_cmd.c @@ -8,6 +8,32 @@ #include "ufshci_private.h" void +ufshci_ctrlr_cmd_send_task_mgmt_request(struct ufshci_controller *ctrlr, + ufshci_cb_fn_t cb_fn, void *cb_arg, uint8_t function, uint8_t lun, + uint8_t task_tag, uint8_t iid) +{ + struct ufshci_request *req; + struct ufshci_task_mgmt_request_upiu *upiu; + + req = ufshci_allocate_request_vaddr(NULL, 0, M_WAITOK, cb_fn, cb_arg); + + req->request_size = sizeof(struct ufshci_task_mgmt_request_upiu); + req->response_size = sizeof(struct ufshci_task_mgmt_response_upiu); + + upiu = (struct ufshci_task_mgmt_request_upiu *)&req->request_upiu; + memset(upiu, 0, req->request_size); + upiu->header.trans_type = + UFSHCI_UPIU_TRANSACTION_CODE_TASK_MANAGEMENT_REQUEST; + upiu->header.lun = lun; + upiu->header.ext_iid_or_function = function; + upiu->input_param1 = lun; + upiu->input_param2 = task_tag; + upiu->input_param3 = iid; + + ufshci_ctrlr_submit_task_mgmt_request(ctrlr, req); +} + +void ufshci_ctrlr_cmd_send_nop(struct ufshci_controller *ctrlr, ufshci_cb_fn_t cb_fn, void *cb_arg) { diff --git a/sys/dev/ufshci/ufshci_private.h b/sys/dev/ufshci/ufshci_private.h index ac58d44102a0..1a2742ae2e80 100644 --- a/sys/dev/ufshci/ufshci_private.h +++ b/sys/dev/ufshci/ufshci_private.h @@ -125,6 +125,8 @@ struct ufshci_qops { struct ufshci_tracker **tr); void (*ring_doorbell)(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr); + bool (*is_doorbell_cleared)(struct ufshci_controller *ctrlr, + uint8_t slot); void (*clear_cpl_ntf)(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr); bool (*process_cpl)(struct ufshci_req_queue *req_queue); @@ -143,7 +145,10 @@ struct ufshci_hw_queue { int domain; int cpu; - struct ufshci_utp_xfer_req_desc *utrd; + union { + struct ufshci_utp_xfer_req_desc *utrd; + struct ufshci_utp_task_mgmt_req_desc *utmrd; + }; bus_dma_tag_t dma_tag_queue; bus_dmamap_t queuemem_map; @@ -333,6 +338,8 @@ int ufshci_ctrlr_reset(struct ufshci_controller *ctrlr); void ufshci_ctrlr_start_config_hook(void *arg); void ufshci_ctrlr_poll(struct ufshci_controller *ctrlr); +int ufshci_ctrlr_submit_task_mgmt_request(struct ufshci_controller *ctrlr, + struct ufshci_request *req); int ufshci_ctrlr_submit_admin_request(struct ufshci_controller *ctrlr, struct ufshci_request *req); int ufshci_ctrlr_submit_io_request(struct ufshci_controller *ctrlr, @@ -351,6 +358,9 @@ int ufshci_dev_init_ufs_power_mode(struct ufshci_controller *ctrlr); int ufshci_dev_get_descriptor(struct ufshci_controller *ctrlr); /* Controller Command */ +void ufshci_ctrlr_cmd_send_task_mgmt_request(struct ufshci_controller *ctrlr, + ufshci_cb_fn_t cb_fn, void *cb_arg, uint8_t function, uint8_t lun, + uint8_t task_tag, uint8_t iid); void ufshci_ctrlr_cmd_send_nop(struct ufshci_controller *ctrlr, ufshci_cb_fn_t cb_fn, void *cb_arg); void ufshci_ctrlr_cmd_send_query_request(struct ufshci_controller *ctrlr, @@ -361,12 +371,12 @@ void ufshci_ctrlr_cmd_send_scsi_command(struct ufshci_controller *ctrlr, /* Request Queue */ bool ufshci_req_queue_process_completions(struct ufshci_req_queue *req_queue); -int ufshci_utm_req_queue_construct(struct ufshci_controller *ctrlr); -int ufshci_ut_req_queue_construct(struct ufshci_controller *ctrlr); -void ufshci_utm_req_queue_destroy(struct ufshci_controller *ctrlr); -void ufshci_ut_req_queue_destroy(struct ufshci_controller *ctrlr); -int ufshci_utm_req_queue_enable(struct ufshci_controller *ctrlr); -int ufshci_ut_req_queue_enable(struct ufshci_controller *ctrlr); +int ufshci_utmr_req_queue_construct(struct ufshci_controller *ctrlr); +int ufshci_utr_req_queue_construct(struct ufshci_controller *ctrlr); +void ufshci_utmr_req_queue_destroy(struct ufshci_controller *ctrlr); +void ufshci_utr_req_queue_destroy(struct ufshci_controller *ctrlr); +int ufshci_utmr_req_queue_enable(struct ufshci_controller *ctrlr); +int ufshci_utr_req_queue_enable(struct ufshci_controller *ctrlr); void ufshci_req_queue_fail(struct ufshci_controller *ctrlr, struct ufshci_hw_queue *hwq); int ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue, @@ -385,9 +395,17 @@ int ufshci_req_sdb_enable(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue); int ufshci_req_sdb_reserve_slot(struct ufshci_req_queue *req_queue, struct ufshci_tracker **tr); -void ufshci_req_sdb_ring_doorbell(struct ufshci_controller *ctrlr, +void ufshci_req_sdb_utmr_ring_doorbell(struct ufshci_controller *ctrlr, + struct ufshci_tracker *tr); +void ufshci_req_sdb_utr_ring_doorbell(struct ufshci_controller *ctrlr, + struct ufshci_tracker *tr); +bool ufshci_req_sdb_utmr_is_doorbell_cleared(struct ufshci_controller *ctrlr, + uint8_t slot); +bool ufshci_req_sdb_utr_is_doorbell_cleared(struct ufshci_controller *ctrlr, + uint8_t slot); +void ufshci_req_sdb_utmr_clear_cpl_ntf(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr); -void ufshci_req_sdb_clear_cpl_ntf(struct ufshci_controller *ctrlr, +void ufshci_req_sdb_utr_clear_cpl_ntf(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr); bool ufshci_req_sdb_process_cpl(struct ufshci_req_queue *req_queue); int ufshci_req_sdb_get_inflight_io(struct ufshci_controller *ctrlr); diff --git a/sys/dev/ufshci/ufshci_req_queue.c b/sys/dev/ufshci/ufshci_req_queue.c index cc9a2ddae768..bb6efa6d2ccc 100644 --- a/sys/dev/ufshci/ufshci_req_queue.c +++ b/sys/dev/ufshci/ufshci_req_queue.c @@ -19,21 +19,36 @@ static void ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue, struct ufshci_tracker *tr, enum ufshci_data_direction data_direction); -static const struct ufshci_qops sdb_qops = { +static const struct ufshci_qops sdb_utmr_qops = { .construct = ufshci_req_sdb_construct, .destroy = ufshci_req_sdb_destroy, .get_hw_queue = ufshci_req_sdb_get_hw_queue, .enable = ufshci_req_sdb_enable, .reserve_slot = ufshci_req_sdb_reserve_slot, .reserve_admin_slot = ufshci_req_sdb_reserve_slot, - .ring_doorbell = ufshci_req_sdb_ring_doorbell, - .clear_cpl_ntf = ufshci_req_sdb_clear_cpl_ntf, + .ring_doorbell = ufshci_req_sdb_utmr_ring_doorbell, + .is_doorbell_cleared = ufshci_req_sdb_utmr_is_doorbell_cleared, + .clear_cpl_ntf = ufshci_req_sdb_utmr_clear_cpl_ntf, + .process_cpl = ufshci_req_sdb_process_cpl, + .get_inflight_io = ufshci_req_sdb_get_inflight_io, +}; + +static const struct ufshci_qops sdb_utr_qops = { + .construct = ufshci_req_sdb_construct, + .destroy = ufshci_req_sdb_destroy, + .get_hw_queue = ufshci_req_sdb_get_hw_queue, + .enable = ufshci_req_sdb_enable, + .reserve_slot = ufshci_req_sdb_reserve_slot, + .reserve_admin_slot = ufshci_req_sdb_reserve_slot, + .ring_doorbell = ufshci_req_sdb_utr_ring_doorbell, + .is_doorbell_cleared = ufshci_req_sdb_utr_is_doorbell_cleared, + .clear_cpl_ntf = ufshci_req_sdb_utr_clear_cpl_ntf, .process_cpl = ufshci_req_sdb_process_cpl, .get_inflight_io = ufshci_req_sdb_get_inflight_io, }; int -ufshci_utm_req_queue_construct(struct ufshci_controller *ctrlr) +ufshci_utmr_req_queue_construct(struct ufshci_controller *ctrlr) { struct ufshci_req_queue *req_queue; int error; @@ -44,7 +59,7 @@ ufshci_utm_req_queue_construct(struct ufshci_controller *ctrlr) */ req_queue = &ctrlr->task_mgmt_req_queue; req_queue->queue_mode = UFSHCI_Q_MODE_SDB; - req_queue->qops = sdb_qops; + req_queue->qops = sdb_utmr_qops; error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTRM_ENTRIES, /*is_task_mgmt*/ true); @@ -53,21 +68,21 @@ ufshci_utm_req_queue_construct(struct ufshci_controller *ctrlr) } void -ufshci_utm_req_queue_destroy(struct ufshci_controller *ctrlr) +ufshci_utmr_req_queue_destroy(struct ufshci_controller *ctrlr) { ctrlr->task_mgmt_req_queue.qops.destroy(ctrlr, &ctrlr->task_mgmt_req_queue); } int -ufshci_utm_req_queue_enable(struct ufshci_controller *ctrlr) +ufshci_utmr_req_queue_enable(struct ufshci_controller *ctrlr) { return (ctrlr->task_mgmt_req_queue.qops.enable(ctrlr, &ctrlr->task_mgmt_req_queue)); } int -ufshci_ut_req_queue_construct(struct ufshci_controller *ctrlr) +ufshci_utr_req_queue_construct(struct ufshci_controller *ctrlr) { struct ufshci_req_queue *req_queue; int error; @@ -79,7 +94,7 @@ ufshci_ut_req_queue_construct(struct ufshci_controller *ctrlr) */ req_queue = &ctrlr->transfer_req_queue; req_queue->queue_mode = UFSHCI_Q_MODE_SDB; - req_queue->qops = sdb_qops; + req_queue->qops = sdb_utr_qops; error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTR_ENTRIES, /*is_task_mgmt*/ false); @@ -88,14 +103,14 @@ ufshci_ut_req_queue_construct(struct ufshci_controller *ctrlr) } void -ufshci_ut_req_queue_destroy(struct ufshci_controller *ctrlr) +ufshci_utr_req_queue_destroy(struct ufshci_controller *ctrlr) { ctrlr->transfer_req_queue.qops.destroy(ctrlr, &ctrlr->transfer_req_queue); } int -ufshci_ut_req_queue_enable(struct ufshci_controller *ctrlr) +ufshci_utr_req_queue_enable(struct ufshci_controller *ctrlr) { return (ctrlr->transfer_req_queue.qops.enable(ctrlr, &ctrlr->transfer_req_queue)); @@ -213,20 +228,30 @@ ufshci_req_queue_complete_tracker(struct ufshci_tracker *tr) struct ufshci_req_queue *req_queue = tr->req_queue; struct ufshci_request *req = tr->req; struct ufshci_completion cpl; - struct ufshci_utp_xfer_req_desc *desc; uint8_t ocs; bool retry, error, retriable; mtx_assert(&tr->hwq->qlock, MA_NOTOWNED); - bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + /* Copy the response from the Request Descriptor or UTP Command + * Descriptor. */ + if (req_queue->is_task_mgmt) { + cpl.size = tr->response_size; + memcpy(&cpl.response_upiu, + (void *)tr->hwq->utmrd[tr->slot_num].response_upiu, + cpl.size); - cpl.size = tr->response_size; - memcpy(&cpl.response_upiu, (void *)tr->ucd->response_upiu, cpl.size); + ocs = tr->hwq->utmrd[tr->slot_num].overall_command_status; + } else { + bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - desc = &tr->hwq->utrd[tr->slot_num]; - ocs = desc->overall_command_status; + cpl.size = tr->response_size; + memcpy(&cpl.response_upiu, (void *)tr->ucd->response_upiu, + cpl.size); + + ocs = tr->hwq->utrd[tr->slot_num].overall_command_status; + } error = ufshci_req_queue_response_is_error(req_queue, ocs, &cpl.response_upiu); @@ -358,7 +383,19 @@ ufshci_req_queue_prepare_prdt(struct ufshci_tracker *tr) } static void -ufshci_req_queue_fill_descriptor(struct ufshci_utp_xfer_req_desc *desc, +ufshci_req_queue_fill_utmr_descriptor( + struct ufshci_utp_task_mgmt_req_desc *desc, struct ufshci_request *req) +{ + memset(desc, 0, sizeof(struct ufshci_utp_task_mgmt_req_desc)); + desc->interrupt = true; + /* Set the initial value to Invalid. */ + desc->overall_command_status = UFSHCI_UTMR_OCS_INVALID; + + memcpy(desc->request_upiu, &req->request_upiu, req->request_size); +} + +static void +ufshci_req_queue_fill_utr_descriptor(struct ufshci_utp_xfer_req_desc *desc, uint8_t data_direction, const uint64_t paddr, const uint16_t response_off, const uint16_t response_len, const uint16_t prdt_off, const uint16_t prdt_entry_cnt) @@ -378,7 +415,7 @@ ufshci_req_queue_fill_descriptor(struct ufshci_utp_xfer_req_desc *desc, desc->data_direction = data_direction; desc->interrupt = true; /* Set the initial value to Invalid. */ - desc->overall_command_status = UFSHCI_OCS_INVALID; + desc->overall_command_status = UFSHCI_UTR_OCS_INVALID; desc->utp_command_descriptor_base_address = (uint32_t)(paddr & 0xffffffff); desc->utp_command_descriptor_base_address_upper = (uint32_t)(paddr >> @@ -407,26 +444,32 @@ ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue, /* TODO: Check timeout */ - request_len = req->request_size; - response_off = UFSHCI_UTP_XFER_REQ_SIZE; - response_len = req->response_size; - - /* Prepare UTP Command Descriptor */ - memcpy(tr->ucd, &req->request_upiu, request_len); - memset((uint8_t *)tr->ucd + response_off, 0, response_len); - - /* Prepare PRDT */ - if (req->payload_valid) - ufshci_req_queue_prepare_prdt(tr); - - bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - /* Prepare UTP Transfer Request Descriptor. */ - ucd_paddr = tr->ucd_bus_addr; - ufshci_req_queue_fill_descriptor(&tr->hwq->utrd[slot_num], - data_direction, ucd_paddr, response_off, response_len, tr->prdt_off, - tr->prdt_entry_cnt); + if (req_queue->is_task_mgmt) { + /* Prepare UTP Task Management Request Descriptor. */ + ufshci_req_queue_fill_utmr_descriptor(&tr->hwq->utmrd[slot_num], + req); + } else { + request_len = req->request_size; + response_off = UFSHCI_UTP_XFER_REQ_SIZE; + response_len = req->response_size; + + /* Prepare UTP Command Descriptor */ + memcpy(tr->ucd, &req->request_upiu, request_len); + memset((uint8_t *)tr->ucd + response_off, 0, response_len); + + /* Prepare PRDT */ + if (req->payload_valid) + ufshci_req_queue_prepare_prdt(tr); + + /* Prepare UTP Transfer Request Descriptor. */ + ucd_paddr = tr->ucd_bus_addr; + ufshci_req_queue_fill_utr_descriptor(&tr->hwq->utrd[slot_num], + data_direction, ucd_paddr, response_off, response_len, + tr->prdt_off, tr->prdt_entry_cnt); + + bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + } bus_dmamap_sync(tr->hwq->dma_tag_queue, tr->hwq->queuemem_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); diff --git a/sys/dev/ufshci/ufshci_req_sdb.c b/sys/dev/ufshci/ufshci_req_sdb.c index b1f303afaef5..834a459d48e3 100644 --- a/sys/dev/ufshci/ufshci_req_sdb.c +++ b/sys/dev/ufshci/ufshci_req_sdb.c @@ -26,12 +26,6 @@ ufshci_req_sdb_cmd_desc_destroy(struct ufshci_req_queue *req_queue) tr = hwq->act_tr[i]; bus_dmamap_destroy(req_queue->dma_tag_payload, tr->payload_dma_map); - free(tr, M_UFSHCI); - } - - if (hwq->act_tr) { - free(hwq->act_tr, M_UFSHCI); - hwq->act_tr = NULL; } if (req_queue->ucd) { @@ -76,7 +70,6 @@ ufshci_req_sdb_cmd_desc_construct(struct ufshci_req_queue *req_queue, uint32_t num_entries, struct ufshci_controller *ctrlr) { struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q]; - struct ufshci_tracker *tr; size_t ucd_allocsz, payload_allocsz; uint8_t *ucdmem; int i, error; @@ -134,27 +127,14 @@ ufshci_req_sdb_cmd_desc_construct(struct ufshci_req_queue *req_queue, goto out; } - hwq->act_tr = malloc_domainset(sizeof(struct ufshci_tracker *) * - req_queue->num_entries, - M_UFSHCI, DOMAINSET_PREF(req_queue->domain), M_ZERO | M_WAITOK); - for (i = 0; i < req_queue->num_trackers; i++) { - tr = malloc_domainset(sizeof(struct ufshci_tracker), M_UFSHCI, - DOMAINSET_PREF(req_queue->domain), M_ZERO | M_WAITOK); - bus_dmamap_create(req_queue->dma_tag_payload, 0, - &tr->payload_dma_map); + &hwq->act_tr[i]->payload_dma_map); - tr->req_queue = req_queue; - tr->slot_num = i; - tr->slot_state = UFSHCI_SLOT_STATE_FREE; - - tr->ucd = (struct ufshci_utp_cmd_desc *)ucdmem; - tr->ucd_bus_addr = hwq->ucd_bus_addr[i]; + hwq->act_tr[i]->ucd = (struct ufshci_utp_cmd_desc *)ucdmem; + hwq->act_tr[i]->ucd_bus_addr = hwq->ucd_bus_addr[i]; ucdmem += sizeof(struct ufshci_utp_cmd_desc); - - hwq->act_tr[i] = tr; } return (0); @@ -163,25 +143,16 @@ out: return (ENOMEM); } -static bool -ufshci_req_sdb_is_doorbell_cleared(struct ufshci_controller *ctrlr, - uint8_t slot) -{ - uint32_t utrldbr; - - utrldbr = ufshci_mmio_read_4(ctrlr, utrldbr); - return (!(utrldbr & (1 << slot))); -} - int ufshci_req_sdb_construct(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue, uint32_t num_entries, bool is_task_mgmt) { struct ufshci_hw_queue *hwq; - size_t allocsz; + size_t desc_size, alloc_size; uint64_t queuemem_phys; uint8_t *queuemem; - int error; + struct ufshci_tracker *tr; + int i, error; req_queue->ctrlr = ctrlr; req_queue->is_task_mgmt = is_task_mgmt; @@ -209,10 +180,13 @@ ufshci_req_sdb_construct(struct ufshci_controller *ctrlr, * Descriptor (UTRD) or UTP Task Management Request Descriptor (UTMRD)) * Note: UTRD/UTMRD format is restricted to 1024-byte alignment. */ - allocsz = num_entries * sizeof(struct ufshci_utp_xfer_req_desc); + desc_size = is_task_mgmt ? + sizeof(struct ufshci_utp_task_mgmt_req_desc) : + sizeof(struct ufshci_utp_xfer_req_desc); + alloc_size = num_entries * desc_size; error = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 1024, ctrlr->page_size, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, - allocsz, 1, allocsz, 0, NULL, NULL, &hwq->dma_tag_queue); + alloc_size, 1, alloc_size, 0, NULL, NULL, &hwq->dma_tag_queue); if (error != 0) { ufshci_printf(ctrlr, "request queue tag create failed %d\n", error); @@ -227,7 +201,7 @@ ufshci_req_sdb_construct(struct ufshci_controller *ctrlr, } if (bus_dmamap_load(hwq->dma_tag_queue, hwq->queuemem_map, queuemem, - allocsz, ufshci_single_map, &queuemem_phys, 0) != 0) { + alloc_size, ufshci_single_map, &queuemem_phys, 0) != 0) { ufshci_printf(ctrlr, "failed to load request queue memory\n"); bus_dmamem_free(hwq->dma_tag_queue, hwq->utrd, hwq->queuemem_map); @@ -238,13 +212,30 @@ ufshci_req_sdb_construct(struct ufshci_controller *ctrlr, hwq->num_intr_handler_calls = 0; hwq->num_retries = 0; hwq->num_failures = 0; - hwq->utrd = (struct ufshci_utp_xfer_req_desc *)queuemem; hwq->req_queue_addr = queuemem_phys; + /* Allocate trackers */ + hwq->act_tr = malloc_domainset(sizeof(struct ufshci_tracker *) * + req_queue->num_entries, + M_UFSHCI, DOMAINSET_PREF(req_queue->domain), M_ZERO | M_WAITOK); + + for (i = 0; i < req_queue->num_trackers; i++) { + tr = malloc_domainset(sizeof(struct ufshci_tracker), M_UFSHCI, + DOMAINSET_PREF(req_queue->domain), M_ZERO | M_WAITOK); + + tr->req_queue = req_queue; + tr->slot_num = i; + tr->slot_state = UFSHCI_SLOT_STATE_FREE; + + hwq->act_tr[i] = tr; + } + if (is_task_mgmt) { /* UTP Task Management Request (UTMR) */ uint32_t utmrlba, utmrlbau; + hwq->utmrd = (struct ufshci_utp_task_mgmt_req_desc *)queuemem; + utmrlba = hwq->req_queue_addr & 0xffffffff; utmrlbau = hwq->req_queue_addr >> 32; ufshci_mmio_write_4(ctrlr, utmrlba, utmrlba); @@ -253,6 +244,8 @@ ufshci_req_sdb_construct(struct ufshci_controller *ctrlr, /* UTP Transfer Request (UTR) */ uint32_t utrlba, utrlbau; + hwq->utrd = (struct ufshci_utp_xfer_req_desc *)queuemem; + /* * Allocate physical memory for the command descriptor. * UTP Transfer Request (UTR) requires memory for a separate @@ -284,10 +277,22 @@ ufshci_req_sdb_destroy(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue) { struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q]; + struct ufshci_tracker *tr; + int i; if (!req_queue->is_task_mgmt) ufshci_req_sdb_cmd_desc_destroy(&ctrlr->transfer_req_queue); + for (i = 0; i < req_queue->num_trackers; i++) { + tr = hwq->act_tr[i]; + free(tr, M_UFSHCI); + } + + if (hwq->act_tr) { + free(hwq->act_tr, M_UFSHCI); + hwq->act_tr = NULL; + } + if (hwq->utrd != NULL) { bus_dmamap_unload(hwq->dma_tag_queue, hwq->queuemem_map); bus_dmamem_free(hwq->dma_tag_queue, hwq->utrd, @@ -389,7 +394,18 @@ ufshci_req_sdb_reserve_slot(struct ufshci_req_queue *req_queue, } void -ufshci_req_sdb_clear_cpl_ntf(struct ufshci_controller *ctrlr, +ufshci_req_sdb_utmr_clear_cpl_ntf(struct ufshci_controller *ctrlr, + struct ufshci_tracker *tr) +{ + /* + * NOP + * UTP Task Management does not have a Completion Notification + * Register. + */ +} + +void +ufshci_req_sdb_utr_clear_cpl_ntf(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr) { uint32_t utrlcnr; @@ -399,7 +415,19 @@ ufshci_req_sdb_clear_cpl_ntf(struct ufshci_controller *ctrlr, } void -ufshci_req_sdb_ring_doorbell(struct ufshci_controller *ctrlr, +ufshci_req_sdb_utmr_ring_doorbell(struct ufshci_controller *ctrlr, + struct ufshci_tracker *tr) +{ + uint32_t utmrldbr = 0; + + utmrldbr |= 1 << tr->slot_num; + ufshci_mmio_write_4(ctrlr, utmrldbr, utmrldbr); + + tr->req_queue->hwq[UFSHCI_SDB_Q].num_cmds++; +} + +void +ufshci_req_sdb_utr_ring_doorbell(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr) { uint32_t utrldbr = 0; @@ -408,9 +436,26 @@ ufshci_req_sdb_ring_doorbell(struct ufshci_controller *ctrlr, ufshci_mmio_write_4(ctrlr, utrldbr, utrldbr); tr->req_queue->hwq[UFSHCI_SDB_Q].num_cmds++; +} + +bool +ufshci_req_sdb_utmr_is_doorbell_cleared(struct ufshci_controller *ctrlr, + uint8_t slot) +{ + uint32_t utmrldbr; + + utmrldbr = ufshci_mmio_read_4(ctrlr, utmrldbr); + return (!(utmrldbr & (1 << slot))); +} - // utrldbr = ufshci_mmio_read_4(ctrlr, utrldbr); - // printf("DB=0x%08x\n", utrldbr); +bool +ufshci_req_sdb_utr_is_doorbell_cleared(struct ufshci_controller *ctrlr, + uint8_t slot) +{ + uint32_t utrldbr; + + utrldbr = ufshci_mmio_read_4(ctrlr, utrldbr); + return (!(utrldbr & (1 << slot))); } bool @@ -435,7 +480,7 @@ ufshci_req_sdb_process_cpl(struct ufshci_req_queue *req_queue) * is cleared. */ if (tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED && - ufshci_req_sdb_is_doorbell_cleared(req_queue->ctrlr, + req_queue->qops.is_doorbell_cleared(req_queue->ctrlr, slot)) { ufshci_req_queue_complete_tracker(tr); done = true; diff --git a/sys/dev/usb/input/uhid.c b/sys/dev/usb/input/uhid.c index a31081663f0c..e2b97f5accac 100644 --- a/sys/dev/usb/input/uhid.c +++ b/sys/dev/usb/input/uhid.c @@ -40,8 +40,6 @@ * HID spec: http://www.usb.org/developers/devclass_docs/HID1_11.pdf */ -#include "opt_hid.h" - #include <sys/stdint.h> #include <sys/stddef.h> #include <sys/param.h> @@ -928,11 +926,7 @@ static device_method_t uhid_methods[] = { }; static driver_t uhid_driver = { -#ifdef HIDRAW_MAKE_UHID_ALIAS - .name = "hidraw", -#else .name = "uhid", -#endif .methods = uhid_methods, .size = sizeof(struct uhid_softc), }; diff --git a/sys/dev/usb/input/usbhid.c b/sys/dev/usb/input/usbhid.c index df810012b3f8..cba3f34053e5 100644 --- a/sys/dev/usb/input/usbhid.c +++ b/sys/dev/usb/input/usbhid.c @@ -114,6 +114,7 @@ struct usbhid_xfer_ctx { void *cb_ctx; int waiters; bool influx; + bool no_readahead; }; struct usbhid_softc { @@ -272,7 +273,7 @@ usbhid_intr_handler_cb(struct usbhid_xfer_ctx *xfer_ctx) sc->sc_intr_handler(sc->sc_intr_ctx, xfer_ctx->buf, xfer_ctx->req.intr.actlen); - return (0); + return (xfer_ctx->no_readahead ? ECANCELED : 0); } static int @@ -430,6 +431,7 @@ usbhid_intr_start(device_t dev, device_t child __unused) .cb = usbhid_intr_handler_cb, .cb_ctx = sc, .buf = sc->sc_intr_buf, + .no_readahead = hid_test_quirk(&sc->sc_hw, HQ_NO_READAHEAD), }; sc->sc_xfer_ctx[POLL_XFER(USBHID_INTR_IN_DT)] = (struct usbhid_xfer_ctx) { .req.intr.maxlen = @@ -705,6 +707,10 @@ usbhid_ioctl(device_t dev, device_t child __unused, unsigned long cmd, if (error == 0) ucr->ucr_actlen = UGETW(req.ctrl.wLength); break; + case USB_GET_DEVICEINFO: + error = usbd_fill_deviceinfo(sc->sc_udev, + (struct usb_device_info *)data); + break; default: error = EINVAL; } diff --git a/sys/dev/usb/net/if_ipheth.c b/sys/dev/usb/net/if_ipheth.c index f70113c53eb4..cfa800707391 100644 --- a/sys/dev/usb/net/if_ipheth.c +++ b/sys/dev/usb/net/if_ipheth.c @@ -55,6 +55,7 @@ #include <net/if_var.h> #include <dev/usb/usb.h> +#include <dev/usb/usb_cdc.h> #include <dev/usb/usbdi.h> #include <dev/usb/usbdi_util.h> #include "usbdevs.h" @@ -81,6 +82,9 @@ static uether_fn_t ipheth_start; static uether_fn_t ipheth_setmulti; static uether_fn_t ipheth_setpromisc; +static ipheth_consumer_t ipheth_consume_read; +static ipheth_consumer_t ipheth_consume_read_ncm; + #ifdef USB_DEBUG static int ipheth_debug = 0; @@ -96,7 +100,31 @@ static const struct usb_config ipheth_config[IPHETH_N_TRANSFER] = { .direction = UE_DIR_RX, .frames = IPHETH_RX_FRAMES_MAX, .bufsize = (IPHETH_RX_FRAMES_MAX * MCLBYTES), - .flags = {.short_frames_ok = 1,.short_xfer_ok = 1,.ext_buffer = 1,}, + .flags = {.short_frames_ok = 1, .short_xfer_ok = 1, .ext_buffer = 1,}, + .callback = ipheth_bulk_read_callback, + .timeout = 0, /* no timeout */ + }, + + [IPHETH_BULK_TX] = { + .type = UE_BULK, + .endpoint = UE_ADDR_ANY, + .direction = UE_DIR_TX, + .frames = IPHETH_TX_FRAMES_MAX, + .bufsize = (IPHETH_TX_FRAMES_MAX * IPHETH_BUF_SIZE), + .flags = {.force_short_xfer = 1,}, + .callback = ipheth_bulk_write_callback, + .timeout = IPHETH_TX_TIMEOUT, + }, +}; + +static const struct usb_config ipheth_config_ncm[IPHETH_N_TRANSFER] = { + [IPHETH_BULK_RX] = { + .type = UE_BULK, + .endpoint = UE_ADDR_ANY, + .direction = UE_DIR_RX, + .frames = 1, + .bufsize = IPHETH_RX_NCM_BUF_SIZE, + .flags = {.short_frames_ok = 1, .short_xfer_ok = 1,}, .callback = ipheth_bulk_read_callback, .timeout = 0, /* no timeout */ }, @@ -204,6 +232,21 @@ ipheth_get_mac_addr(struct ipheth_softc *sc) return (0); } +static bool +ipheth_enable_ncm(struct ipheth_softc *sc) +{ + struct usb_device_request req; + + req.bmRequestType = UT_WRITE_VENDOR_INTERFACE; + req.bRequest = IPHETH_CMD_ENABLE_NCM; + USETW(req.wValue, 0); + req.wIndex[0] = sc->sc_iface_no; + req.wIndex[1] = 0; + USETW(req.wLength, 0); + + return (usbd_do_request(sc->sc_ue.ue_udev, NULL, &req, NULL) == 0); +} + static int ipheth_probe(device_t dev) { @@ -221,6 +264,7 @@ ipheth_attach(device_t dev) struct ipheth_softc *sc = device_get_softc(dev); struct usb_ether *ue = &sc->sc_ue; struct usb_attach_arg *uaa = device_get_ivars(dev); + const struct usb_config *config; int error; sc->sc_iface_no = uaa->info.bIfaceIndex; @@ -235,18 +279,29 @@ ipheth_attach(device_t dev) device_printf(dev, "Cannot set alternate setting\n"); goto detach; } - error = usbd_transfer_setup(uaa->device, &sc->sc_iface_no, - sc->sc_xfer, ipheth_config, IPHETH_N_TRANSFER, sc, &sc->sc_mtx); - if (error) { - device_printf(dev, "Cannot setup USB transfers\n"); - goto detach; - } + ue->ue_sc = sc; ue->ue_dev = dev; ue->ue_udev = uaa->device; ue->ue_mtx = &sc->sc_mtx; ue->ue_methods = &ipheth_ue_methods; + if (ipheth_enable_ncm(sc)) { + config = ipheth_config_ncm; + sc->is_ncm = true; + sc->consume = &ipheth_consume_read_ncm; + } else { + config = ipheth_config; + sc->consume = &ipheth_consume_read; + } + + error = usbd_transfer_setup(uaa->device, &sc->sc_iface_no, sc->sc_xfer, + config, IPHETH_N_TRANSFER, sc, &sc->sc_mtx); + if (error) { + device_printf(dev, "Cannot setup USB transfers\n"); + goto detach; + } + error = ipheth_get_mac_addr(sc); if (error) { device_printf(dev, "Cannot get MAC address\n"); @@ -389,12 +444,9 @@ ipheth_bulk_write_callback(struct usb_xfer *xfer, usb_error_t error) int actlen; int aframes; - usbd_xfer_status(xfer, &actlen, NULL, &aframes, NULL); - - DPRINTFN(1, "\n"); - switch (USB_GET_STATE(xfer)) { case USB_ST_TRANSFERRED: + usbd_xfer_status(xfer, &actlen, NULL, &aframes, NULL); DPRINTFN(11, "transfer complete: %u bytes in %u frames\n", actlen, aframes); @@ -471,53 +523,40 @@ ipheth_bulk_read_callback(struct usb_xfer *xfer, usb_error_t error) uint8_t x; int actlen; int aframes; - int len; - - usbd_xfer_status(xfer, &actlen, NULL, &aframes, NULL); switch (USB_GET_STATE(xfer)) { case USB_ST_TRANSFERRED: - + usbd_xfer_status(xfer, &actlen, NULL, &aframes, NULL); DPRINTF("received %u bytes in %u frames\n", actlen, aframes); - for (x = 0; x != aframes; x++) { - m = sc->sc_rx_buf[x]; - sc->sc_rx_buf[x] = NULL; - len = usbd_xfer_frame_len(xfer, x); - - if (len < (int)(sizeof(struct ether_header) + - IPHETH_RX_ADJ)) { - m_freem(m); - continue; - } - - m_adj(m, IPHETH_RX_ADJ); - - /* queue up mbuf */ - uether_rxmbuf(&sc->sc_ue, m, len - IPHETH_RX_ADJ); - } + for (x = 0; x != aframes; x++) + sc->consume(xfer, x); /* FALLTHROUGH */ case USB_ST_SETUP: - - for (x = 0; x != IPHETH_RX_FRAMES_MAX; x++) { - if (sc->sc_rx_buf[x] == NULL) { - m = uether_newbuf(); - if (m == NULL) - goto tr_stall; - - /* cancel alignment for ethernet */ - m_adj(m, ETHER_ALIGN); - - sc->sc_rx_buf[x] = m; - } else { - m = sc->sc_rx_buf[x]; + if (!sc->is_ncm) { + for (x = 0; x != IPHETH_RX_FRAMES_MAX; x++) { + if (sc->sc_rx_buf[x] == NULL) { + m = uether_newbuf(); + if (m == NULL) + goto tr_stall; + + /* cancel alignment for ethernet */ + m_adj(m, ETHER_ALIGN); + + sc->sc_rx_buf[x] = m; + } else { + m = sc->sc_rx_buf[x]; + } + usbd_xfer_set_frame_data(xfer, x, m->m_data, m->m_len); } - - usbd_xfer_set_frame_data(xfer, x, m->m_data, m->m_len); + usbd_xfer_set_frames(xfer, x); + } else { + usbd_xfer_set_frame_len(xfer, 0, + IPHETH_RX_NCM_BUF_SIZE); + usbd_xfer_set_frames(xfer, 1); } - /* set number of frames and start hardware */ - usbd_xfer_set_frames(xfer, x); + usbd_transfer_submit(xfer); /* flush any received frames */ uether_rxflush(&sc->sc_ue); @@ -539,3 +578,86 @@ ipheth_bulk_read_callback(struct usb_xfer *xfer, usb_error_t error) break; } } + +static void +ipheth_consume_read(struct usb_xfer *xfer, int x) +{ + struct ipheth_softc *sc = usbd_xfer_softc(xfer); + struct mbuf *m = sc->sc_rx_buf[x]; + int len; + + sc->sc_rx_buf[x] = NULL; + len = usbd_xfer_frame_len(xfer, x); + + if (len < (int)(sizeof(struct ether_header) + IPHETH_RX_ADJ)) { + m_freem(m); + return; + } + + m_adj(m, IPHETH_RX_ADJ); + + /* queue up mbuf */ + uether_rxmbuf(&sc->sc_ue, m, len - IPHETH_RX_ADJ); +} + +static void +ipheth_consume_read_ncm(struct usb_xfer *xfer, int x) +{ + struct ipheth_softc *sc = usbd_xfer_softc(xfer); + struct usb_page_cache *pc = usbd_xfer_get_frame(xfer, 0); + struct ncm_data_cache ncm; + if_t ifp = uether_getifp(&sc->sc_ue); + struct mbuf *new_buf; + int i, actlen; + uint16_t dp_offset, dp_len; + + usbd_xfer_status(xfer, &actlen, NULL, NULL, NULL); + + if (actlen < IPHETH_NCM_HEADER_SIZE) + return; + + usbd_copy_out(pc, 0, &ncm.hdr, sizeof(ncm.hdr)); + + if (UGETDW(ncm.hdr.dwSignature) != 0x484D434E) + return; + + /* Dpt follows the hdr on iOS */ + if (UGETW(ncm.hdr.wDptIndex) != (int)(sizeof(struct usb_ncm16_hdr))) + return; + + usbd_copy_out(pc, UGETW(ncm.hdr.wDptIndex), &ncm.dpt, sizeof(ncm.dpt)); + + if (UGETDW(ncm.dpt.dwSignature) != 0x304D434E) + return; + + usbd_copy_out(pc, UGETW(ncm.hdr.wDptIndex) + sizeof(ncm.dpt), &ncm.dp, + sizeof(ncm.dp)); + + for (i = 0; i < IPHETH_NCM_DPT_DP_NUM; ++i) { + dp_offset = UGETW(ncm.dp[i].wFrameIndex); + dp_len = UGETW(ncm.dp[i].wFrameLength); + + /* (3.3.1 USB CDC NCM spec v1.0) */ + if (dp_offset == 0 && dp_len == 0) + break; + + if (dp_offset < IPHETH_NCM_HEADER_SIZE || dp_offset >= actlen || + actlen < (dp_len + dp_offset) || + dp_len < sizeof(struct ether_header)) { + if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); + continue; + } + if (dp_len > (MCLBYTES - ETHER_ALIGN)) { + if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); + continue; + } + + new_buf = uether_newbuf(); + if (new_buf == NULL) { + if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); + continue; + } + usbd_copy_out(pc, dp_offset, new_buf->m_data, dp_len); + uether_rxmbuf(&sc->sc_ue, new_buf, dp_len); + } +} diff --git a/sys/dev/usb/net/if_iphethvar.h b/sys/dev/usb/net/if_iphethvar.h index 203bb96b6f22..d637e8f67d01 100644 --- a/sys/dev/usb/net/if_iphethvar.h +++ b/sys/dev/usb/net/if_iphethvar.h @@ -41,6 +41,7 @@ #define IPHETH_BUF_SIZE 1514 #define IPHETH_TX_TIMEOUT 5000 /* ms */ +#define IPHETH_RX_NCM_BUF_SIZE 65536 #define IPHETH_RX_FRAMES_MAX 1 #define IPHETH_TX_FRAMES_MAX 8 @@ -55,10 +56,20 @@ #define IPHETH_CTRL_TIMEOUT 5000 /* ms */ #define IPHETH_CMD_GET_MACADDR 0x00 +#define IPHETH_CMD_ENABLE_NCM 0x04 #define IPHETH_CMD_CARRIER_CHECK 0x45 #define IPHETH_CARRIER_ON 0x04 +#define IPHETH_NCM_DPT_DP_NUM 22 +#define IPHETH_NCM_DPT_HEADER_SIZE \ + (sizeof(struct usb_ncm16_dpt) + \ + IPHETH_NCM_DPT_DP_NUM * sizeof(struct usb_ncm16_dp)) +#define IPHETH_NCM_HEADER_SIZE \ + (sizeof(struct usb_ncm16_hdr) + IPHETH_NCM_DPT_HEADER_SIZE) + +typedef void (ipheth_consumer_t)(struct usb_xfer *xfer, int idx); + enum { IPHETH_BULK_TX, IPHETH_BULK_RX, @@ -76,6 +87,16 @@ struct ipheth_softc { uint8_t sc_data[IPHETH_CTRL_BUF_SIZE]; uint8_t sc_iface_no; uint8_t sc_carrier_on; + + bool is_ncm; + + ipheth_consumer_t *consume; +}; + +struct ncm_data_cache { + struct usb_ncm16_hdr hdr; + struct usb_ncm16_dpt dpt; + struct usb_ncm16_dp dp[IPHETH_NCM_DPT_DP_NUM]; }; #define IPHETH_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) diff --git a/sys/dev/usb/usb_device.c b/sys/dev/usb/usb_device.c index 60c2d6745b3f..f0989972f49f 100644 --- a/sys/dev/usb/usb_device.c +++ b/sys/dev/usb/usb_device.c @@ -3111,3 +3111,51 @@ usbd_get_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep) { return (ep->ep_mode); } + +/*------------------------------------------------------------------------* + * usbd_fill_deviceinfo + * + * This function dumps information about an USB device to the + * structure pointed to by the "di" argument. + * + * Returns: + * 0: Success + * Else: Failure + *------------------------------------------------------------------------*/ +int +usbd_fill_deviceinfo(struct usb_device *udev, struct usb_device_info *di) +{ + struct usb_device *hub; + + bzero(di, sizeof(di[0])); + + di->udi_bus = device_get_unit(udev->bus->bdev); + di->udi_addr = udev->address; + di->udi_index = udev->device_index; + strlcpy(di->udi_serial, usb_get_serial(udev), sizeof(di->udi_serial)); + strlcpy(di->udi_vendor, usb_get_manufacturer(udev), sizeof(di->udi_vendor)); + strlcpy(di->udi_product, usb_get_product(udev), sizeof(di->udi_product)); + usb_printbcd(di->udi_release, sizeof(di->udi_release), + UGETW(udev->ddesc.bcdDevice)); + di->udi_vendorNo = UGETW(udev->ddesc.idVendor); + di->udi_productNo = UGETW(udev->ddesc.idProduct); + di->udi_releaseNo = UGETW(udev->ddesc.bcdDevice); + di->udi_class = udev->ddesc.bDeviceClass; + di->udi_subclass = udev->ddesc.bDeviceSubClass; + di->udi_protocol = udev->ddesc.bDeviceProtocol; + di->udi_config_no = udev->curr_config_no; + di->udi_config_index = udev->curr_config_index; + di->udi_power = udev->flags.self_powered ? 0 : udev->power; + di->udi_speed = udev->speed; + di->udi_mode = udev->flags.usb_mode; + di->udi_power_mode = udev->power_mode; + di->udi_suspended = udev->flags.peer_suspended; + + hub = udev->parent_hub; + if (hub) { + di->udi_hubaddr = hub->address; + di->udi_hubindex = hub->device_index; + di->udi_hubport = udev->port_no; + } + return (0); +} diff --git a/sys/dev/usb/usb_generic.c b/sys/dev/usb/usb_generic.c index c0af27d77e5d..ccb0b2184ec4 100644 --- a/sys/dev/usb/usb_generic.c +++ b/sys/dev/usb/usb_generic.c @@ -831,42 +831,7 @@ ugen_get_iface_driver(struct usb_fifo *f, struct usb_gen_descriptor *ugd) int ugen_fill_deviceinfo(struct usb_fifo *f, struct usb_device_info *di) { - struct usb_device *udev; - struct usb_device *hub; - - udev = f->udev; - - bzero(di, sizeof(di[0])); - - di->udi_bus = device_get_unit(udev->bus->bdev); - di->udi_addr = udev->address; - di->udi_index = udev->device_index; - strlcpy(di->udi_serial, usb_get_serial(udev), sizeof(di->udi_serial)); - strlcpy(di->udi_vendor, usb_get_manufacturer(udev), sizeof(di->udi_vendor)); - strlcpy(di->udi_product, usb_get_product(udev), sizeof(di->udi_product)); - usb_printbcd(di->udi_release, sizeof(di->udi_release), - UGETW(udev->ddesc.bcdDevice)); - di->udi_vendorNo = UGETW(udev->ddesc.idVendor); - di->udi_productNo = UGETW(udev->ddesc.idProduct); - di->udi_releaseNo = UGETW(udev->ddesc.bcdDevice); - di->udi_class = udev->ddesc.bDeviceClass; - di->udi_subclass = udev->ddesc.bDeviceSubClass; - di->udi_protocol = udev->ddesc.bDeviceProtocol; - di->udi_config_no = udev->curr_config_no; - di->udi_config_index = udev->curr_config_index; - di->udi_power = udev->flags.self_powered ? 0 : udev->power; - di->udi_speed = udev->speed; - di->udi_mode = udev->flags.usb_mode; - di->udi_power_mode = udev->power_mode; - di->udi_suspended = udev->flags.peer_suspended; - - hub = udev->parent_hub; - if (hub) { - di->udi_hubaddr = hub->address; - di->udi_hubindex = hub->device_index; - di->udi_hubport = udev->port_no; - } - return (0); + return (usbd_fill_deviceinfo(f->udev, di)); } int diff --git a/sys/dev/usb/usbdi.h b/sys/dev/usb/usbdi.h index 08d130aa2868..0826d9f078c4 100644 --- a/sys/dev/usb/usbdi.h +++ b/sys/dev/usb/usbdi.h @@ -38,6 +38,7 @@ struct usb_process; struct usb_proc_msg; struct usb_mbuf; struct usb_fs_privdata; +struct usb_device_info; struct mbuf; typedef enum { /* keep in sync with usb_errstr_table */ @@ -587,6 +588,8 @@ usb_error_t usbd_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep, uint8_t ep_mode); uint8_t usbd_get_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep); +int usbd_fill_deviceinfo(struct usb_device *udev, + struct usb_device_info *di); const struct usb_device_id *usbd_lookup_id_by_info( const struct usb_device_id *id, usb_size_t sizeof_id, diff --git a/sys/dev/virtio/mmio/virtio_mmio.c b/sys/dev/virtio/mmio/virtio_mmio.c index 5a81c8a24779..fe531fced998 100644 --- a/sys/dev/virtio/mmio/virtio_mmio.c +++ b/sys/dev/virtio/mmio/virtio_mmio.c @@ -53,7 +53,6 @@ #include <dev/virtio/virtqueue.h> #include <dev/virtio/mmio/virtio_mmio.h> -#include "virtio_mmio_if.h" #include "virtio_bus_if.h" #include "virtio_if.h" @@ -79,7 +78,6 @@ static int vtmmio_alloc_virtqueues(device_t, int, struct vq_alloc_info *); static int vtmmio_setup_intr(device_t, enum intr_type); static void vtmmio_stop(device_t); -static void vtmmio_poll(device_t); static int vtmmio_reinit(device_t, uint64_t); static void vtmmio_reinit_complete(device_t); static void vtmmio_notify_virtqueue(device_t, uint16_t, bus_size_t); @@ -104,29 +102,11 @@ static void vtmmio_vq_intr(void *); * I/O port read/write wrappers. */ #define vtmmio_write_config_1(sc, o, v) \ -do { \ - if (sc->platform != NULL) \ - VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ - bus_write_1((sc)->res[0], (o), (v)); \ - if (sc->platform != NULL) \ - VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ -} while (0) + bus_write_1((sc)->res[0], (o), (v)) #define vtmmio_write_config_2(sc, o, v) \ -do { \ - if (sc->platform != NULL) \ - VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ - bus_write_2((sc)->res[0], (o), (v)); \ - if (sc->platform != NULL) \ - VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ -} while (0) + bus_write_2((sc)->res[0], (o), (v)) #define vtmmio_write_config_4(sc, o, v) \ -do { \ - if (sc->platform != NULL) \ - VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ - bus_write_4((sc)->res[0], (o), (v)); \ - if (sc->platform != NULL) \ - VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ -} while (0) + bus_write_4((sc)->res[0], (o), (v)) #define vtmmio_read_config_1(sc, o) \ bus_read_1((sc)->res[0], (o)) @@ -157,7 +137,6 @@ static device_method_t vtmmio_methods[] = { DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues), DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr), DEVMETHOD(virtio_bus_stop, vtmmio_stop), - DEVMETHOD(virtio_bus_poll, vtmmio_poll), DEVMETHOD(virtio_bus_reinit, vtmmio_reinit), DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete), DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue), @@ -220,19 +199,9 @@ vtmmio_setup_intr(device_t dev, enum intr_type type) { struct vtmmio_softc *sc; int rid; - int err; sc = device_get_softc(dev); - if (sc->platform != NULL) { - err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev, - vtmmio_vq_intr, sc); - if (err == 0) { - /* Okay we have backend-specific interrupts */ - return (0); - } - } - rid = 0; sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); @@ -597,17 +566,6 @@ vtmmio_stop(device_t dev) vtmmio_reset(device_get_softc(dev)); } -static void -vtmmio_poll(device_t dev) -{ - struct vtmmio_softc *sc; - - sc = device_get_softc(dev); - - if (sc->platform != NULL) - VIRTIO_MMIO_POLL(sc->platform); -} - static int vtmmio_reinit(device_t dev, uint64_t features) { diff --git a/sys/dev/virtio/mmio/virtio_mmio.h b/sys/dev/virtio/mmio/virtio_mmio.h index ac6a96c1c7fe..edcbf0519acc 100644 --- a/sys/dev/virtio/mmio/virtio_mmio.h +++ b/sys/dev/virtio/mmio/virtio_mmio.h @@ -37,7 +37,6 @@ struct vtmmio_virtqueue; struct vtmmio_softc { device_t dev; - device_t platform; struct resource *res[2]; uint64_t vtmmio_features; diff --git a/sys/dev/virtio/mmio/virtio_mmio_fdt.c b/sys/dev/virtio/mmio/virtio_mmio_fdt.c index 7fba8aad8db8..bb9ea8efbaeb 100644 --- a/sys/dev/virtio/mmio/virtio_mmio_fdt.c +++ b/sys/dev/virtio/mmio/virtio_mmio_fdt.c @@ -63,12 +63,10 @@ #include <dev/virtio/mmio/virtio_mmio.h> static int vtmmio_fdt_probe(device_t); -static int vtmmio_fdt_attach(device_t); static device_method_t vtmmio_fdt_methods[] = { /* Device interface. */ DEVMETHOD(device_probe, vtmmio_fdt_probe), - DEVMETHOD(device_attach, vtmmio_fdt_attach), DEVMETHOD_END }; @@ -93,48 +91,3 @@ vtmmio_fdt_probe(device_t dev) return (vtmmio_probe(dev)); } - -static int -vtmmio_setup_platform(device_t dev, struct vtmmio_softc *sc) -{ - phandle_t platform_node; - struct fdt_ic *ic; - phandle_t xref; - phandle_t node; - - sc->platform = NULL; - - if ((node = ofw_bus_get_node(dev)) == -1) - return (ENXIO); - - if (OF_searchencprop(node, "platform", &xref, - sizeof(xref)) == -1) { - return (ENXIO); - } - - platform_node = OF_node_from_xref(xref); - - SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) { - if (ic->iph == platform_node) { - sc->platform = ic->dev; - break; - } - } - - if (sc->platform == NULL) { - /* No platform-specific device. Ignore it. */ - } - - return (0); -} - -static int -vtmmio_fdt_attach(device_t dev) -{ - struct vtmmio_softc *sc; - - sc = device_get_softc(dev); - vtmmio_setup_platform(dev, sc); - - return (vtmmio_attach(dev)); -} diff --git a/sys/dev/virtio/mmio/virtio_mmio_if.m b/sys/dev/virtio/mmio/virtio_mmio_if.m deleted file mode 100644 index baebbd9a0b1c..000000000000 --- a/sys/dev/virtio/mmio/virtio_mmio_if.m +++ /dev/null @@ -1,99 +0,0 @@ -#- -# Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> -# All rights reserved. -# -# This software was developed by SRI International and the University of -# Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) -# ("CTSRD"), as part of the DARPA CRASH research programme. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -# SUCH DAMAGE. -# -# - -#include <sys/types.h> - -# -# This is optional interface to virtio mmio backend. -# Useful when backend is implemented not by the hardware but software, e.g. -# by using another cpu core. -# - -INTERFACE virtio_mmio; - -CODE { - static int - virtio_mmio_prewrite(device_t dev, size_t offset, int val) - { - - return (1); - } - - static int - virtio_mmio_note(device_t dev, size_t offset, int val) - { - - return (1); - } - - static int - virtio_mmio_setup_intr(device_t dev, device_t mmio_dev, - void *handler, void *ih_user) - { - - return (1); - } -}; - -# -# Inform backend we are going to write data at offset. -# -METHOD int prewrite { - device_t dev; - size_t offset; - int val; -} DEFAULT virtio_mmio_prewrite; - -# -# Inform backend we have data wrotten to offset. -# -METHOD int note { - device_t dev; - size_t offset; - int val; -} DEFAULT virtio_mmio_note; - -# -# Inform backend we are going to poll virtqueue. -# -METHOD int poll { - device_t dev; -}; - -# -# Setup backend-specific interrupts. -# -METHOD int setup_intr { - device_t dev; - device_t mmio_dev; - void *handler; - void *ih_user; -} DEFAULT virtio_mmio_setup_intr; diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c index 2ff9be9680b8..ecb3dbb370e5 100644 --- a/sys/dev/virtio/network/if_vtnet.c +++ b/sys/dev/virtio/network/if_vtnet.c @@ -28,6 +28,9 @@ /* Driver for VirtIO network devices. */ +#include "opt_inet.h" +#include "opt_inet6.h" + #include <sys/param.h> #include <sys/eventhandler.h> #include <sys/systm.h> @@ -82,9 +85,6 @@ #include <dev/virtio/network/if_vtnetvar.h> #include "virtio_if.h" -#include "opt_inet.h" -#include "opt_inet6.h" - #if defined(INET) || defined(INET6) #include <machine/in_cksum.h> #endif diff --git a/sys/dev/virtio/random/virtio_random.c b/sys/dev/virtio/random/virtio_random.c index f938ba99ae53..3f30c8b68f4c 100644 --- a/sys/dev/virtio/random/virtio_random.c +++ b/sys/dev/virtio/random/virtio_random.c @@ -77,7 +77,7 @@ static struct virtio_feature_desc vtrnd_feature_desc[] = { { 0, NULL } }; -static struct random_source random_vtrnd = { +static const struct random_source random_vtrnd = { .rs_ident = "VirtIO Entropy Adapter", .rs_source = RANDOM_PURE_VIRTIO, .rs_read = vtrnd_read, diff --git a/sys/dev/virtio/virtio_bus_if.m b/sys/dev/virtio/virtio_bus_if.m index 57ae90bdc917..4181b641faad 100644 --- a/sys/dev/virtio/virtio_bus_if.m +++ b/sys/dev/virtio/virtio_bus_if.m @@ -109,7 +109,3 @@ METHOD void write_device_config { int len; }; -METHOD void poll { - device_t dev; -}; - diff --git a/sys/dev/virtio/virtqueue.c b/sys/dev/virtio/virtqueue.c index 8cc3326dc08e..cc7a233d60ee 100644 --- a/sys/dev/virtio/virtqueue.c +++ b/sys/dev/virtio/virtqueue.c @@ -605,10 +605,8 @@ virtqueue_poll(struct virtqueue *vq, uint32_t *len) { void *cookie; - VIRTIO_BUS_POLL(vq->vq_dev); while ((cookie = virtqueue_dequeue(vq, len)) == NULL) { cpu_spinwait(); - VIRTIO_BUS_POLL(vq->vq_dev); } return (cookie); diff --git a/sys/dev/watchdog/watchdog.c b/sys/dev/watchdog/watchdog.c index e6b6dc1eac70..e1b2e08c3f10 100644 --- a/sys/dev/watchdog/watchdog.c +++ b/sys/dev/watchdog/watchdog.c @@ -50,11 +50,20 @@ #include <sys/syscallsubr.h> /* kern_clock_gettime() */ -static int wd_set_pretimeout(int newtimeout, int disableiftoolong); +#ifdef COMPAT_FREEBSD14 +#define WDIOCPATPAT_14 _IOW('W', 42, u_int) /* pat the watchdog */ +#define WDIOC_SETTIMEOUT_14 _IOW('W', 43, int) /* set/reset the timer */ +#define WDIOC_GETTIMEOUT_14 _IOR('W', 44, int) /* get total timeout */ +#define WDIOC_GETTIMELEFT_14 _IOR('W', 45, int) /* get time left */ +#define WDIOC_GETPRETIMEOUT_14 _IOR('W', 46, int) /* get the pre-timeout */ +#define WDIOC_SETPRETIMEOUT_14 _IOW('W', 47, int) /* set the pre-timeout */ +#endif + +static int wd_set_pretimeout(sbintime_t newtimeout, int disableiftoolong); static void wd_timeout_cb(void *arg); static struct callout wd_pretimeo_handle; -static int wd_pretimeout; +static sbintime_t wd_pretimeout; static int wd_pretimeout_act = WD_SOFT_LOG; static struct callout wd_softtimeo_handle; @@ -63,6 +72,8 @@ static int wd_softtimer; /* true = use softtimer instead of hardware static int wd_softtimeout_act = WD_SOFT_LOG; /* action for the software timeout */ static struct cdev *wd_dev; +static volatile sbintime_t wd_last_sbt; /* last timeout value (sbt) */ +static sbintime_t wd_last_sbt_sysctl; /* last timeout value (sbt) */ static volatile u_int wd_last_u; /* last timeout value set by kern_do_pat */ static u_int wd_last_u_sysctl; /* last timeout value set by kern_do_pat */ static u_int wd_last_u_sysctl_secs; /* wd_last_u in seconds */ @@ -73,6 +84,8 @@ SYSCTL_UINT(_hw_watchdog, OID_AUTO, wd_last_u, CTLFLAG_RD, &wd_last_u_sysctl, 0, "Watchdog last update time"); SYSCTL_UINT(_hw_watchdog, OID_AUTO, wd_last_u_secs, CTLFLAG_RD, &wd_last_u_sysctl_secs, 0, "Watchdog last update time"); +SYSCTL_SBINTIME_MSEC(_hw_watchdog, OID_AUTO, wd_last_msecs, CTLFLAG_RD, + &wd_last_sbt_sysctl, "Watchdog last update time (milliseconds)"); static int wd_lastpat_valid = 0; static time_t wd_lastpat = 0; /* when the watchdog was last patted */ @@ -80,105 +93,94 @@ static time_t wd_lastpat = 0; /* when the watchdog was last patted */ /* Hook for external software watchdog to register for use if needed */ void (*wdog_software_attach)(void); -static void -pow2ns_to_ts(int pow2ns, struct timespec *ts) +/* Legacy interface to watchdog. */ +int +wdog_kern_pat(u_int utim) { - uint64_t ns; + sbintime_t sbt; - ns = 1ULL << pow2ns; - ts->tv_sec = ns / 1000000000ULL; - ts->tv_nsec = ns % 1000000000ULL; -} + if ((utim & WD_LASTVAL) != 0 && (utim & WD_INTERVAL) > 0) + return (EINVAL); -static int -pow2ns_to_ticks(int pow2ns) -{ - struct timeval tv; - struct timespec ts; + if ((utim & WD_LASTVAL) != 0) { + return (wdog_control(WD_CTRL_RESET)); + } - pow2ns_to_ts(pow2ns, &ts); - TIMESPEC_TO_TIMEVAL(&tv, &ts); - return (tvtohz(&tv)); + utim &= WD_INTERVAL; + if (utim == WD_TO_NEVER) + sbt = 0; + else + sbt = nstosbt(1 << utim); + + return (wdog_kern_pat_sbt(sbt)); } -static int -seconds_to_pow2ns(int seconds) +int +wdog_control(int ctrl) { - uint64_t power; - uint64_t ns; - uint64_t shifted; - - ns = ((uint64_t)seconds) * 1000000000ULL; - power = flsll(ns); - shifted = 1ULL << power; - if (shifted <= ns) { - power++; + /* Disable takes precedence */ + if (ctrl == WD_CTRL_DISABLE) { + wdog_kern_pat(0); } - return (power); + + if ((ctrl & WD_CTRL_RESET) != 0) { + wdog_kern_pat_sbt(wd_last_sbt); + } else if ((ctrl & WD_CTRL_ENABLE) != 0) { + wdog_kern_pat_sbt(wd_last_sbt); + } + + return (0); } int -wdog_kern_pat(u_int utim) +wdog_kern_pat_sbt(sbintime_t sbt) { - int error; - static int first = 1; - - if ((utim & WD_LASTVAL) != 0 && (utim & WD_INTERVAL) > 0) - return (EINVAL); - - if ((utim & WD_LASTVAL) != 0) { - /* - * if WD_LASTVAL is set, fill in the bits for timeout - * from the saved value in wd_last_u. - */ - MPASS((wd_last_u & ~WD_INTERVAL) == 0); - utim &= ~WD_LASTVAL; - utim |= wd_last_u; - } else { - /* - * Otherwise save the new interval. - * This can be zero (to disable the watchdog) - */ - wd_last_u = (utim & WD_INTERVAL); + sbintime_t error_sbt = 0; + int pow2ns = 0; + int error = 0; + static bool first = true; + + /* legacy uses power-of-2-nanoseconds time. */ + if (sbt != 0) { + pow2ns = flsl(sbttons(sbt)); + } + if (wd_last_sbt != sbt) { + wd_last_u = pow2ns; wd_last_u_sysctl = wd_last_u; - wd_last_u_sysctl_secs = pow2ns_to_ticks(wd_last_u) / hz; + wd_last_u_sysctl_secs = sbt / SBT_1S; + + wd_last_sbt = sbt; } - if ((utim & WD_INTERVAL) == WD_TO_NEVER) { - utim = 0; - /* Assume all is well; watchdog signals failure. */ - error = 0; - } else { - /* Assume no watchdog available; watchdog flags success */ + if (sbt != 0) error = EOPNOTSUPP; - } + if (wd_softtimer) { - if (utim == 0) { + if (sbt == 0) { callout_stop(&wd_softtimeo_handle); } else { - (void) callout_reset(&wd_softtimeo_handle, - pow2ns_to_ticks(utim), wd_timeout_cb, "soft"); + (void) callout_reset_sbt(&wd_softtimeo_handle, + sbt, 0, wd_timeout_cb, "soft", 0); } error = 0; } else { - EVENTHANDLER_INVOKE(watchdog_list, utim, &error); + EVENTHANDLER_INVOKE(watchdog_sbt_list, sbt, &error_sbt, &error); + EVENTHANDLER_INVOKE(watchdog_list, pow2ns, &error); } /* - * If we no hardware watchdog responded, we have not tried to + * If no hardware watchdog responded, we have not tried to * attach an external software watchdog, and one is available, * attach it now and retry. */ - if (error == EOPNOTSUPP && first && *wdog_software_attach != NULL) { + if (error == EOPNOTSUPP && first && wdog_software_attach != NULL) { (*wdog_software_attach)(); - EVENTHANDLER_INVOKE(watchdog_list, utim, &error); + EVENTHANDLER_INVOKE(watchdog_sbt_list, sbt, &error_sbt, &error); + EVENTHANDLER_INVOKE(watchdog_list, pow2ns, &error); } - first = 0; + first = false; + /* TODO: Print a (rate limited?) warning if error_sbt is too far away */ wd_set_pretimeout(wd_pretimeout, true); - /* - * If we were able to arm/strobe the watchdog, then - * update the last time it was strobed for WDIOC_GETTIMELEFT - */ if (!error) { struct timespec ts; @@ -189,6 +191,7 @@ wdog_kern_pat(u_int utim) wd_lastpat_valid = 1; } } + return (error); } @@ -265,16 +268,14 @@ wd_timeout_cb(void *arg) * current actual watchdog timeout. */ static int -wd_set_pretimeout(int newtimeout, int disableiftoolong) +wd_set_pretimeout(sbintime_t newtimeout, int disableiftoolong) { - u_int utime; - struct timespec utime_ts; - int timeout_ticks; + sbintime_t utime; + sbintime_t timeout_left; - utime = wdog_kern_last_timeout(); - pow2ns_to_ts(utime, &utime_ts); + utime = wdog_kern_last_timeout_sbt(); /* do not permit a pre-timeout >= than the timeout. */ - if (newtimeout >= utime_ts.tv_sec) { + if (newtimeout >= utime) { /* * If 'disableiftoolong' then just fall through * so as to disable the pre-watchdog @@ -292,7 +293,7 @@ wd_set_pretimeout(int newtimeout, int disableiftoolong) return 0; } - timeout_ticks = pow2ns_to_ticks(utime) - (hz*newtimeout); + timeout_left = utime - newtimeout; #if 0 printf("wd_set_pretimeout: " "newtimeout: %d, " @@ -306,8 +307,8 @@ wd_set_pretimeout(int newtimeout, int disableiftoolong) #endif /* We determined the value is sane, so reset the callout */ - (void) callout_reset(&wd_pretimeo_handle, - timeout_ticks, wd_timeout_cb, "pre"); + (void) callout_reset_sbt(&wd_pretimeo_handle, + timeout_left, 0, wd_timeout_cb, "pre", 0); wd_pretimeout = newtimeout; return 0; } @@ -316,6 +317,7 @@ static int wd_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t data, int flags __unused, struct thread *td) { + sbintime_t sb; u_int u; time_t timeleft; int error; @@ -351,29 +353,55 @@ wd_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t data, error = EINVAL; } break; - case WDIOC_GETPRETIMEOUT: - *(int *)data = (int)wd_pretimeout; +#ifdef COMPAT_FREEBSD14 + case WDIOC_GETPRETIMEOUT_14: + *(int *)data = (int)(wd_pretimeout / SBT_1S); break; - case WDIOC_SETPRETIMEOUT: - error = wd_set_pretimeout(*(int *)data, false); + case WDIOC_SETPRETIMEOUT_14: + error = wd_set_pretimeout(*(int *)data * SBT_1S, false); break; - case WDIOC_GETTIMELEFT: + case WDIOC_GETTIMELEFT_14: error = wd_get_time_left(td, &timeleft); if (error) break; *(int *)data = (int)timeleft; break; - case WDIOC_SETTIMEOUT: + case WDIOC_SETTIMEOUT_14: u = *(u_int *)data; - error = wdog_kern_pat(seconds_to_pow2ns(u)); + error = wdog_kern_pat_sbt(mstosbt(u * 1000ULL)); break; - case WDIOC_GETTIMEOUT: + case WDIOC_GETTIMEOUT_14: u = wdog_kern_last_timeout(); *(u_int *)data = u; break; - case WDIOCPATPAT: + case WDIOCPATPAT_14: error = wd_ioctl_patpat(data); break; +#endif + + /* New API */ + case WDIOC_CONTROL: + wdog_control(*(int *)data); + break; + case WDIOC_SETTIMEOUT: + sb = *(sbintime_t *)data; + error = wdog_kern_pat_sbt(sb); + break; + case WDIOC_GETTIMEOUT: + *(sbintime_t *)data = wdog_kern_last_timeout_sbt(); + break; + case WDIOC_GETTIMELEFT: + error = wd_get_time_left(td, &timeleft); + if (error) + break; + *(sbintime_t *)data = (sbintime_t)timeleft * SBT_1S; + break; + case WDIOC_GETPRETIMEOUT: + *(sbintime_t *)data = wd_pretimeout; + break; + case WDIOC_SETPRETIMEOUT: + error = wd_set_pretimeout(*(sbintime_t *)data, false); + break; default: error = ENOIOCTL; break; @@ -392,6 +420,12 @@ wdog_kern_last_timeout(void) return (wd_last_u); } +sbintime_t +wdog_kern_last_timeout_sbt(void) +{ + return (wd_last_sbt); +} + static struct cdevsw wd_cdevsw = { .d_version = D_VERSION, .d_ioctl = wd_ioctl, diff --git a/sys/fs/cd9660/cd9660_vnops.c b/sys/fs/cd9660/cd9660_vnops.c index c4d0e6ba7b30..4a2b80a7ccdd 100644 --- a/sys/fs/cd9660/cd9660_vnops.c +++ b/sys/fs/cd9660/cd9660_vnops.c @@ -124,7 +124,7 @@ cd9660_access(struct vop_access_args *ap) uid_t uid; gid_t gid; - if (vp->v_type == VCHR || vp->v_type == VBLK) + if (VN_ISDEV(vp)) return (EOPNOTSUPP); /* @@ -162,7 +162,7 @@ cd9660_open(struct vop_open_args *ap) struct vnode *vp = ap->a_vp; struct iso_node *ip = VTOI(vp); - if (vp->v_type == VCHR || vp->v_type == VBLK) + if (VN_ISDEV(vp)) return (EOPNOTSUPP); vnode_create_vobject(vp, ip->i_size, ap->a_td); @@ -191,7 +191,7 @@ cd9660_getattr(struct vop_getattr_args *ap) vap->va_atime = ip->inode.iso_atime; vap->va_mtime = ip->inode.iso_mtime; vap->va_ctime = ip->inode.iso_ctime; - vap->va_rdev = ip->inode.iso_rdev; + vap->va_rdev = VN_ISDEV(vp) ? ip->inode.iso_rdev : NODEV; vap->va_size = (u_quad_t) ip->i_size; if (ip->i_size == 0 && (vap->va_mode & S_IFMT) == S_IFLNK) { @@ -242,7 +242,7 @@ cd9660_ioctl(struct vop_ioctl_args *ap) VOP_UNLOCK(vp); return (EBADF); } - if (vp->v_type == VCHR || vp->v_type == VBLK) { + if (VN_ISDEV(vp)) { VOP_UNLOCK(vp); return (EOPNOTSUPP); } @@ -280,7 +280,7 @@ cd9660_read(struct vop_read_args *ap) int seqcount; long size, n, on; - if (vp->v_type == VCHR || vp->v_type == VBLK) + if (VN_ISDEV(vp)) return (EOPNOTSUPP); seqcount = ap->a_ioflag >> IO_SEQSHIFT; @@ -711,7 +711,7 @@ cd9660_strategy(struct vop_strategy_args *ap) struct bufobj *bo; ip = VTOI(vp); - if (vp->v_type == VBLK || vp->v_type == VCHR) + if (VN_ISDEV(vp)) panic("cd9660_strategy: spec"); if (bp->b_blkno == bp->b_lblkno) { bp->b_blkno = (ip->iso_start + bp->b_lblkno) << @@ -818,7 +818,7 @@ cd9660_getpages(struct vop_getpages_args *ap) struct vnode *vp; vp = ap->a_vp; - if (vp->v_type == VCHR || vp->v_type == VBLK) + if (VN_ISDEV(vp)) return (EOPNOTSUPP); if (use_buf_pager) diff --git a/sys/fs/ext2fs/ext2_vnops.c b/sys/fs/ext2fs/ext2_vnops.c index 064c10bd18b2..00389c927087 100644 --- a/sys/fs/ext2fs/ext2_vnops.c +++ b/sys/fs/ext2fs/ext2_vnops.c @@ -222,7 +222,7 @@ ext2_itimes_locked(struct vnode *vp) ip = VTOI(vp); if ((ip->i_flag & (IN_ACCESS | IN_CHANGE | IN_UPDATE)) == 0) return; - if ((vp->v_type == VBLK || vp->v_type == VCHR)) + if (VN_ISDEV(vp)) ip->i_flag |= IN_LAZYMOD; else ip->i_flag |= IN_MODIFIED; @@ -276,7 +276,7 @@ static int ext2_open(struct vop_open_args *ap) { - if (ap->a_vp->v_type == VBLK || ap->a_vp->v_type == VCHR) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); /* @@ -360,7 +360,7 @@ ext2_getattr(struct vop_getattr_args *ap) vap->va_nlink = ip->i_nlink; vap->va_uid = ip->i_uid; vap->va_gid = ip->i_gid; - vap->va_rdev = ip->i_rdev; + vap->va_rdev = VN_ISDEV(vp) ? ip->i_rdev : NODEV; vap->va_size = ip->i_size; vap->va_atime.tv_sec = ip->i_atime; vap->va_atime.tv_nsec = E2DI_HAS_XTIME(ip) ? ip->i_atimensec : 0; @@ -1571,7 +1571,7 @@ ext2_strategy(struct vop_strategy_args *ap) daddr_t blkno; int error; - if (vp->v_type == VBLK || vp->v_type == VCHR) + if (VN_ISDEV(vp)) panic("ext2_strategy: spec"); if (bp->b_blkno == bp->b_lblkno) { if (VTOI(ap->a_vp)->i_flag & IN_E4EXTENTS) @@ -1733,7 +1733,7 @@ ext2_deleteextattr(struct vop_deleteextattr_args *ap) if (!EXT2_HAS_COMPAT_FEATURE(ip->i_e2fs, EXT2F_COMPAT_EXT_ATTR)) return (EOPNOTSUPP); - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, @@ -1771,7 +1771,7 @@ ext2_getextattr(struct vop_getextattr_args *ap) if (!EXT2_HAS_COMPAT_FEATURE(ip->i_e2fs, EXT2F_COMPAT_EXT_ATTR)) return (EOPNOTSUPP); - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, @@ -1814,7 +1814,7 @@ ext2_listextattr(struct vop_listextattr_args *ap) if (!EXT2_HAS_COMPAT_FEATURE(ip->i_e2fs, EXT2F_COMPAT_EXT_ATTR)) return (EOPNOTSUPP); - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, @@ -1855,7 +1855,7 @@ ext2_setextattr(struct vop_setextattr_args *ap) if (!EXT2_HAS_COMPAT_FEATURE(ip->i_e2fs, EXT2F_COMPAT_EXT_ATTR)) return (EOPNOTSUPP); - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, diff --git a/sys/fs/fuse/fuse_vnops.c b/sys/fs/fuse/fuse_vnops.c index b782146b7278..de60a4717dd4 100644 --- a/sys/fs/fuse/fuse_vnops.c +++ b/sys/fs/fuse/fuse_vnops.c @@ -1748,7 +1748,7 @@ fuse_vnop_open(struct vop_open_args *ap) if (fuse_isdeadfs(vp)) return (EXTERROR(ENXIO, "This FUSE session is about " "to be closed")); - if (vp->v_type == VCHR || vp->v_type == VBLK || vp->v_type == VFIFO) + if (VN_ISDEV(vp) || vp->v_type == VFIFO) return (EXTERROR(EOPNOTSUPP, "Unsupported vnode type", vp->v_type)); if ((a_mode & (FREAD | FWRITE | FEXEC)) == 0) diff --git a/sys/fs/nfs/nfs_commonkrpc.c b/sys/fs/nfs/nfs_commonkrpc.c index 0ae3b94bef89..1e4e8506790f 100644 --- a/sys/fs/nfs/nfs_commonkrpc.c +++ b/sys/fs/nfs/nfs_commonkrpc.c @@ -239,6 +239,7 @@ static bool nfscl_use_gss[NFSV42_NPROCS] = { true, true, true, + true, }; /* diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c index 67e33193ecec..7f5b29ca2085 100644 --- a/sys/fs/nfs/nfs_commonsubs.c +++ b/sys/fs/nfs/nfs_commonsubs.c @@ -187,7 +187,7 @@ struct nfsv4_opflag nfsv4_opflag[NFSV42_NOPS] = { { 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Read Plus */ { 0, 1, 0, 0, LK_SHARED, 1, 0 }, /* Seek */ { 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Write Same */ - { 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Clone */ + { 2, 1, 1, 0, LK_SHARED, 1, 0 }, /* Clone */ { 0, 1, 0, 0, LK_SHARED, 1, 1 }, /* Getxattr */ { 0, 1, 1, 1, LK_EXCLUSIVE, 1, 1 }, /* Setxattr */ { 0, 1, 0, 0, LK_SHARED, 1, 1 }, /* Listxattrs */ @@ -219,7 +219,7 @@ NFSD_VNET_DEFINE_STATIC(u_char *, nfsrv_dnsname) = NULL; static int nfs_bigreply[NFSV42_NPROCS] = { 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 1, 0, 0, 0, 0, 0, 0 }; + 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 }; /* local functions */ static int nfsrv_skipace(struct nfsrv_descript *nd, int *acesizep); @@ -310,6 +310,7 @@ static struct { { NFSV4OP_LAYOUTERROR, 1, "LayoutError", 11, }, { NFSV4OP_VERIFY, 3, "AppendWrite", 11, }, { NFSV4OP_OPENATTR, 3, "OpenAttr", 8, }, + { NFSV4OP_SAVEFH, 5, "Clone", 5, }, }; /* @@ -319,7 +320,7 @@ static int nfs_bigrequest[NFSV42_NPROCS] = { 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 0, 1, 0 + 0, 1, 0, 0 }; /* @@ -648,7 +649,7 @@ nfscl_fillsattr(struct nfsrv_descript *nd, struct vattr *vap, NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMECREATE); (void) nfsv4_fillattr(nd, vp->v_mount, vp, NULL, vap, NULL, 0, &attrbits, NULL, NULL, 0, 0, 0, 0, (uint64_t)0, NULL, - false, false, false); + false, false, false, 0); break; } } @@ -1302,7 +1303,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp, struct nfsv3_pathconf *pc, struct statfs *sbp, struct nfsstatfs *sfp, struct nfsfsinfo *fsp, NFSACL_T *aclp, int compare, int *retcmpp, u_int32_t *leasep, u_int32_t *rderrp, bool *has_namedattrp, - NFSPROC_T *p, struct ucred *cred) + uint32_t *clone_blksizep, NFSPROC_T *p, struct ucred *cred) { u_int32_t *tl; int i = 0, j, k, l = 0, m, bitpos, attrsum = 0; @@ -1437,6 +1438,13 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp, NFSCLRBIT_ATTRBIT(&checkattrbits, NFSATTRBIT_SYSTEM); } + /* Some filesystems do not support block cloning */ + if (vp == NULL || VOP_PATHCONF(vp, + _PC_CLONE_BLKSIZE, &has_pathconf) != 0) + has_pathconf = 0; + if (has_pathconf == 0) + NFSCLRBIT_ATTRBIT(&checkattrbits, + NFSATTRBIT_CLONEBLKSIZE); if (!NFSEQUAL_ATTRBIT(&retattrbits, &checkattrbits) || retnotsup) *retcmpp = NFSERR_NOTSAME; @@ -2374,6 +2382,23 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp, if (compare && !(*retcmpp) && i != nfs_srvmaxio) *retcmpp = NFSERR_NOTSAME; break; + case NFSATTRBIT_CLONEBLKSIZE: + NFSM_DISSECT(tl, uint32_t *, NFSX_UNSIGNED); + if (compare) { + if (!(*retcmpp)) { + if (vp == NULL || VOP_PATHCONF(vp, + _PC_CLONE_BLKSIZE, &has_pathconf) + != 0) + has_pathconf = 0; + if (has_pathconf != + fxdr_unsigned(uint32_t, *tl)) + *retcmpp = NFSERR_NOTSAME; + } + } else if (clone_blksizep != NULL) { + *clone_blksizep = fxdr_unsigned(uint32_t, *tl); + } + attrsum += NFSX_UNSIGNED; + break; case NFSATTRBIT_CHANGEATTRTYPE: NFSM_DISSECT(tl, uint32_t *, NFSX_UNSIGNED); if (compare) { @@ -2648,7 +2673,7 @@ nfsv4_fillattr(struct nfsrv_descript *nd, struct mount *mp, vnode_t vp, nfsattrbit_t *attrbitp, struct ucred *cred, NFSPROC_T *p, int isdgram, int reterr, int supports_nfsv4acls, int at_root, uint64_t mounted_on_fileno, struct statfs *pnfssf, bool xattrsupp, bool has_hiddensystem, - bool has_namedattr) + bool has_namedattr, uint32_t clone_blksize) { int bitpos, retnum = 0; u_int32_t *tl; @@ -2771,6 +2796,9 @@ nfsv4_fillattr(struct nfsrv_descript *nd, struct mount *mp, vnode_t vp, NFSCLRBIT_ATTRBIT(&attrbits, NFSATTRBIT_HIDDEN); NFSCLRBIT_ATTRBIT(&attrbits, NFSATTRBIT_SYSTEM); } + if (clone_blksize == 0) + NFSCLRBIT_ATTRBIT(&attrbits, + NFSATTRBIT_CLONEBLKSIZE); retnum += nfsrv_putattrbit(nd, &attrbits); break; case NFSATTRBIT_TYPE: @@ -3249,6 +3277,11 @@ nfsv4_fillattr(struct nfsrv_descript *nd, struct mount *mp, vnode_t vp, } retnum += NFSX_UNSIGNED; break; + case NFSATTRBIT_CLONEBLKSIZE: + NFSM_BUILD(tl, uint32_t *, NFSX_UNSIGNED); + *tl = txdr_unsigned(clone_blksize); + retnum += NFSX_UNSIGNED; + break; default: printf("EEK! Bad V4 attribute bitpos=%d\n", bitpos); } diff --git a/sys/fs/nfs/nfs_var.h b/sys/fs/nfs/nfs_var.h index 54f60a753c50..61083ecf2d66 100644 --- a/sys/fs/nfs/nfs_var.h +++ b/sys/fs/nfs/nfs_var.h @@ -286,6 +286,8 @@ int nfsrvd_deallocate(struct nfsrv_descript *, int, vnode_t, struct nfsexstuff *); int nfsrvd_copy_file_range(struct nfsrv_descript *, int, vnode_t, vnode_t, struct nfsexstuff *, struct nfsexstuff *); +int nfsrvd_clone(struct nfsrv_descript *, int, + vnode_t, vnode_t, struct nfsexstuff *, struct nfsexstuff *); int nfsrvd_seek(struct nfsrv_descript *, int, vnode_t, struct nfsexstuff *); int nfsrvd_getxattr(struct nfsrv_descript *, int, @@ -341,7 +343,8 @@ int nfsv4_loadattr(struct nfsrv_descript *, vnode_t, struct nfsvattr *, struct nfsfh **, fhandle_t *, int, struct nfsv3_pathconf *, struct statfs *, struct nfsstatfs *, struct nfsfsinfo *, NFSACL_T *, - int, int *, u_int32_t *, u_int32_t *, bool *, NFSPROC_T *, struct ucred *); + int, int *, u_int32_t *, u_int32_t *, bool *, uint32_t *, NFSPROC_T *, + struct ucred *); int nfsv4_lock(struct nfsv4lock *, int, int *, struct mtx *, struct mount *); void nfsv4_unlock(struct nfsv4lock *, int); void nfsv4_relref(struct nfsv4lock *); @@ -397,7 +400,7 @@ void nfsrv_wcc(struct nfsrv_descript *, int, struct nfsvattr *, int, int nfsv4_fillattr(struct nfsrv_descript *, struct mount *, vnode_t, NFSACL_T *, struct vattr *, fhandle_t *, int, nfsattrbit_t *, struct ucred *, NFSPROC_T *, int, int, int, int, uint64_t, struct statfs *, bool, bool, - bool); + bool, uint32_t); void nfsrv_fillattr(struct nfsrv_descript *, struct nfsvattr *); struct mbuf *nfsrv_adj(struct mbuf *, int, int); void nfsrv_postopattr(struct nfsrv_descript *, int, struct nfsvattr *); @@ -517,10 +520,10 @@ int nfsrpc_lock(struct nfsrv_descript *, struct nfsmount *, vnode_t, u_int8_t *, int, struct nfscllockowner *, int, int, u_int64_t, u_int64_t, short, struct ucred *, NFSPROC_T *, int); int nfsrpc_statfs(vnode_t, struct nfsstatfs *, struct nfsfsinfo *, uint32_t *, - struct ucred *, NFSPROC_T *, struct nfsvattr *, int *); + uint32_t *, struct ucred *, NFSPROC_T *, struct nfsvattr *, int *); int nfsrpc_fsinfo(vnode_t, struct nfsfsinfo *, struct ucred *, NFSPROC_T *, struct nfsvattr *, int *); -int nfsrpc_pathconf(vnode_t, struct nfsv3_pathconf *, bool *, +int nfsrpc_pathconf(vnode_t, struct nfsv3_pathconf *, bool *, uint32_t *, struct ucred *, NFSPROC_T *, struct nfsvattr *, int *); int nfsrpc_renew(struct nfsclclient *, struct nfsclds *, struct ucred *, NFSPROC_T *); @@ -562,6 +565,8 @@ int nfsrpc_deallocate(vnode_t, off_t, off_t, struct nfsvattr *, int *, int nfsrpc_copy_file_range(vnode_t, off_t *, vnode_t, off_t *, size_t *, unsigned int, int *, struct nfsvattr *, int *, struct nfsvattr *, struct ucred *, bool, bool *); +int nfsrpc_clone(vnode_t, off_t *, vnode_t, off_t *, size_t *, bool, + int *, struct nfsvattr *, int *, struct nfsvattr *, struct ucred *); int nfsrpc_seek(vnode_t, off_t *, bool *, int, struct ucred *, struct nfsvattr *, int *); int nfsrpc_getextattr(vnode_t, const char *, struct uio *, ssize_t *, @@ -668,7 +673,7 @@ int nfscl_nget(mount_t, vnode_t, struct nfsfh *, NFSPROC_T *nfscl_getparent(NFSPROC_T *); void nfscl_start_renewthread(struct nfsclclient *); void nfscl_loadsbinfo(struct nfsmount *, struct nfsstatfs *, void *); -void nfscl_loadfsinfo (struct nfsmount *, struct nfsfsinfo *); +void nfscl_loadfsinfo(struct nfsmount *, struct nfsfsinfo *, uint32_t); void nfscl_delegreturn(struct nfscldeleg *, int, struct nfsmount *, struct ucred *, NFSPROC_T *); void nfsrvd_cbinit(int); @@ -737,7 +742,7 @@ int nfsvno_updfilerev(vnode_t, struct nfsvattr *, struct nfsrv_descript *, int nfsvno_fillattr(struct nfsrv_descript *, struct mount *, vnode_t, struct nfsvattr *, fhandle_t *, int, nfsattrbit_t *, struct ucred *, NFSPROC_T *, int, int, int, int, uint64_t, bool, bool, - bool); + bool, uint32_t); int nfsrv_sattr(struct nfsrv_descript *, vnode_t, struct nfsvattr *, nfsattrbit_t *, NFSACL_T *, NFSPROC_T *); int nfsv4_sattr(struct nfsrv_descript *, vnode_t, struct nfsvattr *, nfsattrbit_t *, diff --git a/sys/fs/nfs/nfsport.h b/sys/fs/nfs/nfsport.h index c30b46261df0..4e9aae70da6f 100644 --- a/sys/fs/nfs/nfsport.h +++ b/sys/fs/nfs/nfsport.h @@ -442,10 +442,13 @@ /* Do a NFSv4 Openattr. */ #define NFSPROC_OPENATTR 70 +/* Do a NFSv4.2 Clone. */ +#define NFSPROC_CLONE 71 + /* * Must be defined as one higher than the last NFSv4.2 Proc# above. */ -#define NFSV42_NPROCS 71 +#define NFSV42_NPROCS 72 /* Value of NFSV42_NPROCS for old nfsstats structure. (Always 69) */ #define NFSV42_OLDNPROCS 69 @@ -477,7 +480,7 @@ struct nfsstatsv1 { uint64_t readlink_bios; uint64_t biocache_readdirs; uint64_t readdir_bios; - uint64_t rpccnt[NFSV42_NPROCS + 9]; + uint64_t rpccnt[NFSV42_NPROCS + 8]; uint64_t rpcretries; uint64_t srvrpccnt[NFSV42_NOPS + NFSV4OP_FAKENOPS + 15]; uint64_t srvlayouts; @@ -906,15 +909,6 @@ int nfsmsleep(void *, void *, int, const char *, struct timespec *); #define NFSBZERO(s, l) bzero((s), (l)) /* - * Some queue.h files don't have these dfined in them. - */ -#ifndef LIST_END -#define LIST_END(head) NULL -#define SLIST_END(head) NULL -#define TAILQ_END(head) NULL -#endif - -/* * This must be defined to be a global variable that increments once * per second, but never stops or goes backwards, even when a "date" * command changes the TOD clock. It is used for delta times for @@ -1023,7 +1017,7 @@ MALLOC_DECLARE(M_NEWNFSDSESSION); int nfscl_loadattrcache(struct vnode **, struct nfsvattr *, void *, int, int); int newnfs_realign(struct mbuf **, int); bool ncl_pager_setsize(struct vnode *vp, u_quad_t *nsizep); -void ncl_copy_vattr(struct vattr *dst, struct vattr *src); +void ncl_copy_vattr(struct vnode *vp, struct vattr *dst, struct vattr *src); /* * If the port runs on an SMP box that can enforce Atomic ops with low @@ -1035,9 +1029,6 @@ void ncl_copy_vattr(struct vattr *dst, struct vattr *src); #define NFSDECRGLOBAL(a) ((a)--) /* - * Assorted funky stuff to make things work under Darwin8. - */ -/* * These macros checks for a field in vattr being set. */ #define NFSATTRISSET(t, v, a) ((v)->a != (t)VNOVAL) diff --git a/sys/fs/nfs/nfsproto.h b/sys/fs/nfs/nfsproto.h index cb5a80e8df73..d628108bdc1a 100644 --- a/sys/fs/nfs/nfsproto.h +++ b/sys/fs/nfs/nfsproto.h @@ -411,10 +411,13 @@ /* Do a NFSv4 Openattr. */ #define NFSPROC_OPENATTR 70 +/* Do a NFSv4.2 Clone. */ +#define NFSPROC_CLONE 71 + /* * Must be defined as one higher than the last NFSv4.2 Proc# above. */ -#define NFSV42_NPROCS 71 +#define NFSV42_NPROCS 72 /* Value of NFSV42_NPROCS for old nfsstats structure. (Always 69) */ #define NFSV42_OLDNPROCS 69 @@ -1194,6 +1197,7 @@ struct nfsv3_sattr { NFSATTRBM_LAYOUTBLKSIZE | \ NFSATTRBM_LAYOUTALIGNMENT | \ NFSATTRBM_SUPPATTREXCLCREAT | \ + NFSATTRBM_CLONEBLKSIZE | \ NFSATTRBM_CHANGEATTRTYPE | \ NFSATTRBM_XATTRSUPPORT) @@ -1242,7 +1246,8 @@ struct nfsv3_sattr { * NFSATTRBIT_NFSV42 - Attributes only supported by NFSv4.2. */ #define NFSATTRBIT_NFSV42_2 \ - (NFSATTRBM_CHANGEATTRTYPE | \ + (NFSATTRBM_CLONEBLKSIZE | \ + NFSATTRBM_CHANGEATTRTYPE | \ NFSATTRBM_XATTRSUPPORT | \ NFSATTRBM_MODEUMASK) @@ -1415,7 +1420,7 @@ struct nfsv3_sattr { /* * NFSGETATTRBIT_STATFS2 - bits 64<->95 */ -#define NFSGETATTRBIT_STATFS2 0 +#define NFSGETATTRBIT_STATFS2 (NFSATTRBM_CLONEBLKSIZE) /* * Set of attributes for the equivalent of an nfsv3 pathconf rpc. @@ -1438,7 +1443,7 @@ struct nfsv3_sattr { /* * NFSGETATTRBIT_PATHCONF2 - bits 64<->95 */ -#define NFSGETATTRBIT_PATHCONF2 0 +#define NFSGETATTRBIT_PATHCONF2 (NFSATTRBM_CLONEBLKSIZE) /* * Sets of attributes required by readdir and readdirplus. diff --git a/sys/fs/nfsclient/nfs_clcomsubs.c b/sys/fs/nfsclient/nfs_clcomsubs.c index bca0bdcd0df1..05963074e53d 100644 --- a/sys/fs/nfsclient/nfs_clcomsubs.c +++ b/sys/fs/nfsclient/nfs_clcomsubs.c @@ -272,7 +272,7 @@ nfsm_loadattr(struct nfsrv_descript *nd, struct nfsvattr *nap) if (nd->nd_flag & ND_NFSV4) { error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, - NULL); + NULL, NULL); } else if (nd->nd_flag & ND_NFSV3) { NFSM_DISSECT(fp, struct nfs_fattr *, NFSX_V3FATTR); nap->na_type = nfsv34tov_type(fp->fa_type); diff --git a/sys/fs/nfsclient/nfs_clport.c b/sys/fs/nfsclient/nfs_clport.c index b25d967982a1..e9f1dc23ddbe 100644 --- a/sys/fs/nfsclient/nfs_clport.c +++ b/sys/fs/nfsclient/nfs_clport.c @@ -412,7 +412,7 @@ nfscl_warn_fileid(struct nfsmount *nmp, struct nfsvattr *oldnap, } void -ncl_copy_vattr(struct vattr *dst, struct vattr *src) +ncl_copy_vattr(struct vnode *vp, struct vattr *dst, struct vattr *src) { dst->va_type = src->va_type; dst->va_mode = src->va_mode; @@ -429,7 +429,7 @@ ncl_copy_vattr(struct vattr *dst, struct vattr *src) dst->va_birthtime = src->va_birthtime; dst->va_gen = src->va_gen; dst->va_flags = src->va_flags; - dst->va_rdev = src->va_rdev; + dst->va_rdev = VN_ISDEV(vp) ? src->va_rdev : NODEV; dst->va_bytes = src->va_bytes; dst->va_filerev = src->va_filerev; } @@ -595,7 +595,7 @@ nfscl_loadattrcache(struct vnode **vpp, struct nfsvattr *nap, void *nvaper, KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp); } if (vaper != NULL) { - ncl_copy_vattr(vaper, vap); + ncl_copy_vattr(vp, vaper, vap); if (np->n_flag & NCHG) { if (np->n_flag & NACC) vaper->va_atime = np->n_atim; @@ -828,7 +828,7 @@ nfscl_wcc_data(struct nfsrv_descript *nd, struct vnode *vp, == (ND_NFSV4 | ND_V4WCCATTR)) { error = nfsv4_loadattr(nd, NULL, &nfsva, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, - NULL, NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL, NULL, NULL); if (error) return (error); /* @@ -963,7 +963,8 @@ nfscl_loadsbinfo(struct nfsmount *nmp, struct nfsstatfs *sfp, void *statfs) * Use the fsinfo stuff to update the mount point. */ void -nfscl_loadfsinfo(struct nfsmount *nmp, struct nfsfsinfo *fsp) +nfscl_loadfsinfo(struct nfsmount *nmp, struct nfsfsinfo *fsp, + uint32_t clone_blksize) { if ((nmp->nm_wsize == 0 || fsp->fs_wtpref < nmp->nm_wsize) && @@ -1003,6 +1004,14 @@ nfscl_loadfsinfo(struct nfsmount *nmp, struct nfsfsinfo *fsp) fsp->fs_maxfilesize < nmp->nm_maxfilesize) nmp->nm_maxfilesize = fsp->fs_maxfilesize; nmp->nm_mountp->mnt_stat.f_iosize = newnfs_iosize(nmp); + + /* + * Although ZFS reports a clone_blksize of 16Mbytes, + * 128Kbytes usually works, so set it to that. + */ + if (clone_blksize > 128 * 1024) + clone_blksize = 128 * 1024; + nmp->nm_cloneblksize = clone_blksize; nmp->nm_state |= NFSSTA_GOTFSINFO; } diff --git a/sys/fs/nfsclient/nfs_clrpcops.c b/sys/fs/nfsclient/nfs_clrpcops.c index 920fcf7b8c61..4ec621de2eff 100644 --- a/sys/fs/nfsclient/nfs_clrpcops.c +++ b/sys/fs/nfsclient/nfs_clrpcops.c @@ -225,6 +225,9 @@ static int nfsrpc_layoutgetres(struct nfsmount *, vnode_t, uint8_t *, static int nfsrpc_copyrpc(vnode_t, off_t, vnode_t, off_t, size_t *, nfsv4stateid_t *, nfsv4stateid_t *, struct nfsvattr *, int *, struct nfsvattr *, int *, bool, int *, struct ucred *, NFSPROC_T *); +static int nfsrpc_clonerpc(vnode_t, off_t, vnode_t, off_t, size_t *, bool, + nfsv4stateid_t *, nfsv4stateid_t *, struct nfsvattr *, int *, + struct nfsvattr *, int *, struct ucred *, NFSPROC_T *); static int nfsrpc_seekrpc(vnode_t, off_t *, nfsv4stateid_t *, bool *, int, struct nfsvattr *, int *, struct ucred *); static struct mbuf *nfsm_split(struct mbuf *, uint64_t); @@ -696,7 +699,7 @@ nfsrpc_openrpc(struct nfsmount *nmp, vnode_t vp, u_int8_t *nfhp, int fhlen, ("nfsrpc_openrpc: Getattr repstat")); error = nfsv4_loadattr(nd, NULL, &nfsva, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, - NULL, NULL, NULL, NULL, p, cred); + NULL, NULL, NULL, NULL, NULL, p, cred); if (error) goto nfsmout; } @@ -1355,7 +1358,7 @@ nfsrpc_getattrnovp(struct nfsmount *nmp, u_int8_t *fhp, int fhlen, int syscred, if ((nd->nd_flag & ND_NFSV4) != 0) error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, NULL, leasep, NULL, - NULL, NULL, NULL); + NULL, NULL, NULL, NULL); else error = nfsm_loadattr(nd, nap); } else @@ -3597,7 +3600,7 @@ nfsrpc_readdir(vnode_t vp, struct uio *uiop, nfsuint64 *cookiep, nfsva.na_mntonfileno = UINT64_MAX; error = nfsv4_loadattr(nd, NULL, &nfsva, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, - NULL, NULL, NULL, NULL, p, cred); + NULL, NULL, NULL, NULL, NULL, p, cred); if (error) { dotdotfileid = dotfileid; } else if (gotmnton) { @@ -3847,7 +3850,7 @@ nfsrpc_readdir(vnode_t vp, struct uio *uiop, nfsuint64 *cookiep, nfsva.na_mntonfileno = UINT64_MAX; error = nfsv4_loadattr(nd, NULL, &nfsva, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, - NULL, NULL, &rderr, NULL, p, cred); + NULL, NULL, &rderr, NULL, NULL, p, cred); if (error) goto nfsmout; NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED); @@ -4072,7 +4075,7 @@ nfsrpc_readdirplus(vnode_t vp, struct uio *uiop, nfsuint64 *cookiep, nfsva.na_mntonfileno = UINT64_MAX; error = nfsv4_loadattr(nd, NULL, &nfsva, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, - NULL, NULL, NULL, NULL, p, cred); + NULL, NULL, NULL, NULL, NULL, p, cred); if (error) { dotdotfileid = dotfileid; } else if (gotmnton) { @@ -4346,7 +4349,7 @@ nfsrpc_readdirplus(vnode_t vp, struct uio *uiop, nfsuint64 *cookiep, nfsva.na_mntonfileno = 0xffffffff; error = nfsv4_loadattr(nd, NULL, &nfsva, &nfhp, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, - NULL, NULL, &rderr, NULL, p, cred); + NULL, NULL, &rderr, NULL, NULL, p, cred); if (error) goto nfsmout; } @@ -4981,8 +4984,8 @@ nfsmout: */ int nfsrpc_statfs(vnode_t vp, struct nfsstatfs *sbp, struct nfsfsinfo *fsp, - uint32_t *leasep, struct ucred *cred, NFSPROC_T *p, struct nfsvattr *nap, - int *attrflagp) + uint32_t *leasep, uint32_t *cloneblksizep, struct ucred *cred, NFSPROC_T *p, + struct nfsvattr *nap, int *attrflagp) { u_int32_t *tl = NULL; struct nfsrv_descript nfsd, *nd = &nfsd; @@ -4991,6 +4994,8 @@ nfsrpc_statfs(vnode_t vp, struct nfsstatfs *sbp, struct nfsfsinfo *fsp, int error; *attrflagp = 0; + if (cloneblksizep != NULL) + *cloneblksizep = 0; nmp = VFSTONFS(vp->v_mount); if (NFSHASNFSV4(nmp)) { /* @@ -5009,7 +5014,7 @@ nfsrpc_statfs(vnode_t vp, struct nfsstatfs *sbp, struct nfsfsinfo *fsp, if (nd->nd_repstat == 0) { error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, sbp, fsp, NULL, 0, NULL, leasep, NULL, - NULL, p, cred); + NULL, cloneblksizep, p, cred); if (!error) { nmp->nm_fsid[0] = nap->na_filesid[0]; nmp->nm_fsid[1] = nap->na_filesid[1]; @@ -5063,7 +5068,8 @@ nfsmout: */ int nfsrpc_pathconf(vnode_t vp, struct nfsv3_pathconf *pc, bool *has_namedattrp, - struct ucred *cred, NFSPROC_T *p, struct nfsvattr *nap, int *attrflagp) + uint32_t *clone_blksizep, struct ucred *cred, NFSPROC_T *p, + struct nfsvattr *nap, int *attrflagp) { struct nfsrv_descript nfsd, *nd = &nfsd; struct nfsmount *nmp; @@ -5074,6 +5080,7 @@ nfsrpc_pathconf(vnode_t vp, struct nfsv3_pathconf *pc, bool *has_namedattrp, *has_namedattrp = false; *attrflagp = 0; + *clone_blksizep = 0; nmp = VFSTONFS(vp->v_mount); if (NFSHASNFSV4(nmp)) { np = VTONFS(vp); @@ -5100,7 +5107,7 @@ nfsrpc_pathconf(vnode_t vp, struct nfsv3_pathconf *pc, bool *has_namedattrp, if (nd->nd_repstat == 0) { error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, pc, NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, - has_namedattrp, p, cred); + has_namedattrp, clone_blksizep, p, cred); if (!error) *attrflagp = 1; } else { @@ -5395,7 +5402,8 @@ nfsrpc_getacl(vnode_t vp, struct ucred *cred, NFSPROC_T *p, struct acl *aclp) return (error); if (!nd->nd_repstat) error = nfsv4_loadattr(nd, vp, NULL, NULL, NULL, 0, NULL, - NULL, NULL, NULL, aclp, 0, NULL, NULL, NULL, NULL, p, cred); + NULL, NULL, NULL, aclp, 0, NULL, NULL, NULL, NULL, NULL, p, + cred); else error = nd->nd_repstat; m_freem(nd->nd_mrep); @@ -5437,7 +5445,7 @@ nfsrpc_setaclrpc(vnode_t vp, struct ucred *cred, NFSPROC_T *p, NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_ACL); (void) nfsv4_fillattr(nd, vp->v_mount, vp, aclp, NULL, NULL, 0, &attrbits, NULL, NULL, 0, 0, 0, 0, (uint64_t)0, NULL, false, false, - false); + false, 0); error = nfscl_request(nd, vp, p, cred); if (error) return (error); @@ -8496,7 +8504,7 @@ nfsrpc_openlayoutrpc(struct nfsmount *nmp, vnode_t vp, u_int8_t *nfhp, if (*++tl == 0) { error = nfsv4_loadattr(nd, NULL, &nfsva, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, - NULL, NULL, NULL, NULL, p, cred); + NULL, NULL, NULL, NULL, NULL, p, cred); if (error != 0) goto nfsmout; if (ndp != NULL) { @@ -9168,6 +9176,199 @@ nfsmout: } /* + * nfs clone operation. + */ +int +nfsrpc_clone(vnode_t invp, off_t *inoffp, vnode_t outvp, + off_t *outoffp, size_t *lenp, bool toeof, int *inattrflagp, + struct nfsvattr *innap, int *outattrflagp, struct nfsvattr *outnap, + struct ucred *cred) +{ + int error, expireret = 0, retrycnt; + uint32_t clidrev = 0; + struct nfsmount *nmp = VFSTONFS(invp->v_mount); + struct nfsfh *innfhp = NULL, *outnfhp = NULL; + nfsv4stateid_t instateid, outstateid; + void *inlckp, *outlckp; + + if (nmp->nm_clp != NULL) + clidrev = nmp->nm_clp->nfsc_clientidrev; + innfhp = VTONFS(invp)->n_fhp; + outnfhp = VTONFS(outvp)->n_fhp; + retrycnt = 0; + do { + /* Get both stateids. */ + inlckp = NULL; + nfscl_getstateid(invp, innfhp->nfh_fh, innfhp->nfh_len, + NFSV4OPEN_ACCESSREAD, 0, NULL, curthread, &instateid, + &inlckp); + outlckp = NULL; + nfscl_getstateid(outvp, outnfhp->nfh_fh, outnfhp->nfh_len, + NFSV4OPEN_ACCESSWRITE, 0, NULL, curthread, &outstateid, + &outlckp); + + error = nfsrpc_clonerpc(invp, *inoffp, outvp, *outoffp, lenp, + toeof, &instateid, &outstateid, innap, inattrflagp, outnap, + outattrflagp, cred, curthread); + if (error == 0) { + *inoffp += *lenp; + *outoffp += *lenp; + } else if (error == NFSERR_STALESTATEID) + nfscl_initiate_recovery(nmp->nm_clp); + if (inlckp != NULL) + nfscl_lockderef(inlckp); + if (outlckp != NULL) + nfscl_lockderef(outlckp); + if (error == NFSERR_GRACE || error == NFSERR_STALESTATEID || + error == NFSERR_STALEDONTRECOVER || error == NFSERR_DELAY || + error == NFSERR_OLDSTATEID || error == NFSERR_BADSESSION) { + (void) nfs_catnap(PZERO, error, "nfs_cfr"); + } else if ((error == NFSERR_EXPIRED || (!NFSHASINT(nmp) && + error == NFSERR_BADSTATEID)) && clidrev != 0) { + expireret = nfscl_hasexpired(nmp->nm_clp, clidrev, + curthread); + } else if (error == NFSERR_BADSTATEID && NFSHASINT(nmp)) { + error = EIO; + } + retrycnt++; + } while (error == NFSERR_GRACE || error == NFSERR_DELAY || + error == NFSERR_STALESTATEID || error == NFSERR_BADSESSION || + error == NFSERR_STALEDONTRECOVER || + (error == NFSERR_OLDSTATEID && retrycnt < 20) || + ((error == NFSERR_EXPIRED || error == NFSERR_BADSTATEID) && + expireret == 0 && clidrev != 0 && retrycnt < 4)); + if (error != 0 && (retrycnt >= 4 || + error == NFSERR_STALESTATEID || error == NFSERR_BADSESSION || + error == NFSERR_STALEDONTRECOVER)) + error = EIO; + return (error); +} + +/* + * The clone RPC. + */ +static int +nfsrpc_clonerpc(vnode_t invp, off_t inoff, vnode_t outvp, off_t outoff, + size_t *lenp, bool toeof, nfsv4stateid_t *instateidp, + nfsv4stateid_t *outstateidp, struct nfsvattr *innap, int *inattrflagp, + struct nfsvattr *outnap, int *outattrflagp, struct ucred *cred, + NFSPROC_T *p) +{ + uint32_t *tl, *opcntp; + int error; + struct nfsrv_descript nfsd; + struct nfsrv_descript *nd = &nfsd; + struct nfsmount *nmp; + nfsattrbit_t attrbits; + struct vattr va; + uint64_t len; + + nmp = VFSTONFS(invp->v_mount); + *inattrflagp = *outattrflagp = 0; + len = *lenp; + if (len == 0) + return (0); + if (toeof) + len = 0; + nfscl_reqstart(nd, NFSPROC_CLONE, nmp, VTONFS(invp)->n_fhp->nfh_fh, + VTONFS(invp)->n_fhp->nfh_len, &opcntp, NULL, 0, 0, cred); + /* + * First do a Setattr of atime to the server's clock + * time. The FreeBSD "collective" was of the opinion + * that setting atime was necessary for this syscall. + * Do the Setattr before the Clone, so that it can be + * handled well if the server replies NFSERR_DELAY to + * the Setattr operation. + */ + if ((nmp->nm_mountp->mnt_flag & MNT_NOATIME) == 0) { + NFSM_BUILD(tl, uint32_t *, NFSX_UNSIGNED); + *tl = txdr_unsigned(NFSV4OP_SETATTR); + nfsm_stateidtom(nd, instateidp, NFSSTATEID_PUTSTATEID); + VATTR_NULL(&va); + va.va_atime.tv_sec = va.va_atime.tv_nsec = 0; + va.va_vaflags = VA_UTIMES_NULL; + nfscl_fillsattr(nd, &va, invp, 0, 0); + /* Bump opcnt from 7 to 8. */ + *opcntp = txdr_unsigned(8); + } + + /* Now Getattr the invp attributes. */ + NFSM_BUILD(tl, uint32_t *, NFSX_UNSIGNED); + *tl = txdr_unsigned(NFSV4OP_GETATTR); + NFSGETATTR_ATTRBIT(&attrbits); + nfsrv_putattrbit(nd, &attrbits); + + /* Set outvp. */ + NFSM_BUILD(tl, uint32_t *, NFSX_UNSIGNED); + *tl = txdr_unsigned(NFSV4OP_PUTFH); + (void)nfsm_fhtom(nmp, nd, VTONFS(outvp)->n_fhp->nfh_fh, + VTONFS(outvp)->n_fhp->nfh_len, 0); + + /* Do the Clone. */ + NFSM_BUILD(tl, uint32_t *, NFSX_UNSIGNED); + *tl = txdr_unsigned(NFSV4OP_CLONE); + nfsm_stateidtom(nd, instateidp, NFSSTATEID_PUTSTATEID); + nfsm_stateidtom(nd, outstateidp, NFSSTATEID_PUTSTATEID); + NFSM_BUILD(tl, uint32_t *, 3 * NFSX_HYPER + NFSX_UNSIGNED); + txdr_hyper(inoff, tl); tl += 2; + txdr_hyper(outoff, tl); tl += 2; + txdr_hyper(len, tl); tl += 2; + + /* Get the outvp attributes. */ + *tl = txdr_unsigned(NFSV4OP_GETATTR); + NFSWRITEGETATTR_ATTRBIT(&attrbits); + nfsrv_putattrbit(nd, &attrbits); + + error = nfscl_request(nd, invp, p, cred); + if (error != 0) + return (error); + /* Skip over the Setattr reply. */ + if ((nd->nd_flag & ND_NOMOREDATA) == 0 && + (nmp->nm_mountp->mnt_flag & MNT_NOATIME) == 0) { + NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); + if (*(tl + 1) == 0) { + error = nfsrv_getattrbits(nd, &attrbits, NULL, NULL); + if (error != 0) + goto nfsmout; + } else + nd->nd_flag |= ND_NOMOREDATA; + } + if ((nd->nd_flag & ND_NOMOREDATA) == 0) { + /* Get the input file's attributes. */ + NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); + if (*(tl + 1) == 0) { + error = nfsm_loadattr(nd, innap); + if (error != 0) + goto nfsmout; + *inattrflagp = 1; + } else + nd->nd_flag |= ND_NOMOREDATA; + } + /* Skip over return stat for PutFH. */ + if ((nd->nd_flag & ND_NOMOREDATA) == 0) { + NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); + if (*++tl != 0) + nd->nd_flag |= ND_NOMOREDATA; + } + /* Skip over return stat for Clone. */ + if ((nd->nd_flag & ND_NOMOREDATA) == 0) + NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); + if (nd->nd_repstat == 0) { + NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); + error = nfsm_loadattr(nd, outnap); + if (error == 0) + *outattrflagp = NFS_LATTR_NOSHRINK; + } else { + *lenp = 0; + } + if (error == 0) + error = nd->nd_repstat; +nfsmout: + m_freem(nd->nd_mrep); + return (error); +} + +/* * Seek operation. */ int @@ -9724,13 +9925,13 @@ nfscl_statfs(struct vnode *vp, struct ucred *cred, NFSPROC_T *td) struct nfsstatfs sb; struct mount *mp; struct nfsmount *nmp; - uint32_t lease; + uint32_t clone_blksize, lease; int attrflag, error; mp = vp->v_mount; nmp = VFSTONFS(mp); - error = nfsrpc_statfs(vp, &sb, &fs, &lease, cred, td, &nfsva, - &attrflag); + error = nfsrpc_statfs(vp, &sb, &fs, &lease, &clone_blksize, cred, td, + &nfsva, &attrflag); if (attrflag != 0) (void) nfscl_loadattrcache(&vp, &nfsva, NULL, 0, 1); if (error == 0) { @@ -9739,7 +9940,7 @@ nfscl_statfs(struct vnode *vp, struct ucred *cred, NFSPROC_T *td) nmp->nm_clp->nfsc_renew = NFSCL_RENEW(lease); NFSUNLOCKCLSTATE(); mtx_lock(&nmp->nm_mtx); - nfscl_loadfsinfo(nmp, &fs); + nfscl_loadfsinfo(nmp, &fs, clone_blksize); nfscl_loadsbinfo(nmp, &sb, &mp->mnt_stat); mp->mnt_stat.f_iosize = newnfs_iosize(nmp); mtx_unlock(&nmp->nm_mtx); diff --git a/sys/fs/nfsclient/nfs_clstate.c b/sys/fs/nfsclient/nfs_clstate.c index 99a781640c53..aa9d01fc4632 100644 --- a/sys/fs/nfsclient/nfs_clstate.c +++ b/sys/fs/nfsclient/nfs_clstate.c @@ -3701,7 +3701,7 @@ nfscl_docb(struct nfsrv_descript *nd, NFSPROC_T *p) if (!error) (void) nfsv4_fillattr(nd, NULL, NULL, NULL, &va, NULL, 0, &rattrbits, NULL, p, 0, 0, 0, 0, - (uint64_t)0, NULL, false, false, false); + (uint64_t)0, NULL, false, false, false, 0); break; case NFSV4OP_CBRECALL: NFSCL_DEBUG(4, "cbrecall\n"); diff --git a/sys/fs/nfsclient/nfs_clvfsops.c b/sys/fs/nfsclient/nfs_clvfsops.c index 0bd05c03885b..5ea7eab07632 100644 --- a/sys/fs/nfsclient/nfs_clvfsops.c +++ b/sys/fs/nfsclient/nfs_clvfsops.c @@ -292,8 +292,10 @@ nfs_statfs(struct mount *mp, struct statfs *sbp) int error = 0, attrflag, gotfsinfo = 0, ret; struct nfsnode *np; char *fakefh; + uint32_t clone_blksize; td = curthread; + clone_blksize = 0; error = vfs_busy(mp, MBF_NOWAIT); if (error) @@ -337,8 +339,8 @@ nfs_statfs(struct mount *mp, struct statfs *sbp) } else mtx_unlock(&nmp->nm_mtx); if (!error) - error = nfsrpc_statfs(vp, &sb, &fs, NULL, td->td_ucred, td, - &nfsva, &attrflag); + error = nfsrpc_statfs(vp, &sb, &fs, NULL, &clone_blksize, + td->td_ucred, td, &nfsva, &attrflag); if ((nmp->nm_privflag & NFSMNTP_FAKEROOTFH) != 0 && error == NFSERR_WRONGSEC) { /* Cannot get new stats, so return what is in mnt_stat. */ @@ -375,7 +377,7 @@ nfs_statfs(struct mount *mp, struct statfs *sbp) if (!error) { mtx_lock(&nmp->nm_mtx); if (gotfsinfo || (nmp->nm_flag & NFSMNT_NFSV4)) - nfscl_loadfsinfo(nmp, &fs); + nfscl_loadfsinfo(nmp, &fs, clone_blksize); nfscl_loadsbinfo(nmp, &sb, sbp); sbp->f_iosize = newnfs_iosize(nmp); mtx_unlock(&nmp->nm_mtx); @@ -408,7 +410,7 @@ ncl_fsinfo(struct nfsmount *nmp, struct vnode *vp, struct ucred *cred, if (attrflag) (void) nfscl_loadattrcache(&vp, &nfsva, NULL, 0, 1); mtx_lock(&nmp->nm_mtx); - nfscl_loadfsinfo(nmp, &fs); + nfscl_loadfsinfo(nmp, &fs, 0); mtx_unlock(&nmp->nm_mtx); } return (error); diff --git a/sys/fs/nfsclient/nfs_clvnops.c b/sys/fs/nfsclient/nfs_clvnops.c index fa451887e73e..a8b06fdb261b 100644 --- a/sys/fs/nfsclient/nfs_clvnops.c +++ b/sys/fs/nfsclient/nfs_clvnops.c @@ -1026,7 +1026,7 @@ nfs_getattr(struct vop_getattr_args *ap) * cached attributes should be ignored. */ if (nmp->nm_fhsize > 0 && ncl_getattrcache(vp, &vattr) == 0) { - ncl_copy_vattr(vap, &vattr); + ncl_copy_vattr(vp, vap, &vattr); /* * Get the local modify time for the case of a write @@ -1782,7 +1782,7 @@ nfs_mknodrpc(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp, int error = 0, attrflag, dattrflag; u_int32_t rdev; - if (vap->va_type == VCHR || vap->va_type == VBLK) + if (VATTR_ISDEV(vap)) rdev = vap->va_rdev; else if (vap->va_type == VFIFO || vap->va_type == VSOCK) rdev = 0xffffffff; @@ -4027,31 +4027,51 @@ nfs_copy_file_range(struct vop_copy_file_range_args *ap) struct vattr va, *vap; struct uio io; struct nfsmount *nmp; + struct nfsnode *np; size_t len, len2; ssize_t r; int error, inattrflag, outattrflag, ret, ret2, invp_lock; off_t inoff, outoff; - bool consecutive, must_commit, tryoutcred; + bool consecutive, must_commit, onevp, toeof, tryclone, tryoutcred; + bool mustclone; /* * NFSv4.2 Copy is not permitted for infile == outfile. + * The NFSv4.2 Clone operation does work on non-overlapping + * byte ranges in the same file, but only if offsets + * (and len if not to EOF) are aligned properly. * TODO: copy_file_range() between multiple NFS mountpoints + * --> This is not possible now, since each mount appears to + * the NFSv4.n server as a separate client. */ - if (invp == outvp || invp->v_mount != outvp->v_mount) { + if ((invp == outvp && (ap->a_flags & COPY_FILE_RANGE_CLONE) == 0) || + (invp != outvp && invp->v_mount != outvp->v_mount)) { generic_copy: return (ENOSYS); } - - invp_lock = LK_SHARED; + if (invp == outvp) { + onevp = true; + invp_lock = LK_EXCLUSIVE; + } else { + onevp = false; + invp_lock = LK_SHARED; + } + mustclone = false; + if (onevp || (ap->a_flags & COPY_FILE_RANGE_CLONE) != 0) + mustclone = true; relock: + inoff = *ap->a_inoffp; + outoff = *ap->a_outoffp; - /* Lock both vnodes, avoiding risk of deadlock. */ + /* Lock vnode(s), avoiding risk of deadlock. */ do { mp = NULL; error = vn_start_write(outvp, &mp, V_WAIT); if (error == 0) { error = vn_lock(outvp, LK_EXCLUSIVE); if (error == 0) { + if (onevp) + break; error = vn_lock(invp, invp_lock | LK_NOWAIT); if (error == 0) break; @@ -4071,16 +4091,24 @@ relock: return (error); /* - * More reasons to avoid nfs copy: not NFSv4.2, or explicitly - * disabled. + * More reasons to avoid nfs copy/clone: not NFSv4.2, explicitly + * disabled or requires cloning and unable to clone. + * Only clone if the clone_blksize attribute is supported + * and the clone_blksize is greater than 0. + * Alignment of offsets and length will be checked later. */ nmp = VFSTONFS(invp->v_mount); + np = VTONFS(invp); mtx_lock(&nmp->nm_mtx); + if ((nmp->nm_privflag & NFSMNTP_NOCOPY) != 0) + mustclone = true; if (!NFSHASNFSV4(nmp) || nmp->nm_minorvers < NFSV42_MINORVERSION || - (nmp->nm_privflag & NFSMNTP_NOCOPY) != 0) { + (mustclone && (!NFSISSET_ATTRBIT(&np->n_vattr.na_suppattr, + NFSATTRBIT_CLONEBLKSIZE) || nmp->nm_cloneblksize == 0))) { mtx_unlock(&nmp->nm_mtx); VOP_UNLOCK(invp); - VOP_UNLOCK(outvp); + if (!onevp) + VOP_UNLOCK(outvp); /* For onevp, same as invp. */ if (mp != NULL) vn_finished_write(mp); goto generic_copy; @@ -4111,6 +4139,8 @@ relock: invp_obj = invp->v_object; if (invp_obj != NULL && vm_object_mightbedirty(invp_obj)) { if (invp_lock != LK_EXCLUSIVE) { + KASSERT(!onevp, ("nfs_copy_file_range: " + "invp_lock LK_SHARED for onevp")); invp_lock = LK_EXCLUSIVE; VOP_UNLOCK(invp); VOP_UNLOCK(outvp); @@ -4134,10 +4164,10 @@ relock: else consecutive = false; mtx_unlock(&nmp->nm_mtx); - inoff = *ap->a_inoffp; - outoff = *ap->a_outoffp; tryoutcred = true; must_commit = false; + toeof = false; + if (error == 0) { vap = &VTONFS(invp)->n_vattr.na_vattr; error = VOP_GETATTR(invp, vap, ap->a_incred); @@ -4169,29 +4199,63 @@ relock: if (error == 0 && ret != 0) error = ret; } - } else if (inoff + len > vap->va_size) + } else if (inoff + len >= vap->va_size) { + toeof = true; *ap->a_lenp = len = vap->va_size - inoff; + } } else error = 0; } /* + * For cloning, the offsets must be clone blksize aligned and + * the len must be blksize aligned unless it goes to EOF on + * the input file. + */ + tryclone = false; + if (len > 0) { + if (error == 0 && NFSISSET_ATTRBIT(&np->n_vattr.na_suppattr, + NFSATTRBIT_CLONEBLKSIZE) && nmp->nm_cloneblksize != 0 && + (inoff % nmp->nm_cloneblksize) == 0 && + (outoff % nmp->nm_cloneblksize) == 0 && + (toeof || (len % nmp->nm_cloneblksize) == 0)) + tryclone = true; + else if (mustclone) + error = ENOSYS; + } + + /* * len will be set to 0 upon a successful Copy RPC. - * As such, this only loops when the Copy RPC needs to be retried. + * As such, this only loops when the Copy/Clone RPC needs to be retried. */ while (len > 0 && error == 0) { inattrflag = outattrflag = 0; len2 = len; - if (tryoutcred) - error = nfsrpc_copy_file_range(invp, ap->a_inoffp, - outvp, ap->a_outoffp, &len2, ap->a_flags, - &inattrflag, &innfsva, &outattrflag, &outnfsva, - ap->a_outcred, consecutive, &must_commit); - else - error = nfsrpc_copy_file_range(invp, ap->a_inoffp, - outvp, ap->a_outoffp, &len2, ap->a_flags, - &inattrflag, &innfsva, &outattrflag, &outnfsva, - ap->a_incred, consecutive, &must_commit); + if (tryclone) { + if (tryoutcred) + error = nfsrpc_clone(invp, ap->a_inoffp, outvp, + ap->a_outoffp, &len2, toeof, &inattrflag, + &innfsva, &outattrflag, &outnfsva, + ap->a_outcred); + else + error = nfsrpc_clone(invp, ap->a_inoffp, outvp, + ap->a_outoffp, &len2, toeof, &inattrflag, + &innfsva, &outattrflag, &outnfsva, + ap->a_incred); + } else { + if (tryoutcred) + error = nfsrpc_copy_file_range(invp, + ap->a_inoffp, outvp, ap->a_outoffp, &len2, + ap->a_flags, &inattrflag, &innfsva, + &outattrflag, &outnfsva, + ap->a_outcred, consecutive, &must_commit); + else + error = nfsrpc_copy_file_range(invp, + ap->a_inoffp, outvp, ap->a_outoffp, &len2, + ap->a_flags, &inattrflag, &innfsva, + &outattrflag, &outnfsva, + ap->a_incred, consecutive, &must_commit); + } if (inattrflag != 0) ret = nfscl_loadattrcache(&invp, &innfsva, NULL, 0, 1); if (outattrflag != 0) @@ -4230,6 +4294,13 @@ relock: /* Try again with incred. */ tryoutcred = false; error = 0; + } else if (tryclone && error != 0) { + if (mustclone) { + error = ENOSYS; + } else { + tryclone = false; + error = 0; + } } if (error == NFSERR_STALEWRITEVERF) { /* @@ -4243,11 +4314,12 @@ relock: } } VOP_UNLOCK(invp); - VOP_UNLOCK(outvp); + if (!onevp) + VOP_UNLOCK(outvp); /* For onevp, same as invp. */ if (mp != NULL) vn_finished_write(mp); if (error == NFSERR_NOTSUPP || error == NFSERR_OFFLOADNOREQS || - error == NFSERR_ACCES) { + error == NFSERR_ACCES || error == ENOSYS) { /* * Unlike the NFSv4.2 Copy, vn_generic_copy_file_range() can * use a_incred for the read and a_outcred for the write, so @@ -4255,7 +4327,7 @@ relock: * For NFSERR_NOTSUPP and NFSERR_OFFLOADNOREQS, the Copy can * never succeed, so disable it. */ - if (error != NFSERR_ACCES) { + if (error != NFSERR_ACCES && error != ENOSYS) { /* Can never do Copy on this mount. */ mtx_lock(&nmp->nm_mtx); nmp->nm_privflag |= NFSMNTP_NOCOPY; @@ -4596,6 +4668,7 @@ nfs_pathconf(struct vop_pathconf_args *ap) struct nfsmount *nmp; struct thread *td = curthread; off_t off; + uint32_t clone_blksize; bool eof, has_namedattr, named_enabled; int attrflag, error; struct nfsnode *np; @@ -4604,19 +4677,22 @@ nfs_pathconf(struct vop_pathconf_args *ap) np = VTONFS(vp); named_enabled = false; has_namedattr = false; + clone_blksize = 0; if ((NFS_ISV34(vp) && (ap->a_name == _PC_LINK_MAX || ap->a_name == _PC_NAME_MAX || ap->a_name == _PC_CHOWN_RESTRICTED || ap->a_name == _PC_NO_TRUNC)) || (NFS_ISV4(vp) && (ap->a_name == _PC_ACL_NFS4 || - ap->a_name == _PC_HAS_NAMEDATTR))) { + ap->a_name == _PC_HAS_NAMEDATTR || + ap->a_name == _PC_CLONE_BLKSIZE))) { /* * Since only the above 4 a_names are returned by the NFSv3 * Pathconf RPC, there is no point in doing it for others. * For NFSv4, the Pathconf RPC (actually a Getattr Op.) can - * be used for _PC_ACL_NFS4 and _PC_HAS_NAMEDATTR as well. + * be used for _PC_ACL_NFS4, _PC_HAS_NAMEDATTR and + * _PC_CLONE_BLKSIZE as well. */ - error = nfsrpc_pathconf(vp, &pc, &has_namedattr, td->td_ucred, - td, &nfsva, &attrflag); + error = nfsrpc_pathconf(vp, &pc, &has_namedattr, &clone_blksize, + td->td_ucred, td, &nfsva, &attrflag); if (attrflag != 0) (void) nfscl_loadattrcache(&vp, &nfsva, NULL, 0, 1); if (error != 0) @@ -4771,6 +4847,9 @@ nfs_pathconf(struct vop_pathconf_args *ap) else *ap->a_retval = 0; break; + case _PC_CLONE_BLKSIZE: + *ap->a_retval = clone_blksize; + break; default: error = vop_stdpathconf(ap); diff --git a/sys/fs/nfsclient/nfsmount.h b/sys/fs/nfsclient/nfsmount.h index 37b84a015dab..ef876dd30e59 100644 --- a/sys/fs/nfsclient/nfsmount.h +++ b/sys/fs/nfsclient/nfsmount.h @@ -87,6 +87,7 @@ struct nfsmount { /* unclipped, wraps to 0 */ struct __rpc_client *nm_aconn[NFS_MAXNCONN - 1]; /* Additional nconn */ /* Locked via nm_sockreq.nr_mtx */ + uint32_t nm_cloneblksize; /* Block cloning alignment */ u_int16_t nm_krbnamelen; /* Krb5 host principal, if any */ u_int16_t nm_dirpathlen; /* and mount dirpath, for V4 */ u_int16_t nm_srvkrbnamelen; /* and the server's target name */ diff --git a/sys/fs/nfsserver/nfs_nfsdport.c b/sys/fs/nfsserver/nfs_nfsdport.c index 8c427c66c156..b2966934f9b7 100644 --- a/sys/fs/nfsserver/nfs_nfsdport.c +++ b/sys/fs/nfsserver/nfs_nfsdport.c @@ -2113,7 +2113,8 @@ nfsvno_fillattr(struct nfsrv_descript *nd, struct mount *mp, struct vnode *vp, struct nfsvattr *nvap, fhandle_t *fhp, int rderror, nfsattrbit_t *attrbitp, struct ucred *cred, struct thread *p, int isdgram, int reterr, int supports_nfsv4acls, int at_root, uint64_t mounted_on_fileno, - bool xattrsupp, bool has_hiddensystem, bool has_namedattr) + bool xattrsupp, bool has_hiddensystem, bool has_namedattr, + uint32_t clone_blksize) { struct statfs *sf; int error; @@ -2130,9 +2131,11 @@ nfsvno_fillattr(struct nfsrv_descript *nd, struct mount *mp, struct vnode *vp, sf = NULL; } } + error = nfsv4_fillattr(nd, mp, vp, NULL, &nvap->na_vattr, fhp, rderror, attrbitp, cred, p, isdgram, reterr, supports_nfsv4acls, at_root, - mounted_on_fileno, sf, xattrsupp, has_hiddensystem, has_namedattr); + mounted_on_fileno, sf, xattrsupp, has_hiddensystem, has_namedattr, + clone_blksize); free(sf, M_TEMP); NFSEXITCODE2(0, nd); return (error); @@ -2441,7 +2444,7 @@ nfsrvd_readdirplus(struct nfsrv_descript *nd, int isdgram, struct vnode *vp, struct nfsexstuff *exp) { struct dirent *dp; - u_int32_t *tl; + uint32_t clone_blksize, *tl; int dirlen; char *cpos, *cend, *rbuf; struct vnode *nvp; @@ -2943,6 +2946,7 @@ again: xattrsupp = false; has_hiddensystem = false; has_namedattr = false; + clone_blksize = 0; if (nvp != NULL) { supports_nfsv4acls = nfs_supportsnfsv4acls(nvp); @@ -2966,6 +2970,11 @@ again: &pathval) != 0) pathval = 0; has_namedattr = pathval > 0; + pathval = 0; + if (VOP_PATHCONF(nvp, _PC_CLONE_BLKSIZE, + &pathval) != 0) + pathval = 0; + clone_blksize = pathval; NFSVOPUNLOCK(nvp); } else supports_nfsv4acls = 0; @@ -2986,14 +2995,16 @@ again: nd->nd_cred, p, isdgram, 0, supports_nfsv4acls, at_root, mounted_on_fileno, xattrsupp, - has_hiddensystem, has_namedattr); + has_hiddensystem, has_namedattr, + clone_blksize); } else { dirlen += nfsvno_fillattr(nd, new_mp, nvp, nvap, &nfh, r, &attrbits, nd->nd_cred, p, isdgram, 0, supports_nfsv4acls, at_root, mounted_on_fileno, xattrsupp, - has_hiddensystem, has_namedattr); + has_hiddensystem, has_namedattr, + clone_blksize); } if (nvp != NULL) vrele(nvp); @@ -3740,6 +3751,7 @@ nfsrv_v4rootexport(void *argp, struct ucred *cred, struct thread *p) NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, nfsexargp->fspec); if ((error = namei(&nd)) != 0) goto out; + NDFREE_PNBUF(&nd); error = nfsvno_getfh(nd.ni_vp, &fh, p); vrele(nd.ni_vp); if (!error) { @@ -5690,7 +5702,8 @@ nfsrv_writedsdorpc(struct nfsmount *nmp, fhandle_t *fhp, off_t off, int len, if ((nd->nd_flag & (ND_NOMOREDATA | ND_NFSV4 | ND_V4WCCATTR)) == (ND_NFSV4 | ND_V4WCCATTR)) { error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, - NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL, + NULL); NFSD_DEBUG(4, "nfsrv_writedsdorpc: wcc attr=%d\n", error); if (error != 0) goto nfsmout; @@ -5721,7 +5734,8 @@ nfsrv_writedsdorpc(struct nfsmount *nmp, fhandle_t *fhp, off_t off, int len, if (error == 0) { NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, - NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL, + NULL); } NFSD_DEBUG(4, "nfsrv_writedsdorpc: aft loadattr=%d\n", error); nfsmout: @@ -5887,7 +5901,8 @@ nfsrv_allocatedsdorpc(struct nfsmount *nmp, fhandle_t *fhp, off_t off, if (nd->nd_repstat == 0) { NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, - NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL, + NULL); } else error = nd->nd_repstat; NFSD_DEBUG(4, "nfsrv_allocatedsdorpc: aft loadattr=%d\n", error); @@ -6054,7 +6069,8 @@ nfsrv_deallocatedsdorpc(struct nfsmount *nmp, fhandle_t *fhp, off_t off, if ((nd->nd_flag & (ND_NOMOREDATA | ND_NFSV4 | ND_V4WCCATTR)) == (ND_NFSV4 | ND_V4WCCATTR)) { error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, - NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL, + NULL); NFSD_DEBUG(4, "nfsrv_deallocatedsdorpc: wcc attr=%d\n", error); if (error != 0) goto nfsmout; @@ -6068,7 +6084,8 @@ nfsrv_deallocatedsdorpc(struct nfsmount *nmp, fhandle_t *fhp, off_t off, if (nd->nd_repstat == 0) { NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, - NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL, + NULL); } else error = nd->nd_repstat; NFSD_DEBUG(4, "nfsrv_deallocatedsdorpc: aft loadattr=%d\n", error); @@ -6216,7 +6233,8 @@ nfsrv_setattrdsdorpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, if ((nd->nd_flag & (ND_NOMOREDATA | ND_NFSV4 | ND_V4WCCATTR)) == (ND_NFSV4 | ND_V4WCCATTR)) { error = nfsv4_loadattr(nd, NULL, dsnap, NULL, NULL, 0, NULL, - NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, + NULL, NULL); NFSD_DEBUG(4, "nfsrv_setattrdsdorpc: wcc attr=%d\n", error); if (error != 0) goto nfsmout; @@ -6241,7 +6259,7 @@ nfsrv_setattrdsdorpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_UNSIGNED); error = nfsv4_loadattr(nd, NULL, dsnap, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, - NULL); + NULL, NULL); } NFSD_DEBUG(4, "nfsrv_setattrdsdorpc: aft setattr loadattr=%d\n", error); nfsmout: @@ -6386,7 +6404,7 @@ nfsrv_setacldsdorpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, * the same type (VREG). */ nfsv4_fillattr(nd, NULL, vp, aclp, NULL, NULL, 0, &attrbits, NULL, - NULL, 0, 0, 0, 0, 0, NULL, false, false, false); + NULL, 0, 0, 0, 0, 0, NULL, false, false, false, 0); error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, cred, NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL); if (error != 0) { @@ -6530,7 +6548,7 @@ nfsrv_getattrdsrpc(fhandle_t *fhp, struct ucred *cred, NFSPROC_T *p, if (nd->nd_repstat == 0) { error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, - NULL, NULL, NULL); + NULL, NULL, NULL, NULL); /* * We can only save the updated values in the extended * attribute if the vp is exclusively locked. diff --git a/sys/fs/nfsserver/nfs_nfsdserv.c b/sys/fs/nfsserver/nfs_nfsdserv.c index 9eebcda548c6..a881968b03be 100644 --- a/sys/fs/nfsserver/nfs_nfsdserv.c +++ b/sys/fs/nfsserver/nfs_nfsdserv.c @@ -253,6 +253,7 @@ nfsrvd_getattr(struct nfsrv_descript *nd, int isdgram, size_t atsiz; long pathval; bool has_hiddensystem, has_namedattr, xattrsupp; + uint32_t clone_blksize; if (nd->nd_repstat) goto out; @@ -330,6 +331,11 @@ nfsrvd_getattr(struct nfsrv_descript *nd, int isdgram, &pathval) != 0) pathval = 0; has_namedattr = pathval > 0; + pathval = 0; + if (VOP_PATHCONF(vp, _PC_CLONE_BLKSIZE, + &pathval) != 0) + pathval = 0; + clone_blksize = pathval; mp = vp->v_mount; if (nfsrv_enable_crossmntpt != 0 && vp->v_type == VDIR && @@ -365,7 +371,7 @@ nfsrvd_getattr(struct nfsrv_descript *nd, int isdgram, isdgram, 1, supports_nfsv4acls, at_root, mounted_on_fileno, xattrsupp, has_hiddensystem, - has_namedattr); + has_namedattr, clone_blksize); vfs_unbusy(mp); } vrele(vp); @@ -4347,7 +4353,7 @@ nfsrvd_verify(struct nfsrv_descript *nd, int isdgram, if (!nd->nd_repstat) { nfsvno_getfs(&fs, isdgram); error = nfsv4_loadattr(nd, vp, &nva, NULL, &fh, fhsize, NULL, - sf, NULL, &fs, NULL, 1, &ret, NULL, NULL, NULL, p, + sf, NULL, &fs, NULL, 1, &ret, NULL, NULL, NULL, NULL, p, nd->nd_cred); if (!error) { if (nd->nd_procnum == NFSV4OP_NVERIFY) { @@ -6011,6 +6017,212 @@ nfsmout: } /* + * nfs clone service + */ +int +nfsrvd_clone(struct nfsrv_descript *nd, __unused int isdgram, + vnode_t vp, vnode_t tovp, struct nfsexstuff *exp, struct nfsexstuff *toexp) +{ + uint32_t *tl; + struct nfsvattr at; + int error = 0, ret; + off_t inoff, outoff; + uint64_t len; + size_t xfer; + struct nfsstate inst, outst, *instp = &inst, *outstp = &outst; + struct nfslock inlo, outlo, *inlop = &inlo, *outlop = &outlo; + nfsquad_t clientid; + nfsv4stateid_t stateid; + nfsattrbit_t attrbits; + void *rl_rcookie, *rl_wcookie; + long pathval; + + rl_rcookie = rl_wcookie = NULL; + pathval = 0; + if (nfsrv_maxcopyrange == 0 || nfsrv_devidcnt > 0 || + VOP_PATHCONF(vp, _PC_CLONE_BLKSIZE, &pathval) != 0 || + pathval == 0) { + /* + * For a pNFS server, reply NFSERR_NOTSUPP so that the client + * will not do the clone and will do I/O on the DS(s). + * If vfs.nfsd.maxcopyrange set to 0, disable Clone. + */ + nd->nd_repstat = NFSERR_NOTSUPP; + goto nfsmout; + } + NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_STATEID + 3 * NFSX_HYPER); + instp->ls_flags = (NFSLCK_CHECK | NFSLCK_READACCESS); + inlop->lo_flags = NFSLCK_READ; + instp->ls_ownerlen = 0; + instp->ls_op = NULL; + instp->ls_uid = nd->nd_cred->cr_uid; + instp->ls_stateid.seqid = fxdr_unsigned(uint32_t, *tl++); + clientid.lval[0] = instp->ls_stateid.other[0] = *tl++; + clientid.lval[1] = instp->ls_stateid.other[1] = *tl++; + if ((nd->nd_flag & ND_IMPLIEDCLID) != 0) + clientid.qval = nd->nd_clientid.qval; + instp->ls_stateid.other[2] = *tl++; + outstp->ls_flags = (NFSLCK_CHECK | NFSLCK_WRITEACCESS); + outlop->lo_flags = NFSLCK_WRITE; + outstp->ls_ownerlen = 0; + outstp->ls_op = NULL; + outstp->ls_uid = nd->nd_cred->cr_uid; + outstp->ls_stateid.seqid = fxdr_unsigned(uint32_t, *tl++); + outstp->ls_stateid.other[0] = *tl++; + outstp->ls_stateid.other[1] = *tl++; + outstp->ls_stateid.other[2] = *tl++; + inoff = fxdr_hyper(tl); tl += 2; + inlop->lo_first = inoff; + outoff = fxdr_hyper(tl); tl += 2; + outlop->lo_first = outoff; + len = fxdr_hyper(tl); + if (len == 0) { + /* len == 0 means to EOF. */ + inlop->lo_end = OFF_MAX; + outlop->lo_end = OFF_MAX; + } else { + inlop->lo_end = inlop->lo_first + len; + outlop->lo_end = outlop->lo_first + len; + } + + if ((inoff > OFF_MAX || outoff > OFF_MAX || + inlop->lo_end > OFF_MAX || outlop->lo_end > OFF_MAX || + inlop->lo_end < inlop->lo_first || outlop->lo_end < + outlop->lo_first)) + nd->nd_repstat = NFSERR_INVAL; + + if (nd->nd_repstat == 0 && vp->v_type != VREG) + nd->nd_repstat = NFSERR_WRONGTYPE; + + /* Check permissions for the input file. */ + NFSZERO_ATTRBIT(&attrbits); + NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_OWNER); + ret = nfsvno_getattr(vp, &at, nd, curthread, 1, &attrbits); + if (nd->nd_repstat == 0) + nd->nd_repstat = ret; + if (nd->nd_repstat == 0 && (at.na_uid != nd->nd_cred->cr_uid || + NFSVNO_EXSTRICTACCESS(exp))) + nd->nd_repstat = nfsvno_accchk(vp, VREAD, nd->nd_cred, exp, + curthread, NFSACCCHK_ALLOWOWNER, NFSACCCHK_VPISLOCKED, + NULL); + if (nd->nd_repstat == 0) + nd->nd_repstat = nfsrv_lockctrl(vp, &instp, &inlop, NULL, + clientid, &stateid, exp, nd, curthread); + if (vp != tovp) { + NFSVOPUNLOCK(vp); + if (nd->nd_repstat != 0) + goto out; + + error = NFSVOPLOCK(tovp, LK_SHARED); + if (error != 0) + goto out; + pathval = 0; + if (VOP_PATHCONF(tovp, _PC_CLONE_BLKSIZE, &pathval) != 0 || + pathval == 0) + nd->nd_repstat = NFSERR_NOTSUPP; + else if (tovp->v_type != VREG) + nd->nd_repstat = NFSERR_WRONGTYPE; + } + + /* For the output file, we only need the Owner attribute. */ + ret = nfsvno_getattr(tovp, &at, nd, curthread, 1, &attrbits); + if (nd->nd_repstat == 0) + nd->nd_repstat = ret; + if (nd->nd_repstat == 0 && (at.na_uid != nd->nd_cred->cr_uid || + NFSVNO_EXSTRICTACCESS(exp))) + nd->nd_repstat = nfsvno_accchk(tovp, VWRITE, nd->nd_cred, toexp, + curthread, NFSACCCHK_ALLOWOWNER, NFSACCCHK_VPISLOCKED, + NULL); + if (nd->nd_repstat == 0) + nd->nd_repstat = nfsrv_lockctrl(tovp, &outstp, &outlop, NULL, + clientid, &stateid, toexp, nd, curthread); + NFSVOPUNLOCK(tovp); + + /* Range lock the byte ranges for both invp and outvp. */ + if (nd->nd_repstat == 0) { + for (;;) { + if (len == 0) + rl_wcookie = vn_rangelock_wlock(tovp, outoff, + OFF_MAX); + else + rl_wcookie = vn_rangelock_wlock(tovp, outoff, + outoff + len); + if (vp != tovp) { + if (len == 0) + rl_rcookie = vn_rangelock_tryrlock(vp, + inoff, OFF_MAX); + else + rl_rcookie = vn_rangelock_tryrlock(vp, + inoff, inoff + len); + if (rl_rcookie != NULL) + break; + } else { + rl_rcookie = NULL; + break; + } + vn_rangelock_unlock(tovp, rl_wcookie); + if (len == 0) + rl_rcookie = vn_rangelock_rlock(vp, inoff, + OFF_MAX); + else + rl_rcookie = vn_rangelock_rlock(vp, inoff, + inoff + len); + vn_rangelock_unlock(vp, rl_rcookie); + } + + error = NFSVOPLOCK(vp, LK_SHARED); + if (error == 0) { + ret = nfsvno_getattr(vp, &at, nd, curthread, 1, NULL); + if (ret == 0) { + /* + * Since invp is range locked, na_size should + * not change. + */ + if (len == 0 && at.na_size > inoff) + len = SSIZE_MAX; /* To EOF. */ + else if (inoff + len > at.na_size) + nd->nd_repstat = NFSERR_INVAL; + } + NFSVOPUNLOCK(vp); + if (ret != 0 && nd->nd_repstat == 0) + nd->nd_repstat = ret; + } else if (nd->nd_repstat == 0) + nd->nd_repstat = error; + } + + /* + * Do the actual copy to an upper limit of vfs.nfsd.maxcopyrange. + * This size limit can be set to limit the time a copy RPC will + * take. + */ + xfer = len; + if (nd->nd_repstat == 0) { + nd->nd_repstat = vn_copy_file_range(vp, &inoff, tovp, &outoff, + &xfer, COPY_FILE_RANGE_CLONE, nd->nd_cred, nd->nd_cred, + NULL); + if (nd->nd_repstat == ENOSYS) + nd->nd_repstat = NFSERR_INVAL; + } + + /* Unlock the ranges. */ + if (rl_rcookie != NULL) + vn_rangelock_unlock(vp, rl_rcookie); + if (rl_wcookie != NULL) + vn_rangelock_unlock(tovp, rl_wcookie); + +out: + vrele(vp); + vrele(tovp); + NFSEXITCODE2(error, nd); + return (error); +nfsmout: + vput(vp); + vrele(tovp); + NFSEXITCODE2(error, nd); + return (error); +} + +/* * nfs seek service */ int diff --git a/sys/fs/nfsserver/nfs_nfsdsocket.c b/sys/fs/nfsserver/nfs_nfsdsocket.c index d6832b4f74be..636c4735a131 100644 --- a/sys/fs/nfsserver/nfs_nfsdsocket.c +++ b/sys/fs/nfsserver/nfs_nfsdsocket.c @@ -371,7 +371,7 @@ int (*nfsrv4_ops2[NFSV42_NOPS])(struct nfsrv_descript *, (int (*)(struct nfsrv_descript *, int, vnode_t , vnode_t , struct nfsexstuff *, struct nfsexstuff *))0, (int (*)(struct nfsrv_descript *, int, vnode_t , vnode_t , struct nfsexstuff *, struct nfsexstuff *))0, (int (*)(struct nfsrv_descript *, int, vnode_t , vnode_t , struct nfsexstuff *, struct nfsexstuff *))0, - (int (*)(struct nfsrv_descript *, int, vnode_t , vnode_t , struct nfsexstuff *, struct nfsexstuff *))0, + nfsrvd_clone, (int (*)(struct nfsrv_descript *, int, vnode_t , vnode_t , struct nfsexstuff *, struct nfsexstuff *))0, (int (*)(struct nfsrv_descript *, int, vnode_t , vnode_t , struct nfsexstuff *, struct nfsexstuff *))0, (int (*)(struct nfsrv_descript *, int, vnode_t , vnode_t , struct nfsexstuff *, struct nfsexstuff *))0, diff --git a/sys/fs/nfsserver/nfs_nfsdstate.c b/sys/fs/nfsserver/nfs_nfsdstate.c index 2e27817389dd..111b0f26d0b5 100644 --- a/sys/fs/nfsserver/nfs_nfsdstate.c +++ b/sys/fs/nfsserver/nfs_nfsdstate.c @@ -4675,7 +4675,7 @@ errout: } else if (error == 0 && procnum == NFSV4OP_CBGETATTR) error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, - NULL, p, NULL); + NULL, NULL, p, NULL); m_freem(nd->nd_mrep); } NFSLOCKSTATE(); @@ -7731,6 +7731,7 @@ nfsrv_setdsserver(char *dspathp, char *mdspathp, NFSPROC_T *p, NFSD_DEBUG(4, "lookup=%d\n", error); if (error != 0) return (error); + NDFREE_PNBUF(&nd); if (nd.ni_vp->v_type != VDIR) { vput(nd.ni_vp); NFSD_DEBUG(4, "dspath not dir\n"); @@ -7767,6 +7768,7 @@ nfsrv_setdsserver(char *dspathp, char *mdspathp, NFSPROC_T *p, NFSD_DEBUG(4, "dsdirpath=%s lookup=%d\n", dsdirpath, error); if (error != 0) break; + NDFREE_PNBUF(&nd); if (nd.ni_vp->v_type != VDIR) { vput(nd.ni_vp); error = ENOTDIR; @@ -7795,6 +7797,7 @@ nfsrv_setdsserver(char *dspathp, char *mdspathp, NFSPROC_T *p, NFSD_DEBUG(4, "mds lookup=%d\n", error); if (error != 0) goto out; + NDFREE_PNBUF(&nd); if (nd.ni_vp->v_type != VDIR) { vput(nd.ni_vp); error = ENOTDIR; @@ -8654,6 +8657,7 @@ nfsrv_mdscopymr(char *mdspathp, char *dspathp, char *curdspathp, char *buf, NFSD_DEBUG(4, "lookup=%d\n", error); if (error != 0) return (error); + NDFREE_PNBUF(&nd); if (nd.ni_vp->v_type != VREG) { vput(nd.ni_vp); NFSD_DEBUG(4, "mdspath not reg\n"); @@ -8675,6 +8679,7 @@ nfsrv_mdscopymr(char *mdspathp, char *dspathp, char *curdspathp, char *buf, vput(vp); return (error); } + NDFREE_PNBUF(&nd); if (nd.ni_vp->v_type != VDIR) { vput(nd.ni_vp); vput(vp); @@ -8717,6 +8722,7 @@ nfsrv_mdscopymr(char *mdspathp, char *dspathp, char *curdspathp, char *buf, vput(curvp); return (error); } + NDFREE_PNBUF(&nd); if (nd.ni_vp->v_type != VDIR || nd.ni_vp == curvp) { vput(nd.ni_vp); vput(vp); diff --git a/sys/fs/p9fs/p9fs_vnops.c b/sys/fs/p9fs/p9fs_vnops.c index 227e2b93883e..acb73973d93b 100644 --- a/sys/fs/p9fs/p9fs_vnops.c +++ b/sys/fs/p9fs/p9fs_vnops.c @@ -1326,7 +1326,7 @@ p9fs_read(struct vop_read_args *ap) np = P9FS_VTON(vp); error = 0; - if (vp->v_type == VCHR || vp->v_type == VBLK) + if (VN_ISDEV(vp)) return (EOPNOTSUPP); if (vp->v_type != VREG) return (EISDIR); diff --git a/sys/fs/tarfs/tarfs_vnops.c b/sys/fs/tarfs/tarfs_vnops.c index afb8e05f5929..acf18de5ab51 100644 --- a/sys/fs/tarfs/tarfs_vnops.c +++ b/sys/fs/tarfs/tarfs_vnops.c @@ -208,8 +208,7 @@ tarfs_getattr(struct vop_getattr_args *ap) vap->va_birthtime = tnp->birthtime; vap->va_gen = tnp->gen; vap->va_flags = tnp->flags; - vap->va_rdev = (vp->v_type == VBLK || vp->v_type == VCHR) ? - tnp->rdev : NODEV; + vap->va_rdev = VN_ISDEV(vp) ? tnp->rdev : NODEV; vap->va_bytes = round_page(tnp->physize); vap->va_filerev = 0; @@ -515,7 +514,7 @@ tarfs_read(struct vop_read_args *ap) uiop = ap->a_uio; vp = ap->a_vp; - if (vp->v_type == VCHR || vp->v_type == VBLK) + if (VN_ISDEV(vp)) return (EOPNOTSUPP); if (vp->v_type != VREG) diff --git a/sys/fs/tmpfs/tmpfs_subr.c b/sys/fs/tmpfs/tmpfs_subr.c index 1237f6b92cdb..dd281d18d87d 100644 --- a/sys/fs/tmpfs/tmpfs_subr.c +++ b/sys/fs/tmpfs/tmpfs_subr.c @@ -551,7 +551,7 @@ tmpfs_alloc_node(struct mount *mp, struct tmpfs_mount *tmp, __enum_uint8(vtype) MPASS(IMPLIES(tmp->tm_root == NULL, parent == NULL && type == VDIR)); MPASS((type == VLNK) ^ (target == NULL)); - MPASS((type == VBLK || type == VCHR) ^ (rdev == VNOVAL)); + MPASS(VTYPE_ISDEV(type) ^ (rdev == VNOVAL)); if (tmp->tm_nodes_inuse >= tmp->tm_nodes_max) return (ENOSPC); diff --git a/sys/fs/tmpfs/tmpfs_vnops.c b/sys/fs/tmpfs/tmpfs_vnops.c index 79b6c8b2e6a1..0f4ea2fdc28c 100644 --- a/sys/fs/tmpfs/tmpfs_vnops.c +++ b/sys/fs/tmpfs/tmpfs_vnops.c @@ -280,8 +280,7 @@ tmpfs_mknod(struct vop_mknod_args *v) struct componentname *cnp = v->a_cnp; struct vattr *vap = v->a_vap; - if (vap->va_type != VBLK && vap->va_type != VCHR && - vap->va_type != VFIFO) + if (!VATTR_ISDEV(vap) && vap->va_type != VFIFO) return (EINVAL); return (tmpfs_alloc_file(dvp, vpp, vap, cnp, NULL)); @@ -462,8 +461,7 @@ tmpfs_stat(struct vop_stat_args *v) sb->st_nlink = node->tn_links; sb->st_uid = node->tn_uid; sb->st_gid = node->tn_gid; - sb->st_rdev = (vp->v_type == VBLK || vp->v_type == VCHR) ? - node->tn_rdev : NODEV; + sb->st_rdev = VN_ISDEV(vp) ? node->tn_rdev : NODEV; sb->st_size = node->tn_size; sb->st_atim.tv_sec = node->tn_atime.tv_sec; sb->st_atim.tv_nsec = node->tn_atime.tv_nsec; @@ -521,8 +519,7 @@ tmpfs_getattr(struct vop_getattr_args *v) vap->va_birthtime = node->tn_birthtime; vap->va_gen = node->tn_gen; vap->va_flags = node->tn_flags; - vap->va_rdev = (vp->v_type == VBLK || vp->v_type == VCHR) ? - node->tn_rdev : NODEV; + vap->va_rdev = VN_ISDEV(vp) ? node->tn_rdev : NODEV; if (vp->v_type == VREG) { #ifdef __ILP32__ vm_object_t obj = node->tn_reg.tn_aobj; @@ -1918,7 +1915,7 @@ tmpfs_deleteextattr(struct vop_deleteextattr_args *ap) node = VP_TO_TMPFS_NODE(vp); tmp = VFS_TO_TMPFS(vp->v_mount); - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, ap->a_td, VWRITE); @@ -1956,7 +1953,7 @@ tmpfs_getextattr(struct vop_getextattr_args *ap) int error; node = VP_TO_TMPFS_NODE(vp); - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, ap->a_td, VREAD); @@ -1993,7 +1990,7 @@ tmpfs_listextattr(struct vop_listextattr_args *ap) int error; node = VP_TO_TMPFS_NODE(vp); - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, ap->a_td, VREAD); @@ -2037,7 +2034,7 @@ tmpfs_setextattr(struct vop_setextattr_args *ap) tmp = VFS_TO_TMPFS(vp->v_mount); attr_size = ap->a_uio->uio_resid; diff = 0; - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, ap->a_cred, ap->a_td, VWRITE); diff --git a/sys/geom/part/g_part.c b/sys/geom/part/g_part.c index 41125f6478ac..88e44b335b29 100644 --- a/sys/geom/part/g_part.c +++ b/sys/geom/part/g_part.c @@ -1046,7 +1046,7 @@ g_part_ctl_create(struct gctl_req *req, struct g_part_parms *gpp) /* * Synthesize a disk geometry. Some partitioning schemes * depend on it and since some file systems need it even - * when the partitition scheme doesn't, we do it here in + * when the partition scheme doesn't, we do it here in * scheme-independent code. */ g_part_geometry(table, cp, pp->mediasize / pp->sectorsize); @@ -1539,7 +1539,7 @@ g_part_ctl_undo(struct gctl_req *req, struct g_part_parms *gpp) /* * Synthesize a disk geometry. Some partitioning schemes * depend on it and since some file systems need it even - * when the partitition scheme doesn't, we do it here in + * when the partition scheme doesn't, we do it here in * scheme-independent code. */ pp = cp->provider; @@ -2023,7 +2023,7 @@ g_part_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) /* * Synthesize a disk geometry. Some partitioning schemes * depend on it and since some file systems need it even - * when the partitition scheme doesn't, we do it here in + * when the partition scheme doesn't, we do it here in * scheme-independent code. */ g_part_geometry(table, cp, pp->mediasize / pp->sectorsize); diff --git a/sys/i386/conf/GENERIC b/sys/i386/conf/GENERIC index f577cd07ac7c..88b8967cd693 100644 --- a/sys/i386/conf/GENERIC +++ b/sys/i386/conf/GENERIC @@ -343,3 +343,4 @@ options HID_DEBUG # enable debug msgs device hid # Generic HID support device hidbus # Generic HID Bus options IICHID_SAMPLING # Workaround missing GPIO INTR support +options U2F_MAKE_UHID_ALIAS # install /dev/uhid alias for /dev/u2f/ diff --git a/sys/isa/isareg.h b/sys/isa/isareg.h index e89136c7e1e5..8b2d55608078 100644 --- a/sys/isa/isareg.h +++ b/sys/isa/isareg.h @@ -49,7 +49,7 @@ #define IO_RTC 0x070 /* RTC */ #define IO_ICU2 0x0A0 /* 8259A Interrupt Controller #2 */ -#define IO_MDA 0x3B0 /* Monochome Adapter */ +#define IO_MDA 0x3B0 /* Monochrome Adapter */ #define IO_VGA 0x3C0 /* E/VGA Ports */ #define IO_CGA 0x3D0 /* CGA Ports */ diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c index 2690ad3b2679..5a53fac50f2c 100644 --- a/sys/kern/imgact_elf.c +++ b/sys/kern/imgact_elf.c @@ -84,6 +84,13 @@ #define ELF_NOTE_ROUNDSIZE 4 #define OLD_EI_BRAND 8 +/* + * ELF_ABI_NAME is a string name of the ELF ABI. ELF_ABI_ID is used + * to build variable names. + */ +#define ELF_ABI_NAME __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) +#define ELF_ABI_ID __CONCAT(elf, __ELF_WORD_SIZE) + static int __elfN(check_header)(const Elf_Ehdr *hdr); static Elf_Brandinfo *__elfN(get_brandinfo)(struct image_params *imgp, const char *interp, int32_t *osrel, uint32_t *fctl0); @@ -104,14 +111,15 @@ static Elf_Word __elfN(untrans_prot)(vm_prot_t); static size_t __elfN(prepare_register_notes)(struct thread *td, struct note_info_list *list, struct thread *target_td); -SYSCTL_NODE(_kern, OID_AUTO, __CONCAT(elf, __ELF_WORD_SIZE), - CTLFLAG_RW | CTLFLAG_MPSAFE, 0, +SYSCTL_NODE(_kern, OID_AUTO, ELF_ABI_ID, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, ""); +#define ELF_NODE_OID __CONCAT(_kern_, ELF_ABI_ID) + int __elfN(fallback_brand) = -1; -SYSCTL_INT(__CONCAT(_kern_elf, __ELF_WORD_SIZE), OID_AUTO, +SYSCTL_INT(ELF_NODE_OID, OID_AUTO, fallback_brand, CTLFLAG_RWTUN, &__elfN(fallback_brand), 0, - __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) " brand of last resort"); + ELF_ABI_NAME " brand of last resort"); static int elf_legacy_coredump = 0; SYSCTL_INT(_debug, OID_AUTO, __elfN(legacy_coredump), CTLFLAG_RW, @@ -126,22 +134,22 @@ int __elfN(nxstack) = #else 0; #endif -SYSCTL_INT(__CONCAT(_kern_elf, __ELF_WORD_SIZE), OID_AUTO, +SYSCTL_INT(ELF_NODE_OID, OID_AUTO, nxstack, CTLFLAG_RW, &__elfN(nxstack), 0, - __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) ": support PT_GNU_STACK for non-executable stack control"); + ELF_ABI_NAME ": support PT_GNU_STACK for non-executable stack control"); #if defined(__amd64__) static int __elfN(vdso) = 1; -SYSCTL_INT(__CONCAT(_kern_elf, __ELF_WORD_SIZE), OID_AUTO, +SYSCTL_INT(ELF_NODE_OID, OID_AUTO, vdso, CTLFLAG_RWTUN, &__elfN(vdso), 0, - __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) ": enable vdso preloading"); + ELF_ABI_NAME ": enable vdso preloading"); #else static int __elfN(vdso) = 0; #endif #if __ELF_WORD_SIZE == 32 && (defined(__amd64__) || defined(__i386__)) int i386_read_exec = 0; -SYSCTL_INT(_kern_elf32, OID_AUTO, read_exec, CTLFLAG_RW, &i386_read_exec, 0, +SYSCTL_INT(ELF_NODE_OID, OID_AUTO, read_exec, CTLFLAG_RW, &i386_read_exec, 0, "enable execution from readable segments"); #endif @@ -161,15 +169,15 @@ sysctl_pie_base(SYSCTL_HANDLER_ARGS) __elfN(pie_base) = val; return (0); } -SYSCTL_PROC(__CONCAT(_kern_elf, __ELF_WORD_SIZE), OID_AUTO, pie_base, +SYSCTL_PROC(ELF_NODE_OID, OID_AUTO, pie_base, CTLTYPE_ULONG | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_pie_base, "LU", "PIE load base without randomization"); -SYSCTL_NODE(__CONCAT(_kern_elf, __ELF_WORD_SIZE), OID_AUTO, aslr, +SYSCTL_NODE(ELF_NODE_OID, OID_AUTO, aslr, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, ""); -#define ASLR_NODE_OID __CONCAT(__CONCAT(_kern_elf, __ELF_WORD_SIZE), _aslr) +#define ASLR_NODE_OID __CONCAT(ELF_NODE_OID, _aslr) /* * Enable ASLR by default for 64-bit non-PIE binaries. 32-bit architectures @@ -179,8 +187,7 @@ SYSCTL_NODE(__CONCAT(_kern_elf, __ELF_WORD_SIZE), OID_AUTO, aslr, static int __elfN(aslr_enabled) = __ELF_WORD_SIZE == 64; SYSCTL_INT(ASLR_NODE_OID, OID_AUTO, enable, CTLFLAG_RWTUN, &__elfN(aslr_enabled), 0, - __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) - ": enable address map randomization"); + ELF_ABI_NAME ": enable address map randomization"); /* * Enable ASLR by default for 64-bit PIE binaries. @@ -188,8 +195,7 @@ SYSCTL_INT(ASLR_NODE_OID, OID_AUTO, enable, CTLFLAG_RWTUN, static int __elfN(pie_aslr_enabled) = __ELF_WORD_SIZE == 64; SYSCTL_INT(ASLR_NODE_OID, OID_AUTO, pie_enable, CTLFLAG_RWTUN, &__elfN(pie_aslr_enabled), 0, - __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) - ": enable address map randomization for PIE binaries"); + ELF_ABI_NAME ": enable address map randomization for PIE binaries"); /* * Sbrk is deprecated and it can be assumed that in most cases it will not be @@ -199,27 +205,25 @@ SYSCTL_INT(ASLR_NODE_OID, OID_AUTO, pie_enable, CTLFLAG_RWTUN, static int __elfN(aslr_honor_sbrk) = 0; SYSCTL_INT(ASLR_NODE_OID, OID_AUTO, honor_sbrk, CTLFLAG_RW, &__elfN(aslr_honor_sbrk), 0, - __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) ": assume sbrk is used"); + ELF_ABI_NAME ": assume sbrk is used"); static int __elfN(aslr_stack) = __ELF_WORD_SIZE == 64; SYSCTL_INT(ASLR_NODE_OID, OID_AUTO, stack, CTLFLAG_RWTUN, &__elfN(aslr_stack), 0, - __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) - ": enable stack address randomization"); + ELF_ABI_NAME ": enable stack address randomization"); static int __elfN(aslr_shared_page) = __ELF_WORD_SIZE == 64; SYSCTL_INT(ASLR_NODE_OID, OID_AUTO, shared_page, CTLFLAG_RWTUN, &__elfN(aslr_shared_page), 0, - __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) - ": enable shared page address randomization"); + ELF_ABI_NAME ": enable shared page address randomization"); static int __elfN(sigfastblock) = 1; -SYSCTL_INT(__CONCAT(_kern_elf, __ELF_WORD_SIZE), OID_AUTO, sigfastblock, +SYSCTL_INT(ELF_NODE_OID, OID_AUTO, sigfastblock, CTLFLAG_RWTUN, &__elfN(sigfastblock), 0, "enable sigfastblock for new processes"); static bool __elfN(allow_wx) = true; -SYSCTL_BOOL(__CONCAT(_kern_elf, __ELF_WORD_SIZE), OID_AUTO, allow_wx, +SYSCTL_BOOL(ELF_NODE_OID, OID_AUTO, allow_wx, CTLFLAG_RWTUN, &__elfN(allow_wx), 0, "Allow pages to be mapped simultaneously writable and executable"); @@ -2951,9 +2955,9 @@ __elfN(check_note)(struct image_params *imgp, Elf_Brandnote *brandnote, */ static struct execsw __elfN(execsw) = { .ex_imgact = __CONCAT(exec_, __elfN(imgact)), - .ex_name = __XSTRING(__CONCAT(ELF, __ELF_WORD_SIZE)) + .ex_name = ELF_ABI_NAME }; -EXEC_SET(__CONCAT(elf, __ELF_WORD_SIZE), __elfN(execsw)); +EXEC_SET(ELF_ABI_ID, __elfN(execsw)); static vm_prot_t __elfN(trans_prot)(Elf_Word flags) diff --git a/sys/kern/init_sysent.c b/sys/kern/init_sysent.c index c0a5479c9634..fcd232cde21e 100644 --- a/sys/kern/init_sysent.c +++ b/sys/kern/init_sysent.c @@ -145,8 +145,8 @@ struct sysent sysent[] = { { .sy_narg = 0, .sy_call = (sy_call_t *)nosys, .sy_auevent = AUE_NULL, .sy_flags = 0, .sy_thrcnt = SY_THR_ABSENT }, /* 76 = obsolete vhangup */ { .sy_narg = 0, .sy_call = (sy_call_t *)nosys, .sy_auevent = AUE_NULL, .sy_flags = 0, .sy_thrcnt = SY_THR_ABSENT }, /* 77 = obsolete vlimit */ { .sy_narg = AS(mincore_args), .sy_call = (sy_call_t *)sys_mincore, .sy_auevent = AUE_MINCORE, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 78 = mincore */ - { .sy_narg = AS(getgroups_args), .sy_call = (sy_call_t *)sys_getgroups, .sy_auevent = AUE_GETGROUPS, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 79 = getgroups */ - { .sy_narg = AS(setgroups_args), .sy_call = (sy_call_t *)sys_setgroups, .sy_auevent = AUE_SETGROUPS, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 80 = setgroups */ + { compat14(AS(freebsd14_getgroups_args),getgroups), .sy_auevent = AUE_GETGROUPS, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 79 = freebsd14 getgroups */ + { compat14(AS(freebsd14_setgroups_args),setgroups), .sy_auevent = AUE_SETGROUPS, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 80 = freebsd14 setgroups */ { .sy_narg = 0, .sy_call = (sy_call_t *)sys_getpgrp, .sy_auevent = AUE_GETPGRP, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 81 = getpgrp */ { .sy_narg = AS(setpgid_args), .sy_call = (sy_call_t *)sys_setpgid, .sy_auevent = AUE_SETPGRP, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 82 = setpgid */ { .sy_narg = AS(setitimer_args), .sy_call = (sy_call_t *)sys_setitimer, .sy_auevent = AUE_SETITIMER, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 83 = setitimer */ @@ -661,4 +661,6 @@ struct sysent sysent[] = { { .sy_narg = AS(exterrctl_args), .sy_call = (sy_call_t *)sys_exterrctl, .sy_auevent = AUE_NULL, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 592 = exterrctl */ { .sy_narg = AS(inotify_add_watch_at_args), .sy_call = (sy_call_t *)sys_inotify_add_watch_at, .sy_auevent = AUE_INOTIFY, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 593 = inotify_add_watch_at */ { .sy_narg = AS(inotify_rm_watch_args), .sy_call = (sy_call_t *)sys_inotify_rm_watch, .sy_auevent = AUE_INOTIFY, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 594 = inotify_rm_watch */ + { .sy_narg = AS(getgroups_args), .sy_call = (sy_call_t *)sys_getgroups, .sy_auevent = AUE_GETGROUPS, .sy_flags = SYF_CAPENABLED, .sy_thrcnt = SY_THR_STATIC }, /* 595 = getgroups */ + { .sy_narg = AS(setgroups_args), .sy_call = (sy_call_t *)sys_setgroups, .sy_auevent = AUE_SETGROUPS, .sy_flags = 0, .sy_thrcnt = SY_THR_STATIC }, /* 596 = setgroups */ }; diff --git a/sys/kern/kern_prot.c b/sys/kern/kern_prot.c index 2cd5b7069023..0ca42d640767 100644 --- a/sys/kern/kern_prot.c +++ b/sys/kern/kern_prot.c @@ -310,6 +310,39 @@ sys_getegid(struct thread *td, struct getegid_args *uap) return (0); } +#ifdef COMPAT_FREEBSD14 +int +freebsd14_getgroups(struct thread *td, struct freebsd14_getgroups_args *uap) +{ + struct ucred *cred; + int ngrp, error; + + cred = td->td_ucred; + + /* + * For FreeBSD < 15.0, we account for the egid being placed at the + * beginning of the group list prior to all supplementary groups. + */ + ngrp = cred->cr_ngroups + 1; + if (uap->gidsetsize == 0) { + error = 0; + goto out; + } else if (uap->gidsetsize < ngrp) { + return (EINVAL); + } + + error = copyout(&cred->cr_gid, uap->gidset, sizeof(gid_t)); + if (error == 0) + error = copyout(cred->cr_groups, uap->gidset + 1, + (ngrp - 1) * sizeof(gid_t)); + +out: + td->td_retval[0] = ngrp; + return (error); + +} +#endif /* COMPAT_FREEBSD14 */ + #ifndef _SYS_SYSPROTO_H_ struct getgroups_args { int gidsetsize; @@ -320,18 +353,11 @@ int sys_getgroups(struct thread *td, struct getgroups_args *uap) { struct ucred *cred; - gid_t *ugidset; int ngrp, error; cred = td->td_ucred; - /* - * cr_gid has been moved out of cr_groups, but we'll continue exporting - * the egid as groups[0] for the time being until we audit userland for - * any surprises. - */ - ngrp = cred->cr_ngroups + 1; - + ngrp = cred->cr_ngroups; if (uap->gidsetsize == 0) { error = 0; goto out; @@ -339,14 +365,7 @@ sys_getgroups(struct thread *td, struct getgroups_args *uap) if (uap->gidsetsize < ngrp) return (EINVAL); - ugidset = uap->gidset; - error = copyout(&cred->cr_gid, ugidset, sizeof(*ugidset)); - if (error != 0) - goto out; - - if (ngrp > 1) - error = copyout(cred->cr_groups, ugidset + 1, - (ngrp - 1) * sizeof(*ugidset)); + error = copyout(cred->cr_groups, uap->gidset, ngrp * sizeof(gid_t)); out: td->td_retval[0] = ngrp; return (error); @@ -1186,6 +1205,44 @@ fail: return (error); } +#ifdef COMPAT_FREEBSD14 +int +freebsd14_setgroups(struct thread *td, struct freebsd14_setgroups_args *uap) +{ + gid_t smallgroups[CRED_SMALLGROUPS_NB]; + gid_t *groups; + int gidsetsize, error; + + /* + * Before FreeBSD 15.0, we allow one more group to be supplied to + * account for the egid appearing before the supplementary groups. This + * may technically allow one more supplementary group for systems that + * did use the default NGROUPS_MAX if we round it back up to 1024. + */ + gidsetsize = uap->gidsetsize; + if (gidsetsize > ngroups_max + 1 || gidsetsize < 0) + return (EINVAL); + + if (gidsetsize > CRED_SMALLGROUPS_NB) + groups = malloc(gidsetsize * sizeof(gid_t), M_TEMP, M_WAITOK); + else + groups = smallgroups; + + error = copyin(uap->gidset, groups, gidsetsize * sizeof(gid_t)); + if (error == 0) { + int ngroups = gidsetsize > 0 ? gidsetsize - 1 /* egid */ : 0; + + error = kern_setgroups(td, &ngroups, groups + 1); + if (error == 0 && gidsetsize > 0) + td->td_proc->p_ucred->cr_gid = groups[0]; + } + + if (groups != smallgroups) + free(groups, M_TEMP); + return (error); +} +#endif /* COMPAT_FREEBSD14 */ + #ifndef _SYS_SYSPROTO_H_ struct setgroups_args { int gidsetsize; @@ -1210,8 +1267,7 @@ sys_setgroups(struct thread *td, struct setgroups_args *uap) * setgroups() differ. */ gidsetsize = uap->gidsetsize; - /* XXXKE Limit to ngroups_max when we change the userland interface. */ - if (gidsetsize > ngroups_max + 1 || gidsetsize < 0) + if (gidsetsize > ngroups_max || gidsetsize < 0) return (EINVAL); if (gidsetsize > CRED_SMALLGROUPS_NB) @@ -1238,35 +1294,17 @@ kern_setgroups(struct thread *td, int *ngrpp, gid_t *groups) struct proc *p = td->td_proc; struct ucred *newcred, *oldcred; int ngrp, error; - gid_t egid; ngrp = *ngrpp; /* Sanity check size. */ - /* XXXKE Limit to ngroups_max when we change the userland interface. */ - if (ngrp < 0 || ngrp > ngroups_max + 1) + if (ngrp < 0 || ngrp > ngroups_max) return (EINVAL); AUDIT_ARG_GROUPSET(groups, ngrp); - /* - * setgroups(0, NULL) is a legitimate way of clearing the groups vector - * on non-BSD systems (which generally do not have the egid in the - * groups[0]). We risk security holes when running non-BSD software if - * we do not do the same. So we allow and treat 0 for 'ngrp' specially - * below (twice). - */ - if (ngrp != 0) { - /* - * To maintain userland compat for now, we use the first group - * as our egid and we'll use the rest as our supplemental - * groups. - */ - egid = groups[0]; - ngrp--; - groups++; - groups_normalize(&ngrp, groups); - *ngrpp = ngrp; - } + groups_normalize(&ngrp, groups); + *ngrpp = ngrp; + newcred = crget(); crextend(newcred, ngrp); PROC_LOCK(p); @@ -1289,15 +1327,7 @@ kern_setgroups(struct thread *td, int *ngrpp, gid_t *groups) if (error) goto fail; - /* - * If some groups were passed, the first one is currently the desired - * egid. This code is to be removed (along with some commented block - * above) when setgroups() is changed to take only supplementary groups. - */ - if (ngrp != 0) - newcred->cr_gid = egid; crsetgroups_internal(newcred, ngrp, groups); - setsugid(p); proc_set_cred(p, newcred); PROC_UNLOCK(p); @@ -2891,7 +2921,8 @@ crextend(struct ucred *cr, int n) * Normalizes a set of groups to be applied to a 'struct ucred'. * * Normalization ensures that the supplementary groups are sorted in ascending - * order and do not contain duplicates. + * order and do not contain duplicates. This allows group_is_supplementary + * to do a binary search. */ static void groups_normalize(int *ngrp, gid_t *groups) diff --git a/sys/kern/syscalls.c b/sys/kern/syscalls.c index 09bf4d519927..4122f9261871 100644 --- a/sys/kern/syscalls.c +++ b/sys/kern/syscalls.c @@ -84,8 +84,8 @@ const char *syscallnames[] = { "obs_vhangup", /* 76 = obsolete vhangup */ "obs_vlimit", /* 77 = obsolete vlimit */ "mincore", /* 78 = mincore */ - "getgroups", /* 79 = getgroups */ - "setgroups", /* 80 = setgroups */ + "compat14.getgroups", /* 79 = freebsd14 getgroups */ + "compat14.setgroups", /* 80 = freebsd14 setgroups */ "getpgrp", /* 81 = getpgrp */ "setpgid", /* 82 = setpgid */ "setitimer", /* 83 = setitimer */ @@ -600,4 +600,6 @@ const char *syscallnames[] = { "exterrctl", /* 592 = exterrctl */ "inotify_add_watch_at", /* 593 = inotify_add_watch_at */ "inotify_rm_watch", /* 594 = inotify_rm_watch */ + "getgroups", /* 595 = getgroups */ + "setgroups", /* 596 = setgroups */ }; diff --git a/sys/kern/syscalls.master b/sys/kern/syscalls.master index 53b5d3cbbba9..fa64597d14a5 100644 --- a/sys/kern/syscalls.master +++ b/sys/kern/syscalls.master @@ -552,13 +552,13 @@ _Out_writes_bytes_(len/PAGE_SIZE) char *vec ); } -79 AUE_GETGROUPS STD|CAPENABLED { +79 AUE_GETGROUPS STD|CAPENABLED|COMPAT14 { int getgroups( int gidsetsize, _Out_writes_opt_(gidsetsize) gid_t *gidset ); } -80 AUE_SETGROUPS STD { +80 AUE_SETGROUPS STD|COMPAT14 { int setgroups( int gidsetsize, _In_reads_(gidsetsize) const gid_t *gidset @@ -3371,5 +3371,17 @@ int wd ); } +595 AUE_GETGROUPS STD|CAPENABLED { + int getgroups( + int gidsetsize, + _Out_writes_opt_(gidsetsize) gid_t *gidset + ); + } +596 AUE_SETGROUPS STD { + int setgroups( + int gidsetsize, + _In_reads_(gidsetsize) const gid_t *gidset + ); + } ; vim: syntax=off diff --git a/sys/kern/systrace_args.c b/sys/kern/systrace_args.c index 4dfc63924da9..2b1ea9eed8d4 100644 --- a/sys/kern/systrace_args.c +++ b/sys/kern/systrace_args.c @@ -454,22 +454,6 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args) *n_args = 3; break; } - /* getgroups */ - case 79: { - struct getgroups_args *p = params; - iarg[a++] = p->gidsetsize; /* int */ - uarg[a++] = (intptr_t)p->gidset; /* gid_t * */ - *n_args = 2; - break; - } - /* setgroups */ - case 80: { - struct setgroups_args *p = params; - iarg[a++] = p->gidsetsize; /* int */ - uarg[a++] = (intptr_t)p->gidset; /* const gid_t * */ - *n_args = 2; - break; - } /* getpgrp */ case 81: { *n_args = 0; @@ -3500,6 +3484,22 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args) *n_args = 2; break; } + /* getgroups */ + case 595: { + struct getgroups_args *p = params; + iarg[a++] = p->gidsetsize; /* int */ + uarg[a++] = (intptr_t)p->gidset; /* gid_t * */ + *n_args = 2; + break; + } + /* setgroups */ + case 596: { + struct setgroups_args *p = params; + iarg[a++] = p->gidsetsize; /* int */ + uarg[a++] = (intptr_t)p->gidset; /* const gid_t * */ + *n_args = 2; + break; + } default: *n_args = 0; break; @@ -4199,32 +4199,6 @@ systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) break; }; break; - /* getgroups */ - case 79: - switch (ndx) { - case 0: - p = "int"; - break; - case 1: - p = "userland gid_t *"; - break; - default: - break; - }; - break; - /* setgroups */ - case 80: - switch (ndx) { - case 0: - p = "int"; - break; - case 1: - p = "userland const gid_t *"; - break; - default: - break; - }; - break; /* getpgrp */ case 81: break; @@ -9367,6 +9341,32 @@ systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) break; }; break; + /* getgroups */ + case 595: + switch (ndx) { + case 0: + p = "int"; + break; + case 1: + p = "userland gid_t *"; + break; + default: + break; + }; + break; + /* setgroups */ + case 596: + switch (ndx) { + case 0: + p = "int"; + break; + case 1: + p = "userland const gid_t *"; + break; + default: + break; + }; + break; default: break; }; @@ -9633,16 +9633,6 @@ systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) if (ndx == 0 || ndx == 1) p = "int"; break; - /* getgroups */ - case 79: - if (ndx == 0 || ndx == 1) - p = "int"; - break; - /* setgroups */ - case 80: - if (ndx == 0 || ndx == 1) - p = "int"; - break; /* getpgrp */ case 81: /* setpgid */ @@ -11365,6 +11355,16 @@ systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz) if (ndx == 0 || ndx == 1) p = "int"; break; + /* getgroups */ + case 595: + if (ndx == 0 || ndx == 1) + p = "int"; + break; + /* setgroups */ + case 596: + if (ndx == 0 || ndx == 1) + p = "int"; + break; default: break; }; diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c index 0056dac65c7d..19870e989437 100644 --- a/sys/kern/uipc_usrreq.c +++ b/sys/kern/uipc_usrreq.c @@ -154,15 +154,12 @@ static struct task unp_defer_task; * and don't really want to reserve the sendspace. Their recvspace should be * large enough for at least one max-size datagram plus address. */ -#ifndef PIPSIZ -#define PIPSIZ 8192 -#endif -static u_long unpst_sendspace = PIPSIZ; -static u_long unpst_recvspace = PIPSIZ; +static u_long unpst_sendspace = 64*1024; +static u_long unpst_recvspace = 64*1024; static u_long unpdg_maxdgram = 8*1024; /* support 8KB syslog msgs */ static u_long unpdg_recvspace = 16*1024; -static u_long unpsp_sendspace = PIPSIZ; -static u_long unpsp_recvspace = PIPSIZ; +static u_long unpsp_sendspace = 64*1024; +static u_long unpsp_recvspace = 64*1024; static SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "Local domain"); diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index fa655c43d155..19c39e42bafa 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -5170,7 +5170,7 @@ bufstrategy(struct bufobj *bo, struct buf *bp) vp = bp->b_vp; KASSERT(vp == bo->bo_private, ("Inconsistent vnode bufstrategy")); - KASSERT(vp->v_type != VCHR && vp->v_type != VBLK, + KASSERT(!VN_ISDEV(vp), ("Wrong vnode in bufstrategy(bp=%p, vp=%p)", bp, vp)); i = VOP_STRATEGY(vp, bp); KASSERT(i == 0, ("VOP_STRATEGY failed bp=%p vp=%p", bp, bp->b_vp)); diff --git a/sys/kern/vfs_inotify.c b/sys/kern/vfs_inotify.c index 746a5a39208e..b265a5ff3a62 100644 --- a/sys/kern/vfs_inotify.c +++ b/sys/kern/vfs_inotify.c @@ -801,6 +801,7 @@ vn_inotify_add_watch(struct vnode *vp, struct inotify_softc *sc, uint32_t mask, vn_lock(vp, LK_SHARED | LK_RETRY); if (error != 0) break; + NDFREE_PNBUF(&nd); vn_irflag_set_cond(nd.ni_vp, VIRF_INOTIFY_PARENT); vrele(nd.ni_vp); } diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c index b805e147bd62..bf3ed9d515dc 100644 --- a/sys/kern/vfs_syscalls.c +++ b/sys/kern/vfs_syscalls.c @@ -2839,7 +2839,7 @@ setfflags(struct thread *td, struct vnode *vp, u_long flags) * if they are allowed to set flags and programs assume that * chown can't fail when done as root. */ - if (vp->v_type == VCHR || vp->v_type == VBLK) { + if (VN_ISDEV(vp)) { error = priv_check(td, PRIV_VFS_CHFLAGS_DEV); if (error != 0) return (error); diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c index 93f87ddae4de..a4f41192f684 100644 --- a/sys/kern/vfs_vnops.c +++ b/sys/kern/vfs_vnops.c @@ -3444,7 +3444,7 @@ vn_generic_copy_file_range(struct vnode *invp, off_t *inoffp, dat = NULL; if ((flags & COPY_FILE_RANGE_CLONE) != 0) { - error = ENOSYS; + error = EOPNOTSUPP; goto out; } diff --git a/sys/libkern/qsort.c b/sys/libkern/qsort.c index 0255a3d64d76..342b1525dd8a 100644 --- a/sys/libkern/qsort.c +++ b/sys/libkern/qsort.c @@ -114,11 +114,10 @@ qsort(void *a, size_t n, size_t es, cmp_t *cmp) char *pa, *pb, *pc, *pd, *pl, *pm, *pn; size_t d1, d2; int cmp_result; - int swaptype_long, swaptype_int, swap_cnt; + int swaptype_long, swaptype_int; loop: SWAPINIT(long, a, es); SWAPINIT(int, a, es); - swap_cnt = 0; if (n < 7) { for (pm = (char *)a + es; pm < (char *)a + n * es; pm += es) for (pl = pm; @@ -147,7 +146,6 @@ loop: SWAPINIT(long, a, es); for (;;) { while (pb <= pc && (cmp_result = CMP(thunk, pb, a)) <= 0) { if (cmp_result == 0) { - swap_cnt = 1; swap(pa, pb); pa += es; } @@ -155,7 +153,6 @@ loop: SWAPINIT(long, a, es); } while (pb <= pc && (cmp_result = CMP(thunk, pc, a)) >= 0) { if (cmp_result == 0) { - swap_cnt = 1; swap(pc, pd); pd -= es; } @@ -164,18 +161,9 @@ loop: SWAPINIT(long, a, es); if (pb > pc) break; swap(pb, pc); - swap_cnt = 1; pb += es; pc -= es; } - if (swap_cnt == 0) { /* Switch to insertion sort */ - for (pm = (char *)a + es; pm < (char *)a + n * es; pm += es) - for (pl = pm; - pl > (char *)a && CMP(thunk, pl - es, pl) > 0; - pl -= es) - swap(pl, pl - es); - return; - } pn = (char *)a + n * es; d1 = MIN(pa - (char *)a, pb - pa); diff --git a/sys/modules/e6000sw/Makefile b/sys/modules/e6000sw/Makefile index da08f80b0a29..73cbaea801f0 100644 --- a/sys/modules/e6000sw/Makefile +++ b/sys/modules/e6000sw/Makefile @@ -3,6 +3,6 @@ KMOD= e6000sw SRCS= e6000sw.c -SRCS+= bus_if.h etherswitch_if.h mdio_if.h miibus_if.h ofw_bus_if.h opt_platform.h +SRCS+= bus_if.h device_if.h etherswitch_if.h mdio_if.h miibus_if.h ofw_bus_if.h opt_platform.h .include <bsd.kmod.mk> diff --git a/sys/modules/etherswitch/Makefile b/sys/modules/etherswitch/Makefile index 087231545cd4..0b16a19e5117 100644 --- a/sys/modules/etherswitch/Makefile +++ b/sys/modules/etherswitch/Makefile @@ -3,7 +3,7 @@ KMOD = etherswitch SRCS= etherswitch.c -SRCS+= mdio_if.h miibus_if.h etherswitch_if.h etherswitch_if.c +SRCS+= bus_if.h device_if.h mdio_if.h miibus_if.h etherswitch_if.h etherswitch_if.c CFLAGS+= -I${SRCTOP}/sys/dev/etherswitch .include <bsd.kmod.mk> diff --git a/sys/modules/evdev/Makefile b/sys/modules/evdev/Makefile index bd66013885db..20813b73f6dd 100644 --- a/sys/modules/evdev/Makefile +++ b/sys/modules/evdev/Makefile @@ -2,7 +2,7 @@ KMOD= evdev SRCS= cdev.c evdev.c evdev_mt.c evdev_utils.c -SRCS+= opt_evdev.h bus_if.h device_if.h +SRCS+= opt_evdev.h opt_kbd.h bus_if.h device_if.h EXPORT_SYMS= YES diff --git a/sys/modules/gpio/gpioaei/Makefile b/sys/modules/gpio/gpioaei/Makefile index 8f856af48eb7..1f0f1d0e53a6 100644 --- a/sys/modules/gpio/gpioaei/Makefile +++ b/sys/modules/gpio/gpioaei/Makefile @@ -10,6 +10,8 @@ SRCS+= \ gpio_if.h \ gpiobus_if.h +SRCS+= opt_acpi.h opt_platform.h + CFLAGS+= -I. -I${SRCTOP}/sys/dev/gpio/ .include <bsd.kmod.mk> diff --git a/sys/modules/gve/Makefile b/sys/modules/gve/Makefile index 08b26a994e36..ece275485df7 100644 --- a/sys/modules/gve/Makefile +++ b/sys/modules/gve/Makefile @@ -40,5 +40,5 @@ SRCS= gve_main.c \ gve_tx_dqo.c \ gve_sysctl.c SRCS+= device_if.h bus_if.h pci_if.h - +SRCS+= opt_inet6.h .include <bsd.kmod.mk> diff --git a/sys/modules/hid/Makefile b/sys/modules/hid/Makefile index 56c3267d8684..10720570deb7 100644 --- a/sys/modules/hid/Makefile +++ b/sys/modules/hid/Makefile @@ -17,6 +17,7 @@ SUBDIR += \ hsctrl \ ietp \ ps4dshock \ + u2f \ xb360gp .include <bsd.subdir.mk> diff --git a/sys/modules/hid/u2f/Makefile b/sys/modules/hid/u2f/Makefile new file mode 100644 index 000000000000..227e7154035b --- /dev/null +++ b/sys/modules/hid/u2f/Makefile @@ -0,0 +1,8 @@ +.PATH: ${SRCTOP}/sys/dev/hid + +KMOD= u2f +SRCS= u2f.c +SRCS+= opt_hid.h opt_usb.h +SRCS+= bus_if.h device_if.h + +.include <bsd.kmod.mk> diff --git a/sys/modules/if_infiniband/Makefile b/sys/modules/if_infiniband/Makefile index 01e3164b1271..7ec343999da1 100644 --- a/sys/modules/if_infiniband/Makefile +++ b/sys/modules/if_infiniband/Makefile @@ -3,7 +3,8 @@ KMOD= if_infiniband SRCS= if_infiniband.c \ opt_inet.h \ - opt_inet6.h + opt_inet6.h \ + opt_kbd.h EXPORT_SYMS= YES diff --git a/sys/modules/if_vlan/Makefile b/sys/modules/if_vlan/Makefile index 3077f4289d5a..0cdab3f7653a 100644 --- a/sys/modules/if_vlan/Makefile +++ b/sys/modules/if_vlan/Makefile @@ -2,6 +2,6 @@ KMOD= if_vlan SRCS= if_vlan.c -SRCS+= opt_inet.h opt_inet6.h opt_kern_tls.h opt_vlan.h opt_ratelimit.h +SRCS+= opt_inet.h opt_inet6.h opt_ipsec.h opt_kern_tls.h opt_vlan.h opt_ratelimit.h .include <bsd.kmod.mk> diff --git a/sys/modules/ix/Makefile b/sys/modules/ix/Makefile index ad9f36e054e3..aec6eaabffdd 100644 --- a/sys/modules/ix/Makefile +++ b/sys/modules/ix/Makefile @@ -7,7 +7,7 @@ SRCS += if_ix.c if_bypass.c if_fdir.c if_sriov.c ix_txrx.c ixgbe_osdep.c # Shared source SRCS += ixgbe_common.c ixgbe_api.c ixgbe_phy.c ixgbe_mbx.c ixgbe_vf.c SRCS += ixgbe_dcb.c ixgbe_dcb_82598.c ixgbe_dcb_82599.c -SRCS += ixgbe_82598.c ixgbe_82599.c ixgbe_x540.c ixgbe_x550.c +SRCS += ixgbe_82598.c ixgbe_82599.c ixgbe_x540.c ixgbe_x550.c ixgbe_e610.c CFLAGS+= -I${SRCTOP}/sys/dev/ixgbe .include <bsd.kmod.mk> diff --git a/sys/modules/ixv/Makefile b/sys/modules/ixv/Makefile index 1b4431ac11cd..e7066bb7829b 100644 --- a/sys/modules/ixv/Makefile +++ b/sys/modules/ixv/Makefile @@ -7,7 +7,7 @@ SRCS += if_ixv.c if_fdir.c ix_txrx.c ixgbe_osdep.c # Shared source SRCS += ixgbe_common.c ixgbe_api.c ixgbe_phy.c ixgbe_mbx.c ixgbe_vf.c SRCS += ixgbe_dcb.c ixgbe_dcb_82598.c ixgbe_dcb_82599.c -SRCS += ixgbe_82598.c ixgbe_82599.c ixgbe_x540.c ixgbe_x550.c +SRCS += ixgbe_82598.c ixgbe_82599.c ixgbe_x540.c ixgbe_x550.c ixgbe_e610.c CFLAGS+= -I${SRCTOP}/sys/dev/ixgbe .include <bsd.kmod.mk> diff --git a/sys/modules/linux64/Makefile b/sys/modules/linux64/Makefile index b23891a65a4f..327da11afdaf 100644 --- a/sys/modules/linux64/Makefile +++ b/sys/modules/linux64/Makefile @@ -31,6 +31,7 @@ SRCS= linux_dummy_machdep.c \ opt_ktrace.h \ opt_inet6.h \ opt_posix.h \ + opt_usb.h \ bus_if.h \ device_if.h \ vnode_if.h \ diff --git a/sys/modules/md/Makefile b/sys/modules/md/Makefile index 2b0586c44717..3f16e04860a1 100644 --- a/sys/modules/md/Makefile +++ b/sys/modules/md/Makefile @@ -1,6 +1,6 @@ .PATH: ${SRCTOP}/sys/dev/md KMOD= geom_md -SRCS= md.c opt_md.h opt_geom.h opt_rootdevname.h vnode_if.h +SRCS= bus_if.h device_if.h md.c opt_md.h opt_geom.h opt_rootdevname.h vnode_if.h .include <bsd.kmod.mk> diff --git a/sys/modules/miiproxy/Makefile b/sys/modules/miiproxy/Makefile index 5173358989da..730bef4220cd 100644 --- a/sys/modules/miiproxy/Makefile +++ b/sys/modules/miiproxy/Makefile @@ -3,7 +3,7 @@ KMOD = miiproxy SRCS= miiproxy.c -SRCS+= mdio_if.h miibus_if.h +SRCS+= bus_if.h mdio_if.h miibus_if.h opt_platform.h CFLAGS+= -I${SRCTOP}/sys/dev/etherswitch .include <bsd.kmod.mk> diff --git a/sys/modules/mlx5/Makefile b/sys/modules/mlx5/Makefile index 506c045ab0ce..65341fdfb8aa 100644 --- a/sys/modules/mlx5/Makefile +++ b/sys/modules/mlx5/Makefile @@ -46,7 +46,7 @@ mlx5_ipsec_offload.c \ mlx5_ipsec.c \ mlx5_ipsec_rxtx.c SRCS+= ${LINUXKPI_GENSRCS} -SRCS+= opt_inet.h opt_inet6.h opt_rss.h opt_ratelimit.h +SRCS+= opt_inet.h opt_inet6.h opt_ipsec.h opt_rss.h opt_ratelimit.h CFLAGS+= -I${SRCTOP}/sys/ofed/include CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi diff --git a/sys/modules/mlx5en/Makefile b/sys/modules/mlx5en/Makefile index 03bf174e33b0..3697fa65dc83 100644 --- a/sys/modules/mlx5en/Makefile +++ b/sys/modules/mlx5en/Makefile @@ -15,7 +15,7 @@ mlx5_en_rl.c \ mlx5_en_txrx.c \ mlx5_en_port_buffer.c SRCS+= ${LINUXKPI_GENSRCS} -SRCS+= opt_inet.h opt_inet6.h opt_rss.h opt_ratelimit.h opt_kern_tls.h +SRCS+= opt_inet.h opt_inet6.h opt_ipsec.h opt_rss.h opt_ratelimit.h opt_kern_tls.h .if defined(HAVE_PER_CQ_EVENT_PACKET) CFLAGS+= -DHAVE_PER_CQ_EVENT_PACKET diff --git a/sys/modules/netgraph/ksocket/Makefile b/sys/modules/netgraph/ksocket/Makefile index 395fdbd7b3e3..7099648f6219 100644 --- a/sys/modules/netgraph/ksocket/Makefile +++ b/sys/modules/netgraph/ksocket/Makefile @@ -1,4 +1,6 @@ KMOD= ng_ksocket SRCS= ng_ksocket.c +SRCS+= opt_inet6.h + .include <bsd.kmod.mk> diff --git a/sys/modules/nvmf/nvmf/Makefile b/sys/modules/nvmf/nvmf/Makefile index 7ebe614998bd..21d73d363d2f 100644 --- a/sys/modules/nvmf/nvmf/Makefile +++ b/sys/modules/nvmf/nvmf/Makefile @@ -10,4 +10,7 @@ SRCS= nvmf.c \ nvmf_qpair.c \ nvmf_sim.c +SRCS+= bus_if.h device_if.h +SRCS+= opt_cam.h + .include <bsd.kmod.mk> diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile index 7a92742d6b36..ac2c752e922e 100644 --- a/sys/modules/ossl/Makefile +++ b/sys/modules/ossl/Makefile @@ -29,6 +29,7 @@ SRCS.arm= \ SRCS.aarch64= \ chacha-armv8.S \ + chacha-armv8-sve.S \ poly1305-armv8.S \ sha1-armv8.S \ sha256-armv8.S \ diff --git a/sys/modules/qlnx/qlnxev/Makefile b/sys/modules/qlnx/qlnxev/Makefile index ed62f1f1dd40..766a5a950032 100644 --- a/sys/modules/qlnx/qlnxev/Makefile +++ b/sys/modules/qlnx/qlnxev/Makefile @@ -49,6 +49,7 @@ SRCS+=ecore_vf.c SRCS+=qlnx_ioctl.c SRCS+=qlnx_os.c +SRCS+=opt_inet.h SRCS+= ${LINUXKPI_GENSRCS} diff --git a/sys/modules/rtw88/Makefile b/sys/modules/rtw88/Makefile index 9739ede11073..822be639da43 100644 --- a/sys/modules/rtw88/Makefile +++ b/sys/modules/rtw88/Makefile @@ -43,6 +43,7 @@ SRCS+= ${LINUXKPI_GENSRCS} SRCS+= opt_wlan.h opt_inet6.h opt_inet.h CFLAGS+= -DKBUILD_MODNAME='"rtw88"' +CFLAGS+= -DLINUXKPI_VERSION=61400 CFLAGS+= -I${DEVRTW88DIR} CFLAGS+= ${LINUXKPI_INCLUDES} diff --git a/sys/modules/rtw89/Makefile b/sys/modules/rtw89/Makefile index 09580f288c62..e66f85c3ac17 100644 --- a/sys/modules/rtw89/Makefile +++ b/sys/modules/rtw89/Makefile @@ -39,6 +39,7 @@ SRCS+= ${LINUXKPI_GENSRCS} SRCS+= opt_wlan.h opt_inet6.h opt_inet.h opt_acpi.h CFLAGS+= -DKBUILD_MODNAME='"rtw89"' +CFLAGS+= -DLINUXKPI_VERSION=61400 CFLAGS+= -DLINUXKPI_WANT_LINUX_ACPI CFLAGS+= -I${DEVRTW89DIR} diff --git a/sys/modules/uinput/Makefile b/sys/modules/uinput/Makefile index 66ade2a5bb33..a9e2ec867b91 100644 --- a/sys/modules/uinput/Makefile +++ b/sys/modules/uinput/Makefile @@ -2,6 +2,6 @@ KMOD= uinput SRCS= uinput.c -SRCS+= opt_evdev.h +SRCS+= opt_evdev.h opt_kbd.h .include <bsd.kmod.mk> diff --git a/sys/modules/usb/usie/Makefile b/sys/modules/usb/usie/Makefile index 6a5f79248ff8..9edeed082f8d 100644 --- a/sys/modules/usb/usie/Makefile +++ b/sys/modules/usb/usie/Makefile @@ -29,6 +29,6 @@ KMOD = usie SRCS = if_usie.c SRCS += opt_bus.h opt_usb.h device_if.h bus_if.h \ - usb_if.h usbdevs.h opt_inet.h + usb_if.h usbdevs.h opt_inet.h opt_inet6.h .include <bsd.kmod.mk> diff --git a/sys/modules/usb/wmt/Makefile b/sys/modules/usb/wmt/Makefile index 72cf1d814908..8cb5abd7383e 100644 --- a/sys/modules/usb/wmt/Makefile +++ b/sys/modules/usb/wmt/Makefile @@ -3,6 +3,6 @@ S= ${SRCTOP}/sys .PATH: $S/dev/usb/input KMOD= wmt -SRCS= opt_bus.h opt_usb.h device_if.h bus_if.h usb_if.h usbdevs.h wmt.c +SRCS= opt_bus.h opt_kbd.h opt_usb.h device_if.h bus_if.h usb_if.h usbdevs.h wmt.c .include <bsd.kmod.mk> diff --git a/sys/modules/zfs/Makefile b/sys/modules/zfs/Makefile index 2dd9e2be3f56..ec531ed646a7 100644 --- a/sys/modules/zfs/Makefile +++ b/sys/modules/zfs/Makefile @@ -15,6 +15,7 @@ KMOD= zfs ${SRCDIR}/icp/asm-ppc64/sha2 \ ${SRCDIR}/icp/asm-ppc64/blake3 \ ${SRCDIR}/icp/asm-x86_64/blake3 \ + ${SRCDIR}/icp/asm-x86_64/modes \ ${SRCDIR}/icp/asm-x86_64/sha2 \ ${SRCDIR}/os/freebsd/spl \ ${SRCDIR}/os/freebsd/zfs \ @@ -40,7 +41,8 @@ CFLAGS+= -D__KERNEL__ -DFREEBSD_NAMECACHE -DBUILDING_ZFS \ .if ${MACHINE_ARCH} == "amd64" CFLAGS+= -D__x86_64 -DHAVE_SSE2 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2 \ - -DHAVE_AVX -DHAVE_AVX2 -DHAVE_AVX512F -DHAVE_AVX512VL -DHAVE_AVX512BW + -DHAVE_AVX -DHAVE_AVX2 -DHAVE_AVX512F -DHAVE_AVX512VL -DHAVE_AVX512BW \ + -DHAVE_VAES -DHAVE_VPCLMULQDQ .endif .if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "powerpc" || \ @@ -82,6 +84,9 @@ SRCS+= blake3_avx2.S \ blake3_avx512.S \ blake3_sse2.S \ blake3_sse41.S + +#icp/asm-x86_64/modes +SRCS+= aesni-gcm-avx2-vaes.S .endif #icp/algs/sha2 diff --git a/sys/modules/zfs/zfs_config.h b/sys/modules/zfs/zfs_config.h index c79c9eaa1a5f..12274bcceea1 100644 --- a/sys/modules/zfs/zfs_config.h +++ b/sys/modules/zfs/zfs_config.h @@ -704,6 +704,11 @@ /* iops->setattr() takes struct user_namespace* */ /* #undef HAVE_USERNS_IOPS_SETATTR */ +#ifdef __amd64__ +/* Define if host toolchain supports VAES */ +#define HAVE_VAES 1 +#endif + /* fops->clone_file_range() is available */ /* #undef HAVE_VFS_CLONE_FILE_RANGE */ @@ -743,6 +748,11 @@ /* __vmalloc page flags exists */ /* #undef HAVE_VMALLOC_PAGE_KERNEL */ +#ifdef __amd64__ +/* Define if host toolchain supports VPCLMULQDQ */ +#define HAVE_VPCLMULQDQ 1 +#endif + /* int (*writepage_t)() takes struct folio* */ /* #undef HAVE_WRITEPAGE_T_FOLIO */ @@ -830,7 +840,7 @@ /* #undef ZFS_DEVICE_MINOR */ /* Define the project alias string. */ -#define ZFS_META_ALIAS "zfs-2.3.99-539-FreeBSD_g1d0b94c4e" +#define ZFS_META_ALIAS "zfs-2.3.99-571-FreeBSD_ga9410ccbd" /* Define the project author. */ #define ZFS_META_AUTHOR "OpenZFS" @@ -839,7 +849,7 @@ /* #undef ZFS_META_DATA */ /* Define the maximum compatible kernel version. */ -#define ZFS_META_KVER_MAX "6.15" +#define ZFS_META_KVER_MAX "6.16" /* Define the minimum compatible kernel version. */ #define ZFS_META_KVER_MIN "4.18" @@ -860,7 +870,7 @@ #define ZFS_META_NAME "zfs" /* Define the project release. */ -#define ZFS_META_RELEASE "539-FreeBSD_g1d0b94c4e" +#define ZFS_META_RELEASE "571-FreeBSD_ga9410ccbd" /* Define the project version. */ #define ZFS_META_VERSION "2.3.99" diff --git a/sys/modules/zfs/zfs_gitrev.h b/sys/modules/zfs/zfs_gitrev.h index 20fd58c620b5..5c265cf5b08e 100644 --- a/sys/modules/zfs/zfs_gitrev.h +++ b/sys/modules/zfs/zfs_gitrev.h @@ -1 +1 @@ -#define ZFS_META_GITREV "zfs-2.3.99-539-g1d0b94c4e" +#define ZFS_META_GITREV "zfs-2.3.99-571-ga9410ccbd" diff --git a/sys/net/if_bridgevar.h b/sys/net/if_bridgevar.h index b0f579f688ac..5ed8c19f3128 100644 --- a/sys/net/if_bridgevar.h +++ b/sys/net/if_bridgevar.h @@ -159,7 +159,7 @@ struct ifbreq { uint32_t ifbr_addrexceeded; /* member if addr violations */ ether_vlanid_t ifbr_pvid; /* member if PVID */ uint16_t ifbr_vlanproto; /* member if VLAN protocol */ - uint8_t pad[32]; + uint8_t pad[28]; }; /* BRDGGIFFLAGS, BRDGSIFFLAGS */ diff --git a/sys/net/if_clone.h b/sys/net/if_clone.h index 5a74ffa1cc2f..d780e49af25f 100644 --- a/sys/net/if_clone.h +++ b/sys/net/if_clone.h @@ -153,7 +153,7 @@ int if_clone_destroy(const char *); int if_clone_list(struct if_clonereq *); void if_clone_restoregroup(struct ifnet *); -/* The below interfaces are used only by epair(4). */ +/* The below interfaces are used only by epair(4) and tun(4)/tap(4). */ void if_clone_addif(struct if_clone *, struct ifnet *); int if_clone_destroyif(struct if_clone *, struct ifnet *); diff --git a/sys/net/if_pfsync.h b/sys/net/if_pfsync.h index 1efc220aa8e1..e99df0b85ccf 100644 --- a/sys/net/if_pfsync.h +++ b/sys/net/if_pfsync.h @@ -160,8 +160,8 @@ struct pfsync_ins_ack { struct pfsync_upd_c { u_int64_t id; - struct pfsync_state_peer src; - struct pfsync_state_peer dst; + struct pf_state_peer_export src; + struct pf_state_peer_export dst; u_int32_t creatorid; u_int32_t expire; u_int8_t timeout; diff --git a/sys/net/if_tap.h b/sys/net/if_tap.h index d84cd2eba6f3..8297b8d9e3d2 100644 --- a/sys/net/if_tap.h +++ b/sys/net/if_tap.h @@ -57,6 +57,8 @@ #define TAPGIFNAME TUNGIFNAME #define TAPSVNETHDR _IOW('t', 91, int) #define TAPGVNETHDR _IOR('t', 94, int) +#define TAPSTRANSIENT TUNSTRANSIENT +#define TAPGTRANSIENT TUNGTRANSIENT /* VMware ioctl's */ #define VMIO_SIOCSIFFLAGS _IOWINT('V', 0) diff --git a/sys/net/if_tun.h b/sys/net/if_tun.h index a8fb61db45a2..ccdc25944823 100644 --- a/sys/net/if_tun.h +++ b/sys/net/if_tun.h @@ -43,5 +43,7 @@ struct tuninfo { #define TUNSIFPID _IO('t', 95) #define TUNSIFHEAD _IOW('t', 96, int) #define TUNGIFHEAD _IOR('t', 97, int) +#define TUNSTRANSIENT _IOW('t', 98, int) +#define TUNGTRANSIENT _IOR('t', 99, int) #endif /* !_NET_IF_TUN_H_ */ diff --git a/sys/net/if_tuntap.c b/sys/net/if_tuntap.c index 5e6f65c04b2f..c8dbb6aa8893 100644 --- a/sys/net/if_tuntap.c +++ b/sys/net/if_tuntap.c @@ -132,6 +132,7 @@ struct tuntap_softc { #define TUN_DYING 0x0200 #define TUN_L2 0x0400 #define TUN_VMNET 0x0800 +#define TUN_TRANSIENT 0x1000 #define TUN_DRIVER_IDENT_MASK (TUN_L2 | TUN_VMNET) #define TUN_READY (TUN_OPEN | TUN_INITED) @@ -443,6 +444,18 @@ tuntap_name2info(const char *name, int *outunit, int *outflags) return (0); } +static struct if_clone * +tuntap_cloner_from_flags(int tun_flags) +{ + + for (u_int i = 0; i < NDRV; i++) + if ((tun_flags & TUN_DRIVER_IDENT_MASK) == + tuntap_drivers[i].ident_flags) + return (V_tuntap_driver_cloners[i]); + + return (NULL); +} + /* * Get driver information from a set of flags specified. Masks the identifying * part of the flags and compares it against all of the available @@ -615,19 +628,39 @@ out: CURVNET_RESTORE(); } -static void -tun_destroy(struct tuntap_softc *tp) +static int +tun_destroy(struct tuntap_softc *tp, bool may_intr) { + int error; TUN_LOCK(tp); + + /* + * Transient tunnels may have set TUN_DYING if we're being destroyed as + * a result of the last close, which we'll allow. + */ + MPASS((tp->tun_flags & (TUN_DYING | TUN_TRANSIENT)) != TUN_DYING); tp->tun_flags |= TUN_DYING; - if (tp->tun_busy != 0) - cv_wait_unlock(&tp->tun_cv, &tp->tun_mtx); - else - TUN_UNLOCK(tp); + error = 0; + while (tp->tun_busy != 0) { + if (may_intr) + error = cv_wait_sig(&tp->tun_cv, &tp->tun_mtx); + else + cv_wait(&tp->tun_cv, &tp->tun_mtx); + if (error != 0) { + tp->tun_flags &= ~TUN_DYING; + TUN_UNLOCK(tp); + return (error); + } + } + TUN_UNLOCK(tp); CURVNET_SET(TUN2IFP(tp)->if_vnet); + mtx_lock(&tunmtx); + TAILQ_REMOVE(&tunhead, tp, tun_list); + mtx_unlock(&tunmtx); + /* destroy_dev will take care of any alias. */ destroy_dev(tp->tun_dev); seldrain(&tp->tun_rsel); @@ -648,6 +681,8 @@ tun_destroy(struct tuntap_softc *tp) cv_destroy(&tp->tun_cv); free(tp, M_TUN); CURVNET_RESTORE(); + + return (0); } static int @@ -655,12 +690,7 @@ tun_clone_destroy(struct if_clone *ifc __unused, struct ifnet *ifp, uint32_t fla { struct tuntap_softc *tp = ifp->if_softc; - mtx_lock(&tunmtx); - TAILQ_REMOVE(&tunhead, tp, tun_list); - mtx_unlock(&tunmtx); - tun_destroy(tp); - - return (0); + return (tun_destroy(tp, true)); } static void @@ -702,9 +732,9 @@ tun_uninit(const void *unused __unused) mtx_lock(&tunmtx); while ((tp = TAILQ_FIRST(&tunhead)) != NULL) { - TAILQ_REMOVE(&tunhead, tp, tun_list); mtx_unlock(&tunmtx); - tun_destroy(tp); + /* tun_destroy() will remove it from the tailq. */ + tun_destroy(tp, false); mtx_lock(&tunmtx); } mtx_unlock(&tunmtx); @@ -1217,6 +1247,23 @@ out: tun_vnethdr_set(ifp, 0); tun_unbusy_locked(tp); + if ((tp->tun_flags & TUN_TRANSIENT) != 0) { + struct if_clone *cloner; + int error __diagused; + + /* Mark it busy so that nothing can re-open it. */ + tp->tun_flags |= TUN_DYING; + TUN_UNLOCK(tp); + + CURVNET_SET_QUIET(ifp->if_home_vnet); + cloner = tuntap_cloner_from_flags(tp->tun_flags); + CURVNET_RESTORE(); + + error = if_clone_destroyif(cloner, ifp); + MPASS(error == 0 || error == EINTR || error == ERESTART); + return; + } + TUN_UNLOCK(tp); } @@ -1668,6 +1715,19 @@ tunioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, case TUNGDEBUG: *(int *)data = tundebug; break; + case TUNSTRANSIENT: + TUN_LOCK(tp); + if (*(int *)data) + tp->tun_flags |= TUN_TRANSIENT; + else + tp->tun_flags &= ~TUN_TRANSIENT; + TUN_UNLOCK(tp); + break; + case TUNGTRANSIENT: + TUN_LOCK(tp); + *(int *)data = (tp->tun_flags & TUN_TRANSIENT) != 0; + TUN_UNLOCK(tp); + break; case FIONBIO: break; case FIOASYNC: diff --git a/sys/net/iflib.c b/sys/net/iflib.c index 6638c90882aa..98c59e5de988 100644 --- a/sys/net/iflib.c +++ b/sys/net/iflib.c @@ -142,6 +142,7 @@ struct iflib_ctx; static void iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid); static void iflib_timer(void *arg); static void iflib_tqg_detach(if_ctx_t ctx); +static int iflib_simple_transmit(if_t ifp, struct mbuf *m); typedef struct iflib_filter_info { driver_filter_t *ifi_filter; @@ -198,6 +199,7 @@ struct iflib_ctx { uint8_t ifc_sysctl_use_logical_cores; uint16_t ifc_sysctl_extra_msix_vectors; bool ifc_cpus_are_physical_cores; + bool ifc_sysctl_simple_tx; qidx_t ifc_sysctl_ntxds[8]; qidx_t ifc_sysctl_nrxds[8]; @@ -725,6 +727,7 @@ static void iflib_free_intr_mem(if_ctx_t ctx); #ifndef __NO_STRICT_ALIGNMENT static struct mbuf *iflib_fixup_rx(struct mbuf *m); #endif +static __inline int iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh); static SLIST_HEAD(cpu_offset_list, cpu_offset) cpu_offsets = SLIST_HEAD_INITIALIZER(cpu_offsets); @@ -2624,8 +2627,10 @@ iflib_stop(if_ctx_t ctx) #endif /* DEV_NETMAP */ CALLOUT_UNLOCK(txq); - /* clean any enqueued buffers */ - iflib_ifmp_purge(txq); + if (!ctx->ifc_sysctl_simple_tx) { + /* clean any enqueued buffers */ + iflib_ifmp_purge(txq); + } /* Free any existing tx buffers. */ for (j = 0; j < txq->ift_size; j++) { iflib_txsd_free(ctx, txq, j); @@ -2910,7 +2915,9 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) struct if_rxd_info ri; int err, budget_left, rx_bytes, rx_pkts; iflib_fl_t fl; +#if defined(INET6) || defined(INET) int lro_enabled; +#endif uint8_t retval = 0; /* @@ -2936,7 +2943,9 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) return (retval); } +#if defined(INET6) || defined(INET) lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO); +#endif /* pfil needs the vnet to be set */ CURVNET_SET_QUIET(if_getvnet(ifp)); @@ -3631,13 +3640,16 @@ defrag: * cxgb */ if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) { - txq->ift_no_desc_avail++; - bus_dmamap_unload(buf_tag, map); - DBG_COUNTER_INC(encap_txq_avail_fail); - DBG_COUNTER_INC(encap_txd_encap_fail); - if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0) - GROUPTASK_ENQUEUE(&txq->ift_task); - return (ENOBUFS); + (void)iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx)); + if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) { + txq->ift_no_desc_avail++; + bus_dmamap_unload(buf_tag, map); + DBG_COUNTER_INC(encap_txq_avail_fail); + DBG_COUNTER_INC(encap_txd_encap_fail); + if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0) + GROUPTASK_ENQUEUE(&txq->ift_task); + return (ENOBUFS); + } } /* * On Intel cards we can greatly reduce the number of TX interrupts @@ -4010,6 +4022,12 @@ _task_fn_tx(void *context) netmap_tx_irq(ifp, txq->ift_id)) goto skip_ifmp; #endif + if (ctx->ifc_sysctl_simple_tx) { + mtx_lock(&txq->ift_mtx); + (void)iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx)); + mtx_unlock(&txq->ift_mtx); + goto skip_ifmp; + } #ifdef ALTQ if (if_altq_is_enabled(ifp)) iflib_altq_if_start(ifp); @@ -4023,9 +4041,8 @@ _task_fn_tx(void *context) */ if (abdicate) ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); -#ifdef DEV_NETMAP + skip_ifmp: -#endif if (ctx->ifc_flags & IFC_LEGACY) IFDI_INTR_ENABLE(ctx); else @@ -5127,7 +5144,14 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct scctx = &ctx->ifc_softc_ctx; ifp = ctx->ifc_ifp; - + if (ctx->ifc_sysctl_simple_tx) { +#ifndef ALTQ + if_settransmitfn(ifp, iflib_simple_transmit); + device_printf(dev, "using simple if_transmit\n"); +#else + device_printf(dev, "ALTQ prevents using simple if_transmit\n"); +#endif + } iflib_reset_qvalues(ctx); IFNET_WLOCK(); CTX_LOCK(ctx); @@ -6762,6 +6786,9 @@ iflib_add_device_sysctl_pre(if_ctx_t ctx) SYSCTL_ADD_CONST_STRING(ctx_list, oid_list, OID_AUTO, "driver_version", CTLFLAG_RD, ctx->ifc_sctx->isc_driver_version, "driver version"); + SYSCTL_ADD_BOOL(ctx_list, oid_list, OID_AUTO, "simple_tx", + CTLFLAG_RDTUN, &ctx->ifc_sysctl_simple_tx, 0, + "use simple tx ring"); SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_ntxqs", CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0, "# of txqs to use, 0 => use default #"); @@ -7084,3 +7111,48 @@ iflib_debugnet_poll(if_t ifp, int count) return (0); } #endif /* DEBUGNET */ + + +static inline iflib_txq_t +iflib_simple_select_queue(if_ctx_t ctx, struct mbuf *m) +{ + int qidx; + + if ((NTXQSETS(ctx) > 1) && M_HASHTYPE_GET(m)) + qidx = QIDX(ctx, m); + else + qidx = NTXQSETS(ctx) + FIRST_QSET(ctx) - 1; + return (&ctx->ifc_txqs[qidx]); +} + +static int +iflib_simple_transmit(if_t ifp, struct mbuf *m) +{ + if_ctx_t ctx; + iflib_txq_t txq; + int error; + int bytes_sent = 0, pkt_sent = 0, mcast_sent = 0; + + + ctx = if_getsoftc(ifp); + if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING) + return (EBUSY); + txq = iflib_simple_select_queue(ctx, m); + mtx_lock(&txq->ift_mtx); + error = iflib_encap(txq, &m); + if (error == 0) { + pkt_sent++; + bytes_sent += m->m_pkthdr.len; + mcast_sent += !!(m->m_flags & M_MCAST); + (void)iflib_txd_db_check(txq, true); + } + (void)iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx)); + mtx_unlock(&txq->ift_mtx); + if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent); + if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent); + if (mcast_sent) + if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent); + + return (error); +} diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h index c397f0b67896..d6c13470f2eb 100644 --- a/sys/net/pfvar.h +++ b/sys/net/pfvar.h @@ -1020,7 +1020,7 @@ struct pf_state_scrub_export { #define PF_SCRUB_FLAG_VALID 0x01 uint8_t scrub_flag; uint32_t pfss_ts_mod; /* timestamp modulation */ -}; +} __packed; struct pf_state_key_export { struct pf_addr addr[2]; @@ -1037,7 +1037,7 @@ struct pf_state_peer_export { uint8_t state; /* active state level */ uint8_t wscale; /* window scaling factor */ uint8_t dummy[6]; -}; +} __packed; _Static_assert(sizeof(struct pf_state_peer_export) == 32, "size incorrect"); struct pf_state_export { @@ -1179,26 +1179,6 @@ struct pf_test_ctx { * Unified state structures for pulling states out of the kernel * used by pfsync(4) and the pf(4) ioctl. */ -struct pfsync_state_scrub { - u_int16_t pfss_flags; - u_int8_t pfss_ttl; /* stashed TTL */ -#define PFSYNC_SCRUB_FLAG_VALID 0x01 - u_int8_t scrub_flag; - u_int32_t pfss_ts_mod; /* timestamp modulation */ -} __packed; - -struct pfsync_state_peer { - struct pfsync_state_scrub scrub; /* state is scrubbed */ - u_int32_t seqlo; /* Max sequence number sent */ - u_int32_t seqhi; /* Max the other end ACKd + win */ - u_int32_t seqdiff; /* Sequence number modulator */ - u_int16_t max_win; /* largest window (pre scaling) */ - u_int16_t mss; /* Maximum segment size option */ - u_int8_t state; /* active state level */ - u_int8_t wscale; /* window scaling factor */ - u_int8_t pad[6]; -} __packed; - struct pfsync_state_key { struct pf_addr addr[2]; u_int16_t port[2]; @@ -1208,8 +1188,8 @@ struct pfsync_state_1301 { u_int64_t id; char ifname[IFNAMSIZ]; struct pfsync_state_key key[2]; - struct pfsync_state_peer src; - struct pfsync_state_peer dst; + struct pf_state_peer_export src; + struct pf_state_peer_export dst; struct pf_addr rt_addr; u_int32_t rule; u_int32_t anchor; @@ -1235,8 +1215,8 @@ struct pfsync_state_1400 { u_int64_t id; char ifname[IFNAMSIZ]; struct pfsync_state_key key[2]; - struct pfsync_state_peer src; - struct pfsync_state_peer dst; + struct pf_state_peer_export src; + struct pf_state_peer_export dst; struct pf_addr rt_addr; u_int32_t rule; u_int32_t anchor; @@ -1323,39 +1303,10 @@ extern pflog_packet_t *pflog_packet_ptr; /* for copies to/from network byte order */ /* ioctl interface also uses network byte order */ -#define pf_state_peer_hton(s,d) do { \ - (d)->seqlo = htonl((s)->seqlo); \ - (d)->seqhi = htonl((s)->seqhi); \ - (d)->seqdiff = htonl((s)->seqdiff); \ - (d)->max_win = htons((s)->max_win); \ - (d)->mss = htons((s)->mss); \ - (d)->state = (s)->state; \ - (d)->wscale = (s)->wscale; \ - if ((s)->scrub) { \ - (d)->scrub.pfss_flags = \ - htons((s)->scrub->pfss_flags & PFSS_TIMESTAMP); \ - (d)->scrub.pfss_ttl = (s)->scrub->pfss_ttl; \ - (d)->scrub.pfss_ts_mod = htonl((s)->scrub->pfss_ts_mod);\ - (d)->scrub.scrub_flag = PFSYNC_SCRUB_FLAG_VALID; \ - } \ -} while (0) - -#define pf_state_peer_ntoh(s,d) do { \ - (d)->seqlo = ntohl((s)->seqlo); \ - (d)->seqhi = ntohl((s)->seqhi); \ - (d)->seqdiff = ntohl((s)->seqdiff); \ - (d)->max_win = ntohs((s)->max_win); \ - (d)->mss = ntohs((s)->mss); \ - (d)->state = (s)->state; \ - (d)->wscale = (s)->wscale; \ - if ((s)->scrub.scrub_flag == PFSYNC_SCRUB_FLAG_VALID && \ - (d)->scrub != NULL) { \ - (d)->scrub->pfss_flags = \ - ntohs((s)->scrub.pfss_flags) & PFSS_TIMESTAMP; \ - (d)->scrub->pfss_ttl = (s)->scrub.pfss_ttl; \ - (d)->scrub->pfss_ts_mod = ntohl((s)->scrub.pfss_ts_mod);\ - } \ -} while (0) +void pf_state_peer_hton(const struct pf_state_peer *, + struct pf_state_peer_export *); +void pf_state_peer_ntoh(const struct pf_state_peer_export *, + struct pf_state_peer *); #define pf_state_counter_hton(s,d) do { \ d[0] = htonl((s>>32)&0xffffffff); \ diff --git a/sys/netinet/ip_icmp.c b/sys/netinet/ip_icmp.c index 543b846fbba5..fc0848b2c944 100644 --- a/sys/netinet/ip_icmp.c +++ b/sys/netinet/ip_icmp.c @@ -391,7 +391,6 @@ stdreply: icmpelen = max(8, min(V_icmp_quotelen, ntohs(oip->ip_len) - nip->ip_hl = 5; nip->ip_p = IPPROTO_ICMP; nip->ip_tos = 0; - nip->ip_off = 0; if (V_error_keeptags) m_tag_copy_chain(m, n, M_NOWAIT); @@ -872,6 +871,8 @@ match: mac_netinet_icmp_replyinplace(m); #endif ip->ip_src = t; + /* ip->ip_tos will be reflected. */ + ip->ip_off = htons(0); ip->ip_ttl = V_ip_defttl; if (optlen > 0) { diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c index 2f2f9abf1c83..3e6519118a40 100644 --- a/sys/netinet/udp_usrreq.c +++ b/sys/netinet/udp_usrreq.c @@ -448,7 +448,7 @@ udp_multi_input(struct mbuf *m, int proto, struct sockaddr_in *udp_in) /* * No matching pcb found; discard datagram. (No need * to send an ICMP Port Unreachable for a broadcast - * or multicast datgram.) + * or multicast datagram.) */ UDPSTAT_INC(udps_noport); if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c index 0027cf3bd230..1a32365f5d1d 100644 --- a/sys/netinet6/udp6_usrreq.c +++ b/sys/netinet6/udp6_usrreq.c @@ -341,7 +341,7 @@ udp6_multi_input(struct mbuf *m, int off, int proto, /* * No matching pcb found; discard datagram. (No need * to send an ICMP Port Unreachable for a broadcast - * or multicast datgram.) + * or multicast datagram.) */ UDPSTAT_INC(udps_noport); UDPSTAT_INC(udps_noportmcast); diff --git a/sys/netlink/netlink_io.c b/sys/netlink/netlink_io.c index e7908d6f3a44..2391d8ea752c 100644 --- a/sys/netlink/netlink_io.c +++ b/sys/netlink/netlink_io.c @@ -216,16 +216,17 @@ nl_send(struct nl_writer *nw, struct nlpcb *nlp) hdr->nlmsg_len); } - if (nlp->nl_linux && linux_netlink_p != NULL && - __predict_false(!linux_netlink_p->msgs_to_linux(nw, nlp))) { + if (nlp->nl_linux && linux_netlink_p != NULL) { + nb = linux_netlink_p->msgs_to_linux(nw->buf, nlp); nl_buf_free(nw->buf); nw->buf = NULL; - return (false); + if (nb == NULL) + return (false); + } else { + nb = nw->buf; + nw->buf = NULL; } - nb = nw->buf; - nw->buf = NULL; - SOCK_RECVBUF_LOCK(so); if (!nw->ignore_limit && __predict_false(sb->sb_hiwat <= sb->sb_ccc)) { SOCK_RECVBUF_UNLOCK(so); diff --git a/sys/netlink/netlink_linux.h b/sys/netlink/netlink_linux.h index d4c451d470b2..794065692901 100644 --- a/sys/netlink/netlink_linux.h +++ b/sys/netlink/netlink_linux.h @@ -37,7 +37,7 @@ struct nlpcb; struct nl_pstate; struct nl_writer; -typedef bool msgs_to_linux_cb_t(struct nl_writer *nw, struct nlpcb *nlp); +typedef struct nl_buf * msgs_to_linux_cb_t(struct nl_buf *, struct nlpcb *); typedef int msg_from_linux_cb_t(int netlink_family, struct nlmsghdr **hdr, struct nl_pstate *npt); diff --git a/sys/netlink/route/iface.c b/sys/netlink/route/iface.c index 8b871576d0b2..9beb80792af4 100644 --- a/sys/netlink/route/iface.c +++ b/sys/netlink/route/iface.c @@ -403,6 +403,7 @@ static const struct nlattr_parser nla_p_linfo[] = { NL_DECLARE_ATTR_PARSER(linfo_parser, nla_p_linfo); static const struct nlattr_parser nla_p_if[] = { + { .type = IFLA_ADDRESS, .off = _OUT(ifla_address), .cb = nlattr_get_nla }, { .type = IFLA_IFNAME, .off = _OUT(ifla_ifname), .cb = nlattr_get_string }, { .type = IFLA_MTU, .off = _OUT(ifla_mtu), .cb = nlattr_get_uint32 }, { .type = IFLA_LINK, .off = _OUT(ifla_link), .cb = nlattr_get_uint32 }, diff --git a/sys/netlink/route/iface_drivers.c b/sys/netlink/route/iface_drivers.c index 4bf913d9c978..21db3017df18 100644 --- a/sys/netlink/route/iface_drivers.c +++ b/sys/netlink/route/iface_drivers.c @@ -105,6 +105,24 @@ _nl_modify_ifp_generic(struct ifnet *ifp, struct nl_parsed_link *lattrs, } } + if (lattrs->ifla_address != NULL) { + if (nlp_has_priv(npt->nlp, PRIV_NET_SETIFMAC)) { + error = if_setlladdr(ifp, + NLA_DATA(lattrs->ifla_address), + NLA_DATA_LEN(lattrs->ifla_address)); + if (error != 0) { + nlmsg_report_err_msg(npt, + "setting IFLA_ADDRESS failed with error code: %d", + error); + return (error); + } + } else { + nlmsg_report_err_msg(npt, + "Not enough privileges to set IFLA_ADDRESS"); + return (EPERM); + } + } + return (0); } diff --git a/sys/netlink/route/route_var.h b/sys/netlink/route/route_var.h index b84b34461e35..41f110038b54 100644 --- a/sys/netlink/route/route_var.h +++ b/sys/netlink/route/route_var.h @@ -69,6 +69,7 @@ struct nl_parsed_link { char *ifla_cloner; char *ifla_ifalias; struct nlattr *ifla_idata; + struct nlattr *ifla_address; unsigned short ifi_type; int ifi_index; uint32_t ifla_link; diff --git a/sys/netpfil/pf/if_pfsync.c b/sys/netpfil/pf/if_pfsync.c index e34c08c8c4db..585c196391c0 100644 --- a/sys/netpfil/pf/if_pfsync.c +++ b/sys/netpfil/pf/if_pfsync.c @@ -123,8 +123,8 @@ union inet_template { sizeof(struct pfsync_header) + \ sizeof(struct pfsync_subheader) ) -static int pfsync_upd_tcp(struct pf_kstate *, struct pfsync_state_peer *, - struct pfsync_state_peer *); +static int pfsync_upd_tcp(struct pf_kstate *, struct pf_state_peer_export *, + struct pf_state_peer_export *); static int pfsync_in_clr(struct mbuf *, int, int, int, int); static int pfsync_in_ins(struct mbuf *, int, int, int, int); static int pfsync_in_iack(struct mbuf *, int, int, int, int); @@ -330,7 +330,7 @@ SYSCTL_UINT(_net_pfsync, OID_AUTO, defer_delay, CTLFLAG_VNET | CTLFLAG_RW, static int pfsync_clone_create(struct if_clone *, int, caddr_t); static void pfsync_clone_destroy(struct ifnet *); -static int pfsync_alloc_scrub_memory(struct pfsync_state_peer *, +static int pfsync_alloc_scrub_memory(struct pf_state_peer_export *, struct pf_state_peer *); static int pfsyncoutput(struct ifnet *, struct mbuf *, const struct sockaddr *, struct route *); @@ -502,7 +502,7 @@ pfsync_clone_destroy(struct ifnet *ifp) } static int -pfsync_alloc_scrub_memory(struct pfsync_state_peer *s, +pfsync_alloc_scrub_memory(struct pf_state_peer_export *s, struct pf_state_peer *d) { if (s->scrub.scrub_flag && d->scrub == NULL) { @@ -1172,8 +1172,8 @@ pfsync_in_iack(struct mbuf *m, int offset, int count, int flags, int action) } static int -pfsync_upd_tcp(struct pf_kstate *st, struct pfsync_state_peer *src, - struct pfsync_state_peer *dst) +pfsync_upd_tcp(struct pf_kstate *st, struct pf_state_peer_export *src, + struct pf_state_peer_export *dst) { int sync = 0; diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c index 9d83e7b82e6f..8cd4fff95b15 100644 --- a/sys/netpfil/pf/pf.c +++ b/sys/netpfil/pf/pf.c @@ -2069,6 +2069,44 @@ pf_find_state_all_exists(const struct pf_state_key_cmp *key, u_int dir) return (false); } +void +pf_state_peer_hton(const struct pf_state_peer *s, struct pf_state_peer_export *d) +{ + d->seqlo = htonl(s->seqlo); + d->seqhi = htonl(s->seqhi); + d->seqdiff = htonl(s->seqdiff); + d->max_win = htons(s->max_win); + d->mss = htons(s->mss); + d->state = s->state; + d->wscale = s->wscale; + if (s->scrub) { + d->scrub.pfss_flags = htons( + s->scrub->pfss_flags & PFSS_TIMESTAMP); + d->scrub.pfss_ttl = (s)->scrub->pfss_ttl; + d->scrub.pfss_ts_mod = htonl((s)->scrub->pfss_ts_mod); + d->scrub.scrub_flag = PF_SCRUB_FLAG_VALID; + } +} + +void +pf_state_peer_ntoh(const struct pf_state_peer_export *s, struct pf_state_peer *d) +{ + d->seqlo = ntohl(s->seqlo); + d->seqhi = ntohl(s->seqhi); + d->seqdiff = ntohl(s->seqdiff); + d->max_win = ntohs(s->max_win); + d->mss = ntohs(s->mss); + d->state = s->state; + d->wscale = s->wscale; + if (s->scrub.scrub_flag == PF_SCRUB_FLAG_VALID && + d->scrub != NULL) { + d->scrub->pfss_flags = ntohs(s->scrub.pfss_flags) & + PFSS_TIMESTAMP; + d->scrub->pfss_ttl = s->scrub.pfss_ttl; + d->scrub->pfss_ts_mod = ntohl(s->scrub.pfss_ts_mod); + } +} + struct pf_udp_mapping * pf_udp_mapping_create(sa_family_t af, struct pf_addr *src_addr, uint16_t src_port, struct pf_addr *nat_addr, uint16_t nat_port) diff --git a/sys/netpfil/pf/pf_nl.c b/sys/netpfil/pf/pf_nl.c index 09754359ec2d..45b5b8dd5fef 100644 --- a/sys/netpfil/pf/pf_nl.c +++ b/sys/netpfil/pf/pf_nl.c @@ -118,7 +118,7 @@ dump_state_peer(struct nl_writer *nw, int attr, const struct pf_state_peer *peer nlattr_add_u16(nw, PF_STP_PFSS_FLAGS, pfss_flags); nlattr_add_u32(nw, PF_STP_PFSS_TS_MOD, sc->pfss_ts_mod); nlattr_add_u8(nw, PF_STP_PFSS_TTL, sc->pfss_ttl); - nlattr_add_u8(nw, PF_STP_SCRUB_FLAG, PFSYNC_SCRUB_FLAG_VALID); + nlattr_add_u8(nw, PF_STP_SCRUB_FLAG, PF_SCRUB_FLAG_VALID); } nlattr_set_len(nw, off); diff --git a/sys/powerpc/conf/GENERIC64 b/sys/powerpc/conf/GENERIC64 index 85711c8fc3ff..630c88b97dd7 100644 --- a/sys/powerpc/conf/GENERIC64 +++ b/sys/powerpc/conf/GENERIC64 @@ -293,3 +293,4 @@ device virtio_balloon # VirtIO Memory Balloon device options HID_DEBUG # enable debug msgs device hid # Generic HID support device hidbus # Generic HID Bus +options U2F_MAKE_UHID_ALIAS # install /dev/uhid alias for /dev/u2f/ diff --git a/sys/powerpc/conf/GENERIC64LE b/sys/powerpc/conf/GENERIC64LE index a56feb6574a4..eb9a9441425d 100644 --- a/sys/powerpc/conf/GENERIC64LE +++ b/sys/powerpc/conf/GENERIC64LE @@ -274,3 +274,4 @@ device virtio_balloon # VirtIO Memory Balloon device options HID_DEBUG # enable debug msgs device hid # Generic HID support device hidbus # Generic HID Bus +options U2F_MAKE_UHID_ALIAS # install /dev/uhid alias for /dev/u2f/ diff --git a/sys/riscv/conf/GENERIC b/sys/riscv/conf/GENERIC index a8500fe80019..2ff711e80127 100644 --- a/sys/riscv/conf/GENERIC +++ b/sys/riscv/conf/GENERIC @@ -132,6 +132,7 @@ device umass # Disks/Mass storage - Requires scbus and da options HID_DEBUG # enable debug msgs device hid # Generic HID support device hidbus # Generic HID Bus +options U2F_MAKE_UHID_ALIAS # install /dev/uhid alias for /dev/u2f/ # Serial (COM) ports device uart # Generic UART driver diff --git a/sys/riscv/starfive/jh7110_pcie.c b/sys/riscv/starfive/jh7110_pcie.c index 2d0a4be69b2c..5181252ab2dc 100644 --- a/sys/riscv/starfive/jh7110_pcie.c +++ b/sys/riscv/starfive/jh7110_pcie.c @@ -483,6 +483,16 @@ jh7110_pcie_msi_enable_intr(device_t dev, struct intr_irqsrc *isrc) } static void +jh7110_pcie_msi_post_filter(device_t dev, struct intr_irqsrc *isrc) +{ +} + +static void +jh7110_pcie_msi_post_ithread(device_t dev, struct intr_irqsrc *isrc) +{ +} + +static void jh7110_pcie_msi_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { struct jh7110_pcie_softc *sc; @@ -1008,6 +1018,8 @@ static device_method_t jh7110_pcie_methods[] = { /* Interrupt controller interface */ DEVMETHOD(pic_enable_intr, jh7110_pcie_msi_enable_intr), DEVMETHOD(pic_disable_intr, jh7110_pcie_msi_disable_intr), + DEVMETHOD(pic_post_filter, jh7110_pcie_msi_post_filter), + DEVMETHOD(pic_post_ithread, jh7110_pcie_msi_post_ithread), DEVMETHOD(pic_pre_ithread, jh7110_pcie_msi_pre_ithread), /* OFW bus interface */ diff --git a/sys/rpc/auth.h b/sys/rpc/auth.h index 3d58fb19536b..33c33ffd594d 100644 --- a/sys/rpc/auth.h +++ b/sys/rpc/auth.h @@ -246,6 +246,19 @@ extern AUTH *authunix_create_default(void); /* takes no parameters */ extern AUTH *authnone_create(void); /* takes no parameters */ extern AUTH *authtls_create(void); /* takes no parameters */ __END_DECLS +/* + * DES style authentication + * AUTH *authsecdes_create(servername, window, timehost, ckey) + * char *servername; - network name of server + * u_int window; - time to live + * const char *timehost; - optional hostname to sync with + * des_block *ckey; - optional conversation key to use + */ +__BEGIN_DECLS +extern AUTH *authdes_create (char *, u_int, struct sockaddr *, des_block *); +extern AUTH *authdes_seccreate (const char *, const u_int, const char *, + const des_block *); +__END_DECLS __BEGIN_DECLS extern bool_t xdr_opaque_auth (XDR *, struct opaque_auth *); @@ -267,6 +280,19 @@ extern void passwd2des ( char *, char * ); __END_DECLS /* + * + * These routines interface to the keyserv daemon + * + */ +__BEGIN_DECLS +extern int key_decryptsession(const char *, des_block *); +extern int key_encryptsession(const char *, des_block *); +extern int key_gendes(des_block *); +extern int key_setsecret(const char *); +extern int key_secretkey_is_set(void); +__END_DECLS + +/* * Publickey routines. */ __BEGIN_DECLS diff --git a/sys/security/mac_do/mac_do.c b/sys/security/mac_do/mac_do.c index 8856be5fa1a3..7a5ac2e01f75 100644 --- a/sys/security/mac_do/mac_do.c +++ b/sys/security/mac_do/mac_do.c @@ -44,7 +44,7 @@ SYSCTL_INT(_security_mac_do, OID_AUTO, print_parse_error, CTLFLAG_RWTUN, &print_parse_error, 0, "Print parse errors on setting rules " "(via sysctl(8))."); -static MALLOC_DEFINE(M_DO, "do_rule", "Rules for mac_do"); +static MALLOC_DEFINE(M_MAC_DO, "mac_do", "mac_do(4) security module"); #define MAC_RULE_STRING_LEN 1024 @@ -319,17 +319,17 @@ toast_rules(struct rules *const rules) struct rule *rule, *rule_next; STAILQ_FOREACH_SAFE(rule, head, r_entries, rule_next) { - free(rule->uids, M_DO); - free(rule->gids, M_DO); - free(rule, M_DO); + free(rule->uids, M_MAC_DO); + free(rule->gids, M_MAC_DO); + free(rule, M_MAC_DO); } - free(rules, M_DO); + free(rules, M_MAC_DO); } static struct rules * alloc_rules(void) { - struct rules *const rules = malloc(sizeof(*rules), M_DO, M_WAITOK); + struct rules *const rules = malloc(sizeof(*rules), M_MAC_DO, M_WAITOK); _Static_assert(MAC_RULE_STRING_LEN > 0, "MAC_RULE_STRING_LEN <= 0!"); rules->string[0] = 0; @@ -433,7 +433,7 @@ static void make_parse_error(struct parse_error **const parse_error, const size_t pos, const char *const fmt, ...) { - struct parse_error *const err = malloc(sizeof(*err), M_DO, M_WAITOK); + struct parse_error *const err = malloc(sizeof(*err), M_MAC_DO, M_WAITOK); va_list ap; err->pos = pos; @@ -448,7 +448,7 @@ make_parse_error(struct parse_error **const parse_error, const size_t pos, static void free_parse_error(struct parse_error *const parse_error) { - free(parse_error, M_DO); + free(parse_error, M_MAC_DO); } static int @@ -733,7 +733,7 @@ parse_target_clause(char *to, struct rule *const rule, "Too many target clauses of type '%s'.", to_type); return (EOVERFLOW); } - ie = malloc(sizeof(*ie), M_DO, M_WAITOK); + ie = malloc(sizeof(*ie), M_MAC_DO, M_WAITOK); ie->spec = is; STAILQ_INSERT_TAIL(list, ie, ie_entries); check_type_and_id_spec(type, &is); @@ -784,7 +784,7 @@ pour_list_into_rule(const id_type_t type, struct id_list *const list, STAILQ_FOREACH_SAFE(ie, list, ie_entries, ie_next) { MPASS(idx < *nb); array[idx] = ie->spec; - free(ie, M_DO); + free(ie, M_MAC_DO); ++idx; } MPASS(idx == *nb); @@ -874,7 +874,7 @@ parse_single_rule(char *rule, struct rules *const rules, STAILQ_INIT(&gid_list); /* Freed when the 'struct rules' container is freed. */ - new = malloc(sizeof(*new), M_DO, M_WAITOK | M_ZERO); + new = malloc(sizeof(*new), M_MAC_DO, M_WAITOK | M_ZERO); from_type = strsep_noblanks(&rule, "="); MPASS(from_type != NULL); /* Because 'rule' was not NULL. */ @@ -933,7 +933,7 @@ parse_single_rule(char *rule, struct rules *const rules, } while (to_list != NULL); if (new->uids_nb != 0) { - new->uids = malloc(sizeof(*new->uids) * new->uids_nb, M_DO, + new->uids = malloc(sizeof(*new->uids) * new->uids_nb, M_MAC_DO, M_WAITOK); error = pour_list_into_rule(IT_UID, &uid_list, new->uids, &new->uids_nb, parse_error); @@ -949,7 +949,7 @@ parse_single_rule(char *rule, struct rules *const rules, } if (new->gids_nb != 0) { - new->gids = malloc(sizeof(*new->gids) * new->gids_nb, M_DO, + new->gids = malloc(sizeof(*new->gids) * new->gids_nb, M_MAC_DO, M_WAITOK); error = pour_list_into_rule(IT_GID, &gid_list, new->gids, &new->gids_nb, parse_error); @@ -969,13 +969,13 @@ parse_single_rule(char *rule, struct rules *const rules, return (0); einval: - free(new->gids, M_DO); - free(new->uids, M_DO); - free(new, M_DO); + free(new->gids, M_MAC_DO); + free(new->uids, M_MAC_DO); + free(new, M_MAC_DO); STAILQ_FOREACH_SAFE(ie, &gid_list, ie_entries, ie_next) - free(ie, M_DO); + free(ie, M_MAC_DO); STAILQ_FOREACH_SAFE(ie, &uid_list, ie_entries, ie_next) - free(ie, M_DO); + free(ie, M_MAC_DO); MPASS(*parse_error != NULL); return (EINVAL); } @@ -1028,7 +1028,7 @@ parse_rules(const char *const string, struct rules **const rulesp, bcopy(string, rules->string, len + 1); MPASS(rules->string[len] == '\0'); /* Catch some races. */ - copy = malloc(len + 1, M_DO, M_WAITOK); + copy = malloc(len + 1, M_MAC_DO, M_WAITOK); bcopy(string, copy, len + 1); MPASS(copy[len] == '\0'); /* Catch some races. */ @@ -1046,7 +1046,7 @@ parse_rules(const char *const string, struct rules **const rulesp, *rulesp = rules; out: - free(copy, M_DO); + free(copy, M_MAC_DO); return (error); } @@ -1226,7 +1226,7 @@ parse_and_set_rules(struct prison *const pr, const char *rules_string, static int mac_do_sysctl_rules(SYSCTL_HANDLER_ARGS) { - char *const buf = malloc(MAC_RULE_STRING_LEN, M_DO, M_WAITOK); + char *const buf = malloc(MAC_RULE_STRING_LEN, M_MAC_DO, M_WAITOK); struct prison *const td_pr = req->td->td_ucred->cr_prison; struct prison *pr; struct rules *rules; @@ -1250,7 +1250,7 @@ mac_do_sysctl_rules(SYSCTL_HANDLER_ARGS) free_parse_error(parse_error); } out: - free(buf, M_DO); + free(buf, M_MAC_DO); return (error); } @@ -1573,7 +1573,7 @@ set_data_header(void *const data, const size_t size, const int priv, static void * alloc_data(void *const data, const size_t size) { - struct mac_do_data_header *const hdr = realloc(data, size, M_DO, + struct mac_do_data_header *const hdr = realloc(data, size, M_MAC_DO, M_WAITOK); MPASS(size >= sizeof(struct mac_do_data_header)); @@ -1602,7 +1602,7 @@ alloc_data(void *const data, const size_t size) static void dealloc_thread_osd(void *const value) { - free(value, M_DO); + free(value, M_MAC_DO); } /* diff --git a/sys/sys/conf.h b/sys/sys/conf.h index 1646aa108701..a830c9d4c622 100644 --- a/sys/sys/conf.h +++ b/sys/sys/conf.h @@ -159,6 +159,7 @@ typedef int dumper_hdr_t(struct dumperinfo *di, struct kerneldumpheader *kdh); #define GID_RT_PRIO 47 #define GID_ID_PRIO 48 #define GID_DIALER 68 +#define GID_U2F 116 #define GID_NOGROUP 65533 #define GID_NOBODY 65534 diff --git a/sys/sys/param.h b/sys/sys/param.h index c410a6ee666f..fc2a78883f1e 100644 --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -74,7 +74,7 @@ * cannot include sys/param.h and should only be updated here. */ #undef __FreeBSD_version -#define __FreeBSD_version 1500060 +#define __FreeBSD_version 1500063 /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, diff --git a/sys/sys/syscall.h b/sys/sys/syscall.h index d703a11fda01..2d6903967e15 100644 --- a/sys/sys/syscall.h +++ b/sys/sys/syscall.h @@ -85,8 +85,8 @@ /* 76 is obsolete vhangup */ /* 77 is obsolete vlimit */ #define SYS_mincore 78 -#define SYS_getgroups 79 -#define SYS_setgroups 80 +#define SYS_freebsd14_getgroups 79 +#define SYS_freebsd14_setgroups 80 #define SYS_getpgrp 81 #define SYS_setpgid 82 #define SYS_setitimer 83 @@ -533,4 +533,6 @@ #define SYS_exterrctl 592 #define SYS_inotify_add_watch_at 593 #define SYS_inotify_rm_watch 594 -#define SYS_MAXSYSCALL 595 +#define SYS_getgroups 595 +#define SYS_setgroups 596 +#define SYS_MAXSYSCALL 597 diff --git a/sys/sys/syscall.mk b/sys/sys/syscall.mk index b7ded62cacb4..d1172c2dc7bf 100644 --- a/sys/sys/syscall.mk +++ b/sys/sys/syscall.mk @@ -65,8 +65,8 @@ MIASM = \ mprotect.o \ madvise.o \ mincore.o \ - getgroups.o \ - setgroups.o \ + freebsd14_getgroups.o \ + freebsd14_setgroups.o \ getpgrp.o \ setpgid.o \ setitimer.o \ @@ -436,4 +436,6 @@ MIASM = \ setcred.o \ exterrctl.o \ inotify_add_watch_at.o \ - inotify_rm_watch.o + inotify_rm_watch.o \ + getgroups.o \ + setgroups.o diff --git a/sys/sys/sysproto.h b/sys/sys/sysproto.h index 8d666f9c8ee9..98311a6dbf94 100644 --- a/sys/sys/sysproto.h +++ b/sys/sys/sysproto.h @@ -273,14 +273,6 @@ struct mincore_args { char len_l_[PADL_(size_t)]; size_t len; char len_r_[PADR_(size_t)]; char vec_l_[PADL_(char *)]; char * vec; char vec_r_[PADR_(char *)]; }; -struct getgroups_args { - char gidsetsize_l_[PADL_(int)]; int gidsetsize; char gidsetsize_r_[PADR_(int)]; - char gidset_l_[PADL_(gid_t *)]; gid_t * gidset; char gidset_r_[PADR_(gid_t *)]; -}; -struct setgroups_args { - char gidsetsize_l_[PADL_(int)]; int gidsetsize; char gidsetsize_r_[PADR_(int)]; - char gidset_l_[PADL_(const gid_t *)]; const gid_t * gidset; char gidset_r_[PADR_(const gid_t *)]; -}; struct getpgrp_args { syscallarg_t dummy; }; @@ -1901,6 +1893,14 @@ struct inotify_rm_watch_args { char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)]; char wd_l_[PADL_(int)]; int wd; char wd_r_[PADR_(int)]; }; +struct getgroups_args { + char gidsetsize_l_[PADL_(int)]; int gidsetsize; char gidsetsize_r_[PADR_(int)]; + char gidset_l_[PADL_(gid_t *)]; gid_t * gidset; char gidset_r_[PADR_(gid_t *)]; +}; +struct setgroups_args { + char gidsetsize_l_[PADL_(int)]; int gidsetsize; char gidsetsize_r_[PADR_(int)]; + char gidset_l_[PADL_(const gid_t *)]; const gid_t * gidset; char gidset_r_[PADR_(const gid_t *)]; +}; int sys__exit(struct thread *, struct _exit_args *); int sys_fork(struct thread *, struct fork_args *); int sys_read(struct thread *, struct read_args *); @@ -1957,8 +1957,6 @@ int sys_munmap(struct thread *, struct munmap_args *); int sys_mprotect(struct thread *, struct mprotect_args *); int sys_madvise(struct thread *, struct madvise_args *); int sys_mincore(struct thread *, struct mincore_args *); -int sys_getgroups(struct thread *, struct getgroups_args *); -int sys_setgroups(struct thread *, struct setgroups_args *); int sys_getpgrp(struct thread *, struct getpgrp_args *); int sys_setpgid(struct thread *, struct setpgid_args *); int sys_setitimer(struct thread *, struct setitimer_args *); @@ -2305,6 +2303,8 @@ int sys_setcred(struct thread *, struct setcred_args *); int sys_exterrctl(struct thread *, struct exterrctl_args *); int sys_inotify_add_watch_at(struct thread *, struct inotify_add_watch_at_args *); int sys_inotify_rm_watch(struct thread *, struct inotify_rm_watch_args *); +int sys_getgroups(struct thread *, struct getgroups_args *); +int sys_setgroups(struct thread *, struct setgroups_args *); #ifdef COMPAT_43 @@ -2799,6 +2799,16 @@ int freebsd13_swapoff(struct thread *, struct freebsd13_swapoff_args *); #ifdef COMPAT_FREEBSD14 +struct freebsd14_getgroups_args { + char gidsetsize_l_[PADL_(int)]; int gidsetsize; char gidsetsize_r_[PADR_(int)]; + char gidset_l_[PADL_(gid_t *)]; gid_t * gidset; char gidset_r_[PADR_(gid_t *)]; +}; +struct freebsd14_setgroups_args { + char gidsetsize_l_[PADL_(int)]; int gidsetsize; char gidsetsize_r_[PADR_(int)]; + char gidset_l_[PADL_(const gid_t *)]; const gid_t * gidset; char gidset_r_[PADR_(const gid_t *)]; +}; +int freebsd14_getgroups(struct thread *, struct freebsd14_getgroups_args *); +int freebsd14_setgroups(struct thread *, struct freebsd14_setgroups_args *); #endif /* COMPAT_FREEBSD14 */ @@ -2873,8 +2883,8 @@ int freebsd13_swapoff(struct thread *, struct freebsd13_swapoff_args *); #define SYS_AUE_mprotect AUE_MPROTECT #define SYS_AUE_madvise AUE_MADVISE #define SYS_AUE_mincore AUE_MINCORE -#define SYS_AUE_getgroups AUE_GETGROUPS -#define SYS_AUE_setgroups AUE_SETGROUPS +#define SYS_AUE_freebsd14_getgroups AUE_GETGROUPS +#define SYS_AUE_freebsd14_setgroups AUE_SETGROUPS #define SYS_AUE_getpgrp AUE_GETPGRP #define SYS_AUE_setpgid AUE_SETPGRP #define SYS_AUE_setitimer AUE_SETITIMER @@ -3289,6 +3299,8 @@ int freebsd13_swapoff(struct thread *, struct freebsd13_swapoff_args *); #define SYS_AUE_exterrctl AUE_NULL #define SYS_AUE_inotify_add_watch_at AUE_INOTIFY #define SYS_AUE_inotify_rm_watch AUE_INOTIFY +#define SYS_AUE_getgroups AUE_GETGROUPS +#define SYS_AUE_setgroups AUE_SETGROUPS #undef PAD_ #undef PADL_ diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h index 074769d55c2d..6ef9bbec9446 100644 --- a/sys/sys/vnode.h +++ b/sys/sys/vnode.h @@ -67,6 +67,11 @@ __enum_uint8_decl(vtype) { VLASTTYPE = VMARKER, }; +/* + * We frequently need to test is something is a device node. + */ +#define VTYPE_ISDEV(vtype) ((vtype) == VCHR || (vtype) == VBLK) + __enum_uint8_decl(vstate) { VSTATE_UNINITIALIZED, VSTATE_CONSTRUCTED, @@ -199,6 +204,8 @@ struct vnode { int v_seqc_users; /* i modifications pending */ }; +#define VN_ISDEV(vp) VTYPE_ISDEV((vp)->v_type) + #ifndef DEBUG_LOCKS #ifdef _LP64 /* @@ -309,6 +316,8 @@ struct vattr { long va_spare; /* remain quad aligned */ }; +#define VATTR_ISDEV(vap) VTYPE_ISDEV((vap)->va_type) + /* * Flags for va_vaflags. */ diff --git a/sys/sys/watchdog.h b/sys/sys/watchdog.h index 4a16b18509f5..8401d343a6b7 100644 --- a/sys/sys/watchdog.h +++ b/sys/sys/watchdog.h @@ -32,15 +32,16 @@ #define _SYS_WATCHDOG_H #include <sys/ioccom.h> +#include <sys/_types.h> #define _PATH_WATCHDOG "fido" -#define WDIOCPATPAT _IOW('W', 42, u_int) /* pat the watchdog */ -#define WDIOC_SETTIMEOUT _IOW('W', 43, int) /* set/reset the timer */ -#define WDIOC_GETTIMEOUT _IOR('W', 44, int) /* get total timeout */ -#define WDIOC_GETTIMELEFT _IOR('W', 45, int) /* get time left */ -#define WDIOC_GETPRETIMEOUT _IOR('W', 46, int) /* get the pre-timeout */ -#define WDIOC_SETPRETIMEOUT _IOW('W', 47, int) /* set the pre-timeout */ +#define WDIOC_PATPAT _IOW('W', 52, sbintime_t) /* pat the watchdog */ +#define WDIOC_SETTIMEOUT _IOW('W', 53, sbintime_t) /* set/reset the timer */ +#define WDIOC_GETTIMEOUT _IOR('W', 54, sbintime_t) /* get total timeout */ +#define WDIOC_GETTIMELEFT _IOR('W', 55, sbintime_t) /* get time left */ +#define WDIOC_GETPRETIMEOUT _IOR('W', 56, sbintime_t) /* get the pre-timeout */ +#define WDIOC_SETPRETIMEOUT _IOW('W', 57, sbintime_t) /* set the pre-timeout */ /* set the action when a pre-timeout occurs see: WD_SOFT_* */ #define WDIOC_SETPRETIMEOUTACT _IOW('W', 48, int) @@ -48,6 +49,8 @@ #define WDIOC_SETSOFT _IOW('W', 49, int) #define WDIOC_SETSOFTTIMEOUTACT _IOW('W', 50, int) +#define WDIOC_CONTROL _IOW('W', 51, int) /* configure watchdog */ + #define WD_ACTIVE 0x8000000 /* * Watchdog reset, timeout set to value in WD_INTERVAL field. @@ -93,6 +96,11 @@ #define WD_TO_64SEC 36 #define WD_TO_128SEC 37 +/* Control options for WDIOC_CONTROL */ +#define WD_CTRL_DISABLE 0x00000000 +#define WD_CTRL_ENABLE 0x00000001 +#define WD_CTRL_RESET 0x00000002 + /* action on pre-timeout trigger */ #define WD_SOFT_PANIC 0x01 /* panic */ #define WD_SOFT_DDB 0x02 /* enter debugger */ @@ -105,11 +113,16 @@ #include <sys/_eventhandler.h> typedef void (*watchdog_fn)(void *, u_int, int *); +typedef void (*watchdog_sbt_fn)(void *, sbintime_t, sbintime_t *, int *); EVENTHANDLER_DECLARE(watchdog_list, watchdog_fn); +EVENTHANDLER_DECLARE(watchdog_sbt_list, watchdog_sbt_fn); u_int wdog_kern_last_timeout(void); int wdog_kern_pat(u_int utim); +sbintime_t wdog_kern_last_timeout_sbt(void); +int wdog_kern_pat_sbt(sbintime_t utim); +int wdog_control(int ctrl); /* * The following function pointer is used to attach a software watchdog diff --git a/sys/tools/amd64_ia32_vdso.sh b/sys/tools/amd64_ia32_vdso.sh index 85d2299b45d0..e5865639d398 100644 --- a/sys/tools/amd64_ia32_vdso.sh +++ b/sys/tools/amd64_ia32_vdso.sh @@ -58,7 +58,7 @@ then exit 1 fi -${CC} ${DEBUG} -x assembler-with-cpp -DLOCORE -fPIC -nostdinc -c \ +${CC} -x assembler-with-cpp -DLOCORE -fPIC -nostdinc -c \ -o elf-vdso32.so.o -I. -I"${S}" -include opt_global.h \ -DVDSO_NAME=elf_vdso32_so_1 -DVDSO_FILE=\"elf-vdso32.so.1\" \ "${S}"/tools/vdso_wrap.S diff --git a/sys/tools/amd64_vdso.sh b/sys/tools/amd64_vdso.sh index 2a83ae874ab7..ed91ddc8abb5 100644 --- a/sys/tools/amd64_vdso.sh +++ b/sys/tools/amd64_vdso.sh @@ -67,7 +67,7 @@ then exit 1 fi -${CC} ${DEBUG} -x assembler-with-cpp -DLOCORE -fPIC -nostdinc -c \ +${CC} -x assembler-with-cpp -DLOCORE -fPIC -nostdinc -c \ -o elf-vdso.so.o -I. -I"${S}" -include opt_global.h \ -DVDSO_NAME=elf_vdso_so_1 -DVDSO_FILE=\"elf-vdso.so.1\" \ "${S}"/tools/vdso_wrap.S diff --git a/sys/ufs/ffs/ffs_vnops.c b/sys/ufs/ffs/ffs_vnops.c index 897a21032907..c7e2b3f4b8e6 100644 --- a/sys/ufs/ffs/ffs_vnops.c +++ b/sys/ufs/ffs/ffs_vnops.c @@ -1550,7 +1550,7 @@ ffs_openextattr( } */ *ap) { - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); return (ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td)); @@ -1572,7 +1572,7 @@ ffs_closeextattr( struct vnode *vp; vp = ap->a_vp; - if (vp->v_type == VCHR || vp->v_type == VBLK) + if (VN_ISDEV(vp)) return (EOPNOTSUPP); if (ap->a_commit && (vp->v_mount->mnt_flag & MNT_RDONLY) != 0) return (EROFS); @@ -1610,7 +1610,7 @@ ffs_deleteextattr( vp = ap->a_vp; ip = VTOI(vp); - if (vp->v_type == VCHR || vp->v_type == VBLK) + if (VN_ISDEV(vp)) return (EOPNOTSUPP); if (strlen(ap->a_name) == 0) return (EINVAL); @@ -1688,7 +1688,7 @@ ffs_getextattr( ip = VTOI(ap->a_vp); - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, @@ -1738,7 +1738,7 @@ ffs_listextattr( ip = VTOI(ap->a_vp); - if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK) + if (VN_ISDEV(ap->a_vp)) return (EOPNOTSUPP); error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace, @@ -1803,7 +1803,7 @@ ffs_setextattr( ip = VTOI(vp); fs = ITOFS(ip); - if (vp->v_type == VCHR || vp->v_type == VBLK) + if (VN_ISDEV(vp)) return (EOPNOTSUPP); if (strlen(ap->a_name) == 0) return (EINVAL); diff --git a/sys/ufs/ufs/ufs_vnops.c b/sys/ufs/ufs/ufs_vnops.c index ffc993aef9fc..0921eee92b9d 100644 --- a/sys/ufs/ufs/ufs_vnops.c +++ b/sys/ufs/ufs/ufs_vnops.c @@ -156,30 +156,30 @@ ufs_itimes_locked(struct vnode *vp) if ((ip->i_flag & (IN_ACCESS | IN_CHANGE | IN_UPDATE)) == 0) return; - if ((vp->v_type == VBLK || vp->v_type == VCHR) && !DOINGSOFTDEP(vp)) + if (VN_ISDEV(vp) && !DOINGSOFTDEP(vp)) UFS_INODE_SET_FLAG(ip, IN_LAZYMOD); else if (((vp->v_mount->mnt_kern_flag & - (MNTK_SUSPENDED | MNTK_SUSPEND)) == 0) || - (ip->i_flag & (IN_CHANGE | IN_UPDATE))) + (MNTK_SUSPENDED | MNTK_SUSPEND)) == 0) || + (ip->i_flag & (IN_CHANGE | IN_UPDATE)) != 0) UFS_INODE_SET_FLAG(ip, IN_MODIFIED); - else if (ip->i_flag & IN_ACCESS) + else if ((ip->i_flag & IN_ACCESS) != 0) UFS_INODE_SET_FLAG(ip, IN_LAZYACCESS); vfs_timestamp(&ts); - if (ip->i_flag & IN_ACCESS) { + if ((ip->i_flag & IN_ACCESS) != 0) { DIP_SET(ip, i_atime, ts.tv_sec); DIP_SET(ip, i_atimensec, ts.tv_nsec); } - if (ip->i_flag & IN_UPDATE) { + if ((ip->i_flag & IN_UPDATE) != 0) { DIP_SET(ip, i_mtime, ts.tv_sec); DIP_SET(ip, i_mtimensec, ts.tv_nsec); } - if (ip->i_flag & IN_CHANGE) { + if ((ip->i_flag & IN_CHANGE) != 0) { DIP_SET(ip, i_ctime, ts.tv_sec); DIP_SET(ip, i_ctimensec, ts.tv_nsec); DIP_SET(ip, i_modrev, DIP(ip, i_modrev) + 1); } - out: +out: ip->i_flag &= ~(IN_ACCESS | IN_CHANGE | IN_UPDATE); } @@ -319,7 +319,7 @@ ufs_open(struct vop_open_args *ap) struct vnode *vp = ap->a_vp; struct inode *ip; - if (vp->v_type == VCHR || vp->v_type == VBLK) + if (VN_ISDEV(vp)) return (EOPNOTSUPP); ip = VTOI(vp); @@ -540,7 +540,7 @@ ufs_stat(struct vop_stat_args *ap) sb->st_uid = ip->i_uid; sb->st_gid = ip->i_gid; if (I_IS_UFS1(ip)) { - sb->st_rdev = ip->i_din1->di_rdev; + sb->st_rdev = VN_ISDEV(vp) ? ip->i_din1->di_rdev : NODEV; sb->st_size = ip->i_din1->di_size; sb->st_mtim.tv_sec = ip->i_din1->di_mtime; sb->st_mtim.tv_nsec = ip->i_din1->di_mtimensec; @@ -551,7 +551,7 @@ ufs_stat(struct vop_stat_args *ap) sb->st_blocks = dbtob((uint64_t)ip->i_din1->di_blocks) / S_BLKSIZE; sb->st_filerev = ip->i_din1->di_modrev; } else { - sb->st_rdev = ip->i_din2->di_rdev; + sb->st_rdev = VN_ISDEV(vp) ? ip->i_din2->di_rdev : NODEV; sb->st_size = ip->i_din2->di_size; sb->st_mtim.tv_sec = ip->i_din2->di_mtime; sb->st_mtim.tv_nsec = ip->i_din2->di_mtimensec; @@ -603,7 +603,7 @@ ufs_getattr( vap->va_uid = ip->i_uid; vap->va_gid = ip->i_gid; if (I_IS_UFS1(ip)) { - vap->va_rdev = ip->i_din1->di_rdev; + vap->va_rdev = VN_ISDEV(vp) ? ip->i_din1->di_rdev : NODEV; vap->va_size = ip->i_din1->di_size; vap->va_mtime.tv_sec = ip->i_din1->di_mtime; vap->va_mtime.tv_nsec = ip->i_din1->di_mtimensec; @@ -612,7 +612,7 @@ ufs_getattr( vap->va_bytes = dbtob((uint64_t)ip->i_din1->di_blocks); vap->va_filerev = ip->i_din1->di_modrev; } else { - vap->va_rdev = ip->i_din2->di_rdev; + vap->va_rdev = VN_ISDEV(vp) ? ip->i_din2->di_rdev : NODEV; vap->va_size = ip->i_din2->di_size; vap->va_mtime.tv_sec = ip->i_din2->di_mtime; vap->va_mtime.tv_nsec = ip->i_din2->di_mtimensec; diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index de8a6c52c08f..244aa31ea703 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -901,8 +901,7 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count, int error, before, after, rbehind, rahead, poff, i; int bytecount, secmask; - KASSERT(vp->v_type != VCHR && vp->v_type != VBLK, - ("%s does not support devices", __func__)); + KASSERT(!VN_ISDEV(vp), ("%s does not support devices", __func__)); if (VN_IS_DOOMED(vp)) return (VM_PAGER_BAD); diff --git a/sys/x86/include/apicreg.h b/sys/x86/include/apicreg.h index d610d7f11a1c..1252647fbab3 100644 --- a/sys/x86/include/apicreg.h +++ b/sys/x86/include/apicreg.h @@ -296,6 +296,8 @@ typedef struct IOAPIC ioapic_t; /* constants relating to APIC ID registers */ #define APIC_ID_MASK 0xff000000 #define APIC_ID_SHIFT 24 +#define APIC_EXT_ID_MASK 0x00fe0000 +#define APIC_EXT_ID_SHIFT 17 #define APIC_ID_CLUSTER 0xf0 #define APIC_ID_CLUSTER_ID 0x0f #define APIC_MAX_CLUSTER 0xe |