diff options
Diffstat (limited to 'sys')
91 files changed, 2292 insertions, 717 deletions
diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC index 81427b5b18b6..786edc4125c9 100644 --- a/sys/amd64/conf/GENERIC +++ b/sys/amd64/conf/GENERIC @@ -26,7 +26,7 @@ makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support options SCHED_ULE # ULE scheduler options NUMA # Non-Uniform Memory Architecture support options PREEMPTION # Enable kernel thread preemption -options BLOAT_KERNEL_WITH_EXTERR +options EXTERR_STRINGS options VIMAGE # Subsystem virtualization, e.g. VNET options INET # InterNETworking options INET6 # IPv6 communications protocols diff --git a/sys/arm64/arm64/elf32_machdep.c b/sys/arm64/arm64/elf32_machdep.c index 5c81c6cdce3d..8f8a934ad520 100644 --- a/sys/arm64/arm64/elf32_machdep.c +++ b/sys/arm64/arm64/elf32_machdep.c @@ -225,7 +225,7 @@ freebsd32_fetch_syscall_args(struct thread *td) sa->args[i] = ap[i]; if (narg > nap) { if (narg - nap > nitems(args)) - panic("Too many system call arguiments"); + panic("Too many system call arguments"); error = copyin((void *)td->td_frame->tf_x[13], args, (narg - nap) * sizeof(int)); if (error != 0) diff --git a/sys/arm64/conf/std.arm64 b/sys/arm64/conf/std.arm64 index c83e98c17a33..a0568466cfaf 100644 --- a/sys/arm64/conf/std.arm64 +++ b/sys/arm64/conf/std.arm64 @@ -7,6 +7,7 @@ makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support options SCHED_ULE # ULE scheduler options NUMA # Non-Uniform Memory Architecture support options PREEMPTION # Enable kernel thread preemption +options EXTERR_STRINGS options VIMAGE # Subsystem virtualization, e.g. VNET options INET # InterNETworking options INET6 # IPv6 communications protocols diff --git a/sys/compat/linuxkpi/common/include/linux/ieee80211.h b/sys/compat/linuxkpi/common/include/linux/ieee80211.h index b9161c586d07..17041bb03ce8 100644 --- a/sys/compat/linuxkpi/common/include/linux/ieee80211.h +++ b/sys/compat/linuxkpi/common/include/linux/ieee80211.h @@ -408,6 +408,14 @@ enum ieee80211_sta_state { IEEE80211_STA_AUTHORIZED = 4, /* 802.1x */ }; +enum ieee80211_sta_rx_bandwidth { + IEEE80211_STA_RX_BW_20 = 0, + IEEE80211_STA_RX_BW_40, + IEEE80211_STA_RX_BW_80, + IEEE80211_STA_RX_BW_160, + IEEE80211_STA_RX_BW_320, +}; + enum ieee80211_tx_info_flags { /* XXX TODO .. right shift numbers - not sure where that came from? */ IEEE80211_TX_CTL_AMPDU = BIT(0), @@ -524,24 +532,24 @@ struct ieee80211_mgmt { uint16_t beacon_int; uint16_t capab_info; uint8_t variable[0]; - } beacon; + } __packed beacon; /* 9.3.3.5 Association Request frame format */ struct { uint16_t capab_info; uint16_t listen_interval; uint8_t variable[0]; - } assoc_req; + } __packed assoc_req; /* 9.3.3.10 Probe Request frame format */ struct { uint8_t variable[0]; - } probe_req; + } __packed probe_req; /* 9.3.3.11 Probe Response frame format */ struct { uint64_t timestamp; uint16_t beacon_int; uint16_t capab_info; uint8_t variable[0]; - } probe_resp; + } __packed probe_resp; /* 9.3.3.14 Action frame format */ struct { /* 9.4.1.11 Action field */ @@ -557,7 +565,7 @@ struct ieee80211_mgmt { uint8_t tpc_elem_length; uint8_t tpc_elem_tx_power; uint8_t tpc_elem_link_margin; - } tpc_report; + } __packed tpc_report; /* 9.6.8.33 Fine Timing Measurement frame format */ struct { uint8_t dialog_token; @@ -567,7 +575,7 @@ struct ieee80211_mgmt { uint16_t tod_error; uint16_t toa_error; uint8_t variable[0]; - } ftm; + } __packed ftm; /* 802.11-2016, 9.6.5.2 ADDBA Request frame format */ struct { uint8_t action_code; @@ -577,16 +585,16 @@ struct ieee80211_mgmt { uint16_t start_seq_num; /* Optional follows... */ uint8_t variable[0]; - } addba_req; + } __packed addba_req; /* XXX */ struct { uint8_t dialog_token; - } wnm_timing_msr; + } __packed wnm_timing_msr; } u; - } action; + } __packed action; DECLARE_FLEX_ARRAY(uint8_t, body); } u; -}; +} __packed __aligned(2); struct ieee80211_cts { /* net80211::ieee80211_frame_cts */ __le16 frame_control; diff --git a/sys/compat/linuxkpi/common/include/linux/skbuff.h b/sys/compat/linuxkpi/common/include/linux/skbuff.h index c8ad90281e34..6e41c368a8b8 100644 --- a/sys/compat/linuxkpi/common/include/linux/skbuff.h +++ b/sys/compat/linuxkpi/common/include/linux/skbuff.h @@ -1,6 +1,6 @@ /*- * Copyright (c) 2020-2025 The FreeBSD Foundation - * Copyright (c) 2021-2023 Bjoern A. Zeeb + * Copyright (c) 2021-2025 Bjoern A. Zeeb * * This software was developed by Björn Zeeb under sponsorship from * the FreeBSD Foundation. @@ -47,13 +47,11 @@ #include <linux/ktime.h> #include <linux/compiler.h> -#include "opt_wlan.h" - -/* Currently this is only used for wlan so we can depend on that. */ -#if defined(IEEE80211_DEBUG) && !defined(SKB_DEBUG) -#define SKB_DEBUG -#endif - +/* + * At least the net/intel-irdma-kmod port pulls this header in; likely through + * if_ether.h (see PR289268). This means we no longer can rely on + * IEEE80211_DEBUG (opt_wlan.h) to automatically set SKB_DEBUG. + */ /* #define SKB_DEBUG */ #ifdef SKB_DEBUG @@ -120,7 +118,7 @@ enum sk_checksum_flags { CHECKSUM_NONE = 0x00, CHECKSUM_UNNECESSARY = 0x01, CHECKSUM_PARTIAL = 0x02, - CHECKSUM_COMPLETE = 0x04, + CHECKSUM_COMPLETE = 0x03, }; struct skb_frag { @@ -170,7 +168,7 @@ struct sk_buff { }; }; uint16_t protocol; - uint8_t ip_summed; + uint8_t ip_summed; /* 2 bit only. */ /* uint8_t */ /* "Scratch" area for layers to store metadata. */ diff --git a/sys/compat/linuxkpi/common/include/linux/string_helpers.h b/sys/compat/linuxkpi/common/include/linux/string_helpers.h index 1bdff2730361..2c6fe0b1708d 100644 --- a/sys/compat/linuxkpi/common/include/linux/string_helpers.h +++ b/sys/compat/linuxkpi/common/include/linux/string_helpers.h @@ -66,4 +66,6 @@ str_enable_disable(bool value) return "disable"; } +#define str_disable_enable(_v) str_enable_disable(!(_v)) + #endif diff --git a/sys/compat/linuxkpi/common/include/net/mac80211.h b/sys/compat/linuxkpi/common/include/net/mac80211.h index 0106e6648bd4..8de03410c6b6 100644 --- a/sys/compat/linuxkpi/common/include/net/mac80211.h +++ b/sys/compat/linuxkpi/common/include/net/mac80211.h @@ -737,7 +737,7 @@ struct ieee80211_link_sta { struct ieee80211_he_6ghz_capa he_6ghz_capa; struct ieee80211_sta_eht_cap eht_cap; uint8_t rx_nss; - enum ieee80211_sta_rx_bw bandwidth; + enum ieee80211_sta_rx_bandwidth bandwidth; enum ieee80211_smps_mode smps_mode; struct ieee80211_sta_agg agg; struct ieee80211_sta_txpwr txpwr; diff --git a/sys/compat/linuxkpi/common/src/linux_80211.c b/sys/compat/linuxkpi/common/src/linux_80211.c index e248588dd275..f0881773726f 100644 --- a/sys/compat/linuxkpi/common/src/linux_80211.c +++ b/sys/compat/linuxkpi/common/src/linux_80211.c @@ -77,6 +77,8 @@ #include <linux/rculist.h> #include "linux_80211.h" +/* #define LKPI_80211_USE_SCANLIST */ +/* #define LKPI_80211_BGSCAN */ #define LKPI_80211_WME #define LKPI_80211_HW_CRYPTO #define LKPI_80211_HT @@ -103,6 +105,10 @@ SYSCTL_DECL(_compat_linuxkpi); SYSCTL_NODE(_compat_linuxkpi, OID_AUTO, 80211, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "LinuxKPI 802.11 compatibility layer"); +static bool lkpi_order_scanlist = false; +SYSCTL_BOOL(_compat_linuxkpi_80211, OID_AUTO, order_scanlist, CTLFLAG_RW, + &lkpi_order_scanlist, 0, "Enable LinuxKPI 802.11 scan list shuffeling"); + #if defined(LKPI_80211_HW_CRYPTO) static bool lkpi_hwcrypto = false; SYSCTL_BOOL(_compat_linuxkpi_80211, OID_AUTO, hw_crypto, CTLFLAG_RDTUN, @@ -167,6 +173,7 @@ const struct cfg80211_ops linuxkpi_mac80211cfgops = { static struct lkpi_sta *lkpi_find_lsta_by_ni(struct lkpi_vif *, struct ieee80211_node *); #endif +static void lkpi_sw_scan_task(void *, int); static void lkpi_80211_txq_tx_one(struct lkpi_sta *, struct mbuf *); static void lkpi_80211_txq_task(void *, int); static void lkpi_80211_lhw_rxq_task(void *, int); @@ -394,7 +401,7 @@ lkpi_80211_dump_stas(SYSCTL_HANDLER_ARGS) return (0); } -static enum ieee80211_sta_rx_bw +static enum ieee80211_sta_rx_bandwidth lkpi_cw_to_rx_bw(enum nl80211_chan_width cw) { switch (cw) { @@ -418,7 +425,7 @@ lkpi_cw_to_rx_bw(enum nl80211_chan_width cw) } static enum nl80211_chan_width -lkpi_rx_bw_to_cw(enum ieee80211_sta_rx_bw rx_bw) +lkpi_rx_bw_to_cw(enum ieee80211_sta_rx_bandwidth rx_bw) { switch (rx_bw) { case IEEE80211_STA_RX_BW_20: @@ -439,7 +446,7 @@ lkpi_sync_chanctx_cw_from_rx_bw(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta) { struct ieee80211_chanctx_conf *chanctx_conf; - enum ieee80211_sta_rx_bw old_bw; + enum ieee80211_sta_rx_bandwidth old_bw; uint32_t changed; chanctx_conf = rcu_dereference_protected(vif->bss_conf.chanctx_conf, @@ -544,7 +551,7 @@ static void lkpi_sta_sync_vht_from_ni(struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct ieee80211_node *ni) { - enum ieee80211_sta_rx_bw bw; + enum ieee80211_sta_rx_bandwidth bw; uint32_t width; int rx_nss; uint16_t rx_mcs_map; @@ -955,6 +962,30 @@ lkpi_nl80211_band_to_net80211_band(enum nl80211_band band) return (0x00); } +#ifdef LINUXKPI_DEBUG_80211 +static const char * +lkpi_nl80211_band_name(enum nl80211_band band) +{ + switch (band) { + case NL80211_BAND_2GHZ: + return "2Ghz"; + break; + case NL80211_BAND_5GHZ: + return "5Ghz"; + break; + case NL80211_BAND_60GHZ: + return "60Ghz"; + break; + case NL80211_BAND_6GHZ: + return "6Ghz"; + break; + default: + panic("%s: unsupported band %u\n", __func__, band); + break; + } +} +#endif + #if 0 static enum ieee80211_ac_numbers lkpi_ac_net_to_l80211(int ac) @@ -1319,6 +1350,7 @@ lkpi_iv_key_delete(struct ieee80211vap *vap, const struct ieee80211_key *k) lhw = ic->ic_softc; hw = LHW_TO_HW(lhw); lvif = VAP_TO_LVIF(vap); + vif = LVIF_TO_VIF(lvif); /* * Make sure we do not make it here without going through @@ -1326,6 +1358,23 @@ lkpi_iv_key_delete(struct ieee80211vap *vap, const struct ieee80211_key *k) */ lockdep_assert_wiphy(hw->wiphy); + /* + * While we are assoc we may still send packets. We cannot delete the + * keys as otherwise packets could go out unencrypted. Some firmware + * does not like this and will fire an assert. + * net80211 needs to drive this better but given we want the disassoc + * frame out and have to unlock we are open to a race currently. + * This check should prevent problems. + * How to test: run 800Mbit/s UDP traffic and during that restart your + * supplicant. You want to survive that. + */ + if (vif->cfg.assoc) { + if (linuxkpi_debug_80211 & D80211_TRACE_HW_CRYPTO) + ic_printf(ic, "%d %lu %s: vif still assoc; not deleting keys\n", + curthread->td_tid, jiffies, __func__); + return (0); + } + if (IEEE80211_KEY_UNDEFINED(k)) { ic_printf(ic, "%s: vap %p key %p is undefined: %p %u\n", __func__, vap, k, k->wk_cipher, k->wk_keyix); @@ -1370,7 +1419,6 @@ lkpi_iv_key_delete(struct ieee80211vap *vap, const struct ieee80211_key *k) kc->keyidx, kc->hw_key_idx, kc->flags, IEEE80211_KEY_FLAG_BITS); #endif - vif = LVIF_TO_VIF(lvif); error = lkpi_80211_mo_set_key(hw, DISABLE_KEY, vif, sta, kc); if (error != 0) { ic_printf(ic, "%d %lu %s: set_key cmd %d(%s) for sta %6D failed: %d\n", @@ -1842,13 +1890,31 @@ lkpi_update_dtim_tsf(struct ieee80211_vif *vif, struct ieee80211_node *ni, vif->bss_conf.beacon_int = 16; bss_changed |= BSS_CHANGED_BEACON_INT; } - if (vif->bss_conf.dtim_period != vap->iv_dtim_period && - vap->iv_dtim_period > 0) { - vif->bss_conf.dtim_period = vap->iv_dtim_period; + + /* + * lkpi_iv_sta_recv_mgmt() will directly call into this function. + * iwlwifi(4) in iwl_mvm_bss_info_changed_station_common() will + * stop seesion protection the moment it sees + * BSS_CHANGED_BEACON_INFO (with the expectations that it was + * "a beacon from the associated AP"). It will also update + * the beacon filter in that case. This is the only place + * we set the BSS_CHANGED_BEACON_INFO on the non-teardown + * path so make sure we only do run this check once we are + * assoc. (*iv_recv_mgmt)() will be called before we enter + * here so the ni will be updates with information from the + * beacon via net80211::sta_recv_mgmt(). We also need to + * make sure we do not do it on every beacon we still may + * get so only do if something changed. vif->bss_conf.dtim_period + * should be 0 as we start up (we also reset it on teardown). + */ + if (vif->cfg.assoc && + vif->bss_conf.dtim_period != ni->ni_dtim_period && + ni->ni_dtim_period > 0) { + vif->bss_conf.dtim_period = ni->ni_dtim_period; bss_changed |= BSS_CHANGED_BEACON_INFO; } - vif->bss_conf.sync_dtim_count = vap->iv_dtim_count; + vif->bss_conf.sync_dtim_count = ni->ni_dtim_count; vif->bss_conf.sync_tsf = le64toh(ni->ni_tstamp.tsf); /* vif->bss_conf.sync_device_ts = set in linuxkpi_ieee80211_rx. */ @@ -1876,6 +1942,8 @@ lkpi_stop_hw_scan(struct lkpi_hw *lhw, struct ieee80211_vif *vif) int error; bool cancel; + TRACE_SCAN(lhw->ic, "scan_flags %b", lhw->scan_flags, LKPI_LHW_SCAN_BITS); + LKPI_80211_LHW_SCAN_LOCK(lhw); cancel = (lhw->scan_flags & LKPI_LHW_SCAN_RUNNING) != 0; LKPI_80211_LHW_SCAN_UNLOCK(lhw); @@ -2798,6 +2866,14 @@ _lkpi_sta_assoc_to_down(struct ieee80211vap *vap, enum ieee80211_state nstate, i bss_changed = 0; bss_changed |= lkpi_disassoc(sta, vif, lhw); +#ifdef LKPI_80211_HW_CRYPTO + /* + * In theory we remove keys here but there must not exist any for this + * state change until we clean them up again into small steps and no + * code duplication. + */ +#endif + lkpi_lsta_dump(lsta, ni, __func__, __LINE__); /* Adjust sta and change state (from NONE) to NOTEXIST. */ @@ -3333,6 +3409,16 @@ lkpi_sta_run_to_init(struct ieee80211vap *vap, enum ieee80211_state nstate, int #ifdef LKPI_80211_HW_CRYPTO if (lkpi_hwcrypto) { + /* + * In theory we only need to do this if we changed assoc. + * If we were not assoc, there should be no keys and we + * should not be here. + */ +#ifdef notyet + KASSERT((bss_changed & BSS_CHANGED_ASSOC) != 0, ("%s: " + "trying to remove keys but were not assoc: %#010jx, lvif %p\n", + __func__, (uintmax_t)bss_changed, lvif)); +#endif error = lkpi_sta_del_keys(hw, vif, lsta); if (error != 0) { ic_printf(vap->iv_ic, "%s:%d: lkpi_sta_del_keys " @@ -3394,6 +3480,9 @@ lkpi_sta_run_to_init(struct ieee80211vap *vap, enum ieee80211_state nstate, int * 4) call unassign_vif_chanctx * 5) call lkpi_hw_conf_idle * 6) call remove_chanctx + * + * Note: vif->driver_flags & IEEE80211_VIF_REMOVE_AP_AFTER_DISASSOC + * might change this. */ bss_changed |= lkpi_disassoc(sta, vif, lhw); @@ -3545,7 +3634,7 @@ lkpi_iv_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) vif = LVIF_TO_VIF(lvif); /* No need to replicate this in most state handlers. */ - if (ostate == IEEE80211_S_SCAN && nstate != IEEE80211_S_SCAN) + if (nstate > IEEE80211_S_SCAN) lkpi_stop_hw_scan(lhw, vif); s = sta_state_fsm; @@ -3739,6 +3828,7 @@ lkpi_iv_sta_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0, enum ieee80211_bss_changed bss_changed; lvif = VAP_TO_LVIF(ni->ni_vap); + vif = LVIF_TO_VIF(lvif); lvif->iv_recv_mgmt(ni, m0, subtype, rxs, rssi, nf); @@ -3746,13 +3836,18 @@ lkpi_iv_sta_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0, case IEEE80211_FC0_SUBTYPE_PROBE_RESP: break; case IEEE80211_FC0_SUBTYPE_BEACON: - lvif->beacons++; + /* + * Only count beacons when assoc. SCAN has its own logging. + * This is for connection/beacon loss/session protection almost + * over debugging when trying to get into a stable RUN state. + */ + if (vif->cfg.assoc) + lvif->beacons++; break; default: return; } - vif = LVIF_TO_VIF(lvif); lhw = ni->ni_ic->ic_softc; hw = LHW_TO_HW(lhw); @@ -3824,6 +3919,7 @@ lkpi_ic_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], lvif = malloc(len, M_80211_VAP, M_WAITOK | M_ZERO); mtx_init(&lvif->mtx, "lvif", NULL, MTX_DEF); + TASK_INIT(&lvif->sw_scan_task, 0, lkpi_sw_scan_task, lvif); INIT_LIST_HEAD(&lvif->lsta_list); lvif->lvif_bss = NULL; refcount_init(&lvif->nt_unlocked, 0); @@ -3987,13 +4083,9 @@ lkpi_ic_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], * Modern chipset/fw/drv will do A-MPDU in drv/fw and fail * to do so if they cannot do the crypto too. */ - if (!lkpi_hwcrypto && ieee80211_hw_check(hw, AMPDU_AGGREGATION)) + if (!lkpi_hwcrypto && IEEE80211_CONF_AMPDU_OFFLOAD(ic)) vap->iv_flags_ht &= ~IEEE80211_FHT_AMPDU_RX; #endif -#if defined(LKPI_80211_HT) - /* 20250125-BZ Keep A-MPDU TX cleared until we sorted out AddBA for all drivers. */ - vap->iv_flags_ht &= ~IEEE80211_FHT_AMPDU_TX; -#endif if (hw->max_listen_interval == 0) hw->max_listen_interval = 7 * (ic->ic_lintval / ic->ic_bintval); @@ -4062,6 +4154,8 @@ lkpi_ic_vap_delete(struct ieee80211vap *vap) /* Clear up per-VIF/VAP sysctls. */ sysctl_ctx_free(&lvif->sysctl_ctx); + ieee80211_draintask(ic, &lvif->sw_scan_task); + LKPI_80211_LHW_LVIF_LOCK(lhw); TAILQ_REMOVE(&lhw->lvif_head, lvif, lvif_entry); LKPI_80211_LHW_LVIF_UNLOCK(lhw); @@ -4303,6 +4397,113 @@ lkpi_scan_ies_add(uint8_t *p, struct ieee80211_scan_ies *scan_ies, } static void +lkpi_enable_hw_scan(struct lkpi_hw *lhw) +{ + + if (lhw->ops->hw_scan) { + /* + * Advertise full-offload scanning. + * + * Not limiting to SINGLE_SCAN_ON_ALL_BANDS here as otherwise + * we essentially disable hw_scan for all drivers not setting + * the flag. + */ + lhw->ic->ic_flags_ext |= IEEE80211_FEXT_SCAN_OFFLOAD; + lhw->scan_flags |= LKPI_LHW_SCAN_HW; + } +} + +#ifndef LKPI_80211_USE_SCANLIST +static const uint32_t chan_pri[] = { + 5180, 5500, 5745, + 5260, 5580, 5660, 5825, + 5220, 5300, 5540, 5620, 5700, 5785, 5865, + 2437, 2412, 2422, 2462, 2472, 2432, 2452 +}; + +static int +lkpi_scan_chan_list_idx(const struct linuxkpi_ieee80211_channel *lc) +{ + int i; + + for (i = 0; i < nitems(chan_pri); i++) { + if (lc->center_freq == chan_pri[i]) + return (i); + } + + return (-1); +} + +static int +lkpi_scan_chan_list_comp(const struct linuxkpi_ieee80211_channel *lc1, + const struct linuxkpi_ieee80211_channel *lc2) +{ + int idx1, idx2; + + /* Find index in list. */ + idx1 = lkpi_scan_chan_list_idx(lc1); + idx2 = lkpi_scan_chan_list_idx(lc2); + + if (idx1 == -1 && idx2 != -1) + return (1); + if (idx1 != -1 && idx2 == -1) + return (-1); + + /* Neither on the list, use center_freq. */ + if (idx1 == -1 && idx2 == -1) + return (lc1->center_freq - lc2->center_freq); + + /* Whichever is first in the list. */ + return (idx1 - idx2); +} + +static void +lkpi_scan_chan_list_resort(struct linuxkpi_ieee80211_channel **cpp, size_t nchan) +{ + struct linuxkpi_ieee80211_channel *lc, *nc; + size_t i, j; + int rc; + + for (i = (nchan - 1); i > 0; i--) { + for (j = i; j > 0 ; j--) { + lc = *(cpp + j); + nc = *(cpp + j - 1); + rc = lkpi_scan_chan_list_comp(lc, nc); + if (rc < 0) { + *(cpp + j) = nc; + *(cpp + j - 1) = lc; + } + } + } +} + +static bool +lkpi_scan_chan(struct linuxkpi_ieee80211_channel *c, + struct ieee80211com *ic, bool log) +{ + + if ((c->flags & IEEE80211_CHAN_DISABLED) != 0) { + if (log) + TRACE_SCAN(ic, "Skipping disabled chan " + "on band %s [%#x/%u/%#x]", + lkpi_nl80211_band_name(c->band), c->hw_value, + c->center_freq, c->flags); + return (false); + } + if (isclr(ic->ic_chan_active, ieee80211_mhz2ieee(c->center_freq, + lkpi_nl80211_band_to_net80211_band(c->band)))) { + if (log) + TRACE_SCAN(ic, "Skipping !active chan " + "on band %s [%#x/%u/%#x]", + lkpi_nl80211_band_name(c->band), c->hw_value, + c->center_freq, c->flags); + return (false); + } + return (true); +} +#endif + +static void lkpi_ic_scan_start(struct ieee80211com *ic) { struct lkpi_hw *lhw; @@ -4315,35 +4516,52 @@ lkpi_ic_scan_start(struct ieee80211com *ic) bool is_hw_scan; lhw = ic->ic_softc; + ss = ic->ic_scan; + vap = ss->ss_vap; + TRACE_SCAN(ic, "scan_flags %b", lhw->scan_flags, LKPI_LHW_SCAN_BITS); + LKPI_80211_LHW_SCAN_LOCK(lhw); if ((lhw->scan_flags & LKPI_LHW_SCAN_RUNNING) != 0) { /* A scan is still running. */ LKPI_80211_LHW_SCAN_UNLOCK(lhw); + TRACE_SCAN(ic, "Trying to start new scan while still running; " + "cancelling new net80211 scan; scan_flags %b", + lhw->scan_flags, LKPI_LHW_SCAN_BITS); + ieee80211_cancel_scan(vap); return; } is_hw_scan = (lhw->scan_flags & LKPI_LHW_SCAN_HW) != 0; LKPI_80211_LHW_SCAN_UNLOCK(lhw); - ss = ic->ic_scan; - vap = ss->ss_vap; +#if 0 if (vap->iv_state != IEEE80211_S_SCAN) { - IMPROVE("We need to be able to scan if not in S_SCAN"); + TODO("We need to be able to scan if not in S_SCAN"); + TRACE_SCAN(ic, "scan_flags %b iv_state %d", + lhw->scan_flags, LKPI_LHW_SCAN_BITS, vap->iv_state); + ieee80211_cancel_scan(vap); return; } +#endif hw = LHW_TO_HW(lhw); if (!is_hw_scan) { /* If hw_scan is cleared clear FEXT_SCAN_OFFLOAD too. */ vap->iv_flags_ext &= ~IEEE80211_FEXT_SCAN_OFFLOAD; -sw_scan: + lvif = VAP_TO_LVIF(vap); vif = LVIF_TO_VIF(lvif); if (vap->iv_state == IEEE80211_S_SCAN) lkpi_hw_conf_idle(hw, false); + LKPI_80211_LHW_SCAN_LOCK(lhw); + lhw->scan_flags |= LKPI_LHW_SCAN_RUNNING; + LKPI_80211_LHW_SCAN_UNLOCK(lhw); + lkpi_update_mcast_filter(ic); + TRACE_SCAN(vap->iv_ic, "Starting SW_SCAN: scan_flags %b", + lhw->scan_flags, LKPI_LHW_SCAN_BITS); lkpi_80211_mo_sw_scan_start(hw, vif, vif->addr); /* net80211::scan_start() handled PS for us. */ IMPROVE(); @@ -4358,6 +4576,9 @@ sw_scan: struct cfg80211_scan_6ghz_params *s6gp; size_t chan_len, nchan, ssids_len, s6ghzlen; int band, i, ssid_count, common_ie_len; +#ifndef LKPI_80211_USE_SCANLIST + int n; +#endif uint32_t band_mask; uint8_t *ie, *ieend; bool running; @@ -4369,7 +4590,8 @@ sw_scan: band_mask = 0; nchan = 0; if (ieee80211_hw_check(hw, SINGLE_SCAN_ON_ALL_BANDS)) { -#if 0 /* Avoid net80211 scan lists until it has proper scan offload support. */ +#ifdef LKPI_80211_USE_SCANLIST + /* Avoid net80211 scan lists until it has proper scan offload support. */ for (i = ss->ss_next; i < ss->ss_last; i++) { nchan++; band = lkpi_net80211_chan_to_nl80211_band( @@ -4387,8 +4609,17 @@ sw_scan: continue; } if (hw->wiphy->bands[band] != NULL) { - nchan += hw->wiphy->bands[band]->n_channels; + struct linuxkpi_ieee80211_channel *channels; + int n; + band_mask |= (1 << band); + + channels = hw->wiphy->bands[band]->channels; + n = hw->wiphy->bands[band]->n_channels; + for (i = 0; i < n; i++) { + if (lkpi_scan_chan(&channels[i], ic, true)) + nchan++; + } } } #endif @@ -4427,11 +4658,32 @@ sw_scan: /* hw_req->req.wdev */ hw_req->req.wiphy = hw->wiphy; hw_req->req.no_cck = false; /* XXX */ -#if 0 - /* This seems to pessimise default scanning behaviour. */ - hw_req->req.duration_mandatory = TICKS_2_USEC(ss->ss_mindwell); - hw_req->req.duration = TICKS_2_USEC(ss->ss_maxdwell); -#endif + + /* + * In general setting duration[_mandatory] seems to pessimise + * default scanning behaviour. We only use it for BGSCANnig + * to keep the dwell times small. + * Setting duration_mandatory makes this the maximum dwell + * time (otherwise may be shorter). Duration is in TU. + */ + if ((ic->ic_flags_ext & IEEE80211_FEXT_BGSCAN) != 0) { + unsigned long dwell; + + if ((ic->ic_caps & IEEE80211_C_BGSCAN) == 0 || + (vap->iv_flags & IEEE80211_F_BGSCAN) == 0) + ic_printf(ic, "BGSCAN despite off: %b, %b, %b\n", + ic->ic_flags_ext, IEEE80211_FEXT_BITS, + vap->iv_flags, IEEE80211_F_BITS, + ic->ic_caps, IEEE80211_C_BITS); + + dwell = ss->ss_mindwell; + if (dwell == 0) + dwell = msecs_to_ticks(20); + + hw_req->req.duration_mandatory = true; + hw_req->req.duration = TICKS_2_USEC(dwell) / 1024; + } + #ifdef __notyet__ hw_req->req.flags |= NL80211_SCAN_FLAG_RANDOM_ADDR; memcpy(hw_req->req.mac_addr, xxx, IEEE80211_ADDR_LEN); @@ -4442,11 +4694,12 @@ sw_scan: hw_req->req.n_channels = nchan; cpp = (struct linuxkpi_ieee80211_channel **)(hw_req + 1); lc = (struct linuxkpi_ieee80211_channel *)(cpp + nchan); +#ifdef LKPI_80211_USE_SCANLIST for (i = 0; i < nchan; i++) { *(cpp + i) = (struct linuxkpi_ieee80211_channel *)(lc + i); } -#if 0 /* Avoid net80211 scan lists until it has proper scan offload support. */ + /* Avoid net80211 scan lists until it has proper scan offload support. */ for (i = 0; i < nchan; i++) { struct ieee80211_channel *c; @@ -4459,7 +4712,9 @@ sw_scan: lc++; } #else - for (band = 0; band < NUM_NL80211_BANDS; band++) { + /* Add bands in reverse order for scanning. */ + n = 0; + for (band = NUM_NL80211_BANDS - 1; band >= 0; band--) { struct ieee80211_supported_band *supband; struct linuxkpi_ieee80211_channel *channels; @@ -4474,11 +4729,30 @@ sw_scan: channels = supband->channels; for (i = 0; i < supband->n_channels; i++) { - *lc = channels[i]; - lc++; + if (lkpi_scan_chan(&channels[i], ic, false)) + *(cpp + n++) = &channels[i]; } } + if (lkpi_order_scanlist) + lkpi_scan_chan_list_resort(cpp, nchan); + + if ((linuxkpi_debug_80211 & D80211_SCAN) != 0) { + printf("%s:%d: %s SCAN Channel List (nchan=%zu): ", + __func__, __LINE__, ic->ic_name, nchan); + for (i = 0; i < nchan; i++) { + struct linuxkpi_ieee80211_channel *xc; + + xc = *(cpp + i); + printf(" %d(%d)", + ieee80211_mhz2ieee(xc->center_freq, + lkpi_nl80211_band_to_net80211_band( + xc->band)), + xc->center_freq); + } + printf("\n"); + } #endif + hw_req->req.n_ssids = ssid_count; if (hw_req->req.n_ssids > 0) { ssids = (struct cfg80211_ssid *)lc; @@ -4505,6 +4779,7 @@ sw_scan: ieend = lkpi_scan_ies_add(ie, &hw_req->ies, band_mask, vap, hw); hw_req->req.ie = ie; hw_req->req.ie_len = ieend - ie; + hw_req->req.scan_start = jiffies; lvif = VAP_TO_LVIF(vap); vif = LVIF_TO_VIF(lvif); @@ -4522,13 +4797,30 @@ sw_scan: LKPI_80211_LHW_SCAN_UNLOCK(lhw); if (running) { free(hw_req, M_LKPI80211); + TRACE_SCAN(ic, "Trying to start new scan while still " + "running (2); cancelling new net80211 scan; " + "scan_flags %b", + lhw->scan_flags, LKPI_LHW_SCAN_BITS); + ieee80211_cancel_scan(vap); return; } lkpi_update_mcast_filter(ic); + TRACE_SCAN(ic, "Starting HW_SCAN: scan_flags %b, " + "ie_len %d, n_ssids %d, n_chan %d, common_ie_len %d [%d, %d]", + lhw->scan_flags, LKPI_LHW_SCAN_BITS, hw_req->req.ie_len, + hw_req->req.n_ssids, hw_req->req.n_channels, + hw_req->ies.common_ie_len, + hw_req->ies.len[NL80211_BAND_2GHZ], + hw_req->ies.len[NL80211_BAND_5GHZ]); error = lkpi_80211_mo_hw_scan(hw, vif, hw_req); if (error != 0) { + bool scan_done; + int e; + + TRACE_SCAN(ic, "hw_scan failed; scan_flags %b, error %d", + lhw->scan_flags, LKPI_LHW_SCAN_BITS, error); ieee80211_cancel_scan(vap); /* @@ -4545,14 +4837,35 @@ sw_scan: * So we cannot rely on that behaviour and have to check * and balance between both code paths. */ + e = 0; + scan_done = true; LKPI_80211_LHW_SCAN_LOCK(lhw); if ((lhw->scan_flags & LKPI_LHW_SCAN_RUNNING) != 0) { + free(lhw->hw_req, M_LKPI80211); lhw->hw_req = NULL; + /* + * The ieee80211_cancel_scan() above runs in a + * taskq and it may take ages for the previous + * scan to clear; starting a new one right away + * we run into the problem that the old one is + * still active. + */ + e = msleep(lhw, &lhw->scan_mtx, 0, "lhwscanstop", hz); + scan_done = (lhw->scan_flags & LKPI_LHW_SCAN_RUNNING) != 0; + + /* + * Now we can clear running if no one else did. + */ lhw->scan_flags &= ~LKPI_LHW_SCAN_RUNNING; } LKPI_80211_LHW_SCAN_UNLOCK(lhw); lkpi_update_mcast_filter(ic); + if (!scan_done) { + ic_printf(ic, "ERROR: %s: timeout/error to wait " + "for ieee80211_cancel_scan: %d\n", __func__, e); + return; + } /* * XXX-SIGH magic number. @@ -4560,24 +4873,15 @@ sw_scan: * not possible. Fall back to sw scan in that case. */ if (error == 1) { - LKPI_80211_LHW_SCAN_LOCK(lhw); - lhw->scan_flags &= ~LKPI_LHW_SCAN_HW; - LKPI_80211_LHW_SCAN_UNLOCK(lhw); /* - * XXX If we clear this now and later a driver - * thinks it * can do a hw_scan again, we will - * currently not re-enable it? + * We need to put this into some defered context + * the net80211 scan may not be done yet + * (ic_flags & IEEE80211_F_SCAN) and we cannot + * wait here; if we do scan_curchan_task always + * runs after our timeout to finalize the scan. */ - vap->iv_flags_ext &= ~IEEE80211_FEXT_SCAN_OFFLOAD; - ieee80211_start_scan(vap, - IEEE80211_SCAN_ACTIVE | - IEEE80211_SCAN_NOPICK | - IEEE80211_SCAN_ONCE, - IEEE80211_SCAN_FOREVER, - ss->ss_mindwell ? ss->ss_mindwell : msecs_to_ticks(20), - ss->ss_maxdwell ? ss->ss_maxdwell : msecs_to_ticks(200), - vap->iv_des_nssid, vap->iv_des_ssid); - goto sw_scan; + ieee80211_runtask(ic, &lvif->sw_scan_task); + return; } ic_printf(ic, "ERROR: %s: hw_scan returned %d\n", @@ -4587,12 +4891,50 @@ sw_scan: } static void +lkpi_sw_scan_task(void *arg, int pending __unused) +{ + struct lkpi_hw *lhw; + struct lkpi_vif *lvif; + struct ieee80211vap *vap; + struct ieee80211_scan_state *ss; + + lvif = arg; + vap = LVIF_TO_VAP(lvif); + lhw = vap->iv_ic->ic_softc; + ss = vap->iv_ic->ic_scan; + + LKPI_80211_LHW_SCAN_LOCK(lhw); + /* + * We will re-enable this at scan_end calling lkpi_enable_hw_scan(). + * IEEE80211_FEXT_SCAN_OFFLOAD will be cleared by lkpi_ic_scan_start. + */ + lhw->scan_flags &= ~LKPI_LHW_SCAN_HW; + LKPI_80211_LHW_SCAN_UNLOCK(lhw); + + TRACE_SCAN(vap->iv_ic, "Triggering SW_SCAN: pending %d, scan_flags %b", + pending, lhw->scan_flags, LKPI_LHW_SCAN_BITS); + + /* + * This will call ic_scan_start() and we will get into the right path + * unless other scans started in between. + */ + ieee80211_start_scan(vap, + IEEE80211_SCAN_ONCE, + msecs_to_ticks(10000), /* 10000 ms (=~ 50 chan * 200 ms) */ + ss->ss_mindwell ? ss->ss_mindwell : msecs_to_ticks(20), + ss->ss_maxdwell ? ss->ss_maxdwell : msecs_to_ticks(200), + vap->iv_des_nssid, vap->iv_des_ssid); +} + +static void lkpi_ic_scan_end(struct ieee80211com *ic) { struct lkpi_hw *lhw; bool is_hw_scan; lhw = ic->ic_softc; + TRACE_SCAN(ic, "scan_flags %b", lhw->scan_flags, LKPI_LHW_SCAN_BITS); + LKPI_80211_LHW_SCAN_LOCK(lhw); if ((lhw->scan_flags & LKPI_LHW_SCAN_RUNNING) == 0) { LKPI_80211_LHW_SCAN_UNLOCK(lhw); @@ -4621,6 +4963,16 @@ lkpi_ic_scan_end(struct ieee80211com *ic) if (vap->iv_state == IEEE80211_S_SCAN) lkpi_hw_conf_idle(hw, true); } + + /* + * In case we disabled the hw_scan in lkpi_ic_scan_start() and + * switched to swscan, re-enable hw_scan if available. + */ + lkpi_enable_hw_scan(lhw); + + LKPI_80211_LHW_SCAN_LOCK(lhw); + wakeup(lhw); + LKPI_80211_LHW_SCAN_UNLOCK(lhw); } static void @@ -4631,6 +4983,10 @@ lkpi_ic_scan_curchan(struct ieee80211_scan_state *ss, bool is_hw_scan; lhw = ss->ss_ic->ic_softc; + TRACE_SCAN(ss->ss_ic, "scan_flags %b chan %d maxdwell %lu", + lhw->scan_flags, LKPI_LHW_SCAN_BITS, + ss->ss_ic->ic_curchan->ic_ieee, maxdwell); + LKPI_80211_LHW_SCAN_LOCK(lhw); is_hw_scan = (lhw->scan_flags & LKPI_LHW_SCAN_HW) != 0; LKPI_80211_LHW_SCAN_UNLOCK(lhw); @@ -4645,6 +5001,10 @@ lkpi_ic_scan_mindwell(struct ieee80211_scan_state *ss) bool is_hw_scan; lhw = ss->ss_ic->ic_softc; + TRACE_SCAN(ss->ss_ic, "scan_flags %b chan %d mindwell %lu", + lhw->scan_flags, LKPI_LHW_SCAN_BITS, + ss->ss_ic->ic_curchan->ic_ieee, ss->ss_mindwell); + LKPI_80211_LHW_SCAN_LOCK(lhw); is_hw_scan = (lhw->scan_flags & LKPI_LHW_SCAN_HW) != 0; LKPI_80211_LHW_SCAN_UNLOCK(lhw); @@ -6042,6 +6402,7 @@ linuxkpi_ieee80211_alloc_hw(size_t priv_len, const struct ieee80211_ops *ops) LKPI_80211_LHW_SCAN_LOCK_INIT(lhw); LKPI_80211_LHW_TXQ_LOCK_INIT(lhw); + spin_lock_init(&lhw->txq_lock); sx_init_flags(&lhw->lvif_sx, "lhw-lvif", SX_RECURSE | SX_DUPOK); LKPI_80211_LHW_MC_LOCK_INIT(lhw); TAILQ_INIT(&lhw->lvif_head); @@ -6147,6 +6508,7 @@ linuxkpi_ieee80211_iffree(struct ieee80211_hw *hw) LKPI_80211_LHW_MC_UNLOCK(lhw); /* Cleanup more of lhw here or in wiphy_free()? */ + spin_lock_destroy(&lhw->txq_lock); LKPI_80211_LHW_TXQ_LOCK_DESTROY(lhw); LKPI_80211_LHW_SCAN_LOCK_DESTROY(lhw); sx_destroy(&lhw->lvif_sx); @@ -6255,26 +6617,26 @@ linuxkpi_ieee80211_ifattach(struct ieee80211_hw *hw) IEEE80211_C_SHSLOT | /* short slot time supported */ IEEE80211_C_SHPREAMBLE /* short preamble supported */ ; -#if 0 - /* Scanning is a different kind of beast to re-work. */ - ic->ic_caps |= IEEE80211_C_BGSCAN; + +#ifdef LKPI_80211_BGSCAN + if (lhw->ops->hw_scan) + ic->ic_caps |= IEEE80211_C_BGSCAN; #endif - if (lhw->ops->hw_scan) { - /* - * Advertise full-offload scanning. - * - * Not limiting to SINGLE_SCAN_ON_ALL_BANDS here as otherwise - * we essentially disable hw_scan for all drivers not setting - * the flag. - */ - ic->ic_flags_ext |= IEEE80211_FEXT_SCAN_OFFLOAD; - lhw->scan_flags |= LKPI_LHW_SCAN_HW; - } + + lkpi_enable_hw_scan(lhw); /* Does HW support Fragmentation offload? */ if (ieee80211_hw_check(hw, SUPPORTS_TX_FRAG)) ic->ic_flags_ext |= IEEE80211_FEXT_FRAG_OFFLOAD; + /* Does HW support full AMPDU[-TX] offload? */ + if (ieee80211_hw_check(hw, AMPDU_AGGREGATION)) + ic->ic_flags_ext |= IEEE80211_FEXT_AMPDU_OFFLOAD; +#ifdef __notyet__ + if (ieee80211_hw_check(hw, TX_AMSDU)) + if (ieee80211_hw_check(hw, SUPPORTS_AMSDU_IN_AMPDU)) +#endif + /* * The wiphy variables report bitmasks of avail antennas. * (*get_antenna) get the current bitmask sets which can be @@ -6726,13 +7088,19 @@ linuxkpi_ieee80211_scan_completed(struct ieee80211_hw *hw, ic = lhw->ic; ss = ic->ic_scan; + TRACE_SCAN(ic, "scan_flags %b info { %ju, %6D, aborted %d }", + lhw->scan_flags, LKPI_LHW_SCAN_BITS, + (uintmax_t)info->scan_start_tsf, info->tsf_bssid, ":", + info->aborted); + ieee80211_scan_done(ss->ss_vap); LKPI_80211_LHW_SCAN_LOCK(lhw); free(lhw->hw_req, M_LKPI80211); lhw->hw_req = NULL; lhw->scan_flags &= ~LKPI_LHW_SCAN_RUNNING; - wakeup(lhw); + /* The wakeup(lhw) will be called from lkpi_ic_scan_end(). */ + /* wakeup(lhw); */ LKPI_80211_LHW_SCAN_UNLOCK(lhw); return; @@ -7002,6 +7370,63 @@ lkpi_convert_rx_status(struct ieee80211_hw *hw, struct lkpi_sta *lsta, } } +#ifdef LINUXKPI_DEBUG_80211 +static void +lkpi_rx_log_beacon(struct mbuf *m, struct lkpi_hw *lhw, + struct ieee80211_rx_status *rx_status) +{ + struct ieee80211_mgmt *f; + uint8_t *e; + char ssid[IEEE80211_NWID_LEN * 4 + 1]; + + memset(ssid, '\0', sizeof(ssid)); + + f = mtod(m, struct ieee80211_mgmt *); + e = f->u.beacon.variable; + /* + * Usually SSID is right after the fixed part and for debugging we will + * be fine should we miss it if it is not. + */ + while ((e - (uint8_t *)f) < m->m_len) { + if (*e == IEEE80211_ELEMID_SSID) + break; + e += (2 + *(e + 1)); + } + if (*e == IEEE80211_ELEMID_SSID) { + int i, len; + char *p; + + p = ssid; + len = m->m_len - ((e + 2) - (uint8_t *)f); + if (len > *(e + 1)) + len = *(e + 1); + e += 2; + for (i = 0; i < len; i++) { + /* Printable character? */ + if (*e >= 0x20 && *e < 0x7f) { + *p++ = *e++; + } else { + snprintf(p, 5, "%#04x", *e++); + p += 4; + } + } + *p = '\0'; + } + + /* We print skb, skb->data, m as we are seeing 'ghost beacons'. */ + TRACE_SCAN_BEACON(lhw->ic, "Beacon: scan_flags %b, band %s freq %u chan %-4d " + "len %d { %#06x %#06x %6D %6D %6D %#06x %ju %u %#06x SSID '%s' }", + lhw->scan_flags, LKPI_LHW_SCAN_BITS, + lkpi_nl80211_band_name(rx_status->band), rx_status->freq, + linuxkpi_ieee80211_frequency_to_channel(rx_status->freq, 0), + m->m_pkthdr.len, f->frame_control, f->duration_id, + f->da, ":", f->sa, ":", f->bssid, ":", f->seq_ctrl, + (uintmax_t)le64_to_cpu(f->u.beacon.timestamp), + le16_to_cpu(f->u.beacon.beacon_int), + le16_to_cpu(f->u.beacon.capab_info), ssid); +} +#endif + /* For %list see comment towards the end of the function. */ void linuxkpi_ieee80211_rx(struct ieee80211_hw *hw, struct sk_buff *skb, @@ -7058,7 +7483,15 @@ linuxkpi_ieee80211_rx(struct ieee80211_hw *hw, struct sk_buff *skb, is_beacon = ieee80211_is_beacon(hdr->frame_control); #ifdef LINUXKPI_DEBUG_80211 - if (is_beacon && (linuxkpi_debug_80211 & D80211_TRACE_RX_BEACONS) == 0) + /* + * We use the mbuf here as otherwise the variable part might + * be in skb frags. + */ + if (is_beacon && ((linuxkpi_debug_80211 & D80211_SCAN_BEACON) != 0)) + lkpi_rx_log_beacon(m, lhw, rx_status); + + if (is_beacon && (linuxkpi_debug_80211 & D80211_TRACE_RX_BEACONS) == 0 && + (linuxkpi_debug_80211 & D80211_SCAN_BEACON) == 0) goto no_trace_beacons; if (linuxkpi_debug_80211 & D80211_TRACE_RX) @@ -7073,7 +7506,8 @@ linuxkpi_ieee80211_rx(struct ieee80211_hw *hw, struct sk_buff *skb, hexdump(mtod(m, const void *), m->m_len, "RX (raw) ", 0); /* Implement a dump_rxcb() !!! */ - if (linuxkpi_debug_80211 & D80211_TRACE_RX) + if ((linuxkpi_debug_80211 & D80211_TRACE_RX) != 0 || + (linuxkpi_debug_80211 & D80211_SCAN_BEACON) != 0) printf("TRACE-RX: %s: RXCB: %ju %ju %u, %b, %u, %#0x, %#0x, " "%u band %u, %u { %d %d %d %d }, %d, %#x %#x %#x %#x %u %u %u\n", __func__, @@ -8124,21 +8558,30 @@ lkpi_ieee80211_wake_queues_locked(struct ieee80211_hw *hw) void linuxkpi_ieee80211_wake_queues(struct ieee80211_hw *hw) { - wiphy_lock(hw->wiphy); + struct lkpi_hw *lhw; + unsigned long flags; + + lhw = HW_TO_LHW(hw); + + spin_lock_irqsave(&lhw->txq_lock, flags); lkpi_ieee80211_wake_queues_locked(hw); - wiphy_unlock(hw->wiphy); + spin_unlock_irqrestore(&lhw->txq_lock, flags); } void linuxkpi_ieee80211_wake_queue(struct ieee80211_hw *hw, int qnum) { + struct lkpi_hw *lhw; + unsigned long flags; KASSERT(qnum < hw->queues, ("%s: qnum %d >= hw->queues %d, hw %p\n", __func__, qnum, hw->queues, hw)); - wiphy_lock(hw->wiphy); + lhw = HW_TO_LHW(hw); + + spin_lock_irqsave(&lhw->txq_lock, flags); lkpi_ieee80211_wake_queues(hw, qnum); - wiphy_unlock(hw->wiphy); + spin_unlock_irqrestore(&lhw->txq_lock, flags); } /* This is just hardware queues. */ diff --git a/sys/compat/linuxkpi/common/src/linux_80211.h b/sys/compat/linuxkpi/common/src/linux_80211.h index eaf6d804af4c..0dfcd7646c34 100644 --- a/sys/compat/linuxkpi/common/src/linux_80211.h +++ b/sys/compat/linuxkpi/common/src/linux_80211.h @@ -59,6 +59,8 @@ #define D80211_IMPROVE_TXQ 0x00000004 #define D80211_TRACE 0x00000010 #define D80211_TRACEOK 0x00000020 +#define D80211_SCAN 0x00000040 +#define D80211_SCAN_BEACON 0x00000080 #define D80211_TRACE_TX 0x00000100 #define D80211_TRACE_TX_DUMP 0x00000200 #define D80211_TRACE_RX 0x00001000 @@ -75,6 +77,20 @@ #define D80211_TRACE_MODE_HE 0x04000000 #define D80211_TRACE_MODE_EHT 0x08000000 +#ifdef LINUXKPI_DEBUG_80211 +#define TRACE_SCAN(ic, fmt, ...) \ + if (linuxkpi_debug_80211 & D80211_SCAN) \ + printf("%s:%d: %s SCAN " fmt "\n", \ + __func__, __LINE__, ic->ic_name, ##__VA_ARGS__) +#define TRACE_SCAN_BEACON(ic, fmt, ...) \ + if (linuxkpi_debug_80211 & D80211_SCAN_BEACON) \ + printf("%s:%d: %s SCAN " fmt "\n", \ + __func__, __LINE__, ic->ic_name, ##__VA_ARGS__) +#else +#define TRACE_SCAN(...) do {} while (0) +#define TRACE_SCAN_BEACON(...) do {} while (0) +#endif + #define IMPROVE_TXQ(...) \ if (linuxkpi_debug_80211 & D80211_IMPROVE_TXQ) \ printf("%s:%d: XXX LKPI80211 IMPROVE_TXQ\n", __func__, __LINE__) @@ -191,6 +207,7 @@ struct lkpi_vif { struct mbuf *, int, const struct ieee80211_rx_stats *, int, int); + struct task sw_scan_task; struct list_head lsta_list; @@ -236,6 +253,7 @@ struct lkpi_hw { /* name it mac80211_sc? */ struct mtx txq_mtx; uint32_t txq_generation[IEEE80211_NUM_ACS]; TAILQ_HEAD(, lkpi_txq) scheduled_txqs[IEEE80211_NUM_ACS]; + spinlock_t txq_lock; /* Deferred RX path. */ struct task rxq_task; @@ -298,6 +316,9 @@ struct lkpi_hw { /* name it mac80211_sc? */ #define LHW_TO_HW(_lhw) (&(_lhw)->hw) #define HW_TO_LHW(_hw) container_of(_hw, struct lkpi_hw, hw) +#define LKPI_LHW_SCAN_BITS \ + "\010\1RUNING\2HW" + struct lkpi_chanctx { struct list_head entry; diff --git a/sys/conf/options b/sys/conf/options index 4009ba2b4843..66f7f2ee2d7e 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -53,7 +53,7 @@ DDB_CAPTURE_MAXBUFSIZE opt_ddb.h DDB_CTF opt_ddb.h DDB_NUMSYM opt_ddb.h EARLY_PRINTF opt_global.h -BLOAT_KERNEL_WITH_EXTERR opt_global.h +EXTERR_STRINGS opt_global.h FULL_BUF_TRACKING opt_global.h GDB KDB opt_global.h diff --git a/sys/contrib/dev/rtw88/main.c b/sys/contrib/dev/rtw88/main.c index 021d076808e0..963b73f35350 100644 --- a/sys/contrib/dev/rtw88/main.c +++ b/sys/contrib/dev/rtw88/main.c @@ -57,6 +57,62 @@ module_param_named(support_vht, rtw_vht_support, bool, 0644); MODULE_PARM_DESC(support_vht, "Set to Y to enable VHT support"); #endif +#if defined(__FreeBSD__) +/* Macros based on rtw89::core.c. */ +#define RTW88_DEF_CHAN(_freq, _hw_val, _flags, _band) \ + { .center_freq = _freq, .hw_value = _hw_val, .flags = _flags, .band = _band, } +#define RTW88_DEF_CHAN_2G(_freq, _hw_val) \ + RTW88_DEF_CHAN(_freq, _hw_val, 0, NL80211_BAND_2GHZ) +#define RTW88_DEF_CHAN_5G(_freq, _hw_val) \ + RTW88_DEF_CHAN(_freq, _hw_val, 0, NL80211_BAND_5GHZ) +#define RTW88_DEF_CHAN_5G_NO_HT40MINUS(_freq, _hw_val) \ + RTW88_DEF_CHAN(_freq, _hw_val, IEEE80211_CHAN_NO_HT40MINUS, NL80211_BAND_5GHZ) + +static struct ieee80211_channel rtw_channeltable_2g[] = { + RTW88_DEF_CHAN_2G(2412, 1), + RTW88_DEF_CHAN_2G(2417, 2), + RTW88_DEF_CHAN_2G(2422, 3), + RTW88_DEF_CHAN_2G(2427, 4), + RTW88_DEF_CHAN_2G(2432, 5), + RTW88_DEF_CHAN_2G(2437, 6), + RTW88_DEF_CHAN_2G(2442, 7), + RTW88_DEF_CHAN_2G(2447, 8), + RTW88_DEF_CHAN_2G(2452, 9), + RTW88_DEF_CHAN_2G(2457, 10), + RTW88_DEF_CHAN_2G(2462, 11), + RTW88_DEF_CHAN_2G(2467, 12), + RTW88_DEF_CHAN_2G(2472, 13), + RTW88_DEF_CHAN_2G(2484, 14), +}; + +static struct ieee80211_channel rtw_channeltable_5g[] = { + RTW88_DEF_CHAN_5G(5180, 36), + RTW88_DEF_CHAN_5G(5200, 40), + RTW88_DEF_CHAN_5G(5220, 44), + RTW88_DEF_CHAN_5G(5240, 48), + RTW88_DEF_CHAN_5G(5260, 52), + RTW88_DEF_CHAN_5G(5280, 56), + RTW88_DEF_CHAN_5G(5300, 60), + RTW88_DEF_CHAN_5G(5320, 64), + RTW88_DEF_CHAN_5G(5500, 100), + RTW88_DEF_CHAN_5G(5520, 104), + RTW88_DEF_CHAN_5G(5540, 108), + RTW88_DEF_CHAN_5G(5560, 112), + RTW88_DEF_CHAN_5G(5580, 116), + RTW88_DEF_CHAN_5G(5600, 120), + RTW88_DEF_CHAN_5G(5620, 124), + RTW88_DEF_CHAN_5G(5640, 128), + RTW88_DEF_CHAN_5G(5660, 132), + RTW88_DEF_CHAN_5G(5680, 136), + RTW88_DEF_CHAN_5G(5700, 140), + RTW88_DEF_CHAN_5G(5720, 144), + RTW88_DEF_CHAN_5G(5745, 149), + RTW88_DEF_CHAN_5G(5765, 153), + RTW88_DEF_CHAN_5G(5785, 157), + RTW88_DEF_CHAN_5G(5805, 161), + RTW88_DEF_CHAN_5G_NO_HT40MINUS(5825, 165), +}; +#elif deifned(__linux__) static struct ieee80211_channel rtw_channeltable_2g[] = { {.center_freq = 2412, .hw_value = 1,}, {.center_freq = 2417, .hw_value = 2,}, @@ -102,6 +158,7 @@ static struct ieee80211_channel rtw_channeltable_5g[] = { {.center_freq = 5825, .hw_value = 165, .flags = IEEE80211_CHAN_NO_HT40MINUS}, }; +#endif static struct ieee80211_rate rtw_ratetable[] = { {.bitrate = 10, .hw_value = 0x00,}, diff --git a/sys/dev/ath/ath_rate/sample/sample.c b/sys/dev/ath/ath_rate/sample/sample.c index 291d1ec64ed7..79bf08678249 100644 --- a/sys/dev/ath/ath_rate/sample/sample.c +++ b/sys/dev/ath/ath_rate/sample/sample.c @@ -179,7 +179,7 @@ ath_rate_sample_find_min_pktlength(struct ath_softc *sc, const struct txschedule *sched = &sn->sched[rix0]; int max_pkt_length = 65530; // ATH_AGGR_MAXSIZE // Note: this may not be true in all cases; need to check? - int is_ht40 = (an->an_node.ni_chw == IEEE80211_STA_RX_BW_40); + int is_ht40 = (an->an_node.ni_chw == NET80211_STA_RX_BW_40); // Note: not great, but good enough.. int idx = is_ht40 ? MCS_HT40 : MCS_HT20; @@ -979,7 +979,7 @@ update_stats(struct ath_softc *sc, struct ath_node *an, const int size_bin = size_to_bin(frame_size); const int size = bin_to_size(size_bin); int tt; - int is_ht40 = (an->an_node.ni_chw == IEEE80211_STA_RX_BW_40); + int is_ht40 = (an->an_node.ni_chw == NET80211_STA_RX_BW_40); int pct; if (!IS_RATE_DEFINED(sn, rix0)) @@ -1365,7 +1365,7 @@ ath_rate_ctl_reset(struct ath_softc *sc, struct ieee80211_node *ni) continue; printf(" %d %s/%d", dot11rate(rt, rix), dot11rate_label(rt, rix), calc_usecs_unicast_packet(sc, 1600, rix, 0,0, - (ni->ni_chw == IEEE80211_STA_RX_BW_40))); + (ni->ni_chw == NET80211_STA_RX_BW_40))); } printf("\n"); } @@ -1396,7 +1396,7 @@ ath_rate_ctl_reset(struct ath_softc *sc, struct ieee80211_node *ni) sn->stats[y][rix].perfect_tx_time = calc_usecs_unicast_packet(sc, size, rix, 0, 0, - (ni->ni_chw == IEEE80211_STA_RX_BW_40)); + (ni->ni_chw == NET80211_STA_RX_BW_40)); sn->stats[y][rix].average_tx_time = sn->stats[y][rix].perfect_tx_time; } diff --git a/sys/dev/ath/if_ath_tx_ht.c b/sys/dev/ath/if_ath_tx_ht.c index e7ee029fecf0..f42058bacb0d 100644 --- a/sys/dev/ath/if_ath_tx_ht.c +++ b/sys/dev/ath/if_ath_tx_ht.c @@ -283,7 +283,7 @@ ath_tx_rate_fill_rcflags(struct ath_softc *sc, struct ath_buf *bf) if (IS_HT_RATE(rate)) { rc[i].flags |= ATH_RC_HT_FLAG; - if (ni->ni_chw == IEEE80211_STA_RX_BW_40) + if (ni->ni_chw == NET80211_STA_RX_BW_40) rc[i].flags |= ATH_RC_CW40_FLAG; /* @@ -295,13 +295,13 @@ ath_tx_rate_fill_rcflags(struct ath_softc *sc, struct ath_buf *bf) * and doesn't return the fractional part, so * we are always "out" by some amount. */ - if (ni->ni_chw == IEEE80211_STA_RX_BW_40 && + if (ni->ni_chw == NET80211_STA_RX_BW_40 && ieee80211_ht_check_tx_shortgi_40(ni) && (bf->bf_flags & ATH_BUF_TOA_PROBE) == 0) { rc[i].flags |= ATH_RC_SGI_FLAG; } - if (ni->ni_chw == IEEE80211_STA_RX_BW_20 && + if (ni->ni_chw == NET80211_STA_RX_BW_20 && ieee80211_ht_check_tx_shortgi_20(ni) && (bf->bf_flags & ATH_BUF_TOA_PROBE) == 0) { rc[i].flags |= ATH_RC_SGI_FLAG; diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index 9e91250cb61c..9756a6945384 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -9016,7 +9016,7 @@ sysctl_loadavg(SYSCTL_HANDLER_ARGS) rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4lavg"); if (rc) return (rc); - if (hw_all_ok(sc)) + if (!hw_all_ok(sc)) rc = ENXIO; else { param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | diff --git a/sys/dev/gpio/gpio_if.m b/sys/dev/gpio/gpio_if.m index 5501b2b5c0e7..0b6988ceba79 100644 --- a/sys/dev/gpio/gpio_if.m +++ b/sys/dev/gpio/gpio_if.m @@ -62,6 +62,22 @@ CODE { return (0); } + + static int + gpio_default_get_pin_list(device_t dev, uint32_t *pin_list) + { + uint32_t maxpin; + int err; + + err = GPIO_PIN_MAX(dev, &maxpin); + if (err != 0) + return (ENXIO); + + for (int i = 0; i <= maxpin; i++) + pin_list[i] = i; + + return (0); + } }; HEADER { @@ -185,3 +201,13 @@ METHOD int pin_config_32 { uint32_t num_pins; uint32_t *pin_flags; } DEFAULT gpio_default_nosupport; + +# +# Get the controller's pin numbers. pin_list is expected to be an array with at +# least GPIO_PIN_MAX() elements. Populates pin_list from 0 to GPIO_PIN_MAX() by +# default. +# +METHOD int get_pin_list { + device_t dev; + uint32_t *pin_list; +} DEFAULT gpio_default_get_pin_list; diff --git a/sys/dev/gpio/gpiobus.c b/sys/dev/gpio/gpiobus.c index 5f1f6532a79b..698b5e5fdd01 100644 --- a/sys/dev/gpio/gpiobus.c +++ b/sys/dev/gpio/gpiobus.c @@ -319,10 +319,6 @@ gpiobus_add_bus(device_t dev) busdev = device_add_child(dev, "gpiobus", DEVICE_UNIT_ANY); if (busdev == NULL) return (NULL); - if (device_add_child(dev, "gpioc", DEVICE_UNIT_ANY) == NULL) { - device_delete_child(dev, busdev); - return (NULL); - } #ifdef FDT ofw_gpiobus_register_provider(dev); #endif @@ -372,6 +368,37 @@ gpiobus_init_softc(device_t dev) } int +gpiobus_add_gpioc(device_t dev) +{ + struct gpiobus_ivar *devi; + struct gpiobus_softc *sc; + device_t gpioc; + int err; + + gpioc = BUS_ADD_CHILD(dev, 0, "gpioc", DEVICE_UNIT_ANY); + if (gpioc == NULL) + return (ENXIO); + + sc = device_get_softc(dev); + devi = device_get_ivars(gpioc); + + devi->npins = sc->sc_npins; + err = gpiobus_alloc_ivars(devi); + if (err != 0) { + device_delete_child(dev, gpioc); + return (err); + } + + err = GPIO_GET_PIN_LIST(sc->sc_dev, devi->pins); + if (err != 0) { + device_delete_child(dev, gpioc); + gpiobus_free_ivars(devi); + } + + return (err); +} + +int gpiobus_alloc_ivars(struct gpiobus_ivar *devi) { @@ -562,6 +589,10 @@ gpiobus_attach(device_t dev) if (err != 0) return (err); + err = gpiobus_add_gpioc(dev); + if (err != 0) + return (err); + /* * Get parent's pins and mark them as unmapped */ @@ -961,7 +992,7 @@ gpiobus_pin_getflags(device_t dev, device_t child, uint32_t pin, if (pin >= devi->npins) return (EINVAL); - return GPIO_PIN_GETFLAGS(sc->sc_dev, devi->pins[pin], flags); + return (GPIO_PIN_GETFLAGS(sc->sc_dev, devi->pins[pin], flags)); } static int @@ -974,7 +1005,7 @@ gpiobus_pin_getcaps(device_t dev, device_t child, uint32_t pin, if (pin >= devi->npins) return (EINVAL); - return GPIO_PIN_GETCAPS(sc->sc_dev, devi->pins[pin], caps); + return (GPIO_PIN_GETCAPS(sc->sc_dev, devi->pins[pin], caps)); } static int @@ -987,7 +1018,7 @@ gpiobus_pin_set(device_t dev, device_t child, uint32_t pin, if (pin >= devi->npins) return (EINVAL); - return GPIO_PIN_SET(sc->sc_dev, devi->pins[pin], value); + return (GPIO_PIN_SET(sc->sc_dev, devi->pins[pin], value)); } static int @@ -1000,7 +1031,7 @@ gpiobus_pin_get(device_t dev, device_t child, uint32_t pin, if (pin >= devi->npins) return (EINVAL); - return GPIO_PIN_GET(sc->sc_dev, devi->pins[pin], value); + return (GPIO_PIN_GET(sc->sc_dev, devi->pins[pin], value)); } static int @@ -1012,7 +1043,57 @@ gpiobus_pin_toggle(device_t dev, device_t child, uint32_t pin) if (pin >= devi->npins) return (EINVAL); - return GPIO_PIN_TOGGLE(sc->sc_dev, devi->pins[pin]); + return (GPIO_PIN_TOGGLE(sc->sc_dev, devi->pins[pin])); +} + +/* + * Verify that a child has all the pins they are requesting + * to access in their ivars. + */ +static bool +gpiobus_pin_verify_32(struct gpiobus_ivar *devi, uint32_t first_pin, + uint32_t num_pins) +{ + if (first_pin + num_pins > devi->npins) + return (false); + + /* Make sure the pins are consecutive. */ + for (uint32_t pin = first_pin; pin < first_pin + num_pins - 1; pin++) { + if (devi->pins[pin] + 1 != devi->pins[pin + 1]) + return (false); + } + + return (true); +} + +static int +gpiobus_pin_access_32(device_t dev, device_t child, uint32_t first_pin, + uint32_t clear_pins, uint32_t change_pins, uint32_t *orig_pins) +{ + struct gpiobus_softc *sc = GPIOBUS_SOFTC(dev); + struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); + + if (!gpiobus_pin_verify_32(devi, first_pin, 32)) + return (EINVAL); + + return (GPIO_PIN_ACCESS_32(sc->sc_dev, devi->pins[first_pin], + clear_pins, change_pins, orig_pins)); +} + +static int +gpiobus_pin_config_32(device_t dev, device_t child, uint32_t first_pin, + uint32_t num_pins, uint32_t *pin_flags) +{ + struct gpiobus_softc *sc = GPIOBUS_SOFTC(dev); + struct gpiobus_ivar *devi = GPIOBUS_IVAR(child); + + if (num_pins > 32) + return (EINVAL); + if (!gpiobus_pin_verify_32(devi, first_pin, num_pins)) + return (EINVAL); + + return (GPIO_PIN_CONFIG_32(sc->sc_dev, + devi->pins[first_pin], num_pins, pin_flags)); } static int @@ -1093,6 +1174,8 @@ static device_method_t gpiobus_methods[] = { DEVMETHOD(gpiobus_pin_get, gpiobus_pin_get), DEVMETHOD(gpiobus_pin_set, gpiobus_pin_set), DEVMETHOD(gpiobus_pin_toggle, gpiobus_pin_toggle), + DEVMETHOD(gpiobus_pin_access_32,gpiobus_pin_access_32), + DEVMETHOD(gpiobus_pin_config_32,gpiobus_pin_config_32), DEVMETHOD(gpiobus_pin_getname, gpiobus_pin_getname), DEVMETHOD(gpiobus_pin_setname, gpiobus_pin_setname), diff --git a/sys/dev/gpio/gpiobus_if.m b/sys/dev/gpio/gpiobus_if.m index 8bf29839ef4e..890738c4e809 100644 --- a/sys/dev/gpio/gpiobus_if.m +++ b/sys/dev/gpio/gpiobus_if.m @@ -107,6 +107,36 @@ METHOD int pin_setflags { }; # +# Simultaneously read and/or change up to 32 adjacent pins. +# If the device cannot change the pins simultaneously, returns EOPNOTSUPP. +# +# More details about using this interface can be found in sys/gpio.h +# +METHOD int pin_access_32 { + device_t dev; + device_t child; + uint32_t first_pin; + uint32_t clear_pins; + uint32_t change_pins; + uint32_t *orig_pins; +}; + +# +# Simultaneously configure up to 32 adjacent pins. +# This is intended to change the configuration of all the pins simultaneously, +# but unlike pin_access_32, this will not fail if the hardware can't do so. +# +# More details about using this interface can be found in sys/gpio.h +# +METHOD int pin_config_32 { + device_t dev; + device_t child; + uint32_t first_pin; + uint32_t num_pins; + uint32_t *pin_flags; +}; + +# # Get the pin name # METHOD int pin_getname { diff --git a/sys/dev/gpio/gpiobus_internal.h b/sys/dev/gpio/gpiobus_internal.h index c198e5f79989..58f862343403 100644 --- a/sys/dev/gpio/gpiobus_internal.h +++ b/sys/dev/gpio/gpiobus_internal.h @@ -44,6 +44,7 @@ int gpiobus_acquire_pin(device_t, uint32_t); void gpiobus_release_pin(device_t, uint32_t); int gpiobus_child_location(device_t, device_t, struct sbuf *); device_t gpiobus_add_child_common(device_t, u_int, const char *, int, size_t); +int gpiobus_add_gpioc(device_t); extern driver_t gpiobus_driver; #endif diff --git a/sys/dev/gpio/gpioc.c b/sys/dev/gpio/gpioc.c index 87fed38ebe3e..5a60f939dc78 100644 --- a/sys/dev/gpio/gpioc.c +++ b/sys/dev/gpio/gpioc.c @@ -45,7 +45,6 @@ #include <dev/gpio/gpiobusvar.h> -#include "gpio_if.h" #include "gpiobus_if.h" #undef GPIOC_DEBUG @@ -59,7 +58,7 @@ struct gpioc_softc { device_t sc_dev; /* gpiocX dev */ - device_t sc_pdev; /* gpioX dev */ + device_t sc_pdev; /* gpiobusX dev */ struct cdev *sc_ctl_dev; /* controller device */ int sc_unit; int sc_npins; @@ -69,6 +68,7 @@ struct gpioc_softc { struct gpioc_pin_intr { struct gpioc_softc *sc; gpio_pin_t pin; + uint32_t intr_mode; bool config_locked; int intr_rid; struct resource *intr_res; @@ -112,8 +112,10 @@ struct gpioc_pin_event { static MALLOC_DEFINE(M_GPIOC, "gpioc", "gpioc device data"); -static int gpioc_allocate_pin_intr(struct gpioc_pin_intr*, uint32_t); -static int gpioc_release_pin_intr(struct gpioc_pin_intr*); +static int gpioc_allocate_pin_intr(struct gpioc_softc*, + struct gpioc_pin_intr*, uint32_t, uint32_t); +static int gpioc_release_pin_intr(struct gpioc_softc*, + struct gpioc_pin_intr*); static int gpioc_attach_priv_pin(struct gpioc_cdevpriv*, struct gpioc_pin_intr*); static int gpioc_detach_priv_pin(struct gpioc_cdevpriv*, @@ -191,27 +193,36 @@ number_of_events(struct gpioc_cdevpriv *priv) } static int -gpioc_allocate_pin_intr(struct gpioc_pin_intr *intr_conf, uint32_t flags) +gpioc_allocate_pin_intr(struct gpioc_softc *sc, + struct gpioc_pin_intr *intr_conf, uint32_t pin, uint32_t flags) { int err; intr_conf->config_locked = true; mtx_unlock(&intr_conf->mtx); - intr_conf->intr_res = gpio_alloc_intr_resource(intr_conf->pin->dev, + MPASS(intr_conf->pin == NULL); + err = gpio_pin_get_by_bus_pinnum(sc->sc_pdev, pin, &intr_conf->pin); + if (err != 0) + goto error_exit; + + intr_conf->intr_res = gpio_alloc_intr_resource(sc->sc_dev, &intr_conf->intr_rid, RF_ACTIVE, intr_conf->pin, flags); if (intr_conf->intr_res == NULL) { err = ENXIO; - goto error_exit; + goto error_pin; } - err = bus_setup_intr(intr_conf->pin->dev, intr_conf->intr_res, + err = bus_setup_intr(sc->sc_dev, intr_conf->intr_res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, gpioc_interrupt_handler, intr_conf, &intr_conf->intr_cookie); - if (err != 0) - goto error_exit; + if (err != 0) { + bus_release_resource(sc->sc_dev, intr_conf->intr_res); + intr_conf->intr_res = NULL; + goto error_pin; + } - intr_conf->pin->flags = flags; + intr_conf->intr_mode = flags; error_exit: mtx_lock(&intr_conf->mtx); @@ -219,10 +230,15 @@ error_exit: wakeup(&intr_conf->config_locked); return (err); + +error_pin: + gpio_pin_release(intr_conf->pin); + intr_conf->pin = NULL; + goto error_exit; } static int -gpioc_release_pin_intr(struct gpioc_pin_intr *intr_conf) +gpioc_release_pin_intr(struct gpioc_softc *sc, struct gpioc_pin_intr *intr_conf) { int err; @@ -230,8 +246,8 @@ gpioc_release_pin_intr(struct gpioc_pin_intr *intr_conf) mtx_unlock(&intr_conf->mtx); if (intr_conf->intr_cookie != NULL) { - err = bus_teardown_intr(intr_conf->pin->dev, - intr_conf->intr_res, intr_conf->intr_cookie); + err = bus_teardown_intr(sc->sc_dev, intr_conf->intr_res, + intr_conf->intr_cookie); if (err != 0) goto error_exit; else @@ -239,7 +255,7 @@ gpioc_release_pin_intr(struct gpioc_pin_intr *intr_conf) } if (intr_conf->intr_res != NULL) { - err = bus_release_resource(intr_conf->pin->dev, SYS_RES_IRQ, + err = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, intr_conf->intr_rid, intr_conf->intr_res); if (err != 0) goto error_exit; @@ -249,7 +265,10 @@ gpioc_release_pin_intr(struct gpioc_pin_intr *intr_conf) } } - intr_conf->pin->flags = 0; + gpio_pin_release(intr_conf->pin); + intr_conf->pin = NULL; + + intr_conf->intr_mode = 0; err = 0; error_exit: @@ -386,7 +405,7 @@ gpioc_get_intr_config(struct gpioc_softc *sc, struct gpioc_cdevpriv *priv, struct gpioc_privs *priv_link; uint32_t flags; - flags = intr_conf->pin->flags; + flags = intr_conf->intr_mode; if (flags == 0) return (0); @@ -411,7 +430,7 @@ gpioc_set_intr_config(struct gpioc_softc *sc, struct gpioc_cdevpriv *priv, int res; res = 0; - if (intr_conf->pin->flags == 0 && flags == 0) { + if (intr_conf->intr_mode == 0 && flags == 0) { /* No interrupt configured and none requested: Do nothing. */ return (0); } @@ -419,17 +438,17 @@ gpioc_set_intr_config(struct gpioc_softc *sc, struct gpioc_cdevpriv *priv, while (intr_conf->config_locked == true) mtx_sleep(&intr_conf->config_locked, &intr_conf->mtx, 0, "gpicfg", 0); - if (intr_conf->pin->flags == 0 && flags != 0) { + if (intr_conf->intr_mode == 0 && flags != 0) { /* * No interrupt is configured, but one is requested: Allocate * and setup interrupt on the according pin. */ - res = gpioc_allocate_pin_intr(intr_conf, flags); + res = gpioc_allocate_pin_intr(sc, intr_conf, pin, flags); if (res == 0) res = gpioc_attach_priv_pin(priv, intr_conf); if (res == EEXIST) res = 0; - } else if (intr_conf->pin->flags == flags) { + } else if (intr_conf->intr_mode == flags) { /* * Same interrupt requested as already configured: Attach the * cdevpriv to the corresponding pin. @@ -437,14 +456,14 @@ gpioc_set_intr_config(struct gpioc_softc *sc, struct gpioc_cdevpriv *priv, res = gpioc_attach_priv_pin(priv, intr_conf); if (res == EEXIST) res = 0; - } else if (intr_conf->pin->flags != 0 && flags == 0) { + } else if (intr_conf->intr_mode != 0 && flags == 0) { /* * Interrupt configured, but none requested: Teardown and * release the pin when no other cdevpriv is attached. Otherwise * just detach pin and cdevpriv from each other. */ if (gpioc_intr_reconfig_allowed(priv, intr_conf)) { - res = gpioc_release_pin_intr(intr_conf); + res = gpioc_release_pin_intr(sc, intr_conf); } if (res == 0) res = gpioc_detach_priv_pin(priv, intr_conf); @@ -456,9 +475,10 @@ gpioc_set_intr_config(struct gpioc_softc *sc, struct gpioc_cdevpriv *priv, if (!gpioc_intr_reconfig_allowed(priv, intr_conf)) res = EBUSY; else { - res = gpioc_release_pin_intr(intr_conf); + res = gpioc_release_pin_intr(sc, intr_conf); if (res == 0) - res = gpioc_allocate_pin_intr(intr_conf, flags); + res = gpioc_allocate_pin_intr(sc, intr_conf, + pin, flags); if (res == 0) res = gpioc_attach_priv_pin(priv, intr_conf); if (res == EEXIST) @@ -475,18 +495,16 @@ gpioc_interrupt_handler(void *arg) { struct gpioc_pin_intr *intr_conf; struct gpioc_privs *privs; - struct gpioc_softc *sc; sbintime_t evtime; - uint32_t pin_state; + bool pin_state; intr_conf = arg; - sc = intr_conf->sc; /* Capture time and pin state first. */ evtime = sbinuptime(); - if (intr_conf->pin->flags & GPIO_INTR_EDGE_BOTH) - GPIO_PIN_GET(sc->sc_pdev, intr_conf->pin->pin, &pin_state); - else if (intr_conf->pin->flags & GPIO_INTR_EDGE_RISING) + if (intr_conf->intr_mode & GPIO_INTR_EDGE_BOTH) + gpio_pin_is_active(intr_conf->pin, &pin_state); + else if (intr_conf->intr_mode & GPIO_INTR_EDGE_RISING) pin_state = true; else pin_state = false; @@ -575,18 +593,11 @@ gpioc_attach(device_t dev) sc->sc_pdev = device_get_parent(dev); sc->sc_unit = device_get_unit(dev); - err = GPIO_PIN_MAX(sc->sc_pdev, &sc->sc_npins); - sc->sc_npins++; /* Number of pins is one more than max pin number. */ - if (err != 0) - return (err); + sc->sc_npins = gpiobus_get_npins(dev); sc->sc_pin_intr = malloc(sizeof(struct gpioc_pin_intr) * sc->sc_npins, M_GPIOC, M_WAITOK | M_ZERO); for (int i = 0; i < sc->sc_npins; i++) { - sc->sc_pin_intr[i].pin = malloc(sizeof(struct gpiobus_pin), - M_GPIOC, M_WAITOK | M_ZERO); sc->sc_pin_intr[i].sc = sc; - sc->sc_pin_intr[i].pin->pin = i; - sc->sc_pin_intr[i].pin->dev = sc->sc_pdev; mtx_init(&sc->sc_pin_intr[i].mtx, "gpioc pin", NULL, MTX_DEF); SLIST_INIT(&sc->sc_pin_intr[i].privs); } @@ -610,20 +621,16 @@ static int gpioc_detach(device_t dev) { struct gpioc_softc *sc = device_get_softc(dev); - int err; if (sc->sc_ctl_dev) destroy_dev(sc->sc_ctl_dev); for (int i = 0; i < sc->sc_npins; i++) { mtx_destroy(&sc->sc_pin_intr[i].mtx); - free(sc->sc_pin_intr[i].pin, M_GPIOC); + MPASS(sc->sc_pin_intr[i].pin == NULL); } free(sc->sc_pin_intr, M_GPIOC); - if ((err = bus_generic_detach(dev)) != 0) - return (err); - return (0); } @@ -655,7 +662,7 @@ gpioc_cdevpriv_dtor(void *data) KASSERT(consistency == 1, ("inconsistent links between pin config and cdevpriv")); if (gpioc_intr_reconfig_allowed(priv, pin_link->pin)) { - gpioc_release_pin_intr(pin_link->pin); + gpioc_release_pin_intr(priv->sc, pin_link->pin); } mtx_unlock(&pin_link->pin->mtx); SLIST_REMOVE(&priv->pins, pin_link, gpioc_pins, next); @@ -778,7 +785,6 @@ static int gpioc_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int fflag, struct thread *td) { - device_t bus; int max_pin, res; struct gpioc_softc *sc = cdev->si_drv1; struct gpioc_cdevpriv *priv; @@ -789,30 +795,32 @@ gpioc_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int fflag, struct gpio_event_config *evcfg; uint32_t caps, intrflags; - bus = GPIO_GET_BUS(sc->sc_pdev); - if (bus == NULL) - return (EINVAL); switch (cmd) { case GPIOMAXPIN: - max_pin = -1; - res = GPIO_PIN_MAX(sc->sc_pdev, &max_pin); + res = 0; + max_pin = sc->sc_npins - 1; bcopy(&max_pin, arg, sizeof(max_pin)); break; case GPIOGETCONFIG: bcopy(arg, &pin, sizeof(pin)); dprintf("get config pin %d\n", pin.gp_pin); - res = GPIO_PIN_GETFLAGS(sc->sc_pdev, pin.gp_pin, + res = GPIOBUS_PIN_GETFLAGS(sc->sc_pdev, sc->sc_dev, pin.gp_pin, &pin.gp_flags); /* Fail early */ - if (res) + if (res != 0) break; res = devfs_get_cdevpriv((void **)&priv); - if (res) + if (res != 0) break; pin.gp_flags |= gpioc_get_intr_config(sc, priv, pin.gp_pin); - GPIO_PIN_GETCAPS(sc->sc_pdev, pin.gp_pin, &pin.gp_caps); - GPIOBUS_PIN_GETNAME(bus, pin.gp_pin, pin.gp_name); + res = GPIOBUS_PIN_GETCAPS(sc->sc_pdev, sc->sc_dev, pin.gp_pin, + &pin.gp_caps); + if (res != 0) + break; + res = GPIOBUS_PIN_GETNAME(sc->sc_pdev, pin.gp_pin, pin.gp_name); + if (res != 0) + break; bcopy(&pin, arg, sizeof(pin)); break; case GPIOSETCONFIG: @@ -821,7 +829,8 @@ gpioc_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int fflag, res = devfs_get_cdevpriv((void **)&priv); if (res != 0) break; - res = GPIO_PIN_GETCAPS(sc->sc_pdev, pin.gp_pin, &caps); + res = GPIOBUS_PIN_GETCAPS(sc->sc_pdev, sc->sc_dev, + pin.gp_pin, &caps); if (res != 0) break; res = gpio_check_flags(caps, pin.gp_flags); @@ -847,8 +856,8 @@ gpioc_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int fflag, } if (res != 0) break; - res = GPIO_PIN_SETFLAGS(sc->sc_pdev, pin.gp_pin, - (pin.gp_flags & ~GPIO_INTR_MASK)); + res = GPIOBUS_PIN_SETFLAGS(sc->sc_pdev, sc->sc_dev, pin.gp_pin, + pin.gp_flags & ~GPIO_INTR_MASK); if (res != 0) break; res = gpioc_set_intr_config(sc, priv, pin.gp_pin, @@ -856,40 +865,43 @@ gpioc_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int fflag, break; case GPIOGET: bcopy(arg, &req, sizeof(req)); - res = GPIO_PIN_GET(sc->sc_pdev, req.gp_pin, + res = GPIOBUS_PIN_GET(sc->sc_pdev, sc->sc_dev, req.gp_pin, &req.gp_value); - dprintf("read pin %d -> %d\n", + if (res != 0) + break; + dprintf("read pin %d -> %d\n", req.gp_pin, req.gp_value); bcopy(&req, arg, sizeof(req)); break; case GPIOSET: bcopy(arg, &req, sizeof(req)); - res = GPIO_PIN_SET(sc->sc_pdev, req.gp_pin, + res = GPIOBUS_PIN_SET(sc->sc_pdev, sc->sc_dev, req.gp_pin, req.gp_value); - dprintf("write pin %d -> %d\n", + dprintf("write pin %d -> %d\n", req.gp_pin, req.gp_value); break; case GPIOTOGGLE: bcopy(arg, &req, sizeof(req)); - dprintf("toggle pin %d\n", + dprintf("toggle pin %d\n", req.gp_pin); - res = GPIO_PIN_TOGGLE(sc->sc_pdev, req.gp_pin); + res = GPIOBUS_PIN_TOGGLE(sc->sc_pdev, sc->sc_dev, req.gp_pin); break; case GPIOSETNAME: bcopy(arg, &pin, sizeof(pin)); dprintf("set name on pin %d\n", pin.gp_pin); - res = GPIOBUS_PIN_SETNAME(bus, pin.gp_pin, + res = GPIOBUS_PIN_SETNAME(sc->sc_pdev, pin.gp_pin, pin.gp_name); break; case GPIOACCESS32: a32 = (struct gpio_access_32 *)arg; - res = GPIO_PIN_ACCESS_32(sc->sc_pdev, a32->first_pin, - a32->clear_pins, a32->change_pins, &a32->orig_pins); + res = GPIOBUS_PIN_ACCESS_32(sc->sc_pdev, sc->sc_dev, + a32->first_pin, a32->clear_pins, a32->change_pins, + &a32->orig_pins); break; case GPIOCONFIG32: c32 = (struct gpio_config_32 *)arg; - res = GPIO_PIN_CONFIG_32(sc->sc_pdev, c32->first_pin, - c32->num_pins, c32->pin_flags); + res = GPIOBUS_PIN_CONFIG_32(sc->sc_pdev, sc->sc_dev, + c32->first_pin, c32->num_pins, c32->pin_flags); break; case GPIOCONFIGEVENTS: evcfg = (struct gpio_event_config *)arg; @@ -1050,9 +1062,6 @@ static device_method_t gpioc_methods[] = { DEVMETHOD(device_probe, gpioc_probe), DEVMETHOD(device_attach, gpioc_attach), DEVMETHOD(device_detach, gpioc_detach), - DEVMETHOD(device_shutdown, bus_generic_shutdown), - DEVMETHOD(device_suspend, bus_generic_suspend), - DEVMETHOD(device_resume, bus_generic_resume), DEVMETHOD_END }; @@ -1063,5 +1072,5 @@ driver_t gpioc_driver = { sizeof(struct gpioc_softc) }; -DRIVER_MODULE(gpioc, gpio, gpioc_driver, 0, 0); +DRIVER_MODULE(gpioc, gpiobus, gpioc_driver, 0, 0); MODULE_VERSION(gpioc, 1); diff --git a/sys/dev/gpio/ofw_gpiobus.c b/sys/dev/gpio/ofw_gpiobus.c index b12b78fac18c..da1bfbc268b8 100644 --- a/sys/dev/gpio/ofw_gpiobus.c +++ b/sys/dev/gpio/ofw_gpiobus.c @@ -426,6 +426,9 @@ ofw_gpiobus_attach(device_t dev) err = gpiobus_init_softc(dev); if (err != 0) return (err); + err = gpiobus_add_gpioc(dev); + if (err != 0) + return (err); bus_identify_children(dev); bus_enumerate_hinted_children(dev); /* diff --git a/sys/dev/mpr/mpr.c b/sys/dev/mpr/mpr.c index d1c572e40669..262d6b58b705 100644 --- a/sys/dev/mpr/mpr.c +++ b/sys/dev/mpr/mpr.c @@ -1729,6 +1729,7 @@ mpr_get_tunables(struct mpr_softc *sc) sc->enable_ssu = MPR_SSU_ENABLE_SSD_DISABLE_HDD; sc->spinup_wait_time = DEFAULT_SPINUP_WAIT; sc->use_phynum = 1; + sc->encl_min_slots = 0; sc->max_reqframes = MPR_REQ_FRAMES; sc->max_prireqframes = MPR_PRI_REQ_FRAMES; sc->max_replyframes = MPR_REPLY_FRAMES; @@ -1748,6 +1749,7 @@ mpr_get_tunables(struct mpr_softc *sc) TUNABLE_INT_FETCH("hw.mpr.enable_ssu", &sc->enable_ssu); TUNABLE_INT_FETCH("hw.mpr.spinup_wait_time", &sc->spinup_wait_time); TUNABLE_INT_FETCH("hw.mpr.use_phy_num", &sc->use_phynum); + TUNABLE_INT_FETCH("hw.mpr.encl_min_slots", &sc->encl_min_slots); TUNABLE_INT_FETCH("hw.mpr.max_reqframes", &sc->max_reqframes); TUNABLE_INT_FETCH("hw.mpr.max_prireqframes", &sc->max_prireqframes); TUNABLE_INT_FETCH("hw.mpr.max_replyframes", &sc->max_replyframes); @@ -1797,6 +1799,10 @@ mpr_get_tunables(struct mpr_softc *sc) device_get_unit(sc->mpr_dev)); TUNABLE_INT_FETCH(tmpstr, &sc->use_phynum); + snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.encl_min_slots", + device_get_unit(sc->mpr_dev)); + TUNABLE_INT_FETCH(tmpstr, &sc->encl_min_slots); + snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.max_reqframes", device_get_unit(sc->mpr_dev)); TUNABLE_INT_FETCH(tmpstr, &sc->max_reqframes); @@ -1951,6 +1957,10 @@ mpr_setup_sysctl(struct mpr_softc *sc) SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "prp_page_alloc_fail", CTLFLAG_RD, &sc->prp_page_alloc_fail, "PRP page allocation failures"); + + SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), + OID_AUTO, "encl_min_slots", CTLFLAG_RW, &sc->encl_min_slots, 0, + "force enclosure minimum slots"); } static struct mpr_debug_string { diff --git a/sys/dev/mpr/mpr_mapping.c b/sys/dev/mpr/mpr_mapping.c index f9a9ac1c53d0..38aa4dfc7ef2 100644 --- a/sys/dev/mpr/mpr_mapping.c +++ b/sys/dev/mpr/mpr_mapping.c @@ -2785,6 +2785,8 @@ mpr_mapping_enclosure_dev_status_change_event(struct mpr_softc *sc, * DPM, if it's being used. */ if (enc_idx != MPR_ENCTABLE_BAD_IDX) { + u16 new_num_slots; + et_entry = &sc->enclosure_table[enc_idx]; if (et_entry->init_complete && !et_entry->missing_count) { @@ -2796,6 +2798,17 @@ mpr_mapping_enclosure_dev_status_change_event(struct mpr_softc *sc, et_entry->enc_handle = le16toh(event_data-> EnclosureHandle); et_entry->start_slot = le16toh(event_data->StartSlot); + new_num_slots = le16toh(event_data->NumSlots); + if (new_num_slots < sc->encl_min_slots) { + mpr_dprint(sc, MPR_MAPPING, "%s: Enclosure %d num_slots %d, overriding with %d.\n", + __func__, enc_idx, new_num_slots, sc->encl_min_slots); + new_num_slots = sc->encl_min_slots; + } + if (et_entry->num_slots != new_num_slots) { + mpr_dprint(sc, MPR_MAPPING, "%s: Enclosure %d old num_slots %d, new %d.\n", + __func__, enc_idx, et_entry->num_slots, sc->encl_min_slots); + et_entry->num_slots = new_num_slots; + } saved_phy_bits = et_entry->phy_bits; et_entry->phy_bits |= le32toh(event_data->PhyBits); if (saved_phy_bits != et_entry->phy_bits) @@ -2858,6 +2871,11 @@ mpr_mapping_enclosure_dev_status_change_event(struct mpr_softc *sc, et_entry->start_index = MPR_MAPTABLE_BAD_IDX; et_entry->dpm_entry_num = MPR_DPM_BAD_IDX; et_entry->num_slots = le16toh(event_data->NumSlots); + if (et_entry->num_slots < sc->encl_min_slots) { + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: Enclosure %d num_slots is %d, overriding with %d.\n", + __func__, enc_idx, et_entry->num_slots, sc->encl_min_slots); + et_entry->num_slots = sc->encl_min_slots; + } et_entry->start_slot = le16toh(event_data->StartSlot); et_entry->phy_bits = le32toh(event_data->PhyBits); } diff --git a/sys/dev/mpr/mprvar.h b/sys/dev/mpr/mprvar.h index 0f1743f4266e..93f3fbffe079 100644 --- a/sys/dev/mpr/mprvar.h +++ b/sys/dev/mpr/mprvar.h @@ -366,6 +366,7 @@ struct mpr_softc { int spinup_wait_time; int use_phynum; int dump_reqs_alltypes; + int encl_min_slots; uint64_t chain_alloc_fail; uint64_t prp_page_alloc_fail; struct sysctl_ctx_list sysctl_ctx; diff --git a/sys/dev/mwl/if_mwl.c b/sys/dev/mwl/if_mwl.c index 0e2eb0b2d8fe..c885968dfe15 100644 --- a/sys/dev/mwl/if_mwl.c +++ b/sys/dev/mwl/if_mwl.c @@ -4017,7 +4017,7 @@ mkpeerinfo(MWL_HAL_PEERINFO *pi, const struct ieee80211_node *ni) pi->HTCapabilitiesInfo &= ~IEEE80211_HTCAP_SHORTGI40; if ((vap->iv_flags_ht & IEEE80211_FHT_SHORTGI20) == 0) pi->HTCapabilitiesInfo &= ~IEEE80211_HTCAP_SHORTGI20; - if (ni->ni_chw != IEEE80211_STA_RX_BW_40) + if (ni->ni_chw != NET80211_STA_RX_BW_40) pi->HTCapabilitiesInfo &= ~IEEE80211_HTCAP_CHWIDTH40; } return pi; diff --git a/sys/dev/nvme/nvme.c b/sys/dev/nvme/nvme.c index 84f365024f13..ead91f0d01fe 100644 --- a/sys/dev/nvme/nvme.c +++ b/sys/dev/nvme/nvme.c @@ -295,7 +295,6 @@ nvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn, void nvme_unregister_consumer(struct nvme_consumer *consumer) { - consumer->id = INVALID_CONSUMER_ID; } diff --git a/sys/dev/nvme/nvme_ahci.c b/sys/dev/nvme/nvme_ahci.c index 888207a454f7..b06661226d34 100644 --- a/sys/dev/nvme/nvme_ahci.c +++ b/sys/dev/nvme/nvme_ahci.c @@ -124,6 +124,5 @@ bad: static int nvme_ahci_detach(device_t dev) { - return (nvme_detach(dev)); } diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c index fd7f00ced14b..49960b0f920a 100644 --- a/sys/dev/nvme/nvme_ctrlr.c +++ b/sys/dev/nvme/nvme_ctrlr.c @@ -597,7 +597,6 @@ nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr) static bool is_log_page_id_valid(uint8_t page_id) { - switch (page_id) { case NVME_LOG_ERROR: case NVME_LOG_HEALTH_INFORMATION: @@ -653,7 +652,6 @@ static void nvme_ctrlr_log_critical_warnings(struct nvme_controller *ctrlr, uint8_t state) { - if (state & NVME_CRIT_WARN_ST_AVAILABLE_SPARE) nvme_printf(ctrlr, "SMART WARNING: available spare space below threshold\n"); @@ -781,7 +779,6 @@ nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr) static void nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr) { - ctrlr->int_coal_time = 0; TUNABLE_INT_FETCH("hw.nvme.int_coal_time", &ctrlr->int_coal_time); @@ -1420,8 +1417,8 @@ nvme_ctrlr_linux_passthru_cmd(struct nvme_controller *ctrlr, req->cmd.opc = npc->opcode; req->cmd.fuse = npc->flags; - req->cmd.rsvd2 = htole16(npc->cdw2); - req->cmd.rsvd3 = htole16(npc->cdw3); + req->cmd.rsvd2 = htole32(npc->cdw2); + req->cmd.rsvd3 = htole32(npc->cdw3); req->cmd.cdw10 = htole32(npc->cdw10); req->cmd.cdw11 = htole32(npc->cdw11); req->cmd.cdw12 = htole32(npc->cdw12); @@ -1776,7 +1773,6 @@ void nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr, struct nvme_request *req) { - nvme_qpair_submit_request(&ctrlr->adminq, req); } @@ -1793,14 +1789,12 @@ nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr, device_t nvme_ctrlr_get_device(struct nvme_controller *ctrlr) { - return (ctrlr->dev); } const struct nvme_controller_data * nvme_ctrlr_get_data(struct nvme_controller *ctrlr) { - return (&ctrlr->cdata); } @@ -1853,7 +1847,6 @@ nvme_ctrlr_suspend(struct nvme_controller *ctrlr) int nvme_ctrlr_resume(struct nvme_controller *ctrlr) { - /* * Can't touch failed controllers, so nothing to do to resume. */ diff --git a/sys/dev/nvme/nvme_ctrlr_cmd.c b/sys/dev/nvme/nvme_ctrlr_cmd.c index 993a7718356d..5a44ed425acb 100644 --- a/sys/dev/nvme/nvme_ctrlr_cmd.c +++ b/sys/dev/nvme/nvme_ctrlr_cmd.c @@ -281,7 +281,6 @@ nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr, struct nvme_error_information_entry *payload, uint32_t num_entries, nvme_cb_fn_t cb_fn, void *cb_arg) { - KASSERT(num_entries > 0, ("%s called with num_entries==0\n", __func__)); /* Controller's error log page entries is 0-based. */ @@ -302,7 +301,6 @@ nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr, uint32_t nsid, struct nvme_health_information_page *payload, nvme_cb_fn_t cb_fn, void *cb_arg) { - nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_HEALTH_INFORMATION, nsid, payload, sizeof(*payload), cb_fn, cb_arg); } @@ -311,7 +309,6 @@ void nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr, struct nvme_firmware_page *payload, nvme_cb_fn_t cb_fn, void *cb_arg) { - nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_FIRMWARE_SLOT, NVME_GLOBAL_NAMESPACE_TAG, payload, sizeof(*payload), cb_fn, cb_arg); diff --git a/sys/dev/nvme/nvme_ns.c b/sys/dev/nvme/nvme_ns.c index 3f29382fe42f..e84d2066930e 100644 --- a/sys/dev/nvme/nvme_ns.c +++ b/sys/dev/nvme/nvme_ns.c @@ -129,7 +129,6 @@ static int nvme_ns_close(struct cdev *dev __unused, int flags, int fmt __unused, struct thread *td) { - return (0); } @@ -231,7 +230,6 @@ nvme_ns_get_model_number(struct nvme_namespace *ns) const struct nvme_namespace_data * nvme_ns_get_data(struct nvme_namespace *ns) { - return (&ns->data); } @@ -631,7 +629,6 @@ nvme_ns_construct(struct nvme_namespace *ns, uint32_t id, void nvme_ns_destruct(struct nvme_namespace *ns) { - if (ns->cdev != NULL) { if (ns->cdev->si_drv2 != NULL) destroy_dev(ns->cdev->si_drv2); diff --git a/sys/dev/nvme/nvme_pci.c b/sys/dev/nvme/nvme_pci.c index 29b49b7df403..c07a68d2f0dc 100644 --- a/sys/dev/nvme/nvme_pci.c +++ b/sys/dev/nvme/nvme_pci.c @@ -151,7 +151,6 @@ nvme_pci_probe (device_t device) static int nvme_ctrlr_allocate_bar(struct nvme_controller *ctrlr) { - ctrlr->resource_id = PCIR_BAR(0); ctrlr->resource = bus_alloc_resource_any(ctrlr->dev, SYS_RES_MEMORY, diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h index 36f00fedc48e..52f9e12f8f9a 100644 --- a/sys/dev/nvme/nvme_private.h +++ b/sys/dev/nvme/nvme_private.h @@ -459,8 +459,7 @@ int nvme_detach(device_t dev); * vast majority of these without waiting for a tick plus scheduling delays. Since * these are on startup, this drastically reduces startup time. */ -static __inline -void +static __inline void nvme_completion_poll(struct nvme_completion_poll_status *status) { int timeout = ticks + 10 * hz; diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c index bd8626e32209..4f2c44da3b4f 100644 --- a/sys/dev/nvme/nvme_qpair.c +++ b/sys/dev/nvme/nvme_qpair.c @@ -793,7 +793,6 @@ nvme_admin_qpair_destroy(struct nvme_qpair *qpair) void nvme_io_qpair_destroy(struct nvme_qpair *qpair) { - nvme_qpair_destroy(qpair); } @@ -1202,7 +1201,6 @@ _nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) void nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) { - mtx_lock(&qpair->lock); _nvme_qpair_submit_request(qpair, req); mtx_unlock(&qpair->lock); @@ -1226,7 +1224,6 @@ nvme_qpair_enable(struct nvme_qpair *qpair) void nvme_qpair_reset(struct nvme_qpair *qpair) { - qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0; /* diff --git a/sys/dev/nvme/nvme_sim.c b/sys/dev/nvme/nvme_sim.c index 4974bb718222..a06774a64761 100644 --- a/sys/dev/nvme/nvme_sim.c +++ b/sys/dev/nvme/nvme_sim.c @@ -301,7 +301,6 @@ nvme_sim_action(struct cam_sim *sim, union ccb *ccb) static void nvme_sim_poll(struct cam_sim *sim) { - nvme_ctrlr_poll(sim2ctrlr(sim)); } diff --git a/sys/dev/nvme/nvme_sysctl.c b/sys/dev/nvme/nvme_sysctl.c index a5a44721f9f9..50d19e730a16 100644 --- a/sys/dev/nvme/nvme_sysctl.c +++ b/sys/dev/nvme/nvme_sysctl.c @@ -153,7 +153,6 @@ nvme_sysctl_timeout_period(SYSCTL_HANDLER_ARGS) static void nvme_qpair_reset_stats(struct nvme_qpair *qpair) { - /* * Reset the values. Due to sanity checks in * nvme_qpair_process_completions, we reset the number of interrupt diff --git a/sys/dev/rtwn/rtl8812a/r12a_tx.c b/sys/dev/rtwn/rtl8812a/r12a_tx.c index acb238316559..0ca1ebd37757 100644 --- a/sys/dev/rtwn/rtl8812a/r12a_tx.c +++ b/sys/dev/rtwn/rtl8812a/r12a_tx.c @@ -101,12 +101,12 @@ r12a_tx_set_vht_bw(struct rtwn_softc *sc, void *buf, struct ieee80211_node *ni) prim_chan = r12a_get_primary_channel(sc, ni->ni_chan); - if (ieee80211_vht_check_tx_bw(ni, IEEE80211_STA_RX_BW_80)) { + if (ieee80211_vht_check_tx_bw(ni, NET80211_STA_RX_BW_80)) { txd->txdw5 |= htole32(SM(R12A_TXDW5_DATA_BW, R12A_TXDW5_DATA_BW80)); txd->txdw5 |= htole32(SM(R12A_TXDW5_DATA_PRIM_CHAN, prim_chan)); - } else if (ieee80211_vht_check_tx_bw(ni, IEEE80211_STA_RX_BW_40)) { + } else if (ieee80211_vht_check_tx_bw(ni, NET80211_STA_RX_BW_40)) { txd->txdw5 |= htole32(SM(R12A_TXDW5_DATA_BW, R12A_TXDW5_DATA_BW40)); txd->txdw5 |= htole32(SM(R12A_TXDW5_DATA_PRIM_CHAN, diff --git a/sys/dev/tpm/tpm_tis_core.c b/sys/dev/tpm/tpm_tis_core.c index d8421f8156c9..4159de4daf3b 100644 --- a/sys/dev/tpm/tpm_tis_core.c +++ b/sys/dev/tpm/tpm_tis_core.c @@ -97,6 +97,7 @@ tpmtis_attach(device_t dev) { struct tpm_sc *sc; int result; + int poll = 0; sc = device_get_softc(dev); sc->dev = dev; @@ -105,6 +106,12 @@ tpmtis_attach(device_t dev) sx_init(&sc->dev_lock, "TPM driver lock"); sc->buf = malloc(TPM_BUFSIZE, M_TPM20, M_WAITOK); + resource_int_value("tpm", device_get_unit(dev), "use_polling", &poll); + if (poll != 0) { + device_printf(dev, "Using poll method to get TPM operation status \n"); + goto skip_irq; + } + sc->irq_rid = 0; sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irq_rid, RF_ACTIVE | RF_SHAREABLE); diff --git a/sys/dev/ufshci/ufshci.h b/sys/dev/ufshci/ufshci.h index b96d82ff836e..b055d2d2d769 100644 --- a/sys/dev/ufshci/ufshci.h +++ b/sys/dev/ufshci/ufshci.h @@ -716,6 +716,42 @@ struct ufshci_device_descriptor { _Static_assert(sizeof(struct ufshci_device_descriptor) == 89, "bad size for ufshci_device_descriptor"); +/* Defines the bit field of dExtendedUfsFeaturesSupport. */ +enum ufshci_desc_wb_ext_ufs_feature { + UFSHCI_DESC_EXT_UFS_FEATURE_FFU = (1 << 0), + UFSHCI_DESC_EXT_UFS_FEATURE_PSA = (1 << 1), + UFSHCI_DESC_EXT_UFS_FEATURE_DEV_LIFE_SPAN = (1 << 2), + UFSHCI_DESC_EXT_UFS_FEATURE_REFRESH_OP = (1 << 3), + UFSHCI_DESC_EXT_UFS_FEATURE_TOO_HIGH_TEMP = (1 << 4), + UFSHCI_DESC_EXT_UFS_FEATURE_TOO_LOW_TEMP = (1 << 5), + UFSHCI_DESC_EXT_UFS_FEATURE_EXT_TEMP = (1 << 6), + UFSHCI_DESC_EXT_UFS_FEATURE_HPB_SUPPORT = (1 << 7), + UFSHCI_DESC_EXT_UFS_FEATURE_WRITE_BOOSTER = (1 << 8), + UFSHCI_DESC_EXT_UFS_FEATURE_PERF_THROTTLING = (1 << 9), + UFSHCI_DESC_EXT_UFS_FEATURE_ADVANCED_RPMB = (1 << 10), + UFSHCI_DESC_EXT_UFS_FEATURE_ZONED_UFS_EXTENSION = (1 << 11), + UFSHCI_DESC_EXT_UFS_FEATURE_DEV_LEVEL_EXCEPTION = (1 << 12), + UFSHCI_DESC_EXT_UFS_FEATURE_HID = (1 << 13), + UFSHCI_DESC_EXT_UFS_FEATURE_BARRIER = (1 << 14), + UFSHCI_DESC_EXT_UFS_FEATURE_CLEAR_ERROR_HISTORY = (1 << 15), + UFSHCI_DESC_EXT_UFS_FEATURE_EXT_IID = (1 << 16), + UFSHCI_DESC_EXT_UFS_FEATURE_FBO = (1 << 17), + UFSHCI_DESC_EXT_UFS_FEATURE_FAST_RECOVERY_MODE = (1 << 18), + UFSHCI_DESC_EXT_UFS_FEATURE_RPMB_VENDOR_CMD = (1 << 19), +}; + +/* Defines the bit field of bWriteBoosterBufferType. */ +enum ufshci_desc_wb_buffer_type { + UFSHCI_DESC_WB_BUF_TYPE_LU_DEDICATED = 0x00, + UFSHCI_DESC_WB_BUF_TYPE_SINGLE_SHARED = 0x01, +}; + +/* Defines the bit field of bWriteBoosterBufferPreserveUserSpaceEn. */ +enum ufshci_desc_user_space_config { + UFSHCI_DESC_WB_BUF_USER_SPACE_REDUCTION = 0x00, + UFSHCI_DESC_WB_BUF_PRESERVE_USER_SPACE = 0x01, +}; + /* * UFS Spec 4.1, section 14.1.5.3 "Configuration Descriptor" * ConfigurationDescriptor use big-endian byte ordering. @@ -1014,4 +1050,37 @@ enum ufshci_attributes { UFSHCI_ATTR_B_REFRESH_METHOD = 0x2f, }; +/* bAvailableWriteBoosterBufferSize codes (UFS WriteBooster abailable buffer + * left %) */ +enum ufshci_wb_available_buffer_Size { + UFSHCI_ATTR_WB_AVAILABLE_0 = 0x00, /* 0% buffer remains */ + UFSHCI_ATTR_WB_AVAILABLE_10 = 0x01, /* 10% buffer remains */ + UFSHCI_ATTR_WB_AVAILABLE_20 = 0x02, /* 20% buffer remains */ + UFSHCI_ATTR_WB_AVAILABLE_30 = 0x03, /* 30% buffer remains */ + UFSHCI_ATTR_WB_AVAILABLE_40 = 0x04, /* 40% buffer remains */ + UFSHCI_ATTR_WB_AVAILABLE_50 = 0x05, /* 50% buffer remains */ + UFSHCI_ATTR_WB_AVAILABLE_60 = 0x06, /* 60% buffer remains */ + UFSHCI_ATTR_WB_AVAILABLE_70 = 0x07, /* 70% buffer remains */ + UFSHCI_ATTR_WB_AVAILABLE_80 = 0x08, /* 80% buffer remains */ + UFSHCI_ATTR_WB_AVAILABLE_90 = 0x09, /* 90% buffer remains */ + UFSHCI_ATTR_WB_AVAILABLE_100 = 0x0A, /* 100% buffer remains */ +}; + +/* bWriteBoosterBufferLifeTimeEst codes (UFS WriteBooster buffer life %) */ +enum ufshci_wb_lifetime { + UFSHCI_ATTR_WB_LIFE_DISABLED = 0x00, /* Info not available */ + UFSHCI_ATTR_WB_LIFE_0_10 = 0x01, /* 0%–10% used */ + UFSHCI_ATTR_WB_LIFE_10_20 = 0x02, /* 10%–20% used */ + UFSHCI_ATTR_WB_LIFE_20_30 = 0x03, /* 20%–30% used */ + UFSHCI_ATTR_WB_LIFE_30_40 = 0x04, /* 30%–40% used */ + UFSHCI_ATTR_WB_LIFE_40_50 = 0x05, /* 40%–50% used */ + UFSHCI_ATTR_WB_LIFE_50_60 = 0x06, /* 50%–60% used */ + UFSHCI_ATTR_WB_LIFE_60_70 = 0x07, /* 60%–70% used */ + UFSHCI_ATTR_WB_LIFE_70_80 = 0x08, /* 70%–80% used */ + UFSHCI_ATTR_WB_LIFE_80_90 = 0x09, /* 80%–90% used */ + UFSHCI_ATTR_WB_LIFE_90_100 = 0x0A, /* 90%–100% used */ + UFSHCI_ATTR_WB_LIFE_EXCEEDED = + 0x0B, /* Exceeded estimated life (treat as WB disabled) */ +}; + #endif /* __UFSHCI_H__ */ diff --git a/sys/dev/ufshci/ufshci_ctrlr.c b/sys/dev/ufshci/ufshci_ctrlr.c index 37bd32665b2b..36be94b8b8b7 100644 --- a/sys/dev/ufshci/ufshci_ctrlr.c +++ b/sys/dev/ufshci/ufshci_ctrlr.c @@ -61,7 +61,7 @@ ufshci_ctrlr_enable_host_ctrlr(struct ufshci_controller *ctrlr) int ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev) { - uint32_t ver, cap, hcs, ie; + uint32_t ver, cap, hcs, ie, ahit; uint32_t timeout_period, retry_count; int error; @@ -127,6 +127,13 @@ ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev) if (error) return (error); + /* Read the UECPA register to clear */ + ufshci_mmio_read_4(ctrlr, uecpa); + + /* Diable Auto-hibernate */ + ahit = 0; + ufshci_mmio_write_4(ctrlr, ahit, ahit); + /* * The device_present(UFSHCI_HCS_REG_DP) bit becomes true if the host * controller has successfully received a Link Startup UIC command @@ -139,6 +146,16 @@ ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev) return (ENXIO); } + /* Allocate and initialize UTP Task Management Request List. */ + error = ufshci_utmr_req_queue_construct(ctrlr); + if (error) + return (error); + + /* Allocate and initialize UTP Transfer Request List or SQ/CQ. */ + error = ufshci_utr_req_queue_construct(ctrlr); + if (error) + return (error); + /* Enable additional interrupts by programming the IE register. */ ie = ufshci_mmio_read_4(ctrlr, ie); ie |= UFSHCIM(UFSHCI_IE_REG_UTRCE); /* UTR Completion */ @@ -153,19 +170,12 @@ ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev) /* TODO: Initialize interrupt Aggregation Control Register (UTRIACR) */ - /* Allocate and initialize UTP Task Management Request List. */ - error = ufshci_utmr_req_queue_construct(ctrlr); - if (error) - return (error); - - /* Allocate and initialize UTP Transfer Request List or SQ/CQ. */ - error = ufshci_utr_req_queue_construct(ctrlr); - if (error) - return (error); - /* TODO: Separate IO and Admin slot */ - /* max_hw_pend_io is the number of slots in the transfer_req_queue */ - ctrlr->max_hw_pend_io = ctrlr->transfer_req_queue.num_entries; + /* + * max_hw_pend_io is the number of slots in the transfer_req_queue. + * Reduce num_entries by one to reserve an admin slot. + */ + ctrlr->max_hw_pend_io = ctrlr->transfer_req_queue.num_entries - 1; return (0); } @@ -342,18 +352,19 @@ ufshci_ctrlr_start(struct ufshci_controller *ctrlr) return; } - /* Read Controller Descriptor (Device, Geometry)*/ + /* Read Controller Descriptor (Device, Geometry) */ if (ufshci_dev_get_descriptor(ctrlr) != 0) { ufshci_ctrlr_fail(ctrlr, false); return; } - /* TODO: Configure Write Protect */ + if (ufshci_dev_config_write_booster(ctrlr)) { + ufshci_ctrlr_fail(ctrlr, false); + return; + } /* TODO: Configure Background Operations */ - /* TODO: Configure Write Booster */ - if (ufshci_sim_attach(ctrlr) != 0) { ufshci_ctrlr_fail(ctrlr, false); return; diff --git a/sys/dev/ufshci/ufshci_dev.c b/sys/dev/ufshci/ufshci_dev.c index a0e32914e2aa..dd196b1d638b 100644 --- a/sys/dev/ufshci/ufshci_dev.c +++ b/sys/dev/ufshci/ufshci_dev.c @@ -60,6 +60,14 @@ ufshci_dev_read_geometry_descriptor(struct ufshci_controller *ctrlr, } static int +ufshci_dev_read_unit_descriptor(struct ufshci_controller *ctrlr, uint8_t lun, + struct ufshci_unit_descriptor *desc) +{ + return (ufshci_dev_read_descriptor(ctrlr, UFSHCI_DESC_TYPE_UNIT, lun, 0, + desc, sizeof(struct ufshci_unit_descriptor))); +} + +static int ufshci_dev_read_flag(struct ufshci_controller *ctrlr, enum ufshci_flags flag_type, uint8_t *flag) { @@ -114,6 +122,61 @@ ufshci_dev_set_flag(struct ufshci_controller *ctrlr, } static int +ufshci_dev_clear_flag(struct ufshci_controller *ctrlr, + enum ufshci_flags flag_type) +{ + struct ufshci_completion_poll_status status; + struct ufshci_query_param param; + + param.function = UFSHCI_QUERY_FUNC_STANDARD_WRITE_REQUEST; + param.opcode = UFSHCI_QUERY_OPCODE_CLEAR_FLAG; + param.type = flag_type; + param.index = 0; + param.selector = 0; + param.value = 0; + + status.done = 0; + ufshci_ctrlr_cmd_send_query_request(ctrlr, ufshci_completion_poll_cb, + &status, param); + ufshci_completion_poll(&status); + if (status.error) { + ufshci_printf(ctrlr, "ufshci_dev_clear_flag failed!\n"); + return (ENXIO); + } + + return (0); +} + +static int +ufshci_dev_read_attribute(struct ufshci_controller *ctrlr, + enum ufshci_attributes attr_type, uint8_t index, uint8_t selector, + uint64_t *value) +{ + struct ufshci_completion_poll_status status; + struct ufshci_query_param param; + + param.function = UFSHCI_QUERY_FUNC_STANDARD_READ_REQUEST; + param.opcode = UFSHCI_QUERY_OPCODE_READ_ATTRIBUTE; + param.type = attr_type; + param.index = index; + param.selector = selector; + param.value = 0; + + status.done = 0; + ufshci_ctrlr_cmd_send_query_request(ctrlr, ufshci_completion_poll_cb, + &status, param); + ufshci_completion_poll(&status); + if (status.error) { + ufshci_printf(ctrlr, "ufshci_dev_read_attribute failed!\n"); + return (ENXIO); + } + + *value = status.cpl.response_upiu.query_response_upiu.value_64; + + return (0); +} + +static int ufshci_dev_write_attribute(struct ufshci_controller *ctrlr, enum ufshci_attributes attr_type, uint8_t index, uint8_t selector, uint64_t value) @@ -270,7 +333,7 @@ ufshci_dev_init_uic_power_mode(struct ufshci_controller *ctrlr) */ const uint32_t fast_mode = 1; const uint32_t rx_bit_shift = 4; - const uint32_t power_mode = (fast_mode << rx_bit_shift) | fast_mode; + uint32_t power_mode, peer_granularity; /* Update lanes with available TX/RX lanes */ if (ufshci_uic_send_dme_get(ctrlr, PA_AvailTxDataLanes, @@ -295,6 +358,20 @@ ufshci_dev_init_uic_power_mode(struct ufshci_controller *ctrlr) ctrlr->rx_lanes)) return (ENXIO); + if (ctrlr->quirks & UFSHCI_QUIRK_CHANGE_LANE_AND_GEAR_SEPARATELY) { + /* Before changing gears, first change the number of lanes. */ + if (ufshci_uic_send_dme_get(ctrlr, PA_PWRMode, &power_mode)) + return (ENXIO); + if (ufshci_uic_send_dme_set(ctrlr, PA_PWRMode, power_mode)) + return (ENXIO); + + /* Wait for power mode changed. */ + if (ufshci_uic_power_mode_ready(ctrlr)) { + ufshci_reg_dump(ctrlr); + return (ENXIO); + } + } + /* Set HS-GEAR to max gear */ ctrlr->hs_gear = ctrlr->max_rx_hs_gear; if (ufshci_uic_send_dme_set(ctrlr, PA_TxGear, ctrlr->hs_gear)) @@ -346,6 +423,7 @@ ufshci_dev_init_uic_power_mode(struct ufshci_controller *ctrlr) return (ENXIO); /* Set TX/RX PWRMode */ + power_mode = (fast_mode << rx_bit_shift) | fast_mode; if (ufshci_uic_send_dme_set(ctrlr, PA_PWRMode, power_mode)) return (ENXIO); @@ -366,7 +444,8 @@ ufshci_dev_init_uic_power_mode(struct ufshci_controller *ctrlr) pause_sbt("ufshci", ustosbt(1250), 0, C_PREL(1)); /* Test with dme_peer_get to make sure there are no errors. */ - if (ufshci_uic_send_dme_peer_get(ctrlr, PA_Granularity, NULL)) + if (ufshci_uic_send_dme_peer_get(ctrlr, PA_Granularity, + &peer_granularity)) return (ENXIO); } @@ -398,7 +477,7 @@ ufshci_dev_get_descriptor(struct ufshci_controller *ctrlr) return (error); ver = be16toh(device->dev_desc.wSpecVersion); - ufshci_printf(ctrlr, "UFS device spec version %u.%u%u\n", + ufshci_printf(ctrlr, "UFS device spec version %u.%u.%u\n", UFSHCIV(UFSHCI_VER_REG_MJR, ver), UFSHCIV(UFSHCI_VER_REG_MNR, ver), UFSHCIV(UFSHCI_VER_REG_VS, ver)); ufshci_printf(ctrlr, "%u enabled LUNs found\n", @@ -426,3 +505,273 @@ ufshci_dev_get_descriptor(struct ufshci_controller *ctrlr) return (0); } + +static int +ufshci_dev_enable_write_booster(struct ufshci_controller *ctrlr) +{ + struct ufshci_device *dev = &ctrlr->ufs_dev; + int error; + + /* Enable WriteBooster */ + error = ufshci_dev_set_flag(ctrlr, UFSHCI_FLAG_F_WRITE_BOOSTER_EN); + if (error) { + ufshci_printf(ctrlr, "Failed to enable WriteBooster\n"); + return (error); + } + dev->is_wb_enabled = true; + + /* Enable WriteBooster buffer flush during hibernate */ + error = ufshci_dev_set_flag(ctrlr, + UFSHCI_FLAG_F_WB_BUFFER_FLUSH_DURING_HIBERNATE); + if (error) { + ufshci_printf(ctrlr, + "Failed to enable WriteBooster buffer flush during hibernate\n"); + return (error); + } + + /* Enable WriteBooster buffer flush */ + error = ufshci_dev_set_flag(ctrlr, UFSHCI_FLAG_F_WB_BUFFER_FLUSH_EN); + if (error) { + ufshci_printf(ctrlr, + "Failed to enable WriteBooster buffer flush\n"); + return (error); + } + dev->is_wb_flush_enabled = true; + + return (0); +} + +static int +ufshci_dev_disable_write_booster(struct ufshci_controller *ctrlr) +{ + struct ufshci_device *dev = &ctrlr->ufs_dev; + int error; + + /* Disable WriteBooster buffer flush */ + error = ufshci_dev_clear_flag(ctrlr, UFSHCI_FLAG_F_WB_BUFFER_FLUSH_EN); + if (error) { + ufshci_printf(ctrlr, + "Failed to disable WriteBooster buffer flush\n"); + return (error); + } + dev->is_wb_flush_enabled = false; + + /* Disable WriteBooster buffer flush during hibernate */ + error = ufshci_dev_clear_flag(ctrlr, + UFSHCI_FLAG_F_WB_BUFFER_FLUSH_DURING_HIBERNATE); + if (error) { + ufshci_printf(ctrlr, + "Failed to disable WriteBooster buffer flush during hibernate\n"); + return (error); + } + + /* Disable WriteBooster */ + error = ufshci_dev_clear_flag(ctrlr, UFSHCI_FLAG_F_WRITE_BOOSTER_EN); + if (error) { + ufshci_printf(ctrlr, "Failed to disable WriteBooster\n"); + return (error); + } + dev->is_wb_enabled = false; + + return (0); +} + +static int +ufshci_dev_is_write_booster_buffer_life_time_left( + struct ufshci_controller *ctrlr, bool *is_life_time_left) +{ + struct ufshci_device *dev = &ctrlr->ufs_dev; + uint8_t buffer_lun; + uint64_t life_time; + uint32_t error; + + if (dev->wb_buffer_type == UFSHCI_DESC_WB_BUF_TYPE_LU_DEDICATED) + buffer_lun = dev->wb_dedicated_lu; + else + buffer_lun = 0; + + error = ufshci_dev_read_attribute(ctrlr, + UFSHCI_ATTR_B_WB_BUFFER_LIFE_TIME_EST, buffer_lun, 0, &life_time); + if (error) + return (error); + + *is_life_time_left = (life_time != UFSHCI_ATTR_WB_LIFE_EXCEEDED); + + return (0); +} + +/* + * This function is not yet in use. It will be used when suspend/resume is + * implemented. + */ +static __unused int +ufshci_dev_need_write_booster_buffer_flush(struct ufshci_controller *ctrlr, + bool *need_flush) +{ + struct ufshci_device *dev = &ctrlr->ufs_dev; + bool is_life_time_left = false; + uint64_t available_buffer_size, current_buffer_size; + uint8_t buffer_lun; + uint32_t error; + + *need_flush = false; + + if (!dev->is_wb_enabled) + return (0); + + error = ufshci_dev_is_write_booster_buffer_life_time_left(ctrlr, + &is_life_time_left); + if (error) + return (error); + + if (!is_life_time_left) + return (ufshci_dev_disable_write_booster(ctrlr)); + + if (dev->wb_buffer_type == UFSHCI_DESC_WB_BUF_TYPE_LU_DEDICATED) + buffer_lun = dev->wb_dedicated_lu; + else + buffer_lun = 0; + + error = ufshci_dev_read_attribute(ctrlr, + UFSHCI_ATTR_B_AVAILABLE_WB_BUFFER_SIZE, buffer_lun, 0, + &available_buffer_size); + if (error) + return (error); + + switch (dev->wb_user_space_config_option) { + case UFSHCI_DESC_WB_BUF_USER_SPACE_REDUCTION: + *need_flush = (available_buffer_size <= + UFSHCI_ATTR_WB_AVAILABLE_10); + break; + case UFSHCI_DESC_WB_BUF_PRESERVE_USER_SPACE: + /* + * In PRESERVE USER SPACE mode, flush should be performed when + * the current buffer is greater than 0 and the available buffer + * below write_booster_flush_threshold is left. + */ + error = ufshci_dev_read_attribute(ctrlr, + UFSHCI_ATTR_D_CURRENT_WB_BUFFER_SIZE, buffer_lun, 0, + ¤t_buffer_size); + if (error) + return (error); + + if (current_buffer_size == 0) + return (0); + + *need_flush = (available_buffer_size < + dev->write_booster_flush_threshold); + break; + default: + ufshci_printf(ctrlr, + "Invalid bWriteBoosterBufferPreserveUserSpaceEn value"); + return (EINVAL); + } + + /* + * TODO: Need to handle WRITEBOOSTER_FLUSH_NEEDED exception case from + * wExceptionEventStatus attribute. + */ + + return (0); +} + +int +ufshci_dev_config_write_booster(struct ufshci_controller *ctrlr) +{ + struct ufshci_device *dev = &ctrlr->ufs_dev; + uint32_t extended_ufs_feature_support; + uint32_t alloc_units; + struct ufshci_unit_descriptor unit_desc; + uint8_t lun; + bool is_life_time_left; + uint32_t mega_byte = 1024 * 1024; + uint32_t error = 0; + + extended_ufs_feature_support = be32toh( + dev->dev_desc.dExtendedUfsFeaturesSupport); + if (!(extended_ufs_feature_support & + UFSHCI_DESC_EXT_UFS_FEATURE_WRITE_BOOSTER)) { + /* This device does not support Write Booster */ + return (0); + } + + if (ufshci_dev_enable_write_booster(ctrlr)) + return (0); + + /* Get WriteBooster buffer parameters */ + dev->wb_buffer_type = dev->dev_desc.bWriteBoosterBufferType; + dev->wb_user_space_config_option = + dev->dev_desc.bWriteBoosterBufferPreserveUserSpaceEn; + + /* + * Find the size of the write buffer. + * With LU-dedicated (00h), the WriteBooster buffer is assigned + * exclusively to one chosen LU (not one-per-LU), whereas Shared (01h) + * uses a single device-wide buffer shared by multiple LUs. + */ + if (dev->wb_buffer_type == UFSHCI_DESC_WB_BUF_TYPE_SINGLE_SHARED) { + alloc_units = be32toh( + dev->dev_desc.dNumSharedWriteBoosterBufferAllocUnits); + ufshci_printf(ctrlr, + "WriteBooster buffer type = Shared, alloc_units=%d\n", + alloc_units); + } else if (dev->wb_buffer_type == + UFSHCI_DESC_WB_BUF_TYPE_LU_DEDICATED) { + ufshci_printf(ctrlr, "WriteBooster buffer type = Dedicated\n"); + for (lun = 0; lun < ctrlr->max_lun_count; lun++) { + /* Find a dedicated buffer using a unit descriptor */ + if (ufshci_dev_read_unit_descriptor(ctrlr, lun, + &unit_desc)) + continue; + + alloc_units = be32toh( + unit_desc.dLUNumWriteBoosterBufferAllocUnits); + if (alloc_units) { + dev->wb_dedicated_lu = lun; + break; + } + } + } else { + ufshci_printf(ctrlr, + "Not supported WriteBooster buffer type: 0x%x\n", + dev->wb_buffer_type); + goto out; + } + + if (alloc_units == 0) { + ufshci_printf(ctrlr, "The WriteBooster buffer size is zero\n"); + goto out; + } + + dev->wb_buffer_size_mb = alloc_units * + dev->geo_desc.bAllocationUnitSize * + (be32toh(dev->geo_desc.dSegmentSize)) / + (mega_byte / UFSHCI_SECTOR_SIZE); + + /* Set to flush when 40% of the available buffer size remains */ + dev->write_booster_flush_threshold = UFSHCI_ATTR_WB_AVAILABLE_40; + + /* + * Check if WriteBooster Buffer lifetime is left. + * WriteBooster Buffer lifetime — percent of life used based on P/E + * cycles. If "preserve user space" is enabled, writes to normal user + * space also consume WB life since the area is shared. + */ + error = ufshci_dev_is_write_booster_buffer_life_time_left(ctrlr, + &is_life_time_left); + if (error) + goto out; + + if (!is_life_time_left) { + ufshci_printf(ctrlr, + "There is no WriteBooster buffer life time left.\n"); + goto out; + } + + ufshci_printf(ctrlr, "WriteBooster Enabled\n"); + return (0); +out: + ufshci_dev_disable_write_booster(ctrlr); + return (error); +} + diff --git a/sys/dev/ufshci/ufshci_pci.c b/sys/dev/ufshci/ufshci_pci.c index 65a69ee0b518..d64b7526f713 100644 --- a/sys/dev/ufshci/ufshci_pci.c +++ b/sys/dev/ufshci/ufshci_pci.c @@ -53,7 +53,8 @@ static struct _pcsid { { 0x98fa8086, "Intel Lakefield UFS Host Controller", UFSHCI_REF_CLK_19_2MHz, UFSHCI_QUIRK_LONG_PEER_PA_TACTIVATE | - UFSHCI_QUIRK_WAIT_AFTER_POWER_MODE_CHANGE }, + UFSHCI_QUIRK_WAIT_AFTER_POWER_MODE_CHANGE | + UFSHCI_QUIRK_CHANGE_LANE_AND_GEAR_SEPARATELY }, { 0x54ff8086, "Intel UFS Host Controller", UFSHCI_REF_CLK_19_2MHz }, { 0x00000000, NULL } }; diff --git a/sys/dev/ufshci/ufshci_private.h b/sys/dev/ufshci/ufshci_private.h index 1a2742ae2e80..2e033f84c373 100644 --- a/sys/dev/ufshci/ufshci_private.h +++ b/sys/dev/ufshci/ufshci_private.h @@ -46,6 +46,8 @@ MALLOC_DECLARE(M_UFSHCI); #define UFSHCI_UTR_ENTRIES (32) #define UFSHCI_UTRM_ENTRIES (8) +#define UFSHCI_SECTOR_SIZE (512) + struct ufshci_controller; struct ufshci_completion_poll_status { @@ -214,6 +216,15 @@ struct ufshci_device { struct ufshci_geometry_descriptor geo_desc; uint32_t unipro_version; + + /* WriteBooster */ + bool is_wb_enabled; + bool is_wb_flush_enabled; + uint32_t wb_buffer_type; + uint32_t wb_buffer_size_mb; + uint32_t wb_user_space_config_option; + uint8_t wb_dedicated_lu; + uint32_t write_booster_flush_threshold; }; /* @@ -229,7 +240,8 @@ struct ufshci_controller { 2 /* Need an additional 200 ms of PA_TActivate */ #define UFSHCI_QUIRK_WAIT_AFTER_POWER_MODE_CHANGE \ 4 /* Need to wait 1250us after power mode change */ - +#define UFSHCI_QUIRK_CHANGE_LANE_AND_GEAR_SEPARATELY \ + 8 /* Need to change the number of lanes before changing HS-GEAR. */ uint32_t ref_clk; struct cam_sim *ufshci_sim; @@ -356,6 +368,7 @@ int ufshci_dev_init_unipro(struct ufshci_controller *ctrlr); int ufshci_dev_init_uic_power_mode(struct ufshci_controller *ctrlr); int ufshci_dev_init_ufs_power_mode(struct ufshci_controller *ctrlr); int ufshci_dev_get_descriptor(struct ufshci_controller *ctrlr); +int ufshci_dev_config_write_booster(struct ufshci_controller *ctrlr); /* Controller Command */ void ufshci_ctrlr_cmd_send_task_mgmt_request(struct ufshci_controller *ctrlr, diff --git a/sys/dev/ufshci/ufshci_reg.h b/sys/dev/ufshci/ufshci_reg.h index 6c9b3e2c8c04..6d5768505102 100644 --- a/sys/dev/ufshci/ufshci_reg.h +++ b/sys/dev/ufshci/ufshci_reg.h @@ -274,7 +274,7 @@ struct ufshci_registers { #define UFSHCI_HCS_REG_UTMRLRDY_MASK (0x1) #define UFSHCI_HCS_REG_UCRDY_SHIFT (3) #define UFSHCI_HCS_REG_UCRDY_MASK (0x1) -#define UFSHCI_HCS_REG_UPMCRS_SHIFT (7) +#define UFSHCI_HCS_REG_UPMCRS_SHIFT (8) #define UFSHCI_HCS_REG_UPMCRS_MASK (0x7) #define UFSHCI_HCS_REG_UTPEC_SHIFT (12) #define UFSHCI_HCS_REG_UTPEC_MASK (0xF) diff --git a/sys/dev/ufshci/ufshci_sysctl.c b/sys/dev/ufshci/ufshci_sysctl.c index 5e5069f12e5f..56bc06b13f3c 100644 --- a/sys/dev/ufshci/ufshci_sysctl.c +++ b/sys/dev/ufshci/ufshci_sysctl.c @@ -152,6 +152,7 @@ ufshci_sysctl_initialize_ctrlr(struct ufshci_controller *ctrlr) struct sysctl_ctx_list *ctrlr_ctx; struct sysctl_oid *ctrlr_tree, *que_tree, *ioq_tree; struct sysctl_oid_list *ctrlr_list, *ioq_list; + struct ufshci_device *dev = &ctrlr->ufs_dev; #define QUEUE_NAME_LENGTH 16 char queue_name[QUEUE_NAME_LENGTH]; int i; @@ -177,6 +178,25 @@ ufshci_sysctl_initialize_ctrlr(struct ufshci_controller *ctrlr) SYSCTL_ADD_UINT(ctrlr_ctx, ctrlr_list, OID_AUTO, "cap", CTLFLAG_RD, &ctrlr->cap, 0, "Number of I/O queue pairs"); + SYSCTL_ADD_BOOL(ctrlr_ctx, ctrlr_list, OID_AUTO, "wb_enabled", + CTLFLAG_RD, &dev->is_wb_enabled, 0, "WriteBooster enable/disable"); + + SYSCTL_ADD_BOOL(ctrlr_ctx, ctrlr_list, OID_AUTO, "wb_flush_enabled", + CTLFLAG_RD, &dev->is_wb_flush_enabled, 0, + "WriteBooster flush enable/disable"); + + SYSCTL_ADD_UINT(ctrlr_ctx, ctrlr_list, OID_AUTO, "wb_buffer_type", + CTLFLAG_RD, &dev->wb_buffer_type, 0, "WriteBooster type"); + + SYSCTL_ADD_UINT(ctrlr_ctx, ctrlr_list, OID_AUTO, "wb_buffer_size_mb", + CTLFLAG_RD, &dev->wb_buffer_size_mb, 0, + "WriteBooster buffer size in MB"); + + SYSCTL_ADD_UINT(ctrlr_ctx, ctrlr_list, OID_AUTO, + "wb_user_space_config_option", CTLFLAG_RD, + &dev->wb_user_space_config_option, 0, + "WriteBooster preserve user space mode"); + SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO, "timeout_period", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, &ctrlr->timeout_period, 0, ufshci_sysctl_timeout_period, "IU", diff --git a/sys/dev/ufshci/ufshci_uic_cmd.c b/sys/dev/ufshci/ufshci_uic_cmd.c index 2c5f635dc11e..b9c867ff7065 100644 --- a/sys/dev/ufshci/ufshci_uic_cmd.c +++ b/sys/dev/ufshci/ufshci_uic_cmd.c @@ -14,7 +14,7 @@ int ufshci_uic_power_mode_ready(struct ufshci_controller *ctrlr) { - uint32_t is; + uint32_t is, hcs; int timeout; /* Wait for the IS flag to change */ @@ -40,6 +40,15 @@ ufshci_uic_power_mode_ready(struct ufshci_controller *ctrlr) DELAY(10); } + /* Check HCS power mode change request status */ + hcs = ufshci_mmio_read_4(ctrlr, hcs); + if (UFSHCIV(UFSHCI_HCS_REG_UPMCRS, hcs) != 0x01) { + ufshci_printf(ctrlr, + "Power mode change request status error: 0x%x\n", + UFSHCIV(UFSHCI_HCS_REG_UPMCRS, hcs)); + return (ENXIO); + } + return (0); } @@ -112,6 +121,7 @@ ufshci_uic_send_cmd(struct ufshci_controller *ctrlr, struct ufshci_uic_cmd *uic_cmd, uint32_t *return_value) { int error; + uint32_t config_result_code; mtx_lock(&ctrlr->uic_cmd_lock); @@ -134,6 +144,13 @@ ufshci_uic_send_cmd(struct ufshci_controller *ctrlr, if (error) return (ENXIO); + config_result_code = ufshci_mmio_read_4(ctrlr, ucmdarg2); + if (config_result_code) { + ufshci_printf(ctrlr, + "Failed to send UIC command. (config result code = 0x%x)\n", + config_result_code); + } + if (return_value != NULL) *return_value = ufshci_mmio_read_4(ctrlr, ucmdarg3); diff --git a/sys/dev/usb/controller/xhci.c b/sys/dev/usb/controller/xhci.c index 5be592512196..788b2b718062 100644 --- a/sys/dev/usb/controller/xhci.c +++ b/sys/dev/usb/controller/xhci.c @@ -156,6 +156,7 @@ struct xhci_std_temp { static void xhci_do_poll(struct usb_bus *); static void xhci_device_done(struct usb_xfer *, usb_error_t); +static void xhci_get_xecp(struct xhci_softc *); static void xhci_root_intr(struct xhci_softc *); static void xhci_free_device_ext(struct usb_device *); static struct xhci_endpoint_ext *xhci_get_endpoint_ext(struct usb_device *, @@ -566,6 +567,8 @@ xhci_init(struct xhci_softc *sc, device_t self, uint8_t dma32) device_printf(self, "%d bytes context size, %d-bit DMA\n", sc->sc_ctx_is_64_byte ? 64 : 32, (int)sc->sc_bus.dma_bits); + xhci_get_xecp(sc); + /* enable 64Kbyte control endpoint quirk */ sc->sc_bus.control_ep_quirk = (xhcictlquirk ? 1 : 0); @@ -654,6 +657,88 @@ xhci_uninit(struct xhci_softc *sc) } static void +xhci_get_xecp(struct xhci_softc *sc) +{ + + uint32_t hccp1; + uint32_t eec; + uint32_t eecp; + bool first = true; + + hccp1 = XREAD4(sc, capa, XHCI_HCSPARAMS0); + + if (XHCI_HCS0_XECP(hccp1) == 0) { + device_printf(sc->sc_bus.parent, + "xECP: no capabilities found\n"); + return; + } + + /* + * Parse the xECP Capabilities table and print known caps. + * Implemented, vendor and reserved xECP Capabilities values are + * documented in Table 7.2 of eXtensible Host Controller Interface for + * Universal Serial Bus (xHCI) Rev 1.2b 2023. + */ + device_printf(sc->sc_bus.parent, "xECP capabilities <"); + + eec = -1; + for (eecp = XHCI_HCS0_XECP(hccp1) << 2; + eecp != 0 && XHCI_XECP_NEXT(eec) != 0; + eecp += XHCI_XECP_NEXT(eec) << 2) { + eec = XREAD4(sc, capa, eecp); + + uint8_t xecpid = XHCI_XECP_ID(eec); + + if ((xecpid >= 11 && xecpid <= 16) || + (xecpid >= 19 && xecpid <= 191)) { + if (!first) + printf(","); + printf("RES(%x)", xecpid); + } else if (xecpid > 191) { + if (!first) + printf(","); + printf("VEND(%x)", xecpid); + } else { + if (!first) + printf(","); + switch (xecpid) + { + case XHCI_ID_USB_LEGACY: + printf("LEGACY"); + break; + case XHCI_ID_PROTOCOLS: + printf("PROTO"); + break; + case XHCI_ID_POWER_MGMT: + printf("POWER"); + break; + case XHCI_ID_VIRTUALIZATION: + printf("VIRT"); + break; + case XHCI_ID_MSG_IRQ: + printf("MSG IRQ"); + break; + case XHCI_ID_USB_LOCAL_MEM: + printf("LOCAL MEM"); + break; + case XHCI_ID_USB_DEBUG: + printf("DEBUG"); + break; + case XHCI_ID_EXT_MSI: + printf("EXT MSI"); + break; + case XHCI_ID_USB3_TUN: + printf("TUN"); + break; + + } + } + first = false; + } + printf(">\n"); +} + +static void xhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state) { struct xhci_softc *sc = XHCI_BUS2SC(bus); diff --git a/sys/dev/usb/controller/xhcireg.h b/sys/dev/usb/controller/xhcireg.h index 9d0b6e2f4b4b..821897155544 100644 --- a/sys/dev/usb/controller/xhcireg.h +++ b/sys/dev/usb/controller/xhcireg.h @@ -205,6 +205,11 @@ #define XHCI_ID_VIRTUALIZATION 0x0004 #define XHCI_ID_MSG_IRQ 0x0005 #define XHCI_ID_USB_LOCAL_MEM 0x0006 +/* values 7-9 are reserved */ +#define XHCI_ID_USB_DEBUG 0x000a +/* values 11-16 are reserved */ +#define XHCI_ID_EXT_MSI 0x0011 +#define XHCI_ID_USB3_TUN 0x0012 /* XHCI register R/W wrappers */ #define XREAD1(sc, what, a) \ diff --git a/sys/dev/usb/usb_hub.c b/sys/dev/usb/usb_hub.c index e3509862ef54..ee9d8ab0c9bb 100644 --- a/sys/dev/usb/usb_hub.c +++ b/sys/dev/usb/usb_hub.c @@ -954,7 +954,8 @@ done: * packet. This function is called having the "bus_mtx" locked. *------------------------------------------------------------------------*/ void -uhub_root_intr(struct usb_bus *bus, const uint8_t *ptr, uint8_t len) +uhub_root_intr(struct usb_bus *bus, + const uint8_t *ptr __unused, uint8_t len __unused) { USB_BUS_LOCK_ASSERT(bus, MA_OWNED); diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c index ecb3dbb370e5..867da80a53a8 100644 --- a/sys/dev/virtio/network/if_vtnet.c +++ b/sys/dev/virtio/network/if_vtnet.c @@ -2551,8 +2551,10 @@ vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head, error = sglist_append_mbuf(sg, m); if (error) { m = m_defrag(m, M_NOWAIT); - if (m == NULL) + if (m == NULL) { + sc->vtnet_stats.tx_defrag_failed++; goto fail; + } *m_head = m; sc->vtnet_stats.tx_defragged++; @@ -2568,7 +2570,6 @@ vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head, return (error); fail: - sc->vtnet_stats.tx_defrag_failed++; m_freem(*m_head); *m_head = NULL; @@ -3813,9 +3814,9 @@ vtnet_rx_filter_mac(struct vtnet_softc *sc) if_printf(ifp, "error setting host MAC filter table\n"); out: - if (promisc != 0 && vtnet_set_promisc(sc, true) != 0) + if (promisc && vtnet_set_promisc(sc, true) != 0) if_printf(ifp, "cannot enable promiscuous mode\n"); - if (allmulti != 0 && vtnet_set_allmulti(sc, true) != 0) + if (allmulti && vtnet_set_allmulti(sc, true) != 0) if_printf(ifp, "cannot enable all-multicast mode\n"); } @@ -4100,21 +4101,29 @@ vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *ctx, stats = &rxq->vtnrx_stats; - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vrxs_ipackets, "Receive packets"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vrxs_ibytes, "Receive bytes"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vrxs_iqdrops, "Receive drops"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vrxs_ierrors, "Receive errors"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vrxs_csum, "Receive checksum offloaded"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vrxs_csum_failed, "Receive checksum offload failed"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "host_lro", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "host_lro", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vrxs_host_lro, "Receive host segmentation offloaded"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vrxs_rescheduled, "Receive interrupt handler rescheduled"); } @@ -4135,17 +4144,23 @@ vtnet_setup_txq_sysctl(struct sysctl_ctx_list *ctx, stats = &txq->vtntx_stats; - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vtxs_opackets, "Transmit packets"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vtxs_obytes, "Transmit bytes"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vtxs_omcasts, "Transmit multicasts"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vtxs_csum, "Transmit checksum offloaded"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vtxs_tso, "Transmit TCP segmentation offloaded"); - SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD, + SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", + CTLFLAG_RD | CTLFLAG_STATS, &stats->vtxs_rescheduled, "Transmit interrupt handler rescheduled"); } @@ -4170,6 +4185,102 @@ vtnet_setup_queue_sysctl(struct vtnet_softc *sc) } } +static int +vtnet_sysctl_rx_csum_failed(SYSCTL_HANDLER_ARGS) +{ + struct vtnet_softc *sc = (struct vtnet_softc *)arg1; + struct vtnet_statistics *stats = &sc->vtnet_stats; + struct vtnet_rxq_stats *rxst; + int i; + + stats->rx_csum_failed = 0; + for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { + rxst = &sc->vtnet_rxqs[i].vtnrx_stats; + stats->rx_csum_failed += rxst->vrxs_csum_failed; + } + return (sysctl_handle_64(oidp, NULL, stats->rx_csum_failed, req)); +} + +static int +vtnet_sysctl_rx_csum_offloaded(SYSCTL_HANDLER_ARGS) +{ + struct vtnet_softc *sc = (struct vtnet_softc *)arg1; + struct vtnet_statistics *stats = &sc->vtnet_stats; + struct vtnet_rxq_stats *rxst; + int i; + + stats->rx_csum_offloaded = 0; + for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { + rxst = &sc->vtnet_rxqs[i].vtnrx_stats; + stats->rx_csum_offloaded += rxst->vrxs_csum; + } + return (sysctl_handle_64(oidp, NULL, stats->rx_csum_offloaded, req)); +} + +static int +vtnet_sysctl_rx_task_rescheduled(SYSCTL_HANDLER_ARGS) +{ + struct vtnet_softc *sc = (struct vtnet_softc *)arg1; + struct vtnet_statistics *stats = &sc->vtnet_stats; + struct vtnet_rxq_stats *rxst; + int i; + + stats->rx_task_rescheduled = 0; + for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { + rxst = &sc->vtnet_rxqs[i].vtnrx_stats; + stats->rx_task_rescheduled += rxst->vrxs_rescheduled; + } + return (sysctl_handle_64(oidp, NULL, stats->rx_task_rescheduled, req)); +} + +static int +vtnet_sysctl_tx_csum_offloaded(SYSCTL_HANDLER_ARGS) +{ + struct vtnet_softc *sc = (struct vtnet_softc *)arg1; + struct vtnet_statistics *stats = &sc->vtnet_stats; + struct vtnet_txq_stats *txst; + int i; + + stats->tx_csum_offloaded = 0; + for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { + txst = &sc->vtnet_txqs[i].vtntx_stats; + stats->tx_csum_offloaded += txst->vtxs_csum; + } + return (sysctl_handle_64(oidp, NULL, stats->tx_csum_offloaded, req)); +} + +static int +vtnet_sysctl_tx_tso_offloaded(SYSCTL_HANDLER_ARGS) +{ + struct vtnet_softc *sc = (struct vtnet_softc *)arg1; + struct vtnet_statistics *stats = &sc->vtnet_stats; + struct vtnet_txq_stats *txst; + int i; + + stats->tx_tso_offloaded = 0; + for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { + txst = &sc->vtnet_txqs[i].vtntx_stats; + stats->tx_tso_offloaded += txst->vtxs_tso; + } + return (sysctl_handle_64(oidp, NULL, stats->tx_tso_offloaded, req)); +} + +static int +vtnet_sysctl_tx_task_rescheduled(SYSCTL_HANDLER_ARGS) +{ + struct vtnet_softc *sc = (struct vtnet_softc *)arg1; + struct vtnet_statistics *stats = &sc->vtnet_stats; + struct vtnet_txq_stats *txst; + int i; + + stats->tx_task_rescheduled = 0; + for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { + txst = &sc->vtnet_txqs[i].vtntx_stats; + stats->tx_task_rescheduled += txst->vtxs_rescheduled; + } + return (sysctl_handle_64(oidp, NULL, stats->tx_task_rescheduled, req)); +} + static void vtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child, struct vtnet_softc *sc) @@ -4189,69 +4300,75 @@ vtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx, stats->tx_task_rescheduled = txaccum.vtxs_rescheduled; SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed", - CTLFLAG_RD, &stats->mbuf_alloc_failed, + CTLFLAG_RD | CTLFLAG_STATS, &stats->mbuf_alloc_failed, "Mbuf cluster allocation failures"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large", - CTLFLAG_RD, &stats->rx_frame_too_large, + CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_frame_too_large, "Received frame larger than the mbuf chain"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed", - CTLFLAG_RD, &stats->rx_enq_replacement_failed, + CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_enq_replacement_failed, "Enqueuing the replacement receive mbuf failed"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed", - CTLFLAG_RD, &stats->rx_mergeable_failed, + CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_mergeable_failed, "Mergeable buffers receive failures"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype", - CTLFLAG_RD, &stats->rx_csum_bad_ethtype, + CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_csum_bad_ethtype, "Received checksum offloaded buffer with unsupported " "Ethernet type"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto", - CTLFLAG_RD, &stats->rx_csum_bad_ipproto, + CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_csum_bad_ipproto, "Received checksum offloaded buffer with incorrect IP protocol"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset", - CTLFLAG_RD, &stats->rx_csum_bad_offset, + CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_csum_bad_offset, "Received checksum offloaded buffer with incorrect offset"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_proto", - CTLFLAG_RD, &stats->rx_csum_bad_proto, + CTLFLAG_RD | CTLFLAG_STATS, &stats->rx_csum_bad_proto, "Received checksum offloaded buffer with incorrect protocol"); - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_failed", - CTLFLAG_RD, &stats->rx_csum_failed, + SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_csum_failed", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, + sc, 0, vtnet_sysctl_rx_csum_failed, "QU", "Received buffer checksum offload failed"); - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_offloaded", - CTLFLAG_RD, &stats->rx_csum_offloaded, + SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_csum_offloaded", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, + sc, 0, vtnet_sysctl_rx_csum_offloaded, "QU", "Received buffer checksum offload succeeded"); - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_task_rescheduled", - CTLFLAG_RD, &stats->rx_task_rescheduled, + SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_task_rescheduled", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, + sc, 0, vtnet_sysctl_rx_task_rescheduled, "QU", "Times the receive interrupt task rescheduled itself"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_unknown_ethtype", - CTLFLAG_RD, &stats->tx_csum_unknown_ethtype, + CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_csum_unknown_ethtype, "Aborted transmit of checksum offloaded buffer with unknown " "Ethernet type"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_proto_mismatch", - CTLFLAG_RD, &stats->tx_csum_proto_mismatch, + CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_csum_proto_mismatch, "Aborted transmit of checksum offloaded buffer because mismatched " "protocols"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_not_tcp", - CTLFLAG_RD, &stats->tx_tso_not_tcp, + CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_tso_not_tcp, "Aborted transmit of TSO buffer with non TCP protocol"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_without_csum", - CTLFLAG_RD, &stats->tx_tso_without_csum, + CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_tso_without_csum, "Aborted transmit of TSO buffer without TCP checksum offload"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged", - CTLFLAG_RD, &stats->tx_defragged, + CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_defragged, "Transmit mbufs defragged"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defrag_failed", - CTLFLAG_RD, &stats->tx_defrag_failed, + CTLFLAG_RD | CTLFLAG_STATS, &stats->tx_defrag_failed, "Aborted transmit of buffer because defrag failed"); - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_offloaded", - CTLFLAG_RD, &stats->tx_csum_offloaded, + SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_csum_offloaded", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, + sc, 0, vtnet_sysctl_tx_csum_offloaded, "QU", "Offloaded checksum of transmitted buffer"); - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_offloaded", - CTLFLAG_RD, &stats->tx_tso_offloaded, + SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_tso_offloaded", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, + sc, 0, vtnet_sysctl_tx_tso_offloaded, "QU", "Segmentation offload of transmitted buffer"); - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_task_rescheduled", - CTLFLAG_RD, &stats->tx_task_rescheduled, + SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "tx_task_rescheduled", + CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS, + sc, 0, vtnet_sysctl_tx_task_rescheduled, "QU", "Times the transmit interrupt task rescheduled itself"); } diff --git a/sys/fs/nfsclient/nfs_clvnops.c b/sys/fs/nfsclient/nfs_clvnops.c index a8b06fdb261b..eee571a04821 100644 --- a/sys/fs/nfsclient/nfs_clvnops.c +++ b/sys/fs/nfsclient/nfs_clvnops.c @@ -3474,7 +3474,7 @@ nfs_advlock(struct vop_advlock_args *ap) u_quad_t size; struct nfsmount *nmp; - error = NFSVOPLOCK(vp, LK_SHARED); + error = NFSVOPLOCK(vp, LK_EXCLUSIVE); if (error != 0) return (EBADF); nmp = VFSTONFS(vp->v_mount); @@ -3511,11 +3511,6 @@ nfs_advlock(struct vop_advlock_args *ap) cred = p->p_ucred; else cred = td->td_ucred; - NFSVOPLOCK(vp, LK_UPGRADE | LK_RETRY); - if (VN_IS_DOOMED(vp)) { - error = EBADF; - goto out; - } /* * If this is unlocking a write locked region, flush and diff --git a/sys/fs/tarfs/tarfs_vnops.c b/sys/fs/tarfs/tarfs_vnops.c index acf18de5ab51..c110107bb210 100644 --- a/sys/fs/tarfs/tarfs_vnops.c +++ b/sys/fs/tarfs/tarfs_vnops.c @@ -334,6 +334,10 @@ tarfs_readdir(struct vop_readdir_args *ap) tnp, tnp->name, uio->uio_offset, uio->uio_resid); if (uio->uio_offset == TARFS_COOKIE_EOF) { + if (eofflag != NULL) { + TARFS_DPF(VNODE, "%s: Setting EOF flag\n", __func__); + *eofflag = 1; + } TARFS_DPF(VNODE, "%s: EOF\n", __func__); return (0); } diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index 36ce44b988be..2fa429e6368c 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -296,7 +296,7 @@ mi_startup(void) BOOTTRACE_INIT("sysinit 0x%7x", sip->subsystem); #if defined(VERBOSE_SYSINIT) - if (sip->subsystem > last && verbose_sysinit != 0) { + if (sip->subsystem != last && verbose_sysinit != 0) { verbose = 1; printf("subsystem %x\n", sip->subsystem); } diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index 50b040132396..3180c66cb42b 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -1694,8 +1694,10 @@ thread_single_end(struct proc *p, int mode) thread_unlock(td); } } - KASSERT(mode != SINGLE_BOUNDARY || p->p_boundary_count == 0, - ("inconsistent boundary count %d", p->p_boundary_count)); + KASSERT(mode != SINGLE_BOUNDARY || P_SHOULDSTOP(p) || + p->p_boundary_count == 0, + ("pid %d proc %p flags %#x inconsistent boundary count %d", + p->p_pid, p, p->p_flag, p->p_boundary_count)); PROC_SUNLOCK(p); wakeup(&p->p_flag); } diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index ab47b6ad29a3..a65c3ca128d9 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -57,7 +57,7 @@ * b : public affirmation by word or example of usually * religious faith or conviction <the heroic witness to divine * life -- Pilot> - * 6 capitalized : a member of the Jehovah's Witnesses + * 6 capitalized : a member of the Jehovah's Witnesses */ /* @@ -131,7 +131,7 @@ #define LI_SLEEPABLE 0x00040000 /* Lock may be held while sleeping. */ #ifndef WITNESS_COUNT -#define WITNESS_COUNT 1536 +#define WITNESS_COUNT 1536 #endif #define WITNESS_HASH_SIZE 251 /* Prime, gives load factor < 2 */ #define WITNESS_PENDLIST (512 + (MAXCPU * 4)) @@ -158,20 +158,18 @@ * These flags go in the witness relationship matrix and describe the * relationship between any two struct witness objects. */ -#define WITNESS_UNRELATED 0x00 /* No lock order relation. */ -#define WITNESS_PARENT 0x01 /* Parent, aka direct ancestor. */ -#define WITNESS_ANCESTOR 0x02 /* Direct or indirect ancestor. */ -#define WITNESS_CHILD 0x04 /* Child, aka direct descendant. */ -#define WITNESS_DESCENDANT 0x08 /* Direct or indirect descendant. */ -#define WITNESS_ANCESTOR_MASK (WITNESS_PARENT | WITNESS_ANCESTOR) -#define WITNESS_DESCENDANT_MASK (WITNESS_CHILD | WITNESS_DESCENDANT) -#define WITNESS_RELATED_MASK \ - (WITNESS_ANCESTOR_MASK | WITNESS_DESCENDANT_MASK) -#define WITNESS_REVERSAL 0x10 /* A lock order reversal has been - * observed. */ -#define WITNESS_RESERVED1 0x20 /* Unused flag, reserved. */ -#define WITNESS_RESERVED2 0x40 /* Unused flag, reserved. */ -#define WITNESS_LOCK_ORDER_KNOWN 0x80 /* This lock order is known. */ +#define WITNESS_UNRELATED 0x00 /* No lock order relation. */ +#define WITNESS_PARENT 0x01 /* Parent, aka direct ancestor. */ +#define WITNESS_ANCESTOR 0x02 /* Direct or indirect ancestor. */ +#define WITNESS_CHILD 0x04 /* Child, aka direct descendant. */ +#define WITNESS_DESCENDANT 0x08 /* Direct or indirect descendant. */ +#define WITNESS_ANCESTOR_MASK (WITNESS_PARENT | WITNESS_ANCESTOR) +#define WITNESS_DESCENDANT_MASK (WITNESS_CHILD | WITNESS_DESCENDANT) +#define WITNESS_RELATED_MASK (WITNESS_ANCESTOR_MASK | WITNESS_DESCENDANT_MASK) +#define WITNESS_REVERSAL 0x10 /* A lock order reversal has been observed. */ +#define WITNESS_RESERVED1 0x20 /* Unused flag, reserved. */ +#define WITNESS_RESERVED2 0x40 /* Unused flag, reserved. */ +#define WITNESS_LOCK_ORDER_KNOWN 0x80 /* This lock order is known. */ /* Descendant to ancestor flags */ #define WITNESS_DTOA(x) (((x) & WITNESS_RELATED_MASK) >> 2) @@ -218,20 +216,18 @@ struct lock_list_entry { * (for example, "vnode interlock"). */ struct witness { - char w_name[MAX_W_NAME]; - uint32_t w_index; /* Index in the relationship matrix */ + char w_name[MAX_W_NAME]; + uint32_t w_index; /* Index in the relationship matrix */ struct lock_class *w_class; - STAILQ_ENTRY(witness) w_list; /* List of all witnesses. */ - STAILQ_ENTRY(witness) w_typelist; /* Witnesses of a type. */ - struct witness *w_hash_next; /* Linked list in hash buckets. */ - const char *w_file; /* File where last acquired */ - uint32_t w_line; /* Line where last acquired */ - uint32_t w_refcount; - uint16_t w_num_ancestors; /* direct/indirect - * ancestor count */ - uint16_t w_num_descendants; /* direct/indirect - * descendant count */ - int16_t w_ddb_level; + STAILQ_ENTRY(witness) w_list; /* List of all witnesses. */ + STAILQ_ENTRY(witness) w_typelist; /* Witnesses of a type. */ + struct witness *w_hash_next; /* Linked list in hash buckets. */ + const char *w_file; /* File where last acquired */ + uint32_t w_line; /* Line where last acquired */ + uint32_t w_refcount; + uint16_t w_num_ancestors; /* direct/indirect ancestor count */ + uint16_t w_num_descendants; /* direct/indirect descendant count */ + int16_t w_ddb_level; unsigned w_displayed:1; unsigned w_reversed:1; }; @@ -265,7 +261,7 @@ struct witness_lock_order_data { /* * The witness lock order data hash table. Keys are witness index tuples * (struct witness_lock_order_key), elements are lock order data objects - * (struct witness_lock_order_data). + * (struct witness_lock_order_data). */ struct witness_lock_order_hash { struct witness_lock_order_data *wloh_array[WITNESS_LO_HASH_SIZE]; @@ -295,7 +291,6 @@ struct witness_order_list_entry { static __inline int witness_lock_type_equal(struct witness *w1, struct witness *w2) { - return ((w1->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)) == (w2->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK))); } @@ -304,7 +299,6 @@ static __inline int witness_lock_order_key_equal(const struct witness_lock_order_key *a, const struct witness_lock_order_key *b) { - return (a->from == b->from && a->to == b->to); } @@ -415,7 +409,7 @@ SYSCTL_INT(_debug_witness, OID_AUTO, skipspin, CTLFLAG_RDTUN, &witness_skipspin, int badstack_sbuf_size; int witness_count = WITNESS_COUNT; -SYSCTL_INT(_debug_witness, OID_AUTO, witness_count, CTLFLAG_RDTUN, +SYSCTL_INT(_debug_witness, OID_AUTO, witness_count, CTLFLAG_RDTUN, &witness_count, 0, ""); /* @@ -760,7 +754,6 @@ static int witness_spin_warn = 0; static const char * fixup_filename(const char *file) { - if (file == NULL) return (NULL); while (strncmp(file, "../", 3) == 0) @@ -835,7 +828,7 @@ witness_startup(void *mem) w_free_cnt--; for (i = 0; i < witness_count; i++) { - memset(w_rmatrix[i], 0, sizeof(*w_rmatrix[i]) * + memset(w_rmatrix[i], 0, sizeof(*w_rmatrix[i]) * (witness_count + 1)); } @@ -989,16 +982,16 @@ witness_ddb_display_descendants(int(*prnt)(const char *fmt, ...), { int i; - for (i = 0; i < indent; i++) - prnt(" "); + for (i = 0; i < indent; i++) + prnt(" "); prnt("%s (type: %s, depth: %d, active refs: %d)", w->w_name, w->w_class->lc_name, w->w_ddb_level, w->w_refcount); - if (w->w_displayed) { - prnt(" -- (already displayed)\n"); - return; - } - w->w_displayed = 1; + if (w->w_displayed) { + prnt(" -- (already displayed)\n"); + return; + } + w->w_displayed = 1; if (w->w_file != NULL && w->w_line != 0) prnt(" -- last acquired @ %s:%d\n", fixup_filename(w->w_file), w->w_line); @@ -1079,7 +1072,6 @@ witness_ddb_display(int(*prnt)(const char *fmt, ...)) int witness_defineorder(struct lock_object *lock1, struct lock_object *lock2) { - if (witness_watch == -1 || KERNEL_PANICKED()) return (0); @@ -1257,7 +1249,7 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file, w->w_reversed = 1; mtx_unlock_spin(&w_mtx); witness_output( - "acquiring duplicate lock of same type: \"%s\"\n", + "acquiring duplicate lock of same type: \"%s\"\n", w->w_name); witness_output(" 1st %s @ %s:%d\n", plock->li_lock->lo_name, fixup_filename(plock->li_file), plock->li_line); @@ -1743,7 +1735,7 @@ found: /* * In order to reduce contention on w_mtx, we want to keep always an - * head object into lists so that frequent allocation from the + * head object into lists so that frequent allocation from the * free witness pool (and subsequent locking) is avoided. * In order to maintain the current code simple, when the head * object is totally unloaded it means also that we do not have @@ -1781,7 +1773,7 @@ witness_thread_exit(struct thread *td) n++; witness_list_lock(&lle->ll_children[i], witness_output); - + } kassert_panic( "Thread %p cannot exit while holding sleeplocks\n", td); @@ -1948,7 +1940,6 @@ found: static void depart(struct witness *w) { - MPASS(w->w_refcount == 0); if (w->w_class->lc_flags & LC_SLEEPLOCK) { w_sleep_cnt--; @@ -1999,18 +1990,18 @@ adopt(struct witness *parent, struct witness *child) child->w_num_ancestors++; } - /* - * Find each ancestor of 'pi'. Note that 'pi' itself is counted as + /* + * Find each ancestor of 'pi'. Note that 'pi' itself is counted as * an ancestor of 'pi' during this loop. */ for (i = 1; i <= w_max_used_index; i++) { - if ((w_rmatrix[i][pi] & WITNESS_ANCESTOR_MASK) == 0 && + if ((w_rmatrix[i][pi] & WITNESS_ANCESTOR_MASK) == 0 && (i != pi)) continue; /* Find each descendant of 'i' and mark it as a descendant. */ for (j = 1; j <= w_max_used_index; j++) { - /* + /* * Skip children that are already marked as * descendants of 'i'. */ @@ -2021,7 +2012,7 @@ adopt(struct witness *parent, struct witness *child) * We are only interested in descendants of 'ci'. Note * that 'ci' itself is counted as a descendant of 'ci'. */ - if ((w_rmatrix[ci][j] & WITNESS_ANCESTOR_MASK) == 0 && + if ((w_rmatrix[ci][j] & WITNESS_ANCESTOR_MASK) == 0 && (j != ci)) continue; w_rmatrix[i][j] |= WITNESS_ANCESTOR; @@ -2029,16 +2020,16 @@ adopt(struct witness *parent, struct witness *child) w_data[i].w_num_descendants++; w_data[j].w_num_ancestors++; - /* + /* * Make sure we aren't marking a node as both an - * ancestor and descendant. We should have caught + * ancestor and descendant. We should have caught * this as a lock order reversal earlier. */ if ((w_rmatrix[i][j] & WITNESS_ANCESTOR_MASK) && (w_rmatrix[i][j] & WITNESS_DESCENDANT_MASK)) { printf("witness rmatrix paradox! [%d][%d]=%d " "both ancestor and descendant\n", - i, j, w_rmatrix[i][j]); + i, j, w_rmatrix[i][j]); kdb_backtrace(); printf("Witness disabled.\n"); witness_watch = -1; @@ -2047,7 +2038,7 @@ adopt(struct witness *parent, struct witness *child) (w_rmatrix[j][i] & WITNESS_DESCENDANT_MASK)) { printf("witness rmatrix paradox! [%d][%d]=%d " "both ancestor and descendant\n", - j, i, w_rmatrix[j][i]); + j, i, w_rmatrix[j][i]); kdb_backtrace(); printf("Witness disabled.\n"); witness_watch = -1; @@ -2124,7 +2115,6 @@ _isitmyx(struct witness *w1, struct witness *w2, int rmask, const char *fname) static int isitmychild(struct witness *parent, struct witness *child) { - return (_isitmyx(parent, child, WITNESS_PARENT, __func__)); } @@ -2134,7 +2124,6 @@ isitmychild(struct witness *parent, struct witness *child) static int isitmydescendant(struct witness *ancestor, struct witness *descendant) { - return (_isitmyx(ancestor, descendant, WITNESS_ANCESTOR_MASK, __func__)); } @@ -2182,7 +2171,7 @@ witness_get(void) STAILQ_REMOVE_HEAD(&w_free, w_list); w_free_cnt--; index = w->w_index; - MPASS(index > 0 && index == w_max_used_index+1 && + MPASS(index > 0 && index == w_max_used_index + 1 && index < witness_count); bzero(w, sizeof(*w)); w->w_index = index; @@ -2194,7 +2183,6 @@ witness_get(void) static void witness_free(struct witness *w) { - STAILQ_INSERT_HEAD(&w_free, w, w_list); w_free_cnt++; } @@ -2219,11 +2207,10 @@ witness_lock_list_get(void) bzero(lle, sizeof(*lle)); return (lle); } - + static void witness_lock_list_free(struct lock_list_entry *lle) { - mtx_lock_spin(&w_mtx); lle->ll_next = w_lock_list_free; w_lock_list_free = lle; @@ -2297,7 +2284,6 @@ witness_voutput(const char *fmt, va_list ap) static int witness_thread_has_locks(struct thread *td) { - if (td->td_sleeplocks == NULL) return (0); return (td->td_sleeplocks->ll_count != 0); @@ -2573,14 +2559,12 @@ witness_setflag(struct lock_object *lock, int flag, int set) void witness_norelease(struct lock_object *lock) { - witness_setflag(lock, LI_NORELEASE, 1); } void witness_releaseok(struct lock_object *lock) { - witness_setflag(lock, LI_NORELEASE, 0); } @@ -2588,7 +2572,6 @@ witness_releaseok(struct lock_object *lock) static void witness_ddb_list(struct thread *td) { - KASSERT(witness_cold == 0, ("%s: witness_cold", __func__)); KASSERT(kdb_active, ("%s: not in the debugger", __func__)); @@ -2653,7 +2636,6 @@ DB_SHOW_ALIAS_FLAGS(alllocks, db_witness_list_all, DB_CMD_MEMSAFE); DB_SHOW_COMMAND_FLAGS(witness, db_witness_display, DB_CMD_MEMSAFE) { - witness_ddb_display(db_printf); } #endif @@ -2673,9 +2655,9 @@ sbuf_print_witness_badstacks(struct sbuf *sb, size_t *oldidx) /* Allocate and init temporary storage space. */ tmp_w1 = malloc(sizeof(struct witness), M_TEMP, M_WAITOK | M_ZERO); tmp_w2 = malloc(sizeof(struct witness), M_TEMP, M_WAITOK | M_ZERO); - tmp_data1 = malloc(sizeof(struct witness_lock_order_data), M_TEMP, + tmp_data1 = malloc(sizeof(struct witness_lock_order_data), M_TEMP, M_WAITOK | M_ZERO); - tmp_data2 = malloc(sizeof(struct witness_lock_order_data), M_TEMP, + tmp_data2 = malloc(sizeof(struct witness_lock_order_data), M_TEMP, M_WAITOK | M_ZERO); stack_zero(&tmp_data1->wlod_stack); stack_zero(&tmp_data2->wlod_stack); @@ -2750,12 +2732,12 @@ restart: sbuf_printf(sb, "\nLock order reversal between \"%s\"(%s) and \"%s\"(%s)!\n", - tmp_w1->w_name, tmp_w1->w_class->lc_name, + tmp_w1->w_name, tmp_w1->w_class->lc_name, tmp_w2->w_name, tmp_w2->w_class->lc_name); if (data1) { sbuf_printf(sb, "Lock order \"%s\"(%s) -> \"%s\"(%s) first seen at:\n", - tmp_w1->w_name, tmp_w1->w_class->lc_name, + tmp_w1->w_name, tmp_w1->w_class->lc_name, tmp_w2->w_name, tmp_w2->w_class->lc_name); stack_sbuf_print(sb, &tmp_data1->wlod_stack); sbuf_putc(sb, '\n'); @@ -2763,7 +2745,7 @@ restart: if (data2 && data2 != data1) { sbuf_printf(sb, "Lock order \"%s\"(%s) -> \"%s\"(%s) first seen at:\n", - tmp_w2->w_name, tmp_w2->w_class->lc_name, + tmp_w2->w_name, tmp_w2->w_class->lc_name, tmp_w1->w_name, tmp_w1->w_class->lc_name); stack_sbuf_print(sb, &tmp_data2->wlod_stack); sbuf_putc(sb, '\n'); @@ -2823,7 +2805,6 @@ sysctl_debug_witness_badstacks(SYSCTL_HANDLER_ARGS) static int sbuf_db_printf_drain(void *arg __unused, const char *data, int len) { - return (db_printf("%.*s", len, data)); } @@ -3068,7 +3049,7 @@ witness_lock_order_get(struct witness *parent, struct witness *child) & WITNESS_LOCK_ORDER_KNOWN) == 0) goto out; - hash = witness_hash_djb2((const char*)&key, + hash = witness_hash_djb2((const char *)&key, sizeof(key)) % w_lohash.wloh_size; data = w_lohash.wloh_array[hash]; while (data != NULL) { @@ -3089,7 +3070,6 @@ out: static int witness_lock_order_check(struct witness *parent, struct witness *child) { - if (parent != child && w_rmatrix[parent->w_index][child->w_index] & WITNESS_LOCK_ORDER_KNOWN && @@ -3115,7 +3095,7 @@ witness_lock_order_add(struct witness *parent, struct witness *child) & WITNESS_LOCK_ORDER_KNOWN) return (1); - hash = witness_hash_djb2((const char*)&key, + hash = witness_hash_djb2((const char *)&key, sizeof(key)) % w_lohash.wloh_size; w_rmatrix[parent->w_index][child->w_index] |= WITNESS_LOCK_ORDER_KNOWN; data = w_lofree; @@ -3134,7 +3114,6 @@ witness_lock_order_add(struct witness *parent, struct witness *child) static void witness_increment_graph_generation(void) { - if (witness_cold == 0) mtx_assert(&w_mtx, MA_OWNED); w_generation++; @@ -3143,7 +3122,6 @@ witness_increment_graph_generation(void) static int witness_output_drain(void *arg __unused, const char *data, int len) { - witness_output("%.*s", len, data); return (len); } diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c index 19870e989437..6138e543fae7 100644 --- a/sys/kern/uipc_usrreq.c +++ b/sys/kern/uipc_usrreq.c @@ -1807,9 +1807,7 @@ uipc_filt_sowrite(struct knote *kn, long hint) kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); if (so2->so_rcv.sb_state & SBS_CANTRCVMORE) { - /* - * XXXGL: maybe kn->kn_flags |= EV_EOF ? - */ + kn->kn_flags |= EV_EOF; return (1); } else if (kn->kn_sfflags & NOTE_LOWAT) return (kn->kn_data >= kn->kn_sdata); diff --git a/sys/kern/vfs_init.c b/sys/kern/vfs_init.c index cd30d5cfae47..93ac001af8ad 100644 --- a/sys/kern/vfs_init.c +++ b/sys/kern/vfs_init.c @@ -103,6 +103,16 @@ struct vattr va_null; * Routines having to do with the management of the vnode table. */ +void +vfs_unref_vfsconf(struct vfsconf *vfsp) +{ + vfsconf_lock(); + KASSERT(vfsp->vfc_refcount > 0, + ("vfs %p refcount underflow %d", vfsp, vfsp->vfc_refcount)); + vfsp->vfc_refcount--; + vfsconf_unlock(); +} + static struct vfsconf * vfs_byname_locked(const char *name) { @@ -123,9 +133,11 @@ vfs_byname(const char *name) { struct vfsconf *vfsp; - vfsconf_slock(); + vfsconf_lock(); vfsp = vfs_byname_locked(name); - vfsconf_sunlock(); + if (vfsp != NULL) + vfsp->vfc_refcount++; + vfsconf_unlock(); return (vfsp); } diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c index 8e64a7fe966b..13403acacc08 100644 --- a/sys/kern/vfs_mount.c +++ b/sys/kern/vfs_mount.c @@ -683,7 +683,6 @@ vfs_mount_alloc(struct vnode *vp, struct vfsconf *vfsp, const char *fspath, MPASSERT(mp->mnt_vfs_ops == 1, mp, ("vfs_ops should be 1 but %d found", mp->mnt_vfs_ops)); (void) vfs_busy(mp, MBF_NOWAIT); - atomic_add_acq_int(&vfsp->vfc_refcount, 1); mp->mnt_op = vfsp->vfc_vfsops; mp->mnt_vfc = vfsp; mp->mnt_stat.f_type = vfsp->vfc_typenum; @@ -731,7 +730,6 @@ vfs_mount_destroy(struct mount *mp) __FILE__, __LINE__)); MPPASS(mp->mnt_writeopcount == 0, mp); MPPASS(mp->mnt_secondary_writes == 0, mp); - atomic_subtract_rel_int(&mp->mnt_vfc->vfc_refcount, 1); if (!TAILQ_EMPTY(&mp->mnt_nvnodelist)) { struct vnode *vp; @@ -769,6 +767,9 @@ vfs_mount_destroy(struct mount *mp) vfs_free_addrlist(mp->mnt_export); free(mp->mnt_export, M_MOUNT); } + vfsconf_lock(); + mp->mnt_vfc->vfc_refcount--; + vfsconf_unlock(); crfree(mp->mnt_cred); uma_zfree(mount_zone, mp); } @@ -1133,6 +1134,7 @@ vfs_domount_first( if (jailed(td->td_ucred) && (!prison_allow(td->td_ucred, vfsp->vfc_prison_flag) || vp == td->td_ucred->cr_prison->pr_root)) { vput(vp); + vfs_unref_vfsconf(vfsp); return (EPERM); } @@ -1169,6 +1171,7 @@ vfs_domount_first( } if (error != 0) { vput(vp); + vfs_unref_vfsconf(vfsp); return (error); } vn_seqc_write_begin(vp); diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c index a6e38be89291..57732ddab7d9 100644 --- a/sys/kern/vfs_subr.c +++ b/sys/kern/vfs_subr.c @@ -2186,6 +2186,8 @@ freevnode(struct vnode *vp) { struct bufobj *bo; + ASSERT_VOP_UNLOCKED(vp, __func__); + /* * The vnode has been marked for destruction, so free it. * @@ -2222,12 +2224,16 @@ freevnode(struct vnode *vp) mac_vnode_destroy(vp); #endif if (vp->v_pollinfo != NULL) { + int error __diagused; + /* * Use LK_NOWAIT to shut up witness about the lock. We may get * here while having another vnode locked when trying to * satisfy a lookup and needing to recycle. */ - VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT); + error = VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT); + VNASSERT(error == 0, vp, + ("freevnode: cannot lock vp %p for pollinfo destroy", vp)); destroy_vpollinfo(vp->v_pollinfo); VOP_UNLOCK(vp); vp->v_pollinfo = NULL; diff --git a/sys/modules/irdma/Makefile b/sys/modules/irdma/Makefile index b2ffb67ca66f..a9ef6e63d3f2 100644 --- a/sys/modules/irdma/Makefile +++ b/sys/modules/irdma/Makefile @@ -1,8 +1,8 @@ .include <bsd.own.mk> -OFED_INC_DIR = ${.CURDIR}/../../ofed/include -ICE_DIR = ${.CURDIR}/../../dev/ice -.PATH: ${.CURDIR}/../../dev/irdma +OFED_INC_DIR = ${SRCTOP}/sys/ofed/include +ICE_DIR = ${SRCTOP}/sys/dev/ice +.PATH: ${SRCTOP}/sys/dev/irdma KMOD= irdma SRCS= icrdma.c diff --git a/sys/net/if.c b/sys/net/if.c index 202be4794f6e..0fc30488f1e5 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -74,7 +74,6 @@ #include <vm/uma.h> #include <net/bpf.h> -#include <net/ethernet.h> #include <net/if.h> #include <net/if_arp.h> #include <net/if_clone.h> diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c index 66555fd1feb5..a854bbb96394 100644 --- a/sys/net/if_bridge.c +++ b/sys/net/if_bridge.c @@ -1447,10 +1447,10 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) #endif /* - * If member_ifaddrs is disabled, do not allow an interface with - * assigned IP addresses to be added to a bridge. + * If member_ifaddrs is disabled, do not allow an Ethernet-like + * interface with assigned IP addresses to be added to a bridge. */ - if (!V_member_ifaddrs) { + if (!V_member_ifaddrs && ifs->if_type != IFT_GIF) { struct ifaddr *ifa; CK_STAILQ_FOREACH(ifa, &ifs->if_addrhead, ifa_link) { diff --git a/sys/net/iflib.c b/sys/net/iflib.c index 98c59e5de988..308ecad0a846 100644 --- a/sys/net/iflib.c +++ b/sys/net/iflib.c @@ -712,7 +712,7 @@ static uint32_t iflib_txq_can_drain(struct ifmp_ring *); static void iflib_altq_if_start(if_t ifp); static int iflib_altq_if_transmit(if_t ifp, struct mbuf *m); #endif -static int iflib_register(if_ctx_t); +static void iflib_register(if_ctx_t); static void iflib_deregister(if_ctx_t); static void iflib_unregister_vlan_handlers(if_ctx_t ctx); static uint16_t iflib_get_mbuf_size_for(unsigned int size); @@ -5136,10 +5136,7 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct ctx->ifc_dev = dev; ctx->ifc_softc = sc; - if ((err = iflib_register(ctx)) != 0) { - device_printf(dev, "iflib_register failed %d\n", err); - goto fail_ctx_free; - } + iflib_register(ctx); iflib_add_device_sysctl_pre(ctx); scctx = &ctx->ifc_softc_ctx; @@ -5363,7 +5360,6 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct DEBUGNET_SET(ctx->ifc_ifp, iflib); - if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); iflib_add_device_sysctl_post(ctx); iflib_add_pfil(ctx); ctx->ifc_flags |= IFC_INIT_DONE; @@ -5387,7 +5383,6 @@ fail_unlock: CTX_UNLOCK(ctx); IFNET_WUNLOCK(); iflib_deregister(ctx); -fail_ctx_free: device_set_softc(ctx->ifc_dev, NULL); if (ctx->ifc_flags & IFC_SC_ALLOCATED) free(ctx->ifc_softc, M_IFLIB); @@ -5685,7 +5680,7 @@ _iflib_pre_assert(if_softc_ctx_t scctx) MPASS(scctx->isc_txrx->ift_rxd_flush); } -static int +static void iflib_register(if_ctx_t ctx) { if_shared_ctx_t sctx = ctx->ifc_sctx; @@ -5718,6 +5713,7 @@ iflib_register(if_ctx_t ctx) if_settransmitfn(ifp, iflib_if_transmit); #endif if_setqflushfn(ifp, iflib_if_qflush); + if_setgetcounterfn(ifp, iflib_if_get_counter); if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); ctx->ifc_vlan_attach_event = EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx, @@ -5731,7 +5727,6 @@ iflib_register(if_ctx_t ctx) ifmedia_init(ctx->ifc_mediap, IFM_IMASK, iflib_media_change, iflib_media_status); } - return (0); } static void diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h index d6c13470f2eb..f73420494000 100644 --- a/sys/net/pfvar.h +++ b/sys/net/pfvar.h @@ -326,6 +326,7 @@ pf_counter_u64_zero(struct pf_counter_u64 *pfcu64) _Static_assert(sizeof(time_t) == 4 || sizeof(time_t) == 8, "unexpected time_t size"); SYSCTL_DECL(_net_pf); +MALLOC_DECLARE(M_PF); MALLOC_DECLARE(M_PFHASH); MALLOC_DECLARE(M_PF_RULE_ITEM); @@ -645,6 +646,7 @@ struct pf_kpool { int tblidx; u_int16_t proxy_port[2]; u_int8_t opts; + sa_family_t ipv6_nexthop_af; }; struct pf_rule_actions { @@ -859,8 +861,8 @@ struct pf_krule { u_int8_t keep_state; sa_family_t af; u_int8_t proto; - u_int8_t type; - u_int8_t code; + uint16_t type; + uint16_t code; u_int8_t flags; u_int8_t flagset; u_int8_t min_ttl; @@ -2612,6 +2614,7 @@ struct pf_kruleset *pf_find_kruleset(const char *); struct pf_kruleset *pf_get_leaf_kruleset(char *, char **); struct pf_kruleset *pf_find_or_create_kruleset(const char *); void pf_rs_initialize(void); +void pf_rule_tree_free(struct pf_krule_global *); struct pf_krule *pf_krule_alloc(void); @@ -2680,7 +2683,7 @@ u_short pf_map_addr(sa_family_t, struct pf_krule *, struct pf_addr *, struct pf_kpool *); u_short pf_map_addr_sn(u_int8_t, struct pf_krule *, struct pf_addr *, struct pf_addr *, - sa_family_t *, struct pfi_kkif **nkif, + sa_family_t *, struct pfi_kkif **, struct pf_addr *, struct pf_kpool *, pf_sn_types_t); int pf_get_transaddr_af(struct pf_krule *, diff --git a/sys/net80211/ieee80211_ddb.c b/sys/net80211/ieee80211_ddb.c index d96d7988a864..1dd8e38b9896 100644 --- a/sys/net80211/ieee80211_ddb.c +++ b/sys/net80211/ieee80211_ddb.c @@ -296,7 +296,7 @@ _db_show_sta(const struct ieee80211_node *ni) ni->ni_htparam, ni->ni_htctlchan, ni->ni_ht2ndchan); db_printf("\thtopmode 0x%x htstbc 0x%x chw %d (%s)\n", ni->ni_htopmode, ni->ni_htstbc, - ni->ni_chw, ieee80211_ni_chw_to_str(ni->ni_chw)); + ni->ni_chw, net80211_ni_chw_to_str(ni->ni_chw)); /* XXX ampdu state */ for (i = 0; i < WME_NUM_TID; i++) diff --git a/sys/net80211/ieee80211_freebsd.h b/sys/net80211/ieee80211_freebsd.h index 3684fba52c5c..954801d95787 100644 --- a/sys/net80211/ieee80211_freebsd.h +++ b/sys/net80211/ieee80211_freebsd.h @@ -341,11 +341,16 @@ struct mbuf *ieee80211_getmgtframe(uint8_t **frm, int headroom, int pktlen); #define M_AGE_SUB(m,adj) (m->m_pkthdr.csum_data -= adj) /* - * Store the sequence number. + * Store / retrieve the sequence number in an mbuf. + * + * The sequence number being stored/retreived is the 12 bit + * base sequence number, not the 16 bit sequence number field. + * I.e., it's from 0..4095 inclusive, with no 4 bit padding for + * fragment numbers. */ #define M_SEQNO_SET(m, seqno) \ - ((m)->m_pkthdr.tso_segsz = (seqno)) -#define M_SEQNO_GET(m) ((m)->m_pkthdr.tso_segsz) + ((m)->m_pkthdr.tso_segsz = ((seqno) % IEEE80211_SEQ_RANGE)) +#define M_SEQNO_GET(m) (((m)->m_pkthdr.tso_segsz) % IEEE80211_SEQ_RANGE) #define MTAG_ABI_NET80211 1132948340 /* net80211 ABI */ diff --git a/sys/net80211/ieee80211_ht.c b/sys/net80211/ieee80211_ht.c index c28f124648a1..018927a6dad4 100644 --- a/sys/net80211/ieee80211_ht.c +++ b/sys/net80211/ieee80211_ht.c @@ -1476,7 +1476,7 @@ ieee80211_ht_wds_init(struct ieee80211_node *ni) ni->ni_htcap |= IEEE80211_HTCAP_SHORTGI20; if (IEEE80211_IS_CHAN_HT40(ni->ni_chan)) { ni->ni_htcap |= IEEE80211_HTCAP_CHWIDTH40; - ni->ni_chw = IEEE80211_STA_RX_BW_40; + ni->ni_chw = NET80211_STA_RX_BW_40; if (IEEE80211_IS_CHAN_HT40U(ni->ni_chan)) ni->ni_ht2ndchan = IEEE80211_HTINFO_2NDCHAN_ABOVE; else if (IEEE80211_IS_CHAN_HT40D(ni->ni_chan)) @@ -1484,7 +1484,7 @@ ieee80211_ht_wds_init(struct ieee80211_node *ni) if (vap->iv_flags_ht & IEEE80211_FHT_SHORTGI40) ni->ni_htcap |= IEEE80211_HTCAP_SHORTGI40; } else { - ni->ni_chw = IEEE80211_STA_RX_BW_20; + ni->ni_chw = NET80211_STA_RX_BW_20; ni->ni_ht2ndchan = IEEE80211_HTINFO_2NDCHAN_NONE; } ni->ni_htctlchan = ni->ni_chan->ic_ieee; @@ -1580,7 +1580,7 @@ ieee80211_ht_node_join(struct ieee80211_node *ni) if (ni->ni_flags & IEEE80211_NODE_HT) { vap->iv_ht_sta_assoc++; - if (ni->ni_chw == IEEE80211_STA_RX_BW_40) + if (ni->ni_chw == NET80211_STA_RX_BW_40) vap->iv_ht40_sta_assoc++; } htinfo_update(vap); @@ -1598,7 +1598,7 @@ ieee80211_ht_node_leave(struct ieee80211_node *ni) if (ni->ni_flags & IEEE80211_NODE_HT) { vap->iv_ht_sta_assoc--; - if (ni->ni_chw == IEEE80211_STA_RX_BW_40) + if (ni->ni_chw == NET80211_STA_RX_BW_40) vap->iv_ht40_sta_assoc--; } htinfo_update(vap); @@ -1827,7 +1827,7 @@ htinfo_update_chw(struct ieee80211_node *ni, int htflags, int vhtflags) done: /* update node's (11n) tx channel width */ ni->ni_chw = IEEE80211_IS_CHAN_HT40(ni->ni_chan) ? - IEEE80211_STA_RX_BW_40 : IEEE80211_STA_RX_BW_20; + NET80211_STA_RX_BW_40 : NET80211_STA_RX_BW_20; return (ret); } @@ -2689,11 +2689,11 @@ ht_recv_action_ht_txchwidth(struct ieee80211_node *ni, * here. */ chw = (frm[2] == IEEE80211_A_HT_TXCHWIDTH_2040) ? - IEEE80211_STA_RX_BW_40 : IEEE80211_STA_RX_BW_20; + NET80211_STA_RX_BW_40 : NET80211_STA_RX_BW_20; IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "%s: HT txchwidth, width %d%s (%s)", __func__, - chw, ni->ni_chw != chw ? "*" : "", ieee80211_ni_chw_to_str(chw)); + chw, ni->ni_chw != chw ? "*" : "", net80211_ni_chw_to_str(chw)); if (chw != ni->ni_chw) { /* XXX does this need to change the ht40 station count? */ ni->ni_chw = chw; @@ -3832,5 +3832,5 @@ ieee80211_ht_check_tx_ht40(const struct ieee80211_node *ni) return (IEEE80211_IS_CHAN_HT40(bss_chan) && IEEE80211_IS_CHAN_HT40(ni->ni_chan) && - (ni->ni_chw == IEEE80211_STA_RX_BW_40)); + (ni->ni_chw == NET80211_STA_RX_BW_40)); } diff --git a/sys/net80211/ieee80211_node.c b/sys/net80211/ieee80211_node.c index a201d1b278f0..49ba00299fee 100644 --- a/sys/net80211/ieee80211_node.c +++ b/sys/net80211/ieee80211_node.c @@ -2673,7 +2673,7 @@ ieee80211_dump_node(struct ieee80211_node_table *nt __unused, ni->ni_htctlchan, ni->ni_ht2ndchan); net80211_printf("\thtopmode %x htstbc %x htchw %d (%s)\n", ni->ni_htopmode, ni->ni_htstbc, - ni->ni_chw, ieee80211_ni_chw_to_str(ni->ni_chw)); + ni->ni_chw, net80211_ni_chw_to_str(ni->ni_chw)); net80211_printf("\tvhtcap %x freq1 %d freq2 %d vhtbasicmcs %x\n", ni->ni_vhtcap, (int) ni->ni_vht_chan1, (int) ni->ni_vht_chan2, (int) ni->ni_vht_basicmcs); @@ -2831,7 +2831,7 @@ ieee80211_node_join(struct ieee80211_node *ni, int resp) ni->ni_flags & IEEE80211_NODE_QOS ? ", QoS" : "", /* XXX update for VHT string */ ni->ni_flags & IEEE80211_NODE_HT ? - (ni->ni_chw == IEEE80211_STA_RX_BW_40 ? ", HT40" : ", HT20") : "", + (ni->ni_chw == NET80211_STA_RX_BW_40 ? ", HT40" : ", HT20") : "", ni->ni_flags & IEEE80211_NODE_AMPDU ? " (+AMPDU)" : "", ni->ni_flags & IEEE80211_NODE_AMSDU ? " (+AMSDU)" : "", ni->ni_flags & IEEE80211_NODE_MIMO_RTS ? " (+SMPS-DYN)" : diff --git a/sys/net80211/ieee80211_node.h b/sys/net80211/ieee80211_node.h index ef25fa0d7fdd..f1246dd12419 100644 --- a/sys/net80211/ieee80211_node.h +++ b/sys/net80211/ieee80211_node.h @@ -109,33 +109,33 @@ enum ieee80211_mesh_mlstate { "\20\1IDLE\2OPENSNT\2OPENRCV\3CONFIRMRCV\4ESTABLISHED\5HOLDING" /* - * This structure is shared with LinuxKPI 802.11 code describing up-to - * which channel width the station can receive. + * This enum was shared with the LinuxKPI enum ieee80211_sta_rx_bandwidth + * describing up-to which channel width the station can receive. * Rather than using hardcoded MHz values for the channel width use an enum with * flags. This allows us to keep the uint8_t slot for ni_chw in - * struct ieee80211_node and means we do not have to sync to the value for - * LinuxKPI. + * struct ieee80211_node it means we do not have to sync to the value for + * LinuxKPI (just the names). * * NB: BW_20 needs to 0 and values need to be sorted! Cannot make it * bitfield-alike for use with %b. */ -enum ieee80211_sta_rx_bw { - IEEE80211_STA_RX_BW_20 = 0x00, - IEEE80211_STA_RX_BW_40, - IEEE80211_STA_RX_BW_80, - IEEE80211_STA_RX_BW_160, - IEEE80211_STA_RX_BW_320, +enum net80211_sta_rx_bw { + NET80211_STA_RX_BW_20 = 0x00, + NET80211_STA_RX_BW_40, + NET80211_STA_RX_BW_80, + NET80211_STA_RX_BW_160, + NET80211_STA_RX_BW_320, } __packed; static inline const char * -ieee80211_ni_chw_to_str(enum ieee80211_sta_rx_bw bw) +net80211_ni_chw_to_str(enum net80211_sta_rx_bw bw) { switch (bw) { - case IEEE80211_STA_RX_BW_20: return ("BW_20"); - case IEEE80211_STA_RX_BW_40: return ("BW_40"); - case IEEE80211_STA_RX_BW_80: return ("BW_80"); - case IEEE80211_STA_RX_BW_160: return ("BW_160"); - case IEEE80211_STA_RX_BW_320: return ("BW_320"); + case NET80211_STA_RX_BW_20: return ("BW_20"); + case NET80211_STA_RX_BW_40: return ("BW_40"); + case NET80211_STA_RX_BW_80: return ("BW_80"); + case NET80211_STA_RX_BW_160: return ("BW_160"); + case NET80211_STA_RX_BW_320: return ("BW_320"); } } @@ -285,7 +285,7 @@ struct ieee80211_node { uint8_t ni_ht2ndchan; /* HT 2nd channel */ uint8_t ni_htopmode; /* HT operating mode */ uint8_t ni_htstbc; /* HT */ - enum ieee80211_sta_rx_bw ni_chw; /* negotiated channel width */ + enum net80211_sta_rx_bw ni_chw; /* negotiated channel width */ struct ieee80211_htrateset ni_htrates; /* negotiated ht rate set */ struct ieee80211_tx_ampdu ni_tx_ampdu[WME_NUM_TID]; struct ieee80211_rx_ampdu ni_rx_ampdu[WME_NUM_TID]; diff --git a/sys/net80211/ieee80211_phy.c b/sys/net80211/ieee80211_phy.c index eb96d74a2bd9..7f53c717152b 100644 --- a/sys/net80211/ieee80211_phy.c +++ b/sys/net80211/ieee80211_phy.c @@ -658,26 +658,26 @@ static uint16_t ieee80211_vht_mcs_allowed_list_160[] = { * * See 802.11-2020 21.5 (Parameters for VHT-MCSs) for more details. * - * @param bw channel bandwidth, via enum ieee80211_sta_rx_bw + * @param bw channel bandwidth, via enum net80211_sta_rx_bw * @param nss number of spatial streams, 1..8 * @returns bitmask of valid MCS rates from 0..9 */ uint16_t -ieee80211_phy_vht_get_mcs_mask(enum ieee80211_sta_rx_bw bw, uint8_t nss) +ieee80211_phy_vht_get_mcs_mask(enum net80211_sta_rx_bw bw, uint8_t nss) { if (nss == 0 || nss > 8) return (0); switch (bw) { - case IEEE80211_STA_RX_BW_20: + case NET80211_STA_RX_BW_20: return (ieee80211_vht_mcs_allowed_list_20[nss - 1]); - case IEEE80211_STA_RX_BW_40: + case NET80211_STA_RX_BW_40: return (ieee80211_vht_mcs_allowed_list_40[nss - 1]); - case IEEE80211_STA_RX_BW_80: + case NET80211_STA_RX_BW_80: return (ieee80211_vht_mcs_allowed_list_80[nss - 1]); - case IEEE80211_STA_RX_BW_160: + case NET80211_STA_RX_BW_160: return (ieee80211_vht_mcs_allowed_list_160[nss - 1]); - case IEEE80211_STA_RX_BW_320: + case NET80211_STA_RX_BW_320: /* invalid for VHT */ return (0); } @@ -689,14 +689,14 @@ ieee80211_phy_vht_get_mcs_mask(enum ieee80211_sta_rx_bw bw, uint8_t nss) * * See 802.11-2020 21.5 (Parameters for VHT-MCSs) for more details. * - * @param bw channel bandwidth, via enum ieee80211_sta_rx_bw + * @param bw channel bandwidth, via enum net80211_sta_rx_bw * @param nss number of spatial streams, 1..8 * @param mcs MCS rate, 0..9 * @retval true if the NSS / MCS / bandwidth combination is valid * @retval false if the NSS / MCS / bandwidth combination is not valid */ bool -ieee80211_phy_vht_validate_mcs(enum ieee80211_sta_rx_bw bw, uint8_t nss, +ieee80211_phy_vht_validate_mcs(enum net80211_sta_rx_bw bw, uint8_t nss, uint8_t mcs) { uint16_t mask; @@ -737,7 +737,7 @@ static struct mcs_entry mcs_entries[] = { /** * @brief Calculate the bitrate of the given VHT MCS rate. * - * @param bw Channel bandwidth (enum ieee80211_sta_rx_bw) + * @param bw Channel bandwidth (enum net80211_sta_rx_bw) * @param nss Number of spatial streams, 1..8 * @param mcs MCS, 0..9 * @param is_shortgi True if short guard-interval (400nS) @@ -746,7 +746,7 @@ static struct mcs_entry mcs_entries[] = { * @returns The bitrate in kbit/sec. */ uint32_t -ieee80211_phy_vht_get_mcs_kbit(enum ieee80211_sta_rx_bw bw, +ieee80211_phy_vht_get_mcs_kbit(enum net80211_sta_rx_bw bw, uint8_t nss, uint8_t mcs, bool is_shortgi) { uint32_t sym_len, n_carriers; @@ -773,16 +773,16 @@ ieee80211_phy_vht_get_mcs_kbit(enum ieee80211_sta_rx_bw bw, * See 802.11-2020 Table 21-5 (Timing-related constraints.) */ switch (bw) { - case IEEE80211_STA_RX_BW_20: + case NET80211_STA_RX_BW_20: n_carriers = 52; break; - case IEEE80211_STA_RX_BW_40: + case NET80211_STA_RX_BW_40: n_carriers = 108; break; - case IEEE80211_STA_RX_BW_80: + case NET80211_STA_RX_BW_80: n_carriers = 234; break; - case IEEE80211_STA_RX_BW_160: + case NET80211_STA_RX_BW_160: n_carriers = 468; break; default: diff --git a/sys/net80211/ieee80211_phy.h b/sys/net80211/ieee80211_phy.h index 749b082e34e9..391c8bfc5010 100644 --- a/sys/net80211/ieee80211_phy.h +++ b/sys/net80211/ieee80211_phy.h @@ -221,13 +221,13 @@ uint32_t ieee80211_compute_duration_ht(uint32_t frameLen, uint16_t rate, int streams, int isht40, int isShortGI); -enum ieee80211_sta_rx_bw; +enum net80211_sta_rx_bw; -uint16_t ieee80211_phy_vht_get_mcs_mask(enum ieee80211_sta_rx_bw, +uint16_t ieee80211_phy_vht_get_mcs_mask(enum net80211_sta_rx_bw, uint8_t); -bool ieee80211_phy_vht_validate_mcs(enum ieee80211_sta_rx_bw, +bool ieee80211_phy_vht_validate_mcs(enum net80211_sta_rx_bw, uint8_t, uint8_t); -uint32_t ieee80211_phy_vht_get_mcs_kbit(enum ieee80211_sta_rx_bw, +uint32_t ieee80211_phy_vht_get_mcs_kbit(enum net80211_sta_rx_bw, uint8_t, uint8_t, bool); #endif /* _KERNEL */ diff --git a/sys/net80211/ieee80211_sta.c b/sys/net80211/ieee80211_sta.c index 463a8b16773b..19e5ffe9a367 100644 --- a/sys/net80211/ieee80211_sta.c +++ b/sys/net80211/ieee80211_sta.c @@ -1934,7 +1934,7 @@ sta_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0, int subtype, vap->iv_flags&IEEE80211_F_USEPROT ? ", protection" : "", ni->ni_flags & IEEE80211_NODE_QOS ? ", QoS" : "", ni->ni_flags & IEEE80211_NODE_HT ? - (ni->ni_chw == IEEE80211_STA_RX_BW_40 ? ", HT40" : ", HT20") : "", + (ni->ni_chw == NET80211_STA_RX_BW_40 ? ", HT40" : ", HT20") : "", ni->ni_flags & IEEE80211_NODE_AMPDU ? " (+AMPDU)" : "", ni->ni_flags & IEEE80211_NODE_AMSDU ? " (+AMSDU)" : "", ni->ni_flags & IEEE80211_NODE_MIMO_RTS ? " (+SMPS-DYN)" : diff --git a/sys/net80211/ieee80211_vht.c b/sys/net80211/ieee80211_vht.c index de0b691d4d2a..10a5fc7f08ab 100644 --- a/sys/net80211/ieee80211_vht.c +++ b/sys/net80211/ieee80211_vht.c @@ -974,7 +974,7 @@ ieee80211_vht_check_tx_vht40(const struct ieee80211_node *ni) return (IEEE80211_IS_CHAN_VHT40(bss_chan) && IEEE80211_IS_CHAN_VHT40(ni->ni_chan) && - (ni->ni_chw == IEEE80211_STA_RX_BW_40)); + (ni->ni_chw == NET80211_STA_RX_BW_40)); } /* @@ -1003,7 +1003,7 @@ ieee80211_vht_check_tx_vht80(const struct ieee80211_node *ni) */ return (IEEE80211_IS_CHAN_VHT80(bss_chan) && IEEE80211_IS_CHAN_VHT80(ni->ni_chan) && - (ni->ni_chw != IEEE80211_STA_RX_BW_20)); + (ni->ni_chw != NET80211_STA_RX_BW_20)); } /* @@ -1030,7 +1030,7 @@ ieee80211_vht_check_tx_vht160(const struct ieee80211_node *ni) * If a HT TX width action frame sets it to 20MHz * then reject doing 160MHz. */ - if (ni->ni_chw == IEEE80211_STA_RX_BW_20) + if (ni->ni_chw == NET80211_STA_RX_BW_20) return (false); if (IEEE80211_IS_CHAN_VHT160(bss_chan) && @@ -1062,19 +1062,19 @@ ieee80211_vht_check_tx_vht160(const struct ieee80211_node *ni) */ bool ieee80211_vht_check_tx_bw(const struct ieee80211_node *ni, - enum ieee80211_sta_rx_bw bw) + enum net80211_sta_rx_bw bw) { switch (bw) { - case IEEE80211_STA_RX_BW_20: + case NET80211_STA_RX_BW_20: return (ieee80211_vht_check_tx_vht(ni)); - case IEEE80211_STA_RX_BW_40: + case NET80211_STA_RX_BW_40: return (ieee80211_vht_check_tx_vht40(ni)); - case IEEE80211_STA_RX_BW_80: + case NET80211_STA_RX_BW_80: return (ieee80211_vht_check_tx_vht80(ni)); - case IEEE80211_STA_RX_BW_160: + case NET80211_STA_RX_BW_160: return (ieee80211_vht_check_tx_vht160(ni)); - case IEEE80211_STA_RX_BW_320: + case NET80211_STA_RX_BW_320: return (false); default: return (false); @@ -1096,7 +1096,7 @@ ieee80211_vht_check_tx_bw(const struct ieee80211_node *ni, */ bool ieee80211_vht_node_check_tx_valid_mcs(const struct ieee80211_node *ni, - enum ieee80211_sta_rx_bw bw, uint8_t nss, uint8_t mcs) + enum net80211_sta_rx_bw bw, uint8_t nss, uint8_t mcs) { uint8_t mc; diff --git a/sys/net80211/ieee80211_vht.h b/sys/net80211/ieee80211_vht.h index a1529df4a85b..b9b19fbc6008 100644 --- a/sys/net80211/ieee80211_vht.h +++ b/sys/net80211/ieee80211_vht.h @@ -65,8 +65,8 @@ void ieee80211_vht_get_vhtinfo_ie(struct ieee80211_node *ni, bool ieee80211_vht_check_tx_vht(const struct ieee80211_node *); bool ieee80211_vht_check_tx_bw(const struct ieee80211_node *, - enum ieee80211_sta_rx_bw); + enum net80211_sta_rx_bw); bool ieee80211_vht_node_check_tx_valid_mcs(const struct ieee80211_node *, - enum ieee80211_sta_rx_bw bw, uint8_t, uint8_t); + enum net80211_sta_rx_bw bw, uint8_t, uint8_t); #endif /* _NET80211_IEEE80211_VHT_H_ */ diff --git a/sys/netgraph/ng_parse.c b/sys/netgraph/ng_parse.c index 448ecc92f075..5e1a1bb47ac0 100644 --- a/sys/netgraph/ng_parse.c +++ b/sys/netgraph/ng_parse.c @@ -1199,14 +1199,14 @@ ng_parse_composite(const struct ng_parse_type *type, const char *s, int *off, const u_char *const start, u_char *const buf, int *buflen, const enum comptype ctype) { - const int num = ng_get_composite_len(type, start, buf, ctype); int nextIndex = 0; /* next implicit array index */ u_int index; /* field or element index */ int *foff; /* field value offsets in string */ int align, len, blen, error = 0; /* Initialize */ - if (num < 0) + const int num = ng_get_composite_len(type, start, buf, ctype); + if (num < 0 || num > INT_MAX / sizeof(*foff)) return (EINVAL); foff = malloc(num * sizeof(*foff), M_NETGRAPH_PARSE, M_NOWAIT | M_ZERO); if (foff == NULL) { diff --git a/sys/netinet/in.c b/sys/netinet/in.c index 963449d4b4b1..0e283a7d099d 100644 --- a/sys/netinet/in.c +++ b/sys/netinet/in.c @@ -522,8 +522,8 @@ in_aifaddr_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp, struct ucred *cred /* * Check if bridge wants to allow adding addrs to member interfaces. */ - if (ifp->if_bridge && bridge_member_ifaddrs_p && - !bridge_member_ifaddrs_p()) + if (ifp->if_bridge != NULL && ifp->if_type != IFT_GIF && + bridge_member_ifaddrs_p != NULL && !bridge_member_ifaddrs_p()) return (EINVAL); /* diff --git a/sys/netinet/tcp_syncache.c b/sys/netinet/tcp_syncache.c index 80e6b53d10df..1ee6c6e31f33 100644 --- a/sys/netinet/tcp_syncache.c +++ b/sys/netinet/tcp_syncache.c @@ -102,15 +102,15 @@ #include <security/mac/mac_framework.h> -VNET_DEFINE_STATIC(int, tcp_syncookies) = 1; +VNET_DEFINE_STATIC(bool, tcp_syncookies) = true; #define V_tcp_syncookies VNET(tcp_syncookies) -SYSCTL_INT(_net_inet_tcp, OID_AUTO, syncookies, CTLFLAG_VNET | CTLFLAG_RW, +SYSCTL_BOOL(_net_inet_tcp, OID_AUTO, syncookies, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_syncookies), 0, "Use TCP SYN cookies if the syncache overflows"); -VNET_DEFINE_STATIC(int, tcp_syncookiesonly) = 0; +VNET_DEFINE_STATIC(bool, tcp_syncookiesonly) = false; #define V_tcp_syncookiesonly VNET(tcp_syncookiesonly) -SYSCTL_INT(_net_inet_tcp, OID_AUTO, syncookies_only, CTLFLAG_VNET | CTLFLAG_RW, +SYSCTL_BOOL(_net_inet_tcp, OID_AUTO, syncookies_only, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_syncookiesonly), 0, "Use only TCP SYN cookies"); @@ -553,9 +553,8 @@ syncache_timer(void *xsch) static inline bool syncache_cookiesonly(void) { - - return (V_tcp_syncookies && (V_tcp_syncache.paused || - V_tcp_syncookiesonly)); + return ((V_tcp_syncookies && V_tcp_syncache.paused) || + V_tcp_syncookiesonly); } /* @@ -1083,40 +1082,48 @@ syncache_expand(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, #endif if (sc == NULL) { - /* - * There is no syncache entry, so see if this ACK is - * a returning syncookie. To do this, first: - * A. Check if syncookies are used in case of syncache - * overflows - * B. See if this socket has had a syncache entry dropped in - * the recent past. We don't want to accept a bogus - * syncookie if we've never received a SYN or accept it - * twice. - * C. check that the syncookie is valid. If it is, then - * cobble up a fake syncache entry, and return. - */ - if (locked && !V_tcp_syncookies) { - SCH_UNLOCK(sch); - TCPSTAT_INC(tcps_sc_spurcookie); - if ((s = tcp_log_addrs(inc, th, NULL, NULL))) - log(LOG_DEBUG, "%s; %s: Spurious ACK, " - "segment rejected (syncookies disabled)\n", - s, __func__); - goto failed; - } - if (locked && !V_tcp_syncookiesonly && - sch->sch_last_overflow < time_uptime - SYNCOOKIE_LIFETIME) { + if (locked) { + /* + * The syncache is currently in use (neither disabled, + * nor paused), but no entry was found. + */ + if (!V_tcp_syncookies) { + /* + * Since no syncookies are used in case of + * a bucket overflow, don't even check for + * a valid syncookie. + */ + SCH_UNLOCK(sch); + TCPSTAT_INC(tcps_sc_spurcookie); + if ((s = tcp_log_addrs(inc, th, NULL, NULL))) + log(LOG_DEBUG, "%s; %s: Spurious ACK, " + "segment rejected " + "(syncookies disabled)\n", + s, __func__); + goto failed; + } + if (sch->sch_last_overflow < + time_uptime - SYNCOOKIE_LIFETIME) { + /* + * Since the bucket did not overflow recently, + * don't even check for a valid syncookie. + */ + SCH_UNLOCK(sch); + TCPSTAT_INC(tcps_sc_spurcookie); + if ((s = tcp_log_addrs(inc, th, NULL, NULL))) + log(LOG_DEBUG, "%s; %s: Spurious ACK, " + "segment rejected " + "(no syncache entry)\n", + s, __func__); + goto failed; + } SCH_UNLOCK(sch); - TCPSTAT_INC(tcps_sc_spurcookie); - if ((s = tcp_log_addrs(inc, th, NULL, NULL))) - log(LOG_DEBUG, "%s; %s: Spurious ACK, " - "segment rejected (no syncache entry)\n", - s, __func__); - goto failed; } - if (locked) - SCH_UNLOCK(sch); bzero(&scs, sizeof(scs)); + /* + * Now check, if the syncookie is valid. If it is, create an on + * stack syncache entry. + */ if (syncookie_expand(inc, sch, &scs, th, to, *lsop, port)) { sc = &scs; TCPSTAT_INC(tcps_sc_recvcookie); @@ -1291,10 +1298,9 @@ syncache_expand(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, if (__predict_false(*lsop == NULL)) { TCPSTAT_INC(tcps_sc_aborted); TCPSTATES_DEC(TCPS_SYN_RECEIVED); - } else + } else if (sc != &scs) TCPSTAT_INC(tcps_sc_completed); -/* how do we find the inp for the new socket? */ if (sc != &scs) syncache_free(sc); return (1); @@ -1669,7 +1675,7 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, sc->sc_tsoff = tcp_new_ts_offset(inc); } if ((to->to_flags & TOF_SCALE) && (V_tcp_do_rfc1323 != 3)) { - int wscale = 0; + u_int wscale = 0; /* * Pick the smallest possible scaling factor that @@ -1719,13 +1725,13 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, if (V_tcp_do_ecn && (tp->t_flags2 & TF2_CANNOT_DO_ECN) == 0) sc->sc_flags |= tcp_ecn_syncache_add(tcp_get_flags(th), iptos); - if (V_tcp_syncookies) + if (V_tcp_syncookies || V_tcp_syncookiesonly) sc->sc_iss = syncookie_generate(sch, sc); else sc->sc_iss = arc4random(); #ifdef INET6 if (autoflowlabel) { - if (V_tcp_syncookies) + if (V_tcp_syncookies || V_tcp_syncookiesonly) sc->sc_flowlabel = sc->sc_iss; else sc->sc_flowlabel = ip6_randomflowlabel(); @@ -2265,7 +2271,7 @@ syncookie_expand(struct in_conninfo *inc, const struct syncache_head *sch, uint32_t hash; uint8_t *secbits; tcp_seq ack, seq; - int wnd, wscale = 0; + int wnd; union syncookie cookie; /* @@ -2316,12 +2322,14 @@ syncookie_expand(struct in_conninfo *inc, const struct syncache_head *sch, sc->sc_peer_mss = tcp_sc_msstab[cookie.flags.mss_idx]; - /* We can simply recompute receive window scale we sent earlier. */ - while (wscale < TCP_MAX_WINSHIFT && (TCP_MAXWIN << wscale) < sb_max) - wscale++; - /* Only use wscale if it was enabled in the orignal SYN. */ if (cookie.flags.wscale_idx > 0) { + u_int wscale = 0; + + /* Recompute the receive window scale that was sent earlier. */ + while (wscale < TCP_MAX_WINSHIFT && + (TCP_MAXWIN << wscale) < sb_max) + wscale++; sc->sc_requested_r_scale = wscale; sc->sc_requested_s_scale = tcp_sc_wstab[cookie.flags.wscale_idx]; sc->sc_flags |= SCF_WINSCALE; diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c index 3e6519118a40..cea8a916679b 100644 --- a/sys/netinet/udp_usrreq.c +++ b/sys/netinet/udp_usrreq.c @@ -223,16 +223,18 @@ VNET_SYSUNINIT(udp, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH, udp_destroy, NULL); * udp_append() will convert to a sockaddr_in6 before passing the address * into the socket code. * - * In the normal case udp_append() will return 0, indicating that you - * must unlock the inp. However if a tunneling protocol is in place we increment - * the inpcb refcnt and unlock the inp, on return from the tunneling protocol we - * then decrement the reference count. If the inp_rele returns 1, indicating the - * inp is gone, we return that to the caller to tell them *not* to unlock - * the inp. In the case of multi-cast this will cause the distribution - * to stop (though most tunneling protocols known currently do *not* use - * multicast). + * In the normal case udp_append() will return 'false', indicating that you + * must unlock the inpcb. However if a tunneling protocol is in place we + * increment the inpcb refcnt and unlock the inpcb, on return from the tunneling + * protocol we then decrement the reference count. If in_pcbrele_rlocked() + * returns 'true', indicating the inpcb is gone, we return that to the caller + * to tell them *not* to unlock the inpcb. In the case of multicast this will + * cause the distribution to stop (though most tunneling protocols known + * currently do *not* use multicast). + * + * The mbuf is always consumed. */ -static int +static bool udp_append(struct inpcb *inp, struct ip *ip, struct mbuf *n, int off, struct sockaddr_in *udp_in) { @@ -255,15 +257,16 @@ udp_append(struct inpcb *inp, struct ip *ip, struct mbuf *n, int off, in_pcbref(inp); INP_RUNLOCK(inp); - filtered = (*up->u_tun_func)(n, off, inp, (struct sockaddr *)&udp_in[0], - up->u_tun_ctx); + filtered = (*up->u_tun_func)(n, off, inp, + (struct sockaddr *)&udp_in[0], up->u_tun_ctx); INP_RLOCK(inp); - if (in_pcbrele_rlocked(inp)) - return (1); - if (filtered) { - INP_RUNLOCK(inp); - return (1); + if (in_pcbrele_rlocked(inp)) { + if (!filtered) + m_freem(n); + return (true); } + if (filtered) + return (false); } off += sizeof(struct udphdr); @@ -273,18 +276,18 @@ udp_append(struct inpcb *inp, struct ip *ip, struct mbuf *n, int off, if (IPSEC_ENABLED(ipv4) && IPSEC_CHECK_POLICY(ipv4, n, inp) != 0) { m_freem(n); - return (0); + return (false); } if (up->u_flags & UF_ESPINUDP) {/* IPSec UDP encaps. */ if (IPSEC_ENABLED(ipv4) && UDPENCAP_INPUT(ipv4, n, off, AF_INET) != 0) - return (0); /* Consumed. */ + return (false); } #endif /* IPSEC */ #ifdef MAC if (mac_inpcb_check_deliver(inp, n) != 0) { m_freem(n); - return (0); + return (false); } #endif /* MAC */ if (inp->inp_flags & INP_CONTROLOPTS || @@ -330,7 +333,7 @@ udp_append(struct inpcb *inp, struct ip *ip, struct mbuf *n, int off, UDPSTAT_INC(udps_fullsock); } else sorwakeup_locked(so); - return (0); + return (false); } static bool @@ -699,7 +702,7 @@ udp_input(struct mbuf **mp, int *offp, int proto) UDPLITE_PROBE(receive, NULL, inp, ip, inp, uh); else UDP_PROBE(receive, NULL, inp, ip, inp, uh); - if (udp_append(inp, ip, m, iphlen, udp_in) == 0) + if (!udp_append(inp, ip, m, iphlen, udp_in)) INP_RUNLOCK(inp); return (IPPROTO_DONE); diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c index ce0655408a28..a9e6c4eaa51b 100644 --- a/sys/netinet6/in6.c +++ b/sys/netinet6/in6.c @@ -1236,8 +1236,8 @@ in6_addifaddr(struct ifnet *ifp, struct in6_aliasreq *ifra, struct in6_ifaddr *i int error; /* Check if this interface is a bridge member */ - if (ifp->if_bridge && bridge_member_ifaddrs_p && - !bridge_member_ifaddrs_p()) { + if (ifp->if_bridge != NULL && ifp->if_type != IFT_GIF && + bridge_member_ifaddrs_p != NULL && !bridge_member_ifaddrs_p()) { error = EINVAL; goto out; } diff --git a/sys/netpfil/pf/if_pfsync.c b/sys/netpfil/pf/if_pfsync.c index 585c196391c0..7b9405ee1f8d 100644 --- a/sys/netpfil/pf/if_pfsync.c +++ b/sys/netpfil/pf/if_pfsync.c @@ -605,7 +605,8 @@ pfsync_state_import(union pfsync_state_union *sp, int flags, int msg_version) rt_kif = rpool_first->kif; /* * Guess the AF of the route address, FreeBSD 13 does - * not support af-to so it should be safe. + * not support af-to nor prefer-ipv6-nexthop + * so it should be safe. */ rt_af = r->af; } else if (!PF_AZERO(&sp->pfs_1301.rt_addr, sp->pfs_1301.af)) { @@ -634,8 +635,9 @@ pfsync_state_import(union pfsync_state_union *sp, int flags, int msg_version) } rt = sp->pfs_1400.rt; /* - * Guess the AF of the route address, FreeBSD 13 does - * not support af-to so it should be safe. + * Guess the AF of the route address, FreeBSD 14 does + * not support af-to nor prefer-ipv6-nexthop + * so it should be safe. */ rt_af = sp->pfs_1400.af; } @@ -1741,16 +1743,16 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data) if (ifr->ifr_cap_nv.length > IFR_CAP_NV_MAXBUFSIZE) return (EINVAL); - data = malloc(ifr->ifr_cap_nv.length, M_TEMP, M_WAITOK); + data = malloc(ifr->ifr_cap_nv.length, M_PF, M_WAITOK); if ((error = copyin(ifr->ifr_cap_nv.buffer, data, ifr->ifr_cap_nv.length)) != 0) { - free(data, M_TEMP); + free(data, M_PF); return (error); } if ((nvl = nvlist_unpack(data, ifr->ifr_cap_nv.length, 0)) == NULL) { - free(data, M_TEMP); + free(data, M_PF); return (EINVAL); } @@ -1758,7 +1760,7 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data) pfsync_nvstatus_to_kstatus(nvl, &status); nvlist_destroy(nvl); - free(data, M_TEMP); + free(data, M_PF); error = pfsync_kstatus_to_softc(&status, sc); return (error); diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c index 8cd4fff95b15..d15a16c54f81 100644 --- a/sys/netpfil/pf/pf.c +++ b/sys/netpfil/pf/pf.c @@ -1667,7 +1667,6 @@ pf_state_key_addr_setup(struct pf_pdesc *pd, #ifdef INET6 struct nd_neighbor_solicit nd; struct pf_addr *target; - u_short action, reason; if (pd->af == AF_INET || pd->proto != IPPROTO_ICMPV6) goto copy; @@ -1676,7 +1675,8 @@ pf_state_key_addr_setup(struct pf_pdesc *pd, case ND_NEIGHBOR_SOLICIT: if (multi) return (-1); - if (!pf_pull_hdr(pd->m, pd->off, &nd, sizeof(nd), &action, &reason, pd->af)) + if (!pf_pull_hdr(pd->m, pd->off, &nd, sizeof(nd), NULL, + NULL, pd->af)) return (-1); target = (struct pf_addr *)&nd.nd_ns_target; daddr = target; @@ -1684,7 +1684,8 @@ pf_state_key_addr_setup(struct pf_pdesc *pd, case ND_NEIGHBOR_ADVERT: if (multi) return (-1); - if (!pf_pull_hdr(pd->m, pd->off, &nd, sizeof(nd), &action, &reason, pd->af)) + if (!pf_pull_hdr(pd->m, pd->off, &nd, sizeof(nd), NULL, + NULL, pd->af)) return (-1); target = (struct pf_addr *)&nd.nd_ns_target; saddr = target; @@ -5960,7 +5961,9 @@ pf_test_rule(struct pf_krule **rm, struct pf_kstate **sm, if (r->rt) { /* * Set act.rt here instead of in pf_rule_to_actions() because - * it is applied only from the last pass rule. + * it is applied only from the last pass rule. For rules + * with the prefer-ipv6-nexthop option act.rt_af is a hint + * about AF of the forwarded packet and might be changed. */ pd->act.rt = r->rt; if (r->rt == PF_REPLYTO) @@ -8045,6 +8048,7 @@ pf_test_state_icmp(struct pf_kstate **state, struct pf_pdesc *pd, return (PF_DROP); pd2.tot_len = ntohs(h2.ip_len); + pd2.ttl = h2.ip_ttl; pd2.src = (struct pf_addr *)&h2.ip_src; pd2.dst = (struct pf_addr *)&h2.ip_dst; pd2.ip_sum = &h2.ip_sum; @@ -8067,6 +8071,7 @@ pf_test_state_icmp(struct pf_kstate **state, struct pf_pdesc *pd, pd2.tot_len = ntohs(h2_6.ip6_plen) + sizeof(struct ip6_hdr); + pd2.ttl = h2_6.ip6_hlim; pd2.src = (struct pf_addr *)&h2_6.ip6_src; pd2.dst = (struct pf_addr *)&h2_6.ip6_dst; pd2.ip_sum = NULL; @@ -8974,9 +8979,10 @@ pf_route(struct pf_krule *r, struct ifnet *oifp, struct pf_kstate *s, struct pf_pdesc *pd, struct inpcb *inp) { struct mbuf *m0, *m1, *md; - struct route ro; - const struct sockaddr *gw = &ro.ro_dst; - struct sockaddr_in *dst; + struct route_in6 ro; + union sockaddr_union rt_gw; + const union sockaddr_union *gw = (const union sockaddr_union *)&ro.ro_dst; + union sockaddr_union *dst; struct ip *ip; struct ifnet *ifp = NULL; int error = 0; @@ -9071,10 +9077,35 @@ pf_route(struct pf_krule *r, struct ifnet *oifp, ip = mtod(m0, struct ip *); bzero(&ro, sizeof(ro)); - dst = (struct sockaddr_in *)&ro.ro_dst; - dst->sin_family = AF_INET; - dst->sin_len = sizeof(struct sockaddr_in); - dst->sin_addr.s_addr = pd->act.rt_addr.v4.s_addr; + dst = (union sockaddr_union *)&ro.ro_dst; + dst->sin.sin_family = AF_INET; + dst->sin.sin_len = sizeof(struct sockaddr_in); + dst->sin.sin_addr = ip->ip_dst; + if (ifp) { /* Only needed in forward direction and route-to */ + bzero(&rt_gw, sizeof(rt_gw)); + ro.ro_flags |= RT_HAS_GW; + gw = &rt_gw; + switch (pd->act.rt_af) { +#ifdef INET + case AF_INET: + rt_gw.sin.sin_family = AF_INET; + rt_gw.sin.sin_len = sizeof(struct sockaddr_in); + rt_gw.sin.sin_addr.s_addr = pd->act.rt_addr.v4.s_addr; + break; +#endif /* INET */ +#ifdef INET6 + case AF_INET6: + rt_gw.sin6.sin6_family = AF_INET6; + rt_gw.sin6.sin6_len = sizeof(struct sockaddr_in6); + pf_addrcpy((struct pf_addr *)&rt_gw.sin6.sin6_addr, + &pd->act.rt_addr, AF_INET6); + break; +#endif /* INET6 */ + default: + /* Normal af-to without route-to */ + break; + } + } if (pd->dir == PF_IN) { if (ip->ip_ttl <= IPTTLDEC) { @@ -9098,10 +9129,10 @@ pf_route(struct pf_krule *r, struct ifnet *oifp, /* Use the gateway if needed. */ if (nh->nh_flags & NHF_GATEWAY) { - gw = &nh->gw_sa; + gw = (const union sockaddr_union *)&nh->gw_sa; ro.ro_flags |= RT_HAS_GW; } else { - dst->sin_addr = ip->ip_dst; + dst->sin.sin_addr = ip->ip_dst; } /* @@ -9126,6 +9157,9 @@ pf_route(struct pf_krule *r, struct ifnet *oifp, PF_STATE_UNLOCK(s); } + /* It must have been either set from rt_af or from fib4_lookup */ + KASSERT(gw->sin.sin_family != 0, ("%s: gw address family undetermined", __func__)); + if (ifp == NULL) { m0 = pd->m; pd->m = NULL; @@ -9210,9 +9244,11 @@ pf_route(struct pf_krule *r, struct ifnet *oifp, m_clrprotoflags(m0); /* Avoid confusing lower layers. */ md = m0; - error = pf_dummynet_route(pd, s, r, ifp, gw, &md); + error = pf_dummynet_route(pd, s, r, ifp, + (const struct sockaddr *)gw, &md); if (md != NULL) { - error = (*ifp->if_output)(ifp, md, gw, &ro); + error = (*ifp->if_output)(ifp, md, + (const struct sockaddr *)gw, (struct route *)&ro); SDT_PROBE2(pf, ip, route_to, output, ifp, error); } goto done; @@ -9253,9 +9289,11 @@ pf_route(struct pf_krule *r, struct ifnet *oifp, md = m0; pd->pf_mtag = pf_find_mtag(md); error = pf_dummynet_route(pd, s, r, ifp, - gw, &md); + (const struct sockaddr *)gw, &md); if (md != NULL) { - error = (*ifp->if_output)(ifp, md, gw, &ro); + error = (*ifp->if_output)(ifp, md, + (const struct sockaddr *)gw, + (struct route *)&ro); SDT_PROBE2(pf, ip, route_to, output, ifp, error); } } else @@ -9962,9 +10000,12 @@ pf_walk_header(struct pf_pdesc *pd, struct ip *h, u_short *reason) pd->proto = h->ip_p; /* IGMP packets have router alert options, allow them */ if (pd->proto == IPPROTO_IGMP) { - /* According to RFC 1112 ttl must be set to 1. */ - if ((h->ip_ttl != 1) || - !IN_MULTICAST(ntohl(h->ip_dst.s_addr))) { + /* + * According to RFC 1112 ttl must be set to 1 in all IGMP + * packets sent to 224.0.0.1 + */ + if ((h->ip_ttl != 1) && + (h->ip_dst.s_addr == INADDR_ALLHOSTS_GROUP)) { DPFPRINTF(PF_DEBUG_MISC, "Invalid IGMP"); REASON_SET(reason, PFRES_IPOPTIONS); return (PF_DROP); diff --git a/sys/netpfil/pf/pf.h b/sys/netpfil/pf/pf.h index 51b3fd6390e1..8edd5a5110a1 100644 --- a/sys/netpfil/pf/pf.h +++ b/sys/netpfil/pf/pf.h @@ -131,6 +131,7 @@ enum { PF_ADDR_ADDRMASK, PF_ADDR_NOROUTE, PF_ADDR_DYNIFTL, #define PF_POOL_TYPEMASK 0x0f #define PF_POOL_STICKYADDR 0x20 #define PF_POOL_ENDPI 0x40 +#define PF_POOL_IPV6NH 0x80 #define PF_WSCALE_FLAG 0x80 #define PF_WSCALE_MASK 0x0f diff --git a/sys/netpfil/pf/pf_ioctl.c b/sys/netpfil/pf/pf_ioctl.c index e5da05a958f6..06c40a03f575 100644 --- a/sys/netpfil/pf/pf_ioctl.c +++ b/sys/netpfil/pf/pf_ioctl.c @@ -187,6 +187,7 @@ VNET_DEFINE(uma_zone_t, pf_tag_z); #define V_pf_tag_z VNET(pf_tag_z) static MALLOC_DEFINE(M_PFALTQ, "pf_altq", "pf(4) altq configuration db"); static MALLOC_DEFINE(M_PFRULE, "pf_rule", "pf(4) rules"); +MALLOC_DEFINE(M_PF, "pf", "pf(4)"); #if (PF_QNAME_SIZE != PF_TAG_NAME_SIZE) #error PF_QNAME_SIZE must be equal to PF_TAG_NAME_SIZE @@ -1181,18 +1182,18 @@ pf_rule_tree_alloc(int flags) { struct pf_krule_global *tree; - tree = malloc(sizeof(struct pf_krule_global), M_TEMP, flags); + tree = malloc(sizeof(struct pf_krule_global), M_PF, flags); if (tree == NULL) return (NULL); RB_INIT(tree); return (tree); } -static void +void pf_rule_tree_free(struct pf_krule_global *tree) { - free(tree, M_TEMP); + free(tree, M_PF); } static int @@ -1211,7 +1212,7 @@ pf_begin_rules(u_int32_t *ticket, int rs_num, const char *anchor) return (ENOMEM); rs = pf_find_or_create_kruleset(anchor); if (rs == NULL) { - free(tree, M_TEMP); + pf_rule_tree_free(tree); return (EINVAL); } pf_rule_tree_free(rs->rules[rs_num].inactive.tree); @@ -1432,7 +1433,7 @@ pf_commit_rules(u_int32_t ticket, int rs_num, char *anchor) rs->rules[rs_num].inactive.rcount = 0; rs->rules[rs_num].inactive.open = 0; pf_remove_if_empty_kruleset(rs); - free(old_tree, M_TEMP); + pf_rule_tree_free(old_tree); return (0); } @@ -2276,6 +2277,7 @@ pf_ioctl_addrule(struct pf_krule *rule, uint32_t ticket, rule->nat.cur = TAILQ_FIRST(&rule->nat.list); rule->rdr.cur = TAILQ_FIRST(&rule->rdr.list); rule->route.cur = TAILQ_FIRST(&rule->route.list); + rule->route.ipv6_nexthop_af = AF_INET6; TAILQ_INSERT_TAIL(ruleset->rules[rs_num].inactive.ptr, rule, entries); ruleset->rules[rs_num].inactive.rcount++; @@ -4076,7 +4078,7 @@ DIOCCHANGERULE_error: out = ps->ps_states; pstore = mallocarray(slice_count, - sizeof(struct pfsync_state_1301), M_TEMP, M_WAITOK | M_ZERO); + sizeof(struct pfsync_state_1301), M_PF, M_WAITOK | M_ZERO); nr = 0; for (i = 0; i <= V_pf_hashmask; i++) { @@ -4098,10 +4100,10 @@ DIOCGETSTATES_retry: if (count > slice_count) { PF_HASHROW_UNLOCK(ih); - free(pstore, M_TEMP); + free(pstore, M_PF); slice_count = count * 2; pstore = mallocarray(slice_count, - sizeof(struct pfsync_state_1301), M_TEMP, + sizeof(struct pfsync_state_1301), M_PF, M_WAITOK | M_ZERO); goto DIOCGETSTATES_retry; } @@ -4123,13 +4125,15 @@ DIOCGETSTATES_retry: PF_HASHROW_UNLOCK(ih); error = copyout(pstore, out, sizeof(struct pfsync_state_1301) * count); - if (error) + if (error) { + free(pstore, M_PF); goto fail; + } out = ps->ps_states + nr; } DIOCGETSTATES_full: ps->ps_len = sizeof(struct pfsync_state_1301) * nr; - free(pstore, M_TEMP); + free(pstore, M_PF); break; } @@ -4155,7 +4159,7 @@ DIOCGETSTATES_full: out = ps->ps_states; pstore = mallocarray(slice_count, - sizeof(struct pf_state_export), M_TEMP, M_WAITOK | M_ZERO); + sizeof(struct pf_state_export), M_PF, M_WAITOK | M_ZERO); nr = 0; for (i = 0; i <= V_pf_hashmask; i++) { @@ -4177,10 +4181,10 @@ DIOCGETSTATESV2_retry: if (count > slice_count) { PF_HASHROW_UNLOCK(ih); - free(pstore, M_TEMP); + free(pstore, M_PF); slice_count = count * 2; pstore = mallocarray(slice_count, - sizeof(struct pf_state_export), M_TEMP, + sizeof(struct pf_state_export), M_PF, M_WAITOK | M_ZERO); goto DIOCGETSTATESV2_retry; } @@ -4201,13 +4205,15 @@ DIOCGETSTATESV2_retry: PF_HASHROW_UNLOCK(ih); error = copyout(pstore, out, sizeof(struct pf_state_export) * count); - if (error) + if (error) { + free(pstore, M_PF); goto fail; + } out = ps->ps_states + nr; } DIOCGETSTATESV2_full: ps->ps_len = nr * sizeof(struct pf_state_export); - free(pstore, M_TEMP); + free(pstore, M_PF); break; } @@ -4737,17 +4743,17 @@ DIOCCHANGEADDR_error: totlen = io->pfrio_size * sizeof(struct pfr_table); pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->pfrio_buffer, pfrts, totlen); if (error) { - free(pfrts, M_TEMP); + free(pfrts, M_PF); goto fail; } PF_RULES_WLOCK(); error = pfr_add_tables(pfrts, io->pfrio_size, &io->pfrio_nadd, io->pfrio_flags | PFR_FLAG_USERIOCTL); PF_RULES_WUNLOCK(); - free(pfrts, M_TEMP); + free(pfrts, M_PF); break; } @@ -4769,17 +4775,17 @@ DIOCCHANGEADDR_error: totlen = io->pfrio_size * sizeof(struct pfr_table); pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->pfrio_buffer, pfrts, totlen); if (error) { - free(pfrts, M_TEMP); + free(pfrts, M_PF); goto fail; } PF_RULES_WLOCK(); error = pfr_del_tables(pfrts, io->pfrio_size, &io->pfrio_ndel, io->pfrio_flags | PFR_FLAG_USERIOCTL); PF_RULES_WUNLOCK(); - free(pfrts, M_TEMP); + free(pfrts, M_PF); break; } @@ -4805,7 +4811,7 @@ DIOCCHANGEADDR_error: totlen = io->pfrio_size * sizeof(struct pfr_table); pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), - M_TEMP, M_NOWAIT | M_ZERO); + M_PF, M_NOWAIT | M_ZERO); if (pfrts == NULL) { error = ENOMEM; PF_RULES_RUNLOCK(); @@ -4816,7 +4822,7 @@ DIOCCHANGEADDR_error: PF_RULES_RUNLOCK(); if (error == 0) error = copyout(pfrts, io->pfrio_buffer, totlen); - free(pfrts, M_TEMP); + free(pfrts, M_PF); break; } @@ -4843,7 +4849,7 @@ DIOCCHANGEADDR_error: totlen = io->pfrio_size * sizeof(struct pfr_tstats); pfrtstats = mallocarray(io->pfrio_size, - sizeof(struct pfr_tstats), M_TEMP, M_NOWAIT | M_ZERO); + sizeof(struct pfr_tstats), M_PF, M_NOWAIT | M_ZERO); if (pfrtstats == NULL) { error = ENOMEM; PF_RULES_RUNLOCK(); @@ -4856,7 +4862,7 @@ DIOCCHANGEADDR_error: PF_TABLE_STATS_UNLOCK(); if (error == 0) error = copyout(pfrtstats, io->pfrio_buffer, totlen); - free(pfrtstats, M_TEMP); + free(pfrtstats, M_PF); break; } @@ -4881,10 +4887,10 @@ DIOCCHANGEADDR_error: totlen = io->pfrio_size * sizeof(struct pfr_table); pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->pfrio_buffer, pfrts, totlen); if (error) { - free(pfrts, M_TEMP); + free(pfrts, M_PF); goto fail; } @@ -4894,7 +4900,7 @@ DIOCCHANGEADDR_error: &io->pfrio_nzero, io->pfrio_flags | PFR_FLAG_USERIOCTL); PF_RULES_RUNLOCK(); PF_TABLE_STATS_UNLOCK(); - free(pfrts, M_TEMP); + free(pfrts, M_PF); break; } @@ -4922,10 +4928,10 @@ DIOCCHANGEADDR_error: totlen = io->pfrio_size * sizeof(struct pfr_table); pfrts = mallocarray(io->pfrio_size, sizeof(struct pfr_table), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->pfrio_buffer, pfrts, totlen); if (error) { - free(pfrts, M_TEMP); + free(pfrts, M_PF); goto fail; } PF_RULES_WLOCK(); @@ -4933,7 +4939,7 @@ DIOCCHANGEADDR_error: io->pfrio_setflag, io->pfrio_clrflag, &io->pfrio_nchange, &io->pfrio_ndel, io->pfrio_flags | PFR_FLAG_USERIOCTL); PF_RULES_WUNLOCK(); - free(pfrts, M_TEMP); + free(pfrts, M_PF); break; } @@ -4968,10 +4974,10 @@ DIOCCHANGEADDR_error: } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->pfrio_buffer, pfras, totlen); if (error) { - free(pfras, M_TEMP); + free(pfras, M_PF); goto fail; } PF_RULES_WLOCK(); @@ -4982,7 +4988,7 @@ DIOCCHANGEADDR_error: PF_RULES_WUNLOCK(); if (error == 0 && io->pfrio_flags & PFR_FLAG_FEEDBACK) error = copyout(pfras, io->pfrio_buffer, totlen); - free(pfras, M_TEMP); + free(pfras, M_PF); break; } @@ -5003,10 +5009,10 @@ DIOCCHANGEADDR_error: } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->pfrio_buffer, pfras, totlen); if (error) { - free(pfras, M_TEMP); + free(pfras, M_PF); goto fail; } PF_RULES_WLOCK(); @@ -5016,7 +5022,7 @@ DIOCCHANGEADDR_error: PF_RULES_WUNLOCK(); if (error == 0 && io->pfrio_flags & PFR_FLAG_FEEDBACK) error = copyout(pfras, io->pfrio_buffer, totlen); - free(pfras, M_TEMP); + free(pfras, M_PF); break; } @@ -5040,11 +5046,11 @@ DIOCCHANGEADDR_error: goto fail; } totlen = count * sizeof(struct pfr_addr); - pfras = mallocarray(count, sizeof(struct pfr_addr), M_TEMP, + pfras = mallocarray(count, sizeof(struct pfr_addr), M_PF, M_WAITOK); error = copyin(io->pfrio_buffer, pfras, totlen); if (error) { - free(pfras, M_TEMP); + free(pfras, M_PF); goto fail; } PF_RULES_WLOCK(); @@ -5055,7 +5061,7 @@ DIOCCHANGEADDR_error: PF_RULES_WUNLOCK(); if (error == 0 && io->pfrio_flags & PFR_FLAG_FEEDBACK) error = copyout(pfras, io->pfrio_buffer, totlen); - free(pfras, M_TEMP); + free(pfras, M_PF); break; } @@ -5076,14 +5082,14 @@ DIOCCHANGEADDR_error: } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK | M_ZERO); + M_PF, M_WAITOK | M_ZERO); PF_RULES_RLOCK(); error = pfr_get_addrs(&io->pfrio_table, pfras, &io->pfrio_size, io->pfrio_flags | PFR_FLAG_USERIOCTL); PF_RULES_RUNLOCK(); if (error == 0) error = copyout(pfras, io->pfrio_buffer, totlen); - free(pfras, M_TEMP); + free(pfras, M_PF); break; } @@ -5104,14 +5110,14 @@ DIOCCHANGEADDR_error: } totlen = io->pfrio_size * sizeof(struct pfr_astats); pfrastats = mallocarray(io->pfrio_size, - sizeof(struct pfr_astats), M_TEMP, M_WAITOK | M_ZERO); + sizeof(struct pfr_astats), M_PF, M_WAITOK | M_ZERO); PF_RULES_RLOCK(); error = pfr_get_astats(&io->pfrio_table, pfrastats, &io->pfrio_size, io->pfrio_flags | PFR_FLAG_USERIOCTL); PF_RULES_RUNLOCK(); if (error == 0) error = copyout(pfrastats, io->pfrio_buffer, totlen); - free(pfrastats, M_TEMP); + free(pfrastats, M_PF); break; } @@ -5132,10 +5138,10 @@ DIOCCHANGEADDR_error: } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->pfrio_buffer, pfras, totlen); if (error) { - free(pfras, M_TEMP); + free(pfras, M_PF); goto fail; } PF_RULES_WLOCK(); @@ -5145,7 +5151,7 @@ DIOCCHANGEADDR_error: PF_RULES_WUNLOCK(); if (error == 0 && io->pfrio_flags & PFR_FLAG_FEEDBACK) error = copyout(pfras, io->pfrio_buffer, totlen); - free(pfras, M_TEMP); + free(pfras, M_PF); break; } @@ -5166,10 +5172,10 @@ DIOCCHANGEADDR_error: } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->pfrio_buffer, pfras, totlen); if (error) { - free(pfras, M_TEMP); + free(pfras, M_PF); goto fail; } PF_RULES_RLOCK(); @@ -5179,7 +5185,7 @@ DIOCCHANGEADDR_error: PF_RULES_RUNLOCK(); if (error == 0) error = copyout(pfras, io->pfrio_buffer, totlen); - free(pfras, M_TEMP); + free(pfras, M_PF); break; } @@ -5200,10 +5206,10 @@ DIOCCHANGEADDR_error: } totlen = io->pfrio_size * sizeof(struct pfr_addr); pfras = mallocarray(io->pfrio_size, sizeof(struct pfr_addr), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->pfrio_buffer, pfras, totlen); if (error) { - free(pfras, M_TEMP); + free(pfras, M_PF); goto fail; } PF_RULES_WLOCK(); @@ -5211,7 +5217,7 @@ DIOCCHANGEADDR_error: io->pfrio_size, &io->pfrio_nadd, &io->pfrio_naddr, io->pfrio_ticket, io->pfrio_flags | PFR_FLAG_USERIOCTL); PF_RULES_WUNLOCK(); - free(pfras, M_TEMP); + free(pfras, M_PF); break; } @@ -5249,10 +5255,10 @@ DIOCCHANGEADDR_error: } totlen = sizeof(struct pfioc_trans_e) * io->size; ioes = mallocarray(io->size, sizeof(struct pfioc_trans_e), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->array, ioes, totlen); if (error) { - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; } PF_RULES_WLOCK(); @@ -5262,7 +5268,7 @@ DIOCCHANGEADDR_error: case PF_RULESET_ETH: if ((error = pf_begin_eth(&ioe->ticket, ioe->anchor))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; } break; @@ -5270,13 +5276,13 @@ DIOCCHANGEADDR_error: case PF_RULESET_ALTQ: if (ioe->anchor[0]) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); error = EINVAL; goto fail; } if ((error = pf_begin_altq(&ioe->ticket))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; } break; @@ -5291,7 +5297,7 @@ DIOCCHANGEADDR_error: if ((error = pfr_ina_begin(&table, &ioe->ticket, NULL, 0))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; } break; @@ -5300,7 +5306,7 @@ DIOCCHANGEADDR_error: if ((error = pf_begin_rules(&ioe->ticket, ioe->rs_num, ioe->anchor))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; } break; @@ -5308,7 +5314,7 @@ DIOCCHANGEADDR_error: } PF_RULES_WUNLOCK(); error = copyout(ioes, io->array, totlen); - free(ioes, M_TEMP); + free(ioes, M_PF); break; } @@ -5330,10 +5336,10 @@ DIOCCHANGEADDR_error: } totlen = sizeof(struct pfioc_trans_e) * io->size; ioes = mallocarray(io->size, sizeof(struct pfioc_trans_e), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->array, ioes, totlen); if (error) { - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; } PF_RULES_WLOCK(); @@ -5344,7 +5350,7 @@ DIOCCHANGEADDR_error: if ((error = pf_rollback_eth(ioe->ticket, ioe->anchor))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; /* really bad */ } break; @@ -5352,13 +5358,13 @@ DIOCCHANGEADDR_error: case PF_RULESET_ALTQ: if (ioe->anchor[0]) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); error = EINVAL; goto fail; } if ((error = pf_rollback_altq(ioe->ticket))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; /* really bad */ } break; @@ -5373,7 +5379,7 @@ DIOCCHANGEADDR_error: if ((error = pfr_ina_rollback(&table, ioe->ticket, NULL, 0))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; /* really bad */ } break; @@ -5382,14 +5388,14 @@ DIOCCHANGEADDR_error: if ((error = pf_rollback_rules(ioe->ticket, ioe->rs_num, ioe->anchor))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; /* really bad */ } break; } } PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); break; } @@ -5415,10 +5421,10 @@ DIOCCHANGEADDR_error: totlen = sizeof(struct pfioc_trans_e) * io->size; ioes = mallocarray(io->size, sizeof(struct pfioc_trans_e), - M_TEMP, M_WAITOK); + M_PF, M_WAITOK); error = copyin(io->array, ioes, totlen); if (error) { - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; } PF_RULES_WLOCK(); @@ -5431,7 +5437,7 @@ DIOCCHANGEADDR_error: if (ers == NULL || ioe->ticket == 0 || ioe->ticket != ers->inactive.ticket) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); error = EINVAL; goto fail; } @@ -5440,14 +5446,14 @@ DIOCCHANGEADDR_error: case PF_RULESET_ALTQ: if (ioe->anchor[0]) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); error = EINVAL; goto fail; } if (!V_altqs_inactive_open || ioe->ticket != V_ticket_altqs_inactive) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); error = EBUSY; goto fail; } @@ -5458,7 +5464,7 @@ DIOCCHANGEADDR_error: if (rs == NULL || !rs->topen || ioe->ticket != rs->tticket) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); error = EBUSY; goto fail; } @@ -5467,7 +5473,7 @@ DIOCCHANGEADDR_error: if (ioe->rs_num < 0 || ioe->rs_num >= PF_RULESET_MAX) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); error = EINVAL; goto fail; } @@ -5477,7 +5483,7 @@ DIOCCHANGEADDR_error: rs->rules[ioe->rs_num].inactive.ticket != ioe->ticket) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); error = EBUSY; goto fail; } @@ -5490,7 +5496,7 @@ DIOCCHANGEADDR_error: case PF_RULESET_ETH: if ((error = pf_commit_eth(ioe->ticket, ioe->anchor))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; /* really bad */ } break; @@ -5498,7 +5504,7 @@ DIOCCHANGEADDR_error: case PF_RULESET_ALTQ: if ((error = pf_commit_altq(ioe->ticket))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; /* really bad */ } break; @@ -5513,7 +5519,7 @@ DIOCCHANGEADDR_error: if ((error = pfr_ina_commit(&table, ioe->ticket, NULL, NULL, 0))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; /* really bad */ } break; @@ -5522,7 +5528,7 @@ DIOCCHANGEADDR_error: if ((error = pf_commit_rules(ioe->ticket, ioe->rs_num, ioe->anchor))) { PF_RULES_WUNLOCK(); - free(ioes, M_TEMP); + free(ioes, M_PF); goto fail; /* really bad */ } break; @@ -5536,7 +5542,7 @@ DIOCCHANGEADDR_error: else dehook_pf_eth(); - free(ioes, M_TEMP); + free(ioes, M_PF); break; } @@ -5565,7 +5571,7 @@ DIOCCHANGEADDR_error: nr = 0; - p = pstore = malloc(psn->psn_len, M_TEMP, M_WAITOK | M_ZERO); + p = pstore = malloc(psn->psn_len, M_PF, M_WAITOK | M_ZERO); for (i = 0, sh = V_pf_srchash; i <= V_pf_srchashmask; i++, sh++) { PF_HASHROW_LOCK(sh); @@ -5584,11 +5590,11 @@ DIOCCHANGEADDR_error: error = copyout(pstore, psn->psn_src_nodes, sizeof(struct pf_src_node) * nr); if (error) { - free(pstore, M_TEMP); + free(pstore, M_PF); goto fail; } psn->psn_len = sizeof(struct pf_src_node) * nr; - free(pstore, M_TEMP); + free(pstore, M_PF); break; } @@ -5655,13 +5661,13 @@ DIOCCHANGEADDR_error: bufsiz = io->pfiio_size * sizeof(struct pfi_kif); ifstore = mallocarray(io->pfiio_size, sizeof(struct pfi_kif), - M_TEMP, M_WAITOK | M_ZERO); + M_PF, M_WAITOK | M_ZERO); PF_RULES_RLOCK(); pfi_get_ifaces(io->pfiio_name, ifstore, &io->pfiio_size); PF_RULES_RUNLOCK(); error = copyout(ifstore, io->pfiio_buffer, bufsiz); - free(ifstore, M_TEMP); + free(ifstore, M_PF); break; } diff --git a/sys/netpfil/pf/pf_lb.c b/sys/netpfil/pf/pf_lb.c index bc9e1dc72902..b8b5157c9b15 100644 --- a/sys/netpfil/pf/pf_lb.c +++ b/sys/netpfil/pf/pf_lb.c @@ -545,11 +545,18 @@ pf_map_addr(sa_family_t saf, struct pf_krule *r, struct pf_addr *saddr, uint64_t hashidx; int cnt; sa_family_t wanted_af; + u_int8_t pool_type; + bool prefer_ipv6_nexthop = rpool->opts & PF_POOL_IPV6NH; KASSERT(saf != 0, ("%s: saf == 0", __func__)); KASSERT(naf != NULL, ("%s: naf = NULL", __func__)); KASSERT((*naf) != 0, ("%s: *naf = 0", __func__)); + /* + * Given (*naf) is a hint about AF of the forwarded packet. + * It might be changed if prefer_ipv6_nexthop is enabled and + * the combination of nexthop AF and packet AF allows for it. + */ wanted_af = (*naf); mtx_lock(&rpool->mtx); @@ -594,19 +601,38 @@ pf_map_addr(sa_family_t saf, struct pf_krule *r, struct pf_addr *saddr, } else { raddr = &rpool->cur->addr.v.a.addr; rmask = &rpool->cur->addr.v.a.mask; - /* - * For single addresses check their address family. Unless they - * have none, which happens when addresses are added with - * the old ioctl mechanism. In such case trust that the address - * has the proper AF. - */ - if (rpool->cur->af && rpool->cur->af != wanted_af) { - reason = PFRES_MAPFAILED; - goto done_pool_mtx; + } + + /* + * For pools with a single host with the prefer-ipv6-nexthop option + * we can return pool address of any AF, unless the forwarded packet + * is IPv6, then we can return only if pool address is IPv6. + * For non-prefer-ipv6-nexthop we can return pool address only + * of wanted AF, unless the pool address'es AF is unknown, which + * happens in case old ioctls have been used to set up the pool. + * + * Round-robin pools have their own logic for retrying next addresses. + */ + pool_type = rpool->opts & PF_POOL_TYPEMASK; + if (pool_type == PF_POOL_NONE || pool_type == PF_POOL_BITMASK || + ((pool_type == PF_POOL_RANDOM || pool_type == PF_POOL_SRCHASH) && + rpool->cur->addr.type != PF_ADDR_TABLE && + rpool->cur->addr.type != PF_ADDR_DYNIFTL)) { + if (prefer_ipv6_nexthop) { + if (rpool->cur->af == AF_INET && (*naf) == AF_INET6) { + reason = PFRES_MAPFAILED; + goto done_pool_mtx; + } + wanted_af = rpool->cur->af; + } else { + if (rpool->cur->af != 0 && rpool->cur->af != (*naf)) { + reason = PFRES_MAPFAILED; + goto done_pool_mtx; + } } } - switch (rpool->opts & PF_POOL_TYPEMASK) { + switch (pool_type) { case PF_POOL_NONE: pf_addrcpy(naddr, raddr, wanted_af); break; @@ -631,10 +657,22 @@ pf_map_addr(sa_family_t saf, struct pf_krule *r, struct pf_addr *saddr, else rpool->tblidx = (int)arc4random_uniform(cnt); memset(&rpool->counter, 0, sizeof(rpool->counter)); + if (prefer_ipv6_nexthop) + wanted_af = AF_INET6; + retry_other_af_random: if (pfr_pool_get(kt, &rpool->tblidx, &rpool->counter, wanted_af, pf_islinklocal, false)) { - reason = PFRES_MAPFAILED; - goto done_pool_mtx; /* unsupported */ + /* Retry with IPv4 nexthop for IPv4 traffic */ + if (prefer_ipv6_nexthop && + wanted_af == AF_INET6 && + (*naf) == AF_INET) { + wanted_af = AF_INET; + goto retry_other_af_random; + } else { + /* no hosts in wanted AF */ + reason = PFRES_MAPFAILED; + goto done_pool_mtx; + } } pf_addrcpy(naddr, &rpool->counter, wanted_af); } else if (init_addr != NULL && PF_AZERO(init_addr, @@ -702,10 +740,22 @@ pf_map_addr(sa_family_t saf, struct pf_krule *r, struct pf_addr *saddr, else rpool->tblidx = (int)(hashidx % cnt); memset(&rpool->counter, 0, sizeof(rpool->counter)); + if (prefer_ipv6_nexthop) + wanted_af = AF_INET6; + retry_other_af_srchash: if (pfr_pool_get(kt, &rpool->tblidx, &rpool->counter, wanted_af, pf_islinklocal, false)) { - reason = PFRES_MAPFAILED; - goto done_pool_mtx; /* unsupported */ + /* Retry with IPv4 nexthop for IPv4 traffic */ + if (prefer_ipv6_nexthop && + wanted_af == AF_INET6 && + (*naf) == AF_INET) { + wanted_af = AF_INET; + goto retry_other_af_srchash; + } else { + /* no hosts in wanted AF */ + reason = PFRES_MAPFAILED; + goto done_pool_mtx; + } } pf_addrcpy(naddr, &rpool->counter, wanted_af); } else { @@ -718,6 +768,9 @@ pf_map_addr(sa_family_t saf, struct pf_krule *r, struct pf_addr *saddr, { struct pf_kpooladdr *acur = rpool->cur; + retry_other_af_rr: + if (prefer_ipv6_nexthop) + wanted_af = rpool->ipv6_nexthop_af; if (rpool->cur->addr.type == PF_ADDR_TABLE) { if (!pfr_pool_get(rpool->cur->addr.p.tbl, &rpool->tblidx, &rpool->counter, wanted_af, @@ -728,46 +781,55 @@ pf_map_addr(sa_family_t saf, struct pf_krule *r, struct pf_addr *saddr, &rpool->tblidx, &rpool->counter, wanted_af, pf_islinklocal, true)) goto get_addr; - } else if (pf_match_addr(0, raddr, rmask, &rpool->counter, - wanted_af)) + } else if (rpool->cur->af == wanted_af && + pf_match_addr(0, raddr, rmask, &rpool->counter, wanted_af)) goto get_addr; - + if (prefer_ipv6_nexthop && + (*naf) == AF_INET && wanted_af == AF_INET6) { + /* Reset table index when changing wanted AF. */ + rpool->tblidx = -1; + rpool->ipv6_nexthop_af = AF_INET; + goto retry_other_af_rr; + } try_next: + /* Reset prefer-ipv6-nexthop search to IPv6 when iterating pools. */ + rpool->ipv6_nexthop_af = AF_INET6; if (TAILQ_NEXT(rpool->cur, entries) == NULL) rpool->cur = TAILQ_FIRST(&rpool->list); else rpool->cur = TAILQ_NEXT(rpool->cur, entries); + try_next_ipv6_nexthop_rr: + /* Reset table index when iterating pools or changing wanted AF. */ rpool->tblidx = -1; + if (prefer_ipv6_nexthop) + wanted_af = rpool->ipv6_nexthop_af; if (rpool->cur->addr.type == PF_ADDR_TABLE) { - if (pfr_pool_get(rpool->cur->addr.p.tbl, + if (!pfr_pool_get(rpool->cur->addr.p.tbl, &rpool->tblidx, &rpool->counter, wanted_af, NULL, - true)) { - /* table contains no address of type 'wanted_af' */ - if (rpool->cur != acur) - goto try_next; - reason = PFRES_MAPFAILED; - goto done_pool_mtx; - } + true)) + goto get_addr; } else if (rpool->cur->addr.type == PF_ADDR_DYNIFTL) { - if (pfr_pool_get(rpool->cur->addr.p.dyn->pfid_kt, - &rpool->tblidx, &rpool->counter, wanted_af, - pf_islinklocal, true)) { - /* interface has no address of type 'wanted_af' */ - if (rpool->cur != acur) - goto try_next; - reason = PFRES_MAPFAILED; - goto done_pool_mtx; - } + if (!pfr_pool_get(rpool->cur->addr.p.dyn->pfid_kt, + &rpool->tblidx, &rpool->counter, wanted_af, pf_islinklocal, + true)) + goto get_addr; } else { - raddr = &rpool->cur->addr.v.a.addr; - rmask = &rpool->cur->addr.v.a.mask; - if (rpool->cur->af && rpool->cur->af != wanted_af) { - reason = PFRES_MAPFAILED; - goto done_pool_mtx; + if (rpool->cur->af == wanted_af) { + raddr = &rpool->cur->addr.v.a.addr; + rmask = &rpool->cur->addr.v.a.mask; + pf_addrcpy(&rpool->counter, raddr, wanted_af); + goto get_addr; } - pf_addrcpy(&rpool->counter, raddr, wanted_af); } - + if (prefer_ipv6_nexthop && + (*naf) == AF_INET && wanted_af == AF_INET6) { + rpool->ipv6_nexthop_af = AF_INET; + goto try_next_ipv6_nexthop_rr; + } + if (rpool->cur != acur) + goto try_next; + reason = PFRES_MAPFAILED; + goto done_pool_mtx; get_addr: pf_addrcpy(naddr, &rpool->counter, wanted_af); if (init_addr != NULL && PF_AZERO(init_addr, wanted_af)) @@ -777,9 +839,16 @@ pf_map_addr(sa_family_t saf, struct pf_krule *r, struct pf_addr *saddr, } } + if (wanted_af == 0) { + reason = PFRES_MAPFAILED; + goto done_pool_mtx; + } + if (nkif) *nkif = rpool->cur->kif; + (*naf) = wanted_af; + done_pool_mtx: mtx_unlock(&rpool->mtx); diff --git a/sys/netpfil/pf/pf_nl.c b/sys/netpfil/pf/pf_nl.c index 45b5b8dd5fef..73f018db0266 100644 --- a/sys/netpfil/pf/pf_nl.c +++ b/sys/netpfil/pf/pf_nl.c @@ -763,6 +763,8 @@ static const struct nlattr_parser nla_p_rule[] = { { .type = PF_RT_RCV_IFNOT, .off = _OUT(rcvifnot), .cb = nlattr_get_bool }, { .type = PF_RT_PKTRATE, .off = _OUT(pktrate), .arg = &threshold_parser, .cb = nlattr_get_nested }, { .type = PF_RT_MAX_PKT_SIZE, .off = _OUT(max_pkt_size), .cb = nlattr_get_uint16 }, + { .type = PF_RT_TYPE_2, .off = _OUT(type), .cb = nlattr_get_uint16 }, + { .type = PF_RT_CODE_2, .off = _OUT(code), .cb = nlattr_get_uint16 }, }; NL_DECLARE_ATTR_PARSER(rule_parser, nla_p_rule); #undef _OUT @@ -984,8 +986,12 @@ pf_handle_getrule(struct nlmsghdr *hdr, struct nl_pstate *npt) nlattr_add_u8(nw, PF_RT_AF, rule->af); nlattr_add_u8(nw, PF_RT_NAF, rule->naf); nlattr_add_u8(nw, PF_RT_PROTO, rule->proto); + nlattr_add_u8(nw, PF_RT_TYPE, rule->type); nlattr_add_u8(nw, PF_RT_CODE, rule->code); + nlattr_add_u16(nw, PF_RT_TYPE_2, rule->type); + nlattr_add_u16(nw, PF_RT_CODE_2, rule->code); + nlattr_add_u8(nw, PF_RT_FLAGS, rule->flags); nlattr_add_u8(nw, PF_RT_FLAGSET, rule->flagset); nlattr_add_u8(nw, PF_RT_MIN_TTL, rule->min_ttl); @@ -1945,7 +1951,7 @@ pf_handle_get_tstats(struct nlmsghdr *hdr, struct nl_pstate *npt) n = pfr_table_count(&attrs.pfrio_table, attrs.pfrio_flags); pfrtstats = mallocarray(n, - sizeof(struct pfr_tstats), M_TEMP, M_NOWAIT | M_ZERO); + sizeof(struct pfr_tstats), M_PF, M_NOWAIT | M_ZERO); error = pfr_get_tstats(&attrs.pfrio_table, pfrtstats, &n, attrs.pfrio_flags | PFR_FLAG_USERIOCTL); @@ -1997,7 +2003,7 @@ pf_handle_get_tstats(struct nlmsghdr *hdr, struct nl_pstate *npt) } } } - free(pfrtstats, M_TEMP); + free(pfrtstats, M_PF); if (!nlmsg_end_dump(npt->nw, error, hdr)) { NL_LOG(LOG_DEBUG, "Unable to finalize the dump"); diff --git a/sys/netpfil/pf/pf_nl.h b/sys/netpfil/pf/pf_nl.h index 87daac393821..b60d3d4797c6 100644 --- a/sys/netpfil/pf/pf_nl.h +++ b/sys/netpfil/pf/pf_nl.h @@ -283,6 +283,8 @@ enum pf_rule_type_t { PF_RT_SRC_NODES_ROUTE = 81, /* u64 */ PF_RT_PKTRATE = 82, /* nested, pf_threshold_type_t */ PF_RT_MAX_PKT_SIZE = 83, /* u16 */ + PF_RT_TYPE_2 = 84, /* u16 */ + PF_RT_CODE_2 = 85, /* u16 */ }; enum pf_addrule_type_t { diff --git a/sys/netpfil/pf/pf_nv.c b/sys/netpfil/pf/pf_nv.c index 89486928e6e1..2f484e2dabc6 100644 --- a/sys/netpfil/pf/pf_nv.c +++ b/sys/netpfil/pf/pf_nv.c @@ -505,6 +505,7 @@ int pf_nvrule_to_krule(const nvlist_t *nvl, struct pf_krule *rule) { int error = 0; + uint8_t tmp; #define ERROUT(x) ERROUT_FUNCTION(errout, x) @@ -610,8 +611,10 @@ pf_nvrule_to_krule(const nvlist_t *nvl, struct pf_krule *rule) PFNV_CHK(pf_nvuint8(nvl, "keep_state", &rule->keep_state)); PFNV_CHK(pf_nvuint8(nvl, "af", &rule->af)); PFNV_CHK(pf_nvuint8(nvl, "proto", &rule->proto)); - PFNV_CHK(pf_nvuint8(nvl, "type", &rule->type)); - PFNV_CHK(pf_nvuint8(nvl, "code", &rule->code)); + PFNV_CHK(pf_nvuint8(nvl, "type", &tmp)); + rule->type = tmp; + PFNV_CHK(pf_nvuint8(nvl, "code", &tmp)); + rule->code = tmp; PFNV_CHK(pf_nvuint8(nvl, "flags", &rule->flags)); PFNV_CHK(pf_nvuint8(nvl, "flagset", &rule->flagset)); PFNV_CHK(pf_nvuint8(nvl, "min_ttl", &rule->min_ttl)); diff --git a/sys/netpfil/pf/pf_ruleset.c b/sys/netpfil/pf/pf_ruleset.c index 039908a53126..1711e690f6bb 100644 --- a/sys/netpfil/pf/pf_ruleset.c +++ b/sys/netpfil/pf/pf_ruleset.c @@ -59,8 +59,8 @@ #error "Kernel only file. Please use sbin/pfctl/pf_ruleset.c instead." #endif -#define rs_malloc(x) malloc(x, M_TEMP, M_NOWAIT|M_ZERO) -#define rs_free(x) free(x, M_TEMP) +#define rs_malloc(x) malloc(x, M_PF, M_NOWAIT|M_ZERO) +#define rs_free(x) free(x, M_PF) VNET_DEFINE(struct pf_kanchor_global, pf_anchors); VNET_DEFINE(struct pf_kanchor, pf_main_anchor); @@ -336,6 +336,12 @@ pf_remove_if_empty_kruleset(struct pf_kruleset *ruleset) int i; while (ruleset != NULL) { + for (int i = 0; i < PF_RULESET_MAX; i++) { + pf_rule_tree_free(ruleset->rules[i].active.tree); + ruleset->rules[i].active.tree = NULL; + pf_rule_tree_free(ruleset->rules[i].inactive.tree); + ruleset->rules[i].inactive.tree = NULL; + } if (ruleset == &pf_main_ruleset || !RB_EMPTY(&ruleset->anchor->children) || ruleset->anchor->refcnt > 0 || ruleset->tables > 0 || diff --git a/sys/sys/exterrvar.h b/sys/sys/exterrvar.h index 7bf1d264ff5e..6783a0d2d84f 100644 --- a/sys/sys/exterrvar.h +++ b/sys/sys/exterrvar.h @@ -31,7 +31,7 @@ #error "Specify error category before including sys/exterrvar.h" #endif -#ifdef BLOAT_KERNEL_WITH_EXTERR +#ifdef EXTERR_STRINGS #define SET_ERROR_MSG(mmsg) (mmsg) #else #define SET_ERROR_MSG(mmsg) NULL diff --git a/sys/sys/mount.h b/sys/sys/mount.h index f6480b173a5c..18f85192f6c3 100644 --- a/sys/sys/mount.h +++ b/sys/sys/mount.h @@ -1007,6 +1007,7 @@ struct mntarg *mount_argsu(struct mntarg *ma, const char *name, const void *val, void statfs_scale_blocks(struct statfs *sf, long max_size); struct vfsconf *vfs_byname(const char *); struct vfsconf *vfs_byname_kld(const char *, struct thread *td, int *); +void vfs_unref_vfsconf(struct vfsconf *vfsp); void vfs_mount_destroy(struct mount *); void vfs_event_signal(fsid_t *, u_int32_t, intptr_t); void vfs_freeopts(struct vfsoptlist *opts); diff --git a/sys/sys/ttycom.h b/sys/sys/ttycom.h index d7ddc66b09fb..43e8b98a5bc4 100644 --- a/sys/sys/ttycom.h +++ b/sys/sys/ttycom.h @@ -69,8 +69,8 @@ /* 89-91 conflicts: tun and tap */ #define TIOCTIMESTAMP _IOR('t', 89, struct timeval) /* enable/get timestamp * of last input event */ -#define TIOCMGDTRWAIT _IOR('t', 90, int) /* modem: get wait on close */ -#define TIOCMSDTRWAIT _IOW('t', 91, int) /* modem: set wait on close */ +/* TIOCMGDTRWAIT _IOR('t', 90, int) * was modem: get wait on close */ +/* TIOCMSDTRWAIT _IOW('t', 91, int) * was modem: set wait on close */ /* 92-93 tun and tap */ /* 94-97 conflicts: tun and tap */ #define TIOCDRAIN _IO('t', 94) /* wait till output drained */ diff --git a/sys/vm/vm_extern.h b/sys/vm/vm_extern.h index 93ec6014c27d..1fd6518cf4ed 100644 --- a/sys/vm/vm_extern.h +++ b/sys/vm/vm_extern.h @@ -91,6 +91,8 @@ void vm_fault_copy_entry(vm_map_t, vm_map_t, vm_map_entry_t, vm_map_entry_t, vm_ooffset_t *); int vm_fault_disable_pagefaults(void); void vm_fault_enable_pagefaults(int save); +int vm_fault_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len, + vm_prot_t prot, vm_page_t *ma, int max_count, int *ppages_count); int vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len, vm_prot_t prot, vm_page_t *ma, int max_count); int vm_fault_trap(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index 3e57e8d4f1d0..58f8ac16fa0c 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -1995,32 +1995,43 @@ vm_fault_prefault(const struct faultstate *fs, vm_offset_t addra, } /* - * Hold each of the physical pages that are mapped by the specified range of - * virtual addresses, ["addr", "addr" + "len"), if those mappings are valid - * and allow the specified types of access, "prot". If all of the implied - * pages are successfully held, then the number of held pages is returned - * together with pointers to those pages in the array "ma". However, if any - * of the pages cannot be held, -1 is returned. + * Hold each of the physical pages that are mapped by the specified + * range of virtual addresses, ["addr", "addr" + "len"), if those + * mappings are valid and allow the specified types of access, "prot". + * If all of the implied pages are successfully held, then the number + * of held pages is assigned to *ppages_count, together with pointers + * to those pages in the array "ma". The returned value is zero. + * + * However, if any of the pages cannot be held, an error is returned, + * and no pages are held. + * Error values: + * ENOMEM - the range is not valid + * EINVAL - the provided vm_page array is too small to hold all pages + * EAGAIN - a page was not mapped, and the thread is in nofaulting mode + * EFAULT - a page with requested permissions cannot be mapped + * (more detailed result from vm_fault() is lost) */ int -vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len, - vm_prot_t prot, vm_page_t *ma, int max_count) +vm_fault_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len, + vm_prot_t prot, vm_page_t *ma, int max_count, int *ppages_count) { vm_offset_t end, va; vm_page_t *mp; - int count; + int count, error; boolean_t pmap_failed; - if (len == 0) + if (len == 0) { + *ppages_count = 0; return (0); + } end = round_page(addr + len); addr = trunc_page(addr); if (!vm_map_range_valid(map, addr, end)) - return (-1); + return (ENOMEM); if (atop(end - addr) > max_count) - panic("vm_fault_quick_hold_pages: count > max_count"); + return (EINVAL); count = atop(end - addr); /* @@ -2062,19 +2073,49 @@ vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len, * the proper behaviour explicitly. */ if ((prot & VM_PROT_QUICK_NOFAULT) != 0 && - (curthread->td_pflags & TDP_NOFAULTING) != 0) - goto error; - for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) + (curthread->td_pflags & TDP_NOFAULTING) != 0) { + error = EAGAIN; + goto fail; + } + for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) { if (*mp == NULL && vm_fault(map, va, prot, - VM_FAULT_NORMAL, mp) != KERN_SUCCESS) - goto error; + VM_FAULT_NORMAL, mp) != KERN_SUCCESS) { + error = EFAULT; + goto fail; + } + } } - return (count); -error: + *ppages_count = count; + return (0); +fail: for (mp = ma; mp < ma + count; mp++) if (*mp != NULL) vm_page_unwire(*mp, PQ_INACTIVE); - return (-1); + return (error); +} + + /* + * Hold each of the physical pages that are mapped by the specified range of + * virtual addresses, ["addr", "addr" + "len"), if those mappings are valid + * and allow the specified types of access, "prot". If all of the implied + * pages are successfully held, then the number of held pages is returned + * together with pointers to those pages in the array "ma". However, if any + * of the pages cannot be held, -1 is returned. + */ +int +vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len, + vm_prot_t prot, vm_page_t *ma, int max_count) +{ + int error, pages_count; + + error = vm_fault_hold_pages(map, addr, len, prot, ma, + max_count, &pages_count); + if (error != 0) { + if (error == EINVAL) + panic("vm_fault_quick_hold_pages: count > max_count"); + return (-1); + } + return (pages_count); } /* |