diff options
44 files changed, 794 insertions, 864 deletions
diff --git a/sys/dev/cxgbe/cxgbei/cxgbei.c b/sys/dev/cxgbe/cxgbei/cxgbei.c index 31e910795d78..0a4f4ab304c6 100644 --- a/sys/dev/cxgbe/cxgbei/cxgbei.c +++ b/sys/dev/cxgbe/cxgbei/cxgbei.c @@ -343,6 +343,7 @@ do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) struct icl_cxgbei_pdu *icp = toep->ulpcb2; struct icl_pdu *ip; u_int pdu_len, val; + struct epoch_tracker et; MPASS(m == NULL); @@ -411,12 +412,12 @@ do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) SOCKBUF_UNLOCK(sb); INP_WUNLOCK(inp); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); tp = tcp_drop(tp, ECONNRESET); if (tp) INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); icl_cxgbei_conn_pdu_free(NULL, ip); #ifdef INVARIANTS diff --git a/sys/dev/cxgbe/tom/t4_connect.c b/sys/dev/cxgbe/tom/t4_connect.c index 23dfcae2c39a..635dc497009e 100644 --- a/sys/dev/cxgbe/tom/t4_connect.c +++ b/sys/dev/cxgbe/tom/t4_connect.c @@ -115,18 +115,19 @@ act_open_failure_cleanup(struct adapter *sc, u_int atid, u_int status) struct toepcb *toep = lookup_atid(sc, atid); struct inpcb *inp = toep->inp; struct toedev *tod = &toep->td->tod; + struct epoch_tracker et; free_atid(sc, atid); toep->tid = -1; CURVNET_SET(toep->vnet); if (status != EAGAIN) - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); toe_connect_failed(tod, inp, status); final_cpl_received(toep); /* unlocks inp */ if (status != EAGAIN) - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); } diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index 247a61dbc1c5..f9524b544f7a 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -1235,6 +1235,7 @@ do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) struct inpcb *inp = toep->inp; struct tcpcb *tp = NULL; struct socket *so; + struct epoch_tracker et; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif @@ -1268,7 +1269,7 @@ do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); CURVNET_SET(toep->vnet); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); tp = intotcpcb(inp); @@ -1312,7 +1313,7 @@ do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) case TCPS_FIN_WAIT_2: tcp_twstart(tp); INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); INP_WLOCK(inp); @@ -1325,7 +1326,7 @@ do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) } done: INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); return (0); } @@ -1344,6 +1345,7 @@ do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss, struct inpcb *inp = toep->inp; struct tcpcb *tp = NULL; struct socket *so = NULL; + struct epoch_tracker et; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif @@ -1354,7 +1356,7 @@ do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss, KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); CURVNET_SET(toep->vnet); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); tp = intotcpcb(inp); @@ -1372,7 +1374,7 @@ do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss, tcp_twstart(tp); release: INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); INP_WLOCK(inp); @@ -1397,7 +1399,7 @@ release: } done: INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); return (0); } @@ -1452,6 +1454,7 @@ do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) struct sge_wrq *ofld_txq = toep->ofld_txq; struct inpcb *inp; struct tcpcb *tp; + struct epoch_tracker et; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif @@ -1473,7 +1476,7 @@ do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) inp = toep->inp; CURVNET_SET(toep->vnet); - INP_INFO_RLOCK(&V_tcbinfo); /* for tcp_close */ + INP_INFO_RLOCK_ET(&V_tcbinfo, et); /* for tcp_close */ INP_WLOCK(inp); tp = intotcpcb(inp); @@ -1507,7 +1510,7 @@ do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) final_cpl_received(toep); done: - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST); return (0); @@ -1560,6 +1563,7 @@ do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) struct tcpcb *tp; struct socket *so; struct sockbuf *sb; + struct epoch_tracker et; int len; uint32_t ddp_placed = 0; @@ -1631,12 +1635,12 @@ do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) INP_WUNLOCK(inp); CURVNET_SET(toep->vnet); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); tp = tcp_drop(tp, ECONNRESET); if (tp) INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); return (0); diff --git a/sys/dev/cxgbe/tom/t4_listen.c b/sys/dev/cxgbe/tom/t4_listen.c index 7571c353dc3b..6ca33af726c6 100644 --- a/sys/dev/cxgbe/tom/t4_listen.c +++ b/sys/dev/cxgbe/tom/t4_listen.c @@ -1255,6 +1255,7 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss, int reject_reason, v, ntids; uint16_t vid; u_int wnd; + struct epoch_tracker et; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); #endif @@ -1369,15 +1370,15 @@ found: REJECT_PASS_ACCEPT(); rpl = wrtod(wr); - INP_INFO_RLOCK(&V_tcbinfo); /* for 4-tuple check */ + INP_INFO_RLOCK_ET(&V_tcbinfo, et); /* for 4-tuple check */ /* Don't offload if the 4-tuple is already in use */ if (toe_4tuple_check(&inc, &th, ifp) != 0) { - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); free(wr, M_CXGBE); REJECT_PASS_ACCEPT(); } - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); inp = lctx->inp; /* listening socket, not owned by TOE */ INP_WLOCK(inp); @@ -1574,6 +1575,7 @@ do_pass_establish(struct sge_iq *iq, const struct rss_header *rss, struct tcpopt to; struct in_conninfo inc; struct toepcb *toep; + struct epoch_tracker et; u_int txqid, rxqid; #ifdef INVARIANTS unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); @@ -1587,7 +1589,7 @@ do_pass_establish(struct sge_iq *iq, const struct rss_header *rss, ("%s: tid %u (ctx %p) not a synqe", __func__, tid, synqe)); CURVNET_SET(lctx->vnet); - INP_INFO_RLOCK(&V_tcbinfo); /* for syncache_expand */ + INP_INFO_RLOCK_ET(&V_tcbinfo, et); /* for syncache_expand */ INP_WLOCK(inp); CTR6(KTR_CXGBE, @@ -1603,7 +1605,7 @@ do_pass_establish(struct sge_iq *iq, const struct rss_header *rss, } INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); return (0); } @@ -1629,7 +1631,7 @@ reset: */ send_reset_synqe(TOEDEV(ifp), synqe); INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); return (0); } @@ -1695,7 +1697,7 @@ reset: inp = release_lctx(sc, lctx); if (inp != NULL) INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); release_synqe(synqe); diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c index fbf005159a1d..4a9a629f1a38 100644 --- a/sys/dev/cxgbe/tom/t4_tls.c +++ b/sys/dev/cxgbe/tom/t4_tls.c @@ -1559,6 +1559,8 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) SOCKBUF_LOCK(sb); if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { + struct epoch_tracker et; + CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", __func__, tid, pdu_length); m_freem(m); @@ -1566,12 +1568,12 @@ do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) INP_WUNLOCK(inp); CURVNET_SET(toep->vnet); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); tp = tcp_drop(tp, ECONNRESET); if (tp) INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); return (0); diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c index e09151a6d339..4d7a4535d27b 100644 --- a/sys/dev/hwpmc/hwpmc_mod.c +++ b/sys/dev/hwpmc/hwpmc_mod.c @@ -85,6 +85,9 @@ __FBSDID("$FreeBSD$"); #define free_domain(addr, type) free(addr, type) #endif +#define PMC_EPOCH_ENTER() struct epoch_tracker pmc_et; epoch_enter_preempt(global_epoch_preempt, &pmc_et) +#define PMC_EPOCH_EXIT() epoch_exit_preempt(global_epoch_preempt, &pmc_et) + /* * Types */ @@ -1752,12 +1755,12 @@ pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm) const struct pmc_process *pp; freepath = fullpath = NULL; - MPASS(!in_epoch()); + MPASS(!in_epoch(global_epoch_preempt)); pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath); pid = td->td_proc->p_pid; - epoch_enter_preempt(global_epoch_preempt); + PMC_EPOCH_ENTER(); /* Inform owners of all system-wide sampling PMCs. */ CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) @@ -1778,7 +1781,7 @@ pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm) done: if (freepath) free(freepath, M_TEMP); - epoch_exit_preempt(global_epoch_preempt); + PMC_EPOCH_EXIT(); } @@ -1797,12 +1800,12 @@ pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm) pid = td->td_proc->p_pid; - epoch_enter_preempt(global_epoch_preempt); + PMC_EPOCH_ENTER(); CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_map_out(po, pid, pkm->pm_address, pkm->pm_address + pkm->pm_size); - epoch_exit_preempt(global_epoch_preempt); + PMC_EPOCH_EXIT(); if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) return; @@ -1824,7 +1827,7 @@ pmc_log_kernel_mappings(struct pmc *pm) struct pmc_owner *po; struct pmckern_map_in *km, *kmbase; - MPASS(in_epoch() || sx_xlocked(&pmc_sx)); + MPASS(in_epoch(global_epoch_preempt) || sx_xlocked(&pmc_sx)); KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), ("[pmc,%d] non-sampling PMC (%p) desires mapping information", __LINE__, (void *) pm)); @@ -2106,13 +2109,13 @@ pmc_hook_handler(struct thread *td, int function, void *arg) pk = (struct pmckern_procexec *) arg; - epoch_enter_preempt(global_epoch_preempt); + PMC_EPOCH_ENTER(); /* Inform owners of SS mode PMCs of the exec event. */ CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_procexec(po, PMC_ID_INVALID, p->p_pid, pk->pm_entryaddr, fullpath); - epoch_exit_preempt(global_epoch_preempt); + PMC_EPOCH_EXIT(); PROC_LOCK(p); is_using_hwpmcs = p->p_flag & P_HWPMC; @@ -2242,7 +2245,7 @@ pmc_hook_handler(struct thread *td, int function, void *arg) break; case PMC_FN_MUNMAP: - MPASS(in_epoch() || sx_xlocked(&pmc_sx)); + MPASS(in_epoch(global_epoch_preempt) || sx_xlocked(&pmc_sx)); pmc_process_munmap(td, (struct pmckern_map_out *) arg); break; @@ -2479,7 +2482,7 @@ pmc_find_thread_descriptor(struct pmc_process *pp, struct thread *td, if (mode & PMC_FLAG_ALLOCATE) { if ((ptnew = pmc_thread_descriptor_pool_alloc()) == NULL) { wait_flag = M_WAITOK; - if ((mode & PMC_FLAG_NOWAIT) || in_epoch()) + if ((mode & PMC_FLAG_NOWAIT) || in_epoch(global_epoch_preempt)) wait_flag = M_NOWAIT; ptnew = malloc(THREADENTRY_SIZE, M_PMC, @@ -5070,11 +5073,11 @@ pmc_process_exit(void *arg __unused, struct proc *p) /* * Log a sysexit event to all SS PMC owners. */ - epoch_enter_preempt(global_epoch_preempt); + PMC_EPOCH_ENTER(); CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_sysexit(po, p->p_pid); - epoch_exit_preempt(global_epoch_preempt); + PMC_EPOCH_EXIT(); if (!is_using_hwpmcs) return; @@ -5255,13 +5258,13 @@ pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc, * If there are system-wide sampling PMCs active, we need to * log all fork events to their owner's logs. */ - epoch_enter_preempt(global_epoch_preempt); + PMC_EPOCH_ENTER(); CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) { pmclog_process_procfork(po, p1->p_pid, newproc->p_pid); pmclog_process_proccreate(po, newproc, 1); } - epoch_exit_preempt(global_epoch_preempt); + PMC_EPOCH_EXIT(); if (!is_using_hwpmcs) return; @@ -5327,11 +5330,11 @@ pmc_process_threadcreate(struct thread *td) { struct pmc_owner *po; - epoch_enter_preempt(global_epoch_preempt); + PMC_EPOCH_ENTER(); CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_threadcreate(po, td, 1); - epoch_exit_preempt(global_epoch_preempt); + PMC_EPOCH_EXIT(); } static void @@ -5339,11 +5342,11 @@ pmc_process_threadexit(struct thread *td) { struct pmc_owner *po; - epoch_enter_preempt(global_epoch_preempt); + PMC_EPOCH_ENTER(); CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_threadexit(po, td); - epoch_exit_preempt(global_epoch_preempt); + PMC_EPOCH_EXIT(); } static void @@ -5351,11 +5354,11 @@ pmc_process_proccreate(struct proc *p) { struct pmc_owner *po; - epoch_enter_preempt(global_epoch_preempt); + PMC_EPOCH_ENTER(); CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_proccreate(po, p, 1 /* sync */); - epoch_exit_preempt(global_epoch_preempt); + PMC_EPOCH_EXIT(); } static void @@ -5388,12 +5391,12 @@ pmc_kld_load(void *arg __unused, linker_file_t lf) /* * Notify owners of system sampling PMCs about KLD operations. */ - epoch_enter_preempt(global_epoch_preempt); + PMC_EPOCH_ENTER(); CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_map_in(po, (pid_t) -1, (uintfptr_t) lf->address, lf->filename); - epoch_exit_preempt(global_epoch_preempt); + PMC_EPOCH_EXIT(); /* * TODO: Notify owners of (all) process-sampling PMCs too. @@ -5406,12 +5409,12 @@ pmc_kld_unload(void *arg __unused, const char *filename __unused, { struct pmc_owner *po; - epoch_enter_preempt(global_epoch_preempt); + PMC_EPOCH_ENTER(); CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_map_out(po, (pid_t) -1, (uintfptr_t) address, (uintfptr_t) address + size); - epoch_exit_preempt(global_epoch_preempt); + PMC_EPOCH_EXIT(); /* * TODO: Notify owners of process-sampling PMCs. diff --git a/sys/kern/subr_epoch.c b/sys/kern/subr_epoch.c index b5484c36e021..5b561d5ae51e 100644 --- a/sys/kern/subr_epoch.c +++ b/sys/kern/subr_epoch.c @@ -58,18 +58,10 @@ static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation"); #define MAX_ADAPTIVE_SPIN 1000 #define MAX_EPOCHS 64 -#ifdef __amd64__ -#define EPOCH_ALIGN CACHE_LINE_SIZE*2 -#else -#define EPOCH_ALIGN CACHE_LINE_SIZE -#endif - -CTASSERT(sizeof(epoch_section_t) == sizeof(ck_epoch_section_t)); CTASSERT(sizeof(ck_epoch_entry_t) == sizeof(struct epoch_context)); SYSCTL_NODE(_kern, OID_AUTO, epoch, CTLFLAG_RW, 0, "epoch information"); SYSCTL_NODE(_kern_epoch, OID_AUTO, stats, CTLFLAG_RW, 0, "epoch stats"); - /* Stats. */ static counter_u64_t block_count; @@ -100,26 +92,8 @@ TAILQ_HEAD (threadlist, thread); CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry, ck_epoch_entry_container) -typedef struct epoch_record { - ck_epoch_record_t er_record; - volatile struct threadlist er_tdlist; - volatile uint32_t er_gen; - uint32_t er_cpuid; -} *epoch_record_t; - -struct epoch_pcpu_state { - struct epoch_record eps_record; -} __aligned(EPOCH_ALIGN); - -struct epoch { - struct ck_epoch e_epoch __aligned(EPOCH_ALIGN); - struct epoch_pcpu_state *e_pcpu_dom[MAXMEMDOM] __aligned(EPOCH_ALIGN); - int e_idx; - int e_flags; - struct epoch_pcpu_state *e_pcpu[0]; -}; - -epoch_t allepochs[MAX_EPOCHS]; + + epoch_t allepochs[MAX_EPOCHS]; DPCPU_DEFINE(struct grouptask, epoch_cb_task); DPCPU_DEFINE(int, epoch_cb_count); @@ -192,17 +166,15 @@ static void epoch_init_numa(epoch_t epoch) { int domain, cpu_offset; - struct epoch_pcpu_state *eps; epoch_record_t er; for (domain = 0; domain < vm_ndomains; domain++) { - eps = malloc_domain(sizeof(*eps) * domcount[domain], M_EPOCH, + er = malloc_domain(sizeof(*er) * domcount[domain], M_EPOCH, domain, M_ZERO | M_WAITOK); - epoch->e_pcpu_dom[domain] = eps; + epoch->e_pcpu_dom[domain] = er; cpu_offset = domoffsets[domain]; - for (int i = 0; i < domcount[domain]; i++, eps++) { - epoch->e_pcpu[cpu_offset + i] = eps; - er = &eps->eps_record; + for (int i = 0; i < domcount[domain]; i++, er++) { + epoch->e_pcpu[cpu_offset + i] = er; ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL); TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist); er->er_cpuid = cpu_offset + i; @@ -213,14 +185,12 @@ epoch_init_numa(epoch_t epoch) static void epoch_init_legacy(epoch_t epoch) { - struct epoch_pcpu_state *eps; epoch_record_t er; - eps = malloc(sizeof(*eps) * mp_ncpus, M_EPOCH, M_ZERO | M_WAITOK); - epoch->e_pcpu_dom[0] = eps; - for (int i = 0; i < mp_ncpus; i++, eps++) { - epoch->e_pcpu[i] = eps; - er = &eps->eps_record; + er = malloc(sizeof(*er) * mp_ncpus, M_EPOCH, M_ZERO | M_WAITOK); + epoch->e_pcpu_dom[0] = er; + for (int i = 0; i < mp_ncpus; i++, er++) { + epoch->e_pcpu[i] = er; ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL); TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist); er->er_cpuid = i; @@ -253,12 +223,12 @@ epoch_free(epoch_t epoch) { int domain; #ifdef INVARIANTS - struct epoch_pcpu_state *eps; + struct epoch_record *er; int cpu; CPU_FOREACH(cpu) { - eps = epoch->e_pcpu[cpu]; - MPASS(TAILQ_EMPTY(&eps->eps_record.er_tdlist)); + er = epoch->e_pcpu[cpu]; + MPASS(TAILQ_EMPTY(&er->er_tdlist)); } #endif allepochs[epoch->e_idx] = NULL; @@ -271,95 +241,32 @@ epoch_free(epoch_t epoch) free(epoch, M_EPOCH); } -#define INIT_CHECK(epoch) \ - do { \ - if (__predict_false((epoch) == NULL)) \ - return; \ - } while (0) - void -epoch_enter_preempt_internal(epoch_t epoch, struct thread *td) +epoch_enter_preempt_KBI(epoch_t epoch, epoch_tracker_t et) { - struct epoch_pcpu_state *eps; - MPASS(cold || epoch != NULL); - INIT_CHECK(epoch); - MPASS(epoch->e_flags & EPOCH_PREEMPT); - critical_enter(); - td->td_pre_epoch_prio = td->td_priority; - eps = epoch->e_pcpu[curcpu]; -#ifdef INVARIANTS - MPASS(td->td_epochnest < UCHAR_MAX - 2); - if (td->td_epochnest > 1) { - struct thread *curtd; - int found = 0; - - TAILQ_FOREACH(curtd, &eps->eps_record.er_tdlist, td_epochq) - if (curtd == td) - found = 1; - KASSERT(found, ("recursing on a second epoch")); - critical_exit(); - return; - } -#endif - TAILQ_INSERT_TAIL(&eps->eps_record.er_tdlist, td, td_epochq); - sched_pin(); - ck_epoch_begin(&eps->eps_record.er_record, (ck_epoch_section_t *)&td->td_epoch_section); - critical_exit(); + epoch_enter_preempt(epoch, et); } - void -epoch_enter(epoch_t epoch) +epoch_exit_preempt_KBI(epoch_t epoch, epoch_tracker_t et) { - ck_epoch_record_t *record; - struct thread *td; - - MPASS(cold || epoch != NULL); - INIT_CHECK(epoch); - td = curthread; - critical_enter(); - td->td_epochnest++; - record = &epoch->e_pcpu[curcpu]->eps_record.er_record; - ck_epoch_begin(record, NULL); + epoch_exit_preempt(epoch, et); } void -epoch_exit_preempt_internal(epoch_t epoch, struct thread *td) +epoch_enter_KBI(epoch_t epoch) { - struct epoch_pcpu_state *eps; - MPASS(td->td_epochnest == 0); - INIT_CHECK(epoch); - critical_enter(); - eps = epoch->e_pcpu[curcpu]; - - MPASS(epoch->e_flags & EPOCH_PREEMPT); - ck_epoch_end(&eps->eps_record.er_record, (ck_epoch_section_t *)&td->td_epoch_section); - TAILQ_REMOVE(&eps->eps_record.er_tdlist, td, td_epochq); - eps->eps_record.er_gen++; - sched_unpin(); - if (__predict_false(td->td_pre_epoch_prio != td->td_priority)) { - thread_lock(td); - sched_prio(td, td->td_pre_epoch_prio); - thread_unlock(td); - } - critical_exit(); + epoch_enter(epoch); } void -epoch_exit(epoch_t epoch) +epoch_exit_KBI(epoch_t epoch) { - ck_epoch_record_t *record; - struct thread *td; - INIT_CHECK(epoch); - td = curthread; - td->td_epochnest--; - record = &epoch->e_pcpu[curcpu]->eps_record.er_record; - ck_epoch_end(record, NULL); - critical_exit(); + epoch_exit(epoch); } /* @@ -371,7 +278,8 @@ epoch_block_handler_preempt(struct ck_epoch *global __unused, ck_epoch_record_t void *arg __unused) { epoch_record_t record; - struct thread *td, *tdwait, *owner; + struct thread *td, *owner, *curwaittd; + struct epoch_thread *tdwait; struct turnstile *ts; struct lock_object *lock; int spincount, gen; @@ -389,13 +297,13 @@ epoch_block_handler_preempt(struct ck_epoch *global __unused, ck_epoch_record_t * overhead of a migration */ if ((tdwait = TAILQ_FIRST(&record->er_tdlist)) != NULL && - TD_IS_RUNNING(tdwait)) { + TD_IS_RUNNING(tdwait->et_td)) { gen = record->er_gen; thread_unlock(td); do { cpu_spinwait(); } while (tdwait == TAILQ_FIRST(&record->er_tdlist) && - gen == record->er_gen && TD_IS_RUNNING(tdwait) && + gen == record->er_gen && TD_IS_RUNNING(tdwait->et_td) && spincount++ < MAX_ADAPTIVE_SPIN); thread_lock(td); return; @@ -426,28 +334,29 @@ epoch_block_handler_preempt(struct ck_epoch *global __unused, ck_epoch_record_t * priority thread (highest prio value) and drop our priority * to match to allow it to run. */ - TAILQ_FOREACH(tdwait, &record->er_tdlist, td_epochq) { + TAILQ_FOREACH(tdwait, &record->er_tdlist, et_link) { /* * Propagate our priority to any other waiters to prevent us * from starving them. They will have their original priority * restore on exit from epoch_wait(). */ - if (!TD_IS_INHIBITED(tdwait) && tdwait->td_priority > td->td_priority) { + curwaittd = tdwait->et_td; + if (!TD_IS_INHIBITED(curwaittd) && curwaittd->td_priority > td->td_priority) { critical_enter(); thread_unlock(td); - thread_lock(tdwait); - sched_prio(tdwait, td->td_priority); - thread_unlock(tdwait); + thread_lock(curwaittd); + sched_prio(curwaittd, td->td_priority); + thread_unlock(curwaittd); thread_lock(td); critical_exit(); } - if (TD_IS_INHIBITED(tdwait) && TD_ON_LOCK(tdwait) && - ((ts = tdwait->td_blocked) != NULL)) { + if (TD_IS_INHIBITED(curwaittd) && TD_ON_LOCK(curwaittd) && + ((ts = curwaittd->td_blocked) != NULL)) { /* * We unlock td to allow turnstile_wait to reacquire the * the thread lock. Before unlocking it we enter a critical * section to prevent preemption after we reenable interrupts - * by dropping the thread lock in order to prevent tdwait + * by dropping the thread lock in order to prevent curwaittd * from getting to run. */ critical_enter(); @@ -456,15 +365,15 @@ epoch_block_handler_preempt(struct ck_epoch *global __unused, ck_epoch_record_t /* * The owner pointer indicates that the lock succeeded. Only * in case we hold the lock and the turnstile we locked is still - * the one that tdwait is blocked on can we continue. Otherwise + * the one that curwaittd is blocked on can we continue. Otherwise * The turnstile pointer has been changed out from underneath - * us, as in the case where the lock holder has signalled tdwait, + * us, as in the case where the lock holder has signalled curwaittd, * and we need to continue. */ - if (owner != NULL && ts == tdwait->td_blocked) { - MPASS(TD_IS_INHIBITED(tdwait) && TD_ON_LOCK(tdwait)); + if (owner != NULL && ts == curwaittd->td_blocked) { + MPASS(TD_IS_INHIBITED(curwaittd) && TD_ON_LOCK(curwaittd)); critical_exit(); - turnstile_wait(ts, owner, tdwait->td_tsqueue); + turnstile_wait(ts, owner, curwaittd->td_tsqueue); counter_u64_add(turnstile_count, 1); thread_lock(td); return; @@ -569,7 +478,7 @@ epoch_wait(epoch_t epoch) void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t)) { - struct epoch_pcpu_state *eps; + epoch_record_t er; ck_epoch_entry_t *cb; cb = (void *)ctx; @@ -585,8 +494,8 @@ epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t critical_enter(); *DPCPU_PTR(epoch_cb_count) += 1; - eps = epoch->e_pcpu[curcpu]; - ck_epoch_call(&eps->eps_record.er_record, cb, (ck_epoch_cb_t *)callback); + er = epoch->e_pcpu[curcpu]; + ck_epoch_call(&er->er_record, cb, (ck_epoch_cb_t *)callback); critical_exit(); return; boottime: @@ -608,7 +517,7 @@ epoch_call_task(void *arg __unused) for (total = i = 0; i < epoch_count; i++) { if (__predict_false((epoch = allepochs[i]) == NULL)) continue; - record = &epoch->e_pcpu[curcpu]->eps_record.er_record; + record = &epoch->e_pcpu[curcpu]->er_record; if ((npending = record->n_pending) == 0) continue; ck_epoch_poll_deferred(record, &cb_stack); @@ -632,7 +541,47 @@ epoch_call_task(void *arg __unused) } int -in_epoch(void) +in_epoch_verbose(epoch_t epoch, int dump_onfail) +{ + struct epoch_thread *tdwait; + struct thread *td; + epoch_record_t er; + + td = curthread; + if (td->td_epochnest == 0) + return (0); + if (__predict_false((epoch) == NULL)) + return (0); + critical_enter(); + er = epoch->e_pcpu[curcpu]; + TAILQ_FOREACH(tdwait, &er->er_tdlist, et_link) + if (tdwait->et_td == td) { + critical_exit(); + return (1); + } +#ifdef INVARIANTS + if (dump_onfail) { + MPASS(td->td_pinned); + printf("cpu: %d id: %d\n", curcpu, td->td_tid); + TAILQ_FOREACH(tdwait, &er->er_tdlist, et_link) + printf("td_tid: %d ", tdwait->et_td->td_tid); + printf("\n"); + } +#endif + critical_exit(); + return (0); +} + +int +in_epoch(epoch_t epoch) { - return (curthread->td_epochnest != 0); + return (in_epoch_verbose(epoch, 0)); +} + +void +epoch_adjust_prio(struct thread *td, u_char prio) +{ + thread_lock(td); + sched_prio(td, prio); + thread_unlock(td); } diff --git a/sys/net/if.c b/sys/net/if.c index 6d85a0f1165d..caa7b066b4e8 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -1760,29 +1760,35 @@ if_data_copy(struct ifnet *ifp, struct if_data *ifd) void if_addr_rlock(struct ifnet *ifp) { - - IF_ADDR_RLOCK(ifp); + MPASS(*(uint64_t *)&ifp->if_addr_et == 0); + epoch_enter_preempt(net_epoch_preempt, &ifp->if_addr_et); } void if_addr_runlock(struct ifnet *ifp) { - - IF_ADDR_RUNLOCK(ifp); + epoch_exit_preempt(net_epoch_preempt, &ifp->if_addr_et); +#ifdef INVARIANTS + bzero(&ifp->if_addr_et, sizeof(struct epoch_tracker)); +#endif } void if_maddr_rlock(if_t ifp) { - IF_ADDR_RLOCK((struct ifnet *)ifp); + MPASS(*(uint64_t *)&ifp->if_maddr_et == 0); + epoch_enter_preempt(net_epoch_preempt, &ifp->if_maddr_et); } void if_maddr_runlock(if_t ifp) { - IF_ADDR_RUNLOCK((struct ifnet *)ifp); + epoch_exit_preempt(net_epoch_preempt, &ifp->if_maddr_et); +#ifdef INVARIANTS + bzero(&ifp->if_maddr_et, sizeof(struct epoch_tracker)); +#endif } /* @@ -1926,7 +1932,7 @@ ifa_ifwithaddr(const struct sockaddr *addr) struct ifnet *ifp; struct ifaddr *ifa; - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) @@ -1969,7 +1975,7 @@ ifa_ifwithbroadaddr(const struct sockaddr *addr, int fibnum) struct ifnet *ifp; struct ifaddr *ifa; - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if ((fibnum != RT_ALL_FIBS) && (ifp->if_fib != fibnum)) continue; @@ -1999,7 +2005,7 @@ ifa_ifwithdstaddr(const struct sockaddr *addr, int fibnum) struct ifnet *ifp; struct ifaddr *ifa; - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if ((ifp->if_flags & IFF_POINTOPOINT) == 0) continue; @@ -2032,7 +2038,7 @@ ifa_ifwithnet(const struct sockaddr *addr, int ignore_ptp, int fibnum) u_int af = addr->sa_family; const char *addr_data = addr->sa_data, *cplim; - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); /* * AF_LINK addresses can be looked up directly by their index number, * so do that if we can. @@ -2069,7 +2075,6 @@ next: continue; */ if (ifa->ifa_dstaddr != NULL && sa_equal(addr, ifa->ifa_dstaddr)) { - IF_ADDR_RUNLOCK(ifp); goto done; } } else { @@ -2128,7 +2133,8 @@ ifaof_ifpforaddr(const struct sockaddr *addr, struct ifnet *ifp) if (af >= AF_MAX) return (NULL); - MPASS(in_epoch()); + + MPASS(in_epoch(net_epoch_preempt)); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != af) continue; diff --git a/sys/net/if_gif.h b/sys/net/if_gif.h index 217b5fe7b16b..501a4e5d1789 100644 --- a/sys/net/if_gif.h +++ b/sys/net/if_gif.h @@ -96,8 +96,8 @@ struct etherip_header { /* mbuf adjust factor to force 32-bit alignment of IP header */ #define ETHERIP_ALIGN 2 -#define GIF_RLOCK() epoch_enter_preempt(net_epoch_preempt) -#define GIF_RUNLOCK() epoch_exit_preempt(net_epoch_preempt) +#define GIF_RLOCK() struct epoch_tracker gif_et; epoch_enter_preempt(net_epoch_preempt, &gif_et) +#define GIF_RUNLOCK() epoch_exit_preempt(net_epoch_preempt, &gif_et) #define GIF_WAIT() epoch_wait_preempt(net_epoch_preempt) /* Prototypes */ diff --git a/sys/net/if_gre.h b/sys/net/if_gre.h index da71fbbf7d50..cc8b08f98396 100644 --- a/sys/net/if_gre.h +++ b/sys/net/if_gre.h @@ -91,8 +91,8 @@ MALLOC_DECLARE(M_GRE); #endif #define GRE2IFP(sc) ((sc)->gre_ifp) -#define GRE_RLOCK() epoch_enter_preempt(net_epoch_preempt) -#define GRE_RUNLOCK() epoch_exit_preempt(net_epoch_preempt) +#define GRE_RLOCK() struct epoch_tracker gre_et; epoch_enter_preempt(net_epoch_preempt, &gre_et) +#define GRE_RUNLOCK() epoch_exit_preempt(net_epoch_preempt, &gre_et) #define GRE_WAIT() epoch_wait_preempt(net_epoch_preempt) #define gre_hdr gre_uhdr.hdr diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c index ff0892414612..077e93163713 100644 --- a/sys/net/if_lagg.c +++ b/sys/net/if_lagg.c @@ -73,10 +73,10 @@ __FBSDID("$FreeBSD$"); #include <net/if_lagg.h> #include <net/ieee8023ad_lacp.h> -#define LAGG_RLOCK() epoch_enter_preempt(net_epoch_preempt) -#define LAGG_RUNLOCK() epoch_exit_preempt(net_epoch_preempt) -#define LAGG_RLOCK_ASSERT() MPASS(in_epoch()) -#define LAGG_UNLOCK_ASSERT() MPASS(!in_epoch()) +#define LAGG_RLOCK() struct epoch_tracker lagg_et; epoch_enter_preempt(net_epoch_preempt, &lagg_et) +#define LAGG_RUNLOCK() epoch_exit_preempt(net_epoch_preempt, &lagg_et) +#define LAGG_RLOCK_ASSERT() MPASS(in_epoch(net_epoch_preempt)) +#define LAGG_UNLOCK_ASSERT() MPASS(!in_epoch(net_epoch_preempt)) #define LAGG_SX_INIT(_sc) sx_init(&(_sc)->sc_sx, "if_lagg sx") #define LAGG_SX_DESTROY(_sc) sx_destroy(&(_sc)->sc_sx) @@ -1791,6 +1791,7 @@ struct lagg_port * lagg_link_active(struct lagg_softc *sc, struct lagg_port *lp) { struct lagg_port *lp_next, *rval = NULL; + struct epoch_tracker net_et; /* * Search a port which reports an active link state. @@ -1809,15 +1810,14 @@ lagg_link_active(struct lagg_softc *sc, struct lagg_port *lp) } search: - LAGG_RLOCK(); + epoch_enter_preempt(net_epoch_preempt, &net_et); CK_SLIST_FOREACH(lp_next, &sc->sc_ports, lp_entries) { if (LAGG_PORTACTIVE(lp_next)) { - LAGG_RUNLOCK(); - rval = lp_next; - goto found; + epoch_exit_preempt(net_epoch_preempt, &net_et); + return (lp_next); } } - LAGG_RUNLOCK(); + epoch_exit_preempt(net_epoch_preempt, &net_et); found: return (rval); } diff --git a/sys/net/if_me.c b/sys/net/if_me.c index 9ba44b0cb215..534d7030a8f5 100644 --- a/sys/net/if_me.c +++ b/sys/net/if_me.c @@ -87,8 +87,8 @@ struct me_softc { CK_LIST_HEAD(me_list, me_softc); #define ME2IFP(sc) ((sc)->me_ifp) #define ME_READY(sc) ((sc)->me_src.s_addr != 0) -#define ME_RLOCK() epoch_enter_preempt(net_epoch_preempt) -#define ME_RUNLOCK() epoch_exit_preempt(net_epoch_preempt) +#define ME_RLOCK() struct epoch_tracker me_et; epoch_enter_preempt(net_epoch_preempt, &me_et) +#define ME_RUNLOCK() epoch_exit_preempt(net_epoch_preempt, &me_et) #define ME_WAIT() epoch_wait_preempt(net_epoch_preempt) #ifndef ME_HASH_SIZE @@ -315,7 +315,7 @@ me_lookup(const struct mbuf *m, int off, int proto, void **arg) if (V_me_hashtbl == NULL) return (0); - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); ip = mtod(m, const struct ip *); CK_LIST_FOREACH(sc, &ME_HASH(ip->ip_dst.s_addr, ip->ip_src.s_addr), chain) { diff --git a/sys/net/if_var.h b/sys/net/if_var.h index bd4756cb508e..43893b401545 100644 --- a/sys/net/if_var.h +++ b/sys/net/if_var.h @@ -381,6 +381,8 @@ struct ifnet { */ struct netdump_methods *if_netdump_methods; struct epoch_context if_epoch_ctx; + struct epoch_tracker if_addr_et; + struct epoch_tracker if_maddr_et; /* * Spare fields to be added before branching a stable branch, so @@ -398,15 +400,17 @@ struct ifnet { */ #define IF_ADDR_LOCK_INIT(if) mtx_init(&(if)->if_addr_lock, "if_addr_lock", NULL, MTX_DEF) #define IF_ADDR_LOCK_DESTROY(if) mtx_destroy(&(if)->if_addr_lock) -#define IF_ADDR_RLOCK(if) epoch_enter_preempt(net_epoch_preempt); -#define IF_ADDR_RUNLOCK(if) epoch_exit_preempt(net_epoch_preempt); +#define IF_ADDR_RLOCK(if) struct epoch_tracker if_addr_et; epoch_enter_preempt(net_epoch_preempt, &if_addr_et); +#define IF_ADDR_RUNLOCK(if) epoch_exit_preempt(net_epoch_preempt, &if_addr_et); #define IF_ADDR_WLOCK(if) mtx_lock(&(if)->if_addr_lock) #define IF_ADDR_WUNLOCK(if) mtx_unlock(&(if)->if_addr_lock) -#define IF_ADDR_LOCK_ASSERT(if) MPASS(in_epoch() || mtx_owned(&(if)->if_addr_lock)) +#define IF_ADDR_LOCK_ASSERT(if) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(if)->if_addr_lock)) #define IF_ADDR_WLOCK_ASSERT(if) mtx_assert(&(if)->if_addr_lock, MA_OWNED) -#define NET_EPOCH_ENTER() epoch_enter_preempt(net_epoch_preempt) -#define NET_EPOCH_EXIT() epoch_exit_preempt(net_epoch_preempt) +#define NET_EPOCH_ENTER() struct epoch_tracker nep_et; epoch_enter_preempt(net_epoch_preempt, &nep_et) +#define NET_EPOCH_ENTER_ET(et) epoch_enter_preempt(net_epoch_preempt, &(et)) +#define NET_EPOCH_EXIT() epoch_exit_preempt(net_epoch_preempt, &nep_et) +#define NET_EPOCH_EXIT_ET(et) epoch_exit_preempt(net_epoch_preempt, &(et)) /* @@ -482,16 +486,16 @@ EVENTHANDLER_DECLARE(group_change_event, group_change_event_handler_t); mtx_init(&(ifp)->if_afdata_lock, "if_afdata", NULL, MTX_DEF) #define IF_AFDATA_WLOCK(ifp) mtx_lock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_RLOCK(ifp) epoch_enter_preempt(net_epoch_preempt) +#define IF_AFDATA_RLOCK(ifp) struct epoch_tracker if_afdata_et; epoch_enter_preempt(net_epoch_preempt, &if_afdata_et) #define IF_AFDATA_WUNLOCK(ifp) mtx_unlock(&(ifp)->if_afdata_lock) -#define IF_AFDATA_RUNLOCK(ifp) epoch_exit_preempt(net_epoch_preempt) +#define IF_AFDATA_RUNLOCK(ifp) epoch_exit_preempt(net_epoch_preempt, &if_afdata_et) #define IF_AFDATA_LOCK(ifp) IF_AFDATA_WLOCK(ifp) #define IF_AFDATA_UNLOCK(ifp) IF_AFDATA_WUNLOCK(ifp) #define IF_AFDATA_TRYLOCK(ifp) mtx_trylock(&(ifp)->if_afdata_lock) #define IF_AFDATA_DESTROY(ifp) mtx_destroy(&(ifp)->if_afdata_lock) -#define IF_AFDATA_LOCK_ASSERT(ifp) MPASS(in_epoch() || mtx_owned(&(ifp)->if_afdata_lock)) -#define IF_AFDATA_RLOCK_ASSERT(ifp) MPASS(in_epoch()); +#define IF_AFDATA_LOCK_ASSERT(ifp) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(ifp)->if_afdata_lock)) +#define IF_AFDATA_RLOCK_ASSERT(ifp) MPASS(in_epoch(net_epoch_preempt)); #define IF_AFDATA_WLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_OWNED) #define IF_AFDATA_UNLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_NOTOWNED) @@ -573,16 +577,16 @@ extern struct sx ifnet_sxlock; * write, but also whether it was acquired with sleep support or not. */ #define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED) -#define IFNET_RLOCK_NOSLEEP_ASSERT() MPASS(in_epoch()) +#define IFNET_RLOCK_NOSLEEP_ASSERT() MPASS(in_epoch(net_epoch_preempt)) #define IFNET_WLOCK_ASSERT() do { \ sx_assert(&ifnet_sxlock, SA_XLOCKED); \ rw_assert(&ifnet_rwlock, RA_WLOCKED); \ } while (0) #define IFNET_RLOCK() sx_slock(&ifnet_sxlock) -#define IFNET_RLOCK_NOSLEEP() epoch_enter_preempt(net_epoch_preempt) +#define IFNET_RLOCK_NOSLEEP() struct epoch_tracker ifnet_rlock_et; epoch_enter_preempt(net_epoch_preempt, &ifnet_rlock_et) #define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock) -#define IFNET_RUNLOCK_NOSLEEP() epoch_exit_preempt(net_epoch_preempt) +#define IFNET_RUNLOCK_NOSLEEP() epoch_exit_preempt(net_epoch_preempt, &ifnet_rlock_et) /* * Look up an ifnet given its index; the _ref variant also acquires a diff --git a/sys/net/route.c b/sys/net/route.c index 797f1e4d6eed..399a2a9ede03 100644 --- a/sys/net/route.c +++ b/sys/net/route.c @@ -733,7 +733,7 @@ ifa_ifwithroute(int flags, const struct sockaddr *dst, struct sockaddr *gateway, struct ifaddr *ifa; int not_found = 0; - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); if ((flags & RTF_GATEWAY) == 0) { /* * If we are adding a route to an interface, diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c index 2daa8c337902..91c846dc8216 100644 --- a/sys/net/rtsock.c +++ b/sys/net/rtsock.c @@ -1736,15 +1736,15 @@ sysctl_iflist(int af, struct walkarg *w) struct rt_addrinfo info; int len, error = 0; struct sockaddr_storage ss; + struct epoch_tracker et; bzero((caddr_t)&info, sizeof(info)); bzero(&ifd, sizeof(ifd)); - IFNET_RLOCK_NOSLEEP(); + NET_EPOCH_ENTER_ET(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; if_data_copy(ifp, &ifd); - IF_ADDR_RLOCK(ifp); ifa = ifp->if_addr; info.rti_info[RTAX_IFP] = ifa->ifa_addr; error = rtsock_msg_buffer(RTM_IFINFO, &info, w, &len); @@ -1785,15 +1785,12 @@ sysctl_iflist(int af, struct walkarg *w) goto done; } } - IF_ADDR_RUNLOCK(ifp); info.rti_info[RTAX_IFA] = NULL; info.rti_info[RTAX_NETMASK] = NULL; info.rti_info[RTAX_BRD] = NULL; } done: - if (ifp != NULL) - IF_ADDR_RUNLOCK(ifp); - IFNET_RUNLOCK_NOSLEEP(); + NET_EPOCH_EXIT_ET(et); return (error); } diff --git a/sys/netinet/in_gif.c b/sys/netinet/in_gif.c index 06b567400297..4b609294cbe8 100644 --- a/sys/netinet/in_gif.c +++ b/sys/netinet/in_gif.c @@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/sysctl.h> #include <sys/malloc.h> +#include <sys/proc.h> #include <net/ethernet.h> #include <net/if.h> @@ -224,7 +225,7 @@ in_gif_output(struct ifnet *ifp, struct mbuf *m, int proto, uint8_t ecn) int len; /* prepend new IP header */ - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); len = sizeof(struct ip); #ifndef __NO_STRICT_ALIGNMENT if (proto == IPPROTO_ETHERIP) @@ -263,7 +264,7 @@ in_gif_input(struct mbuf *m, int off, int proto, void *arg) struct ip *ip; uint8_t ecn; - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); if (sc == NULL) { m_freem(m); KMOD_IPSTAT_INC(ips_nogif); @@ -292,7 +293,7 @@ in_gif_lookup(const struct mbuf *m, int off, int proto, void **arg) if (V_ipv4_hashtbl == NULL) return (0); - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); ip = mtod(m, const struct ip *); /* * NOTE: it is safe to iterate without any locking here, because softc diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c index 3b2d1f030f0c..ba88a0f30ca0 100644 --- a/sys/netinet/in_pcb.c +++ b/sys/netinet/in_pcb.c @@ -1084,7 +1084,6 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, ifp = ia->ia_ifp; ia = NULL; - IF_ADDR_RLOCK(ifp); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { sa = ifa->ifa_addr; @@ -1098,10 +1097,8 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, } if (ia != NULL) { laddr->s_addr = ia->ia_addr.sin_addr.s_addr; - IF_ADDR_RUNLOCK(ifp); goto done; } - IF_ADDR_RUNLOCK(ifp); /* 3. As a last resort return the 'default' jail address. */ error = prison_get_ip4(cred, laddr); @@ -1143,7 +1140,6 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, */ ia = NULL; ifp = sro.ro_rt->rt_ifp; - IF_ADDR_RLOCK(ifp); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { sa = ifa->ifa_addr; if (sa->sa_family != AF_INET) @@ -1156,10 +1152,8 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, } if (ia != NULL) { laddr->s_addr = ia->ia_addr.sin_addr.s_addr; - IF_ADDR_RUNLOCK(ifp); goto done; } - IF_ADDR_RUNLOCK(ifp); /* 3. As a last resort return the 'default' jail address. */ error = prison_get_ip4(cred, laddr); @@ -1207,9 +1201,7 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, ifp = ia->ia_ifp; ia = NULL; - IF_ADDR_RLOCK(ifp); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { - sa = ifa->ifa_addr; if (sa->sa_family != AF_INET) continue; @@ -1222,10 +1214,8 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr, } if (ia != NULL) { laddr->s_addr = ia->ia_addr.sin_addr.s_addr; - IF_ADDR_RUNLOCK(ifp); goto done; } - IF_ADDR_RUNLOCK(ifp); } /* 3. As a last resort return the 'default' jail address. */ @@ -1673,6 +1663,10 @@ in_pcbdrop(struct inpcb *inp) { INP_WLOCK_ASSERT(inp); +#ifdef INVARIANTS + if (inp->inp_socket != NULL && inp->inp_ppcb != NULL) + MPASS(inp->inp_refcount > 1); +#endif /* * XXXRW: Possibly we should protect the setting of INP_DROPPED with @@ -2251,11 +2245,12 @@ in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr, struct inpcb *inp, *tmpinp; u_short fport = fport_arg, lport = lport_arg; +#ifdef INVARIANTS KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0, ("%s: invalid lookup flags %d", __func__, lookupflags)); - - INP_HASH_LOCK_ASSERT(pcbinfo); - + if (!mtx_owned(&pcbinfo->ipi_hash_lock)) + MPASS(in_epoch_verbose(net_epoch_preempt, 1)); +#endif /* * First look for an exact match. */ diff --git a/sys/netinet/in_pcb.h b/sys/netinet/in_pcb.h index 5028571d32e4..873ba07d0aba 100644 --- a/sys/netinet/in_pcb.h +++ b/sys/netinet/in_pcb.h @@ -632,16 +632,17 @@ int inp_so_options(const struct inpcb *inp); #define INP_INFO_LOCK_INIT(ipi, d) \ mtx_init(&(ipi)->ipi_lock, (d), NULL, MTX_DEF| MTX_RECURSE) #define INP_INFO_LOCK_DESTROY(ipi) mtx_destroy(&(ipi)->ipi_lock) -#define INP_INFO_RLOCK(ipi) NET_EPOCH_ENTER() +#define INP_INFO_RLOCK_ET(ipi, et) NET_EPOCH_ENTER_ET((et)) #define INP_INFO_WLOCK(ipi) mtx_lock(&(ipi)->ipi_lock) #define INP_INFO_TRY_WLOCK(ipi) mtx_trylock(&(ipi)->ipi_lock) #define INP_INFO_WLOCKED(ipi) mtx_owned(&(ipi)->ipi_lock) -#define INP_INFO_RUNLOCK(ipi) NET_EPOCH_EXIT() +#define INP_INFO_RUNLOCK_ET(ipi, et) NET_EPOCH_EXIT_ET((et)) +#define INP_INFO_RUNLOCK_TP(ipi, tp) NET_EPOCH_EXIT_ET(*(tp)->t_inpcb->inp_et) #define INP_INFO_WUNLOCK(ipi) mtx_unlock(&(ipi)->ipi_lock) -#define INP_INFO_LOCK_ASSERT(ipi) MPASS(in_epoch() || mtx_owned(&(ipi)->ipi_lock)) -#define INP_INFO_RLOCK_ASSERT(ipi) MPASS(in_epoch()) +#define INP_INFO_LOCK_ASSERT(ipi) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(ipi)->ipi_lock)) +#define INP_INFO_RLOCK_ASSERT(ipi) MPASS(in_epoch(net_epoch_preempt)) #define INP_INFO_WLOCK_ASSERT(ipi) mtx_assert(&(ipi)->ipi_lock, MA_OWNED) -#define INP_INFO_UNLOCK_ASSERT(ipi) MPASS(!in_epoch() && !mtx_owned(&(ipi)->ipi_lock)) +#define INP_INFO_UNLOCK_ASSERT(ipi) MPASS(!in_epoch(net_epoch_preempt) && !mtx_owned(&(ipi)->ipi_lock)) #define INP_LIST_LOCK_INIT(ipi, d) \ rw_init_flags(&(ipi)->ipi_list_lock, (d), 0) @@ -664,11 +665,13 @@ int inp_so_options(const struct inpcb *inp); #define INP_HASH_LOCK_INIT(ipi, d) mtx_init(&(ipi)->ipi_hash_lock, (d), NULL, MTX_DEF) #define INP_HASH_LOCK_DESTROY(ipi) mtx_destroy(&(ipi)->ipi_hash_lock) -#define INP_HASH_RLOCK(ipi) NET_EPOCH_ENTER() +#define INP_HASH_RLOCK(ipi) struct epoch_tracker inp_hash_et; epoch_enter_preempt(net_epoch_preempt, &inp_hash_et) +#define INP_HASH_RLOCK_ET(ipi, et) epoch_enter_preempt(net_epoch_preempt, &(et)) #define INP_HASH_WLOCK(ipi) mtx_lock(&(ipi)->ipi_hash_lock) -#define INP_HASH_RUNLOCK(ipi) NET_EPOCH_EXIT() +#define INP_HASH_RUNLOCK(ipi) NET_EPOCH_EXIT_ET(inp_hash_et) +#define INP_HASH_RUNLOCK_ET(ipi, et) NET_EPOCH_EXIT_ET((et)) #define INP_HASH_WUNLOCK(ipi) mtx_unlock(&(ipi)->ipi_hash_lock) -#define INP_HASH_LOCK_ASSERT(ipi) MPASS(in_epoch() || mtx_owned(&(ipi)->ipi_hash_lock)) +#define INP_HASH_LOCK_ASSERT(ipi) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(ipi)->ipi_hash_lock)) #define INP_HASH_WLOCK_ASSERT(ipi) mtx_assert(&(ipi)->ipi_hash_lock, MA_OWNED); #define INP_GROUP_LOCK_INIT(ipg, d) mtx_init(&(ipg)->ipg_lock, (d), NULL, \ diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c index f32c808367f4..9c42c8838642 100644 --- a/sys/netinet/ip_divert.c +++ b/sys/netinet/ip_divert.c @@ -192,6 +192,7 @@ divert_packet(struct mbuf *m, int incoming) u_int16_t nport; struct sockaddr_in divsrc; struct m_tag *mtag; + struct epoch_tracker et; mtag = m_tag_locate(m, MTAG_IPFW_RULE, 0, NULL); if (mtag == NULL) { @@ -272,7 +273,7 @@ divert_packet(struct mbuf *m, int incoming) /* Put packet on socket queue, if any */ sa = NULL; nport = htons((u_int16_t)(((struct ipfw_rule_ref *)(mtag+1))->info)); - INP_INFO_RLOCK(&V_divcbinfo); + INP_INFO_RLOCK_ET(&V_divcbinfo, et); CK_LIST_FOREACH(inp, &V_divcb, inp_list) { /* XXX why does only one socket match? */ if (inp->inp_lport == nport) { @@ -290,7 +291,7 @@ divert_packet(struct mbuf *m, int incoming) break; } } - INP_INFO_RUNLOCK(&V_divcbinfo); + INP_INFO_RUNLOCK_ET(&V_divcbinfo, et); if (sa == NULL) { m_freem(m); KMOD_IPSTAT_INC(ips_noproto); @@ -634,6 +635,7 @@ div_pcblist(SYSCTL_HANDLER_ARGS) struct inpcb *inp, **inp_list; inp_gen_t gencnt; struct xinpgen xig; + struct epoch_tracker et; /* * The process of preparing the TCB list is too time-consuming and @@ -652,10 +654,10 @@ div_pcblist(SYSCTL_HANDLER_ARGS) /* * OK, now we're committed to doing something. */ - INP_INFO_RLOCK(&V_divcbinfo); + INP_INFO_WLOCK(&V_divcbinfo); gencnt = V_divcbinfo.ipi_gencnt; n = V_divcbinfo.ipi_count; - INP_INFO_RUNLOCK(&V_divcbinfo); + INP_INFO_WUNLOCK(&V_divcbinfo); error = sysctl_wire_old_buffer(req, 2 * sizeof(xig) + n*sizeof(struct xinpcb)); @@ -674,7 +676,7 @@ div_pcblist(SYSCTL_HANDLER_ARGS) if (inp_list == NULL) return ENOMEM; - INP_INFO_RLOCK(&V_divcbinfo); + INP_INFO_RLOCK_ET(&V_divcbinfo, et); for (inp = CK_LIST_FIRST(V_divcbinfo.ipi_listhead), i = 0; inp && i < n; inp = CK_LIST_NEXT(inp, inp_list)) { INP_WLOCK(inp); @@ -685,7 +687,7 @@ div_pcblist(SYSCTL_HANDLER_ARGS) } INP_WUNLOCK(inp); } - INP_INFO_RUNLOCK(&V_divcbinfo); + INP_INFO_RUNLOCK_ET(&V_divcbinfo, et); n = i; error = 0; @@ -711,6 +713,7 @@ div_pcblist(SYSCTL_HANDLER_ARGS) INP_INFO_WUNLOCK(&V_divcbinfo); if (!error) { + struct epoch_tracker et; /* * Give the user an updated idea of our state. * If the generation differs from what we told @@ -718,11 +721,11 @@ div_pcblist(SYSCTL_HANDLER_ARGS) * while we were processing this request, and it * might be necessary to retry. */ - INP_INFO_RLOCK(&V_divcbinfo); + INP_INFO_RLOCK_ET(&V_divcbinfo, et); xig.xig_gen = V_divcbinfo.ipi_gencnt; xig.xig_sogen = so_gencnt; xig.xig_count = V_divcbinfo.ipi_count; - INP_INFO_RUNLOCK(&V_divcbinfo); + INP_INFO_RUNLOCK_ET(&V_divcbinfo, et); error = SYSCTL_OUT(req, &xig, sizeof xig); } free(inp_list, M_TEMP); diff --git a/sys/netinet/ip_encap.c b/sys/netinet/ip_encap.c index 7679d4202bae..9d98fcfe9cff 100644 --- a/sys/netinet/ip_encap.c +++ b/sys/netinet/ip_encap.c @@ -112,8 +112,8 @@ static struct mtx encapmtx; MTX_SYSINIT(encapmtx, &encapmtx, "encapmtx", MTX_DEF); #define ENCAP_WLOCK() mtx_lock(&encapmtx) #define ENCAP_WUNLOCK() mtx_unlock(&encapmtx) -#define ENCAP_RLOCK() epoch_enter_preempt(net_epoch_preempt) -#define ENCAP_RUNLOCK() epoch_exit_preempt(net_epoch_preempt) +#define ENCAP_RLOCK() struct epoch_tracker encap_et; epoch_enter_preempt(net_epoch_preempt, &encap_et) +#define ENCAP_RUNLOCK() epoch_exit_preempt(net_epoch_preempt, &encap_et) #define ENCAP_WAIT() epoch_wait_preempt(net_epoch_preempt) static struct encaptab * diff --git a/sys/netinet/ip_gre.c b/sys/netinet/ip_gre.c index 9ee09f1e28d5..4a7184ce3f31 100644 --- a/sys/netinet/ip_gre.c +++ b/sys/netinet/ip_gre.c @@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/sysctl.h> #include <sys/malloc.h> +#include <sys/proc.h> #include <net/if.h> #include <net/if_var.h> @@ -118,7 +119,7 @@ in_gre_lookup(const struct mbuf *m, int off, int proto, void **arg) if (V_ipv4_hashtbl == NULL) return (0); - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); ip = mtod(m, const struct ip *); CK_LIST_FOREACH(sc, &GRE_HASH(ip->ip_dst.s_addr, ip->ip_src.s_addr), chain) { diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c index 220488b4f126..27ed81f63850 100644 --- a/sys/netinet/raw_ip.c +++ b/sys/netinet/raw_ip.c @@ -285,6 +285,7 @@ rip_input(struct mbuf **mp, int *offp, int proto) struct ip *ip = mtod(m, struct ip *); struct inpcb *inp, *last; struct sockaddr_in ripsrc; + struct epoch_tracker et; int hash; *mp = NULL; @@ -299,7 +300,7 @@ rip_input(struct mbuf **mp, int *offp, int proto) hash = INP_PCBHASH_RAW(proto, ip->ip_src.s_addr, ip->ip_dst.s_addr, V_ripcbinfo.ipi_hashmask); - INP_INFO_RLOCK(&V_ripcbinfo); + INP_INFO_RLOCK_ET(&V_ripcbinfo, et); CK_LIST_FOREACH(inp, &V_ripcbinfo.ipi_hashbase[hash], inp_hash) { if (inp->inp_ip_p != proto) continue; @@ -422,7 +423,7 @@ rip_input(struct mbuf **mp, int *offp, int proto) skip_2: INP_RUNLOCK(inp); } - INP_INFO_RUNLOCK(&V_ripcbinfo); + INP_INFO_RUNLOCK_ET(&V_ripcbinfo, et); if (last != NULL) { if (rip_append(last, ip, m, &ripsrc) != 0) IPSTAT_INC(ips_delivered); @@ -1035,6 +1036,7 @@ rip_pcblist(SYSCTL_HANDLER_ARGS) struct inpcb *inp, **inp_list; inp_gen_t gencnt; struct xinpgen xig; + struct epoch_tracker et; /* * The process of preparing the TCB list is too time-consuming and @@ -1053,10 +1055,10 @@ rip_pcblist(SYSCTL_HANDLER_ARGS) /* * OK, now we're committed to doing something. */ - INP_INFO_RLOCK(&V_ripcbinfo); + INP_INFO_WLOCK(&V_ripcbinfo); gencnt = V_ripcbinfo.ipi_gencnt; n = V_ripcbinfo.ipi_count; - INP_INFO_RUNLOCK(&V_ripcbinfo); + INP_INFO_WUNLOCK(&V_ripcbinfo); xig.xig_len = sizeof xig; xig.xig_count = n; @@ -1070,7 +1072,7 @@ rip_pcblist(SYSCTL_HANDLER_ARGS) if (inp_list == NULL) return (ENOMEM); - INP_INFO_RLOCK(&V_ripcbinfo); + INP_INFO_RLOCK_ET(&V_ripcbinfo, et); for (inp = CK_LIST_FIRST(V_ripcbinfo.ipi_listhead), i = 0; inp && i < n; inp = CK_LIST_NEXT(inp, inp_list)) { INP_WLOCK(inp); @@ -1081,7 +1083,7 @@ rip_pcblist(SYSCTL_HANDLER_ARGS) } INP_WUNLOCK(inp); } - INP_INFO_RUNLOCK(&V_ripcbinfo); + INP_INFO_RUNLOCK_ET(&V_ripcbinfo, et); n = i; error = 0; @@ -1107,17 +1109,18 @@ rip_pcblist(SYSCTL_HANDLER_ARGS) INP_INFO_WUNLOCK(&V_ripcbinfo); if (!error) { + struct epoch_tracker et; /* * Give the user an updated idea of our state. If the * generation differs from what we told her before, she knows * that something happened while we were processing this * request, and it might be necessary to retry. */ - INP_INFO_RLOCK(&V_ripcbinfo); + INP_INFO_RLOCK_ET(&V_ripcbinfo, et); xig.xig_gen = V_ripcbinfo.ipi_gencnt; xig.xig_sogen = so_gencnt; xig.xig_count = V_ripcbinfo.ipi_count; - INP_INFO_RUNLOCK(&V_ripcbinfo); + INP_INFO_RUNLOCK_ET(&V_ripcbinfo, et); error = SYSCTL_OUT(req, &xig, sizeof xig); } free(inp_list, M_TEMP); diff --git a/sys/netinet/tcp_hpts.c b/sys/netinet/tcp_hpts.c index 15dc95de6d56..efdc99bfc821 100644 --- a/sys/netinet/tcp_hpts.c +++ b/sys/netinet/tcp_hpts.c @@ -998,7 +998,7 @@ __tcp_queue_to_input_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int3 void tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, - int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked) + int32_t tlen, int32_t drop_hdrlen, uint8_t iptos) { /* Setup packet for input first */ INP_WLOCK_ASSERT(tp->t_inpcb); @@ -1006,7 +1006,7 @@ tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, m->m_pkthdr.pace_tlen = (uint16_t) tlen; m->m_pkthdr.pace_drphdrlen = drop_hdrlen; m->m_pkthdr.pace_tos = iptos; - m->m_pkthdr.pace_lock = (uint8_t) ti_locked; + m->m_pkthdr.pace_lock = (curthread->td_epochnest != 0); if (tp->t_in_pkt == NULL) { tp->t_in_pkt = m; tp->t_tail_pkt = m; @@ -1019,11 +1019,11 @@ tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, int32_t __tcp_queue_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, - int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked, int32_t line){ + int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, int32_t line){ struct tcp_hpts_entry *hpts; int32_t ret; - tcp_queue_pkt_to_input(tp, m, th, tlen, drop_hdrlen, iptos, ti_locked); + tcp_queue_pkt_to_input(tp, m, th, tlen, drop_hdrlen, iptos); hpts = tcp_input_lock(tp->t_inpcb); ret = __tcp_queue_to_input_locked(tp->t_inpcb, hpts, line); mtx_unlock(&hpts->p_mtx); @@ -1145,6 +1145,7 @@ tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv) int16_t set_cpu; uint32_t did_prefetch = 0; int32_t ti_locked = TI_UNLOCKED; + struct epoch_tracker et; HPTS_MTX_ASSERT(hpts); while ((inp = TAILQ_FIRST(&hpts->p_input)) != NULL) { @@ -1161,7 +1162,7 @@ tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv) mtx_unlock(&hpts->p_mtx); CURVNET_SET(inp->inp_vnet); if (drop_reason) { - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); ti_locked = TI_RLOCKED; } else { ti_locked = TI_UNLOCKED; @@ -1172,7 +1173,7 @@ tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv) out: hpts->p_inp = NULL; if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); } if (in_pcbrele_wlocked(inp) == 0) { INP_WUNLOCK(inp); @@ -1201,7 +1202,7 @@ out: n = m->m_nextpkt; } tp = tcp_drop(tp, drop_reason); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); if (tp == NULL) { INP_WLOCK(inp); } @@ -1234,7 +1235,7 @@ out: (m->m_pkthdr.pace_lock == TI_RLOCKED || tp->t_state != TCPS_ESTABLISHED)) { ti_locked = TI_RLOCKED; - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); m = tp->t_in_pkt; } if (in_newts_every_tcb) { @@ -1270,13 +1271,15 @@ out: /* Use the hpts specific do_segment */ (*tp->t_fb->tfb_tcp_hpts_do_segment) (m, th, inp->inp_socket, tp, drop_hdrlen, - tlen, iptos, ti_locked, nxt_pkt, tv); + tlen, iptos, nxt_pkt, tv); } else { /* Use the default do_segment */ (*tp->t_fb->tfb_tcp_do_segment) (m, th, inp->inp_socket, tp, drop_hdrlen, - tlen, iptos, ti_locked); + tlen, iptos); } + if (ti_locked == TI_RLOCKED) + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); /* * Do segment returns unlocked we need the * lock again but we also need some kasserts @@ -1289,7 +1292,7 @@ out: n = m->m_nextpkt; if (m != NULL && m->m_pkthdr.pace_lock == TI_RLOCKED) { - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); ti_locked = TI_RLOCKED; } else ti_locked = TI_UNLOCKED; @@ -1316,14 +1319,14 @@ out: if (ti_locked == TI_UNLOCKED && (tp->t_state != TCPS_ESTABLISHED)) { ti_locked = TI_RLOCKED; - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); } } /** end while(m) */ } /** end if ((m != NULL) && (m == tp->t_in_pkt)) */ if (in_pcbrele_wlocked(inp) == 0) INP_WUNLOCK(inp); if (ti_locked == TI_RLOCKED) - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_UNLOCK_ASSERT(inp); ti_locked = TI_UNLOCKED; diff --git a/sys/netinet/tcp_hpts.h b/sys/netinet/tcp_hpts.h index c52a1d78bb10..04c867698915 100644 --- a/sys/netinet/tcp_hpts.h +++ b/sys/netinet/tcp_hpts.h @@ -238,10 +238,10 @@ int #define tcp_queue_to_input_locked(a, b) __tcp_queue_to_input_locked(a, b, __LINE__); void tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, - int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked); + int32_t tlen, int32_t drop_hdrlen, uint8_t iptos); int __tcp_queue_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, - int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked, int32_t line); + int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, int32_t line); #define tcp_queue_to_input(a, b, c, d, e, f, g) __tcp_queue_to_input(a, b, c, d, e, f, g, __LINE__) uint16_t tcp_hpts_delayedby(struct inpcb *inp); diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c index f0d39ffc3943..44008740d103 100644 --- a/sys/netinet/tcp_input.c +++ b/sys/netinet/tcp_input.c @@ -583,6 +583,7 @@ tcp_input(struct mbuf **mp, int *offp, int proto) int rstreason = 0; /* For badport_bandlim accounting purposes */ uint8_t iptos; struct m_tag *fwd_tag = NULL; + struct epoch_tracker et; #ifdef INET6 struct ip6_hdr *ip6 = NULL; int isipv6; @@ -773,7 +774,7 @@ tcp_input(struct mbuf **mp, int *offp, int proto) * connection in TIMEWAIT and SYNs not targeting a listening socket. */ if ((thflags & (TH_FIN | TH_RST)) != 0) { - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); ti_locked = TI_RLOCKED; } else ti_locked = TI_UNLOCKED; @@ -962,7 +963,7 @@ findpcb: */ if (inp->inp_flags & INP_TIMEWAIT) { if (ti_locked == TI_UNLOCKED) { - INP_INFO_RLOCK(); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); ti_locked = TI_RLOCKED; } INP_INFO_RLOCK_ASSERT(&V_tcbinfo); @@ -974,7 +975,7 @@ findpcb: */ if (tcp_twcheck(inp, &to, th, m, tlen)) goto findpcb; - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); return (IPPROTO_DONE); } /* @@ -1011,7 +1012,7 @@ findpcb: (tp->t_state == TCPS_LISTEN && (thflags & TH_SYN) && !IS_FASTOPEN(tp->t_flags)))) { if (ti_locked == TI_UNLOCKED) { - INP_INFO_RLOCK(); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); ti_locked = TI_RLOCKED; } INP_INFO_RLOCK_ASSERT(&V_tcbinfo); @@ -1145,8 +1146,9 @@ tfo_socket_result: * the mbuf chain and unlocks the inpcb. */ tp->t_fb->tfb_tcp_do_segment(m, th, so, tp, drop_hdrlen, tlen, - iptos, ti_locked); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); + iptos); + if (ti_locked == TI_RLOCKED) + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); return (IPPROTO_DONE); } /* @@ -1350,7 +1352,7 @@ tfo_socket_result: * Only the listen socket is unlocked by syncache_add(). */ if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); ti_locked = TI_UNLOCKED; } INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); @@ -1384,15 +1386,16 @@ tfo_socket_result: * state. tcp_do_segment() always consumes the mbuf chain, unlocks * the inpcb, and unlocks pcbinfo. */ - tp->t_fb->tfb_tcp_do_segment(m, th, so, tp, drop_hdrlen, tlen, iptos, ti_locked); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); + tp->t_fb->tfb_tcp_do_segment(m, th, so, tp, drop_hdrlen, tlen, iptos); + if (ti_locked == TI_RLOCKED) + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); return (IPPROTO_DONE); dropwithreset: TCP_PROBE5(receive, NULL, tp, m, tp, th); if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); ti_locked = TI_UNLOCKED; } #ifdef INVARIANTS @@ -1416,7 +1419,7 @@ dropunlock: TCP_PROBE5(receive, NULL, tp, m, tp, th); if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); ti_locked = TI_UNLOCKED; } #ifdef INVARIANTS @@ -1503,8 +1506,7 @@ tcp_autorcvbuf(struct mbuf *m, struct tcphdr *th, struct socket *so, void tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, - struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos, - int ti_locked) + struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos) { int thflags, acked, ourfinisacked, needoutput = 0, sack_changed; int rstreason, todrop, win; @@ -1530,7 +1532,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, tp->sackhint.last_sack_ack = 0; sack_changed = 0; nsegs = max(1, m->m_pkthdr.lro_nsegs); - /* * If this is either a state-changing packet or current state isn't * established, we require a write lock on tcbinfo. Otherwise, we @@ -1539,19 +1540,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((thflags & (TH_SYN | TH_FIN | TH_RST)) != 0 || tp->t_state != TCPS_ESTABLISHED) { - KASSERT(ti_locked == TI_RLOCKED, ("%s ti_locked %d for " - "SYN/FIN/RST/!EST", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - } else { -#ifdef INVARIANTS - if (ti_locked == TI_RLOCKED) - INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - else { - KASSERT(ti_locked == TI_UNLOCKED, ("%s: EST " - "ti_locked: %d", __func__, ti_locked)); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); - } -#endif } INP_WLOCK_ASSERT(tp->t_inpcb); KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", @@ -1760,10 +1749,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, /* * This is a pure ack for outstanding data. */ - if (ti_locked == TI_RLOCKED) - INP_INFO_RUNLOCK(&V_tcbinfo); - ti_locked = TI_UNLOCKED; - TCPSTAT_INC(tcps_predack); /* @@ -1867,10 +1852,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, * nothing on the reassembly queue and we have enough * buffer space to take it. */ - if (ti_locked == TI_RLOCKED) - INP_INFO_RUNLOCK(&V_tcbinfo); - ti_locked = TI_UNLOCKED; - /* Clean receiver SACK report if present */ if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks) tcp_clean_sackreport(tp); @@ -2072,8 +2053,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, tcp_state_change(tp, TCPS_SYN_RECEIVED); } - KASSERT(ti_locked == TI_RLOCKED, ("%s: trimthenstep6: " - "ti_locked %d", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); @@ -2148,9 +2127,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, (tp->rcv_wnd == 0 && tp->last_ack_sent == th->th_seq)) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - KASSERT(ti_locked == TI_RLOCKED, - ("%s: TH_RST ti_locked %d, th %p tp %p", - __func__, ti_locked, th, tp)); KASSERT(tp->t_state != TCPS_SYN_SENT, ("%s: TH_RST for TCPS_SYN_SENT th %p tp %p", __func__, th, tp)); @@ -2193,8 +2169,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((thflags & TH_SYN) && tp->t_state != TCPS_SYN_SENT && tp->t_state != TCPS_SYN_RECEIVED) { - KASSERT(ti_locked == TI_RLOCKED, - ("tcp_do_segment: TH_SYN ti_locked %d", ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); TCPSTAT_INC(tcps_badsyn); @@ -2308,8 +2282,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((so->so_state & SS_NOFDREF) && tp->t_state > TCPS_CLOSE_WAIT && tlen) { - KASSERT(ti_locked == TI_RLOCKED, ("%s: SS_NOFDEREF && " - "CLOSE_WAIT && tlen ti_locked %d", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { @@ -2899,7 +2871,6 @@ process_ACK: if (ourfinisacked) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); tcp_twstart(tp); - INP_INFO_RUNLOCK(&V_tcbinfo); m_freem(m); return; } @@ -3131,19 +3102,11 @@ dodata: /* XXX */ */ case TCPS_FIN_WAIT_2: INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - KASSERT(ti_locked == TI_RLOCKED, ("%s: dodata " - "TCP_FIN_WAIT_2 ti_locked: %d", __func__, - ti_locked)); tcp_twstart(tp); - INP_INFO_RUNLOCK(&V_tcbinfo); return; } } - if (ti_locked == TI_RLOCKED) - INP_INFO_RUNLOCK(&V_tcbinfo); - ti_locked = TI_UNLOCKED; - #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, @@ -3158,9 +3121,6 @@ dodata: /* XXX */ (void) tp->t_fb->tfb_tcp_output(tp); check_delack: - KASSERT(ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", - __func__, ti_locked)); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); if (tp->t_flags & TF_DELACK) { @@ -3198,10 +3158,6 @@ dropafterack: &tcp_savetcp, 0); #endif TCP_PROBE3(debug__input, tp, th, m); - if (ti_locked == TI_RLOCKED) - INP_INFO_RUNLOCK(&V_tcbinfo); - ti_locked = TI_UNLOCKED; - tp->t_flags |= TF_ACKNOW; (void) tp->t_fb->tfb_tcp_output(tp); INP_WUNLOCK(tp->t_inpcb); @@ -3209,10 +3165,6 @@ dropafterack: return; dropwithreset: - if (ti_locked == TI_RLOCKED) - INP_INFO_RUNLOCK(&V_tcbinfo); - ti_locked = TI_UNLOCKED; - if (tp != NULL) { tcp_dropwithreset(m, th, tp, tlen, rstreason); INP_WUNLOCK(tp->t_inpcb); @@ -3221,15 +3173,6 @@ dropwithreset: return; drop: - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - ti_locked = TI_UNLOCKED; - } -#ifdef INVARIANTS - else - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); -#endif - /* * Drop space held by incoming segment and return. */ diff --git a/sys/netinet/tcp_stacks/fastpath.c b/sys/netinet/tcp_stacks/fastpath.c index d5be90fdb1a9..7d5ac10544aa 100644 --- a/sys/netinet/tcp_stacks/fastpath.c +++ b/sys/netinet/tcp_stacks/fastpath.c @@ -121,12 +121,10 @@ __FBSDID("$FreeBSD$"); #include <security/mac/mac_framework.h> static void tcp_do_segment_fastslow(struct mbuf *, struct tcphdr *, - struct socket *, struct tcpcb *, int, int, uint8_t, - int); + struct socket *, struct tcpcb *, int, int, uint8_t); static void tcp_do_segment_fastack(struct mbuf *, struct tcphdr *, - struct socket *, struct tcpcb *, int, int, uint8_t, - int); + struct socket *, struct tcpcb *, int, int, uint8_t); /* * Indicate whether this ack should be delayed. We can delay the ack if @@ -154,7 +152,7 @@ static void tcp_do_segment_fastack(struct mbuf *, struct tcphdr *, static void tcp_do_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen, - int ti_locked, uint32_t tiwin) + uint32_t tiwin) { int acked; uint16_t nsegs; @@ -170,6 +168,7 @@ tcp_do_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcphdr tcp_savetcp; short ostate = 0; #endif + /* * The following if statement will be true if * we are doing the win_up_in_fp <and> @@ -207,11 +206,6 @@ tcp_do_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, /* * This is a pure ack for outstanding data. */ - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - } - ti_locked = TI_UNLOCKED; - TCPSTAT_INC(tcps_predack); /* @@ -310,9 +304,6 @@ tcp_do_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, sowwakeup(so); if (sbavail(&so->so_snd)) (void) tcp_output(tp); - KASSERT(ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", - __func__, ti_locked)); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); if (tp->t_flags & TF_DELACK) { @@ -330,7 +321,7 @@ tcp_do_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, static void tcp_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen, - int ti_locked, uint32_t tiwin) + uint32_t tiwin) { int newsize = 0; /* automatic sockbuf scaling */ #ifdef TCPDEBUG @@ -354,16 +345,6 @@ tcp_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, tp->ts_recent = to->to_tsval; } - /* - * This is a pure, in-sequence data packet with - * nothing on the reassembly queue and we have enough - * buffer space to take it. - */ - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - } - ti_locked = TI_UNLOCKED; - /* Clean receiver SACK report if present */ if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks) tcp_clean_sackreport(tp); @@ -413,9 +394,6 @@ tcp_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, tp->t_flags |= TF_ACKNOW; tcp_output(tp); } - KASSERT(ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", - __func__, ti_locked)); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); if (tp->t_flags & TF_DELACK) { @@ -434,7 +412,7 @@ tcp_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, static void tcp_do_slowpath(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen, - int ti_locked, uint32_t tiwin, int thflags) + uint32_t tiwin, int thflags) { int acked, ourfinisacked, needoutput = 0; int rstreason, todrop, win; @@ -464,7 +442,6 @@ tcp_do_slowpath(struct mbuf *m, struct tcphdr *th, struct socket *so, if (win < 0) win = 0; tp->rcv_wnd = imax(win, (int)(tp->rcv_adv - tp->rcv_nxt)); - switch (tp->t_state) { /* @@ -569,8 +546,6 @@ tcp_do_slowpath(struct mbuf *m, struct tcphdr *th, struct socket *so, tcp_state_change(tp, TCPS_SYN_RECEIVED); } - KASSERT(ti_locked == TI_RLOCKED, ("%s: trimthenstep6: " - "ti_locked %d", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); @@ -644,9 +619,6 @@ tcp_do_slowpath(struct mbuf *m, struct tcphdr *th, struct socket *so, SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) || (tp->rcv_wnd == 0 && tp->last_ack_sent == th->th_seq)) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - KASSERT(ti_locked == TI_RLOCKED, - ("%s: TH_RST ti_locked %d, th %p tp %p", - __func__, ti_locked, th, tp)); KASSERT(tp->t_state != TCPS_SYN_SENT, ("%s: TH_RST for TCPS_SYN_SENT th %p tp %p", __func__, th, tp)); @@ -688,8 +660,6 @@ tcp_do_slowpath(struct mbuf *m, struct tcphdr *th, struct socket *so, * Send challenge ACK for any SYN in synchronized state. */ if ((thflags & TH_SYN) && tp->t_state != TCPS_SYN_SENT) { - KASSERT(ti_locked == TI_RLOCKED, - ("tcp_do_segment: TH_SYN ti_locked %d", ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); TCPSTAT_INC(tcps_badsyn); @@ -803,8 +773,6 @@ tcp_do_slowpath(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((so->so_state & SS_NOFDREF) && tp->t_state > TCPS_CLOSE_WAIT && tlen) { - KASSERT(ti_locked == TI_RLOCKED, ("%s: SS_NOFDEREF && " - "CLOSE_WAIT && tlen ti_locked %d", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); if ((s = tcp_log_addrs(inc, th, NULL, NULL))) { @@ -1333,7 +1301,6 @@ process_ACK: if (ourfinisacked) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); tcp_twstart(tp); - INP_INFO_RUNLOCK(&V_tcbinfo); m_freem(m); return; } @@ -1562,20 +1529,10 @@ dodata: /* XXX */ */ case TCPS_FIN_WAIT_2: INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - KASSERT(ti_locked == TI_RLOCKED, ("%s: dodata " - "TCP_FIN_WAIT_2 ti_locked: %d", __func__, - ti_locked)); - tcp_twstart(tp); - INP_INFO_RUNLOCK(&V_tcbinfo); return; } } - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - } - ti_locked = TI_UNLOCKED; - #ifdef TCPDEBUG if (so->so_options & SO_DEBUG) tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen, @@ -1589,9 +1546,6 @@ dodata: /* XXX */ if (needoutput || (tp->t_flags & TF_ACKNOW)) (void) tp->t_fb->tfb_tcp_output(tp); - KASSERT(ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", - __func__, ti_locked)); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); if (tp->t_flags & TF_DELACK) { @@ -1629,11 +1583,6 @@ dropafterack: &tcp_savetcp, 0); #endif TCP_PROBE3(debug__drop, tp, th, m); - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - } - ti_locked = TI_UNLOCKED; - tp->t_flags |= TF_ACKNOW; (void) tp->t_fb->tfb_tcp_output(tp); INP_WUNLOCK(tp->t_inpcb); @@ -1641,11 +1590,6 @@ dropafterack: return; dropwithreset: - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - } - ti_locked = TI_UNLOCKED; - if (tp != NULL) { tcp_dropwithreset(m, th, tp, tlen, rstreason); INP_WUNLOCK(tp->t_inpcb); @@ -1654,15 +1598,6 @@ dropwithreset: return; drop: - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - ti_locked = TI_UNLOCKED; - } -#ifdef INVARIANTS - else - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); -#endif - /* * Drop space held by incoming segment and return. */ @@ -1687,8 +1622,7 @@ drop: */ void tcp_do_segment_fastslow(struct mbuf *m, struct tcphdr *th, struct socket *so, - struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos, - int ti_locked) + struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos) { int thflags; uint32_t tiwin; @@ -1709,19 +1643,7 @@ tcp_do_segment_fastslow(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((thflags & (TH_SYN | TH_FIN | TH_RST)) != 0 || tp->t_state != TCPS_ESTABLISHED) { - KASSERT(ti_locked == TI_RLOCKED, ("%s ti_locked %d for " - "SYN/FIN/RST/!EST", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - } else { -#ifdef INVARIANTS - if (ti_locked == TI_RLOCKED) { - INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - } else { - KASSERT(ti_locked == TI_UNLOCKED, ("%s: EST " - "ti_locked: %d", __func__, ti_locked)); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); - } -#endif } INP_WLOCK_ASSERT(tp->t_inpcb); KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", @@ -1736,9 +1658,6 @@ tcp_do_segment_fastslow(struct mbuf *m, struct tcphdr *th, struct socket *so, "sysctl setting)\n", s, __func__); free(s, M_TCPLOG); } - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - } INP_WUNLOCK(tp->t_inpcb); m_freem(m); return; @@ -1751,9 +1670,6 @@ tcp_do_segment_fastslow(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) && (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { tcp_dropwithreset(m, th, tp, tlen, BANDLIM_UNLIMITED); - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - } INP_WUNLOCK(tp->t_inpcb); return; } @@ -1919,19 +1835,19 @@ tcp_do_segment_fastslow(struct mbuf *m, struct tcphdr *th, struct socket *so, TAILQ_EMPTY(&tp->snd_holes)))) { /* We are done */ tcp_do_fastack(m, th, so, tp, &to, drop_hdrlen, tlen, - ti_locked, tiwin); + tiwin); return; } else if ((tlen) && (th->th_ack == tp->snd_una && tlen <= sbspace(&so->so_rcv))) { tcp_do_fastnewdata(m, th, so, tp, &to, drop_hdrlen, tlen, - ti_locked, tiwin); + tiwin); /* We are done */ return; } } tcp_do_slowpath(m, th, so, tp, &to, drop_hdrlen, tlen, - ti_locked, tiwin, thflags); + tiwin, thflags); } @@ -1947,7 +1863,7 @@ tcp_do_segment_fastslow(struct mbuf *m, struct tcphdr *th, struct socket *so, static int tcp_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int drop_hdrlen, int tlen, - int ti_locked, uint32_t tiwin) + uint32_t tiwin) { int acked; uint16_t nsegs; @@ -2039,11 +1955,6 @@ tcp_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, /* * This is a pure ack for outstanding data. */ - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - } - ti_locked = TI_UNLOCKED; - TCPSTAT_INC(tcps_predack); /* @@ -2138,9 +2049,6 @@ tcp_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, } if (sbavail(&so->so_snd)) (void) tcp_output(tp); - KASSERT(ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", - __func__, ti_locked)); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); if (tp->t_flags & TF_DELACK) { @@ -2167,8 +2075,7 @@ tcp_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, */ void tcp_do_segment_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, - struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos, - int ti_locked) + struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos) { int thflags; uint32_t tiwin; @@ -2186,19 +2093,7 @@ tcp_do_segment_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((thflags & (TH_SYN | TH_FIN | TH_RST)) != 0 || tp->t_state != TCPS_ESTABLISHED) { - KASSERT(ti_locked == TI_RLOCKED, ("%s ti_locked %d for " - "SYN/FIN/RST/!EST", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - } else { -#ifdef INVARIANTS - if (ti_locked == TI_RLOCKED) { - INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - } else { - KASSERT(ti_locked == TI_UNLOCKED, ("%s: EST " - "ti_locked: %d", __func__, ti_locked)); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); - } -#endif } INP_WLOCK_ASSERT(tp->t_inpcb); KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", @@ -2213,9 +2108,6 @@ tcp_do_segment_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, "sysctl setting)\n", s, __func__); free(s, M_TCPLOG); } - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - } INP_WUNLOCK(tp->t_inpcb); m_freem(m); return; @@ -2228,9 +2120,6 @@ tcp_do_segment_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) && (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { tcp_dropwithreset(m, th, tp, tlen, BANDLIM_UNLIMITED); - if (ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - } INP_WUNLOCK(tp->t_inpcb); return; } @@ -2367,12 +2256,12 @@ tcp_do_segment_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, __predict_true(LIST_EMPTY(&tp->t_segq)) && __predict_true(th->th_seq == tp->rcv_nxt)) { if (tcp_fastack(m, th, so, tp, &to, drop_hdrlen, tlen, - ti_locked, tiwin)) { + tiwin)) { return; } } tcp_do_slowpath(m, th, so, tp, &to, drop_hdrlen, tlen, - ti_locked, tiwin, thflags); + tiwin, thflags); } struct tcp_function_block __tcp_fastslow = { diff --git a/sys/netinet/tcp_stacks/rack.c b/sys/netinet/tcp_stacks/rack.c index e905f4ae1465..99dee76c9c36 100644 --- a/sys/netinet/tcp_stacks/rack.c +++ b/sys/netinet/tcp_stacks/rack.c @@ -251,12 +251,12 @@ rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, static int rack_process_ack(struct mbuf *m, struct tcphdr *th, - struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t * ti_locked, + struct socket *so, struct tcpcb *tp, struct tcpopt *to, uint32_t tiwin, int32_t tlen, int32_t * ofia, int32_t thflags, int32_t * ret_val); static int rack_process_data(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); + uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); static void rack_ack_received(struct tcpcb *tp, struct tcp_rack *rack, struct tcphdr *th, uint16_t nsegs, uint16_t type, int32_t recovery); @@ -275,7 +275,7 @@ static int32_t rack_ctor(void *mem, int32_t size, void *arg, int32_t how); static void rack_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, - uint8_t iptos, int32_t ti_locked); + uint8_t iptos); static void rack_dtor(void *mem, int32_t size, void *arg); static void rack_earlier_retran(struct tcpcb *tp, struct rack_sendmap *rsm, @@ -307,7 +307,7 @@ static int32_t rack_output(struct tcpcb *tp); static void rack_hpts_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, - uint8_t iptos, int32_t ti_locked, int32_t nxt_pkt, struct timeval *tv); + uint8_t iptos, int32_t nxt_pkt, struct timeval *tv); static uint32_t rack_proc_sack_blk(struct tcpcb *tp, struct tcp_rack *rack, @@ -338,57 +338,58 @@ rack_update_rtt(struct tcpcb *tp, struct tcp_rack *rack, static int32_t tcp_addrack(module_t mod, int32_t type, void *data); static void rack_challenge_ack(struct mbuf *m, struct tcphdr *th, - struct tcpcb *tp, int32_t * ti_locked, int32_t * ret_val); + struct tcpcb *tp, int32_t * ret_val); static int rack_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, - int32_t tlen, int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); + int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); static int rack_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, - int32_t tlen, int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); -static void rack_do_drop(struct mbuf *m, struct tcpcb *tp, int32_t * ti_locked); + int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); +static void +rack_do_drop(struct mbuf *m, struct tcpcb *tp); static void rack_do_dropafterack(struct mbuf *m, struct tcpcb *tp, - struct tcphdr *th, int32_t * ti_locked, int32_t thflags, int32_t tlen, int32_t * ret_val); + struct tcphdr *th, int32_t thflags, int32_t tlen, int32_t * ret_val); static void rack_do_dropwithreset(struct mbuf *m, struct tcpcb *tp, - struct tcphdr *th, int32_t * ti_locked, int32_t rstreason, int32_t tlen); + struct tcphdr *th, int32_t rstreason, int32_t tlen); static int rack_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, - int32_t tlen, int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); + int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); static int rack_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, - int32_t tlen, int32_t * ti_locked, uint32_t tiwin, int32_t nxt_pkt); + int32_t tlen, uint32_t tiwin, int32_t nxt_pkt); static int rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, - int32_t tlen, int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); + int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); static int rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, - int32_t tlen, int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); + int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); static int rack_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, - int32_t tlen, int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); + int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); static int rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, - int32_t tlen, int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); + int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); static int rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, - int32_t tlen, int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); + int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); static int rack_drop_checks(struct tcpopt *to, struct mbuf *m, - struct tcphdr *th, struct tcpcb *tp, int32_t * tlenp, int32_t * ti_locked, int32_t * thf, + struct tcphdr *th, struct tcpcb *tp, int32_t * tlenp, int32_t * thf, int32_t * drop_hdrlen, int32_t * ret_val); static int rack_process_rst(struct mbuf *m, struct tcphdr *th, - struct socket *so, struct tcpcb *tp, int32_t * ti_locked); + struct socket *so, struct tcpcb *tp); struct rack_sendmap * tcp_rack_output(struct tcpcb *tp, struct tcp_rack *rack, uint32_t tsused); @@ -398,7 +399,7 @@ static void static int rack_ts_check(struct mbuf *m, struct tcphdr *th, - struct tcpcb *tp, int32_t * ti_locked, int32_t tlen, int32_t thflags, int32_t * ret_val); + struct tcpcb *tp, int32_t tlen, int32_t thflags, int32_t * ret_val); int32_t rack_clear_counter=0; @@ -1492,12 +1493,8 @@ rack_calc_rwin(struct socket *so, struct tcpcb *tp) } static void -rack_do_drop(struct mbuf *m, struct tcpcb *tp, int32_t * ti_locked) +rack_do_drop(struct mbuf *m, struct tcpcb *tp) { - if (*ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - *ti_locked = TI_UNLOCKED; - } /* * Drop space held by incoming segment and return. */ @@ -1508,12 +1505,9 @@ rack_do_drop(struct mbuf *m, struct tcpcb *tp, int32_t * ti_locked) } static void -rack_do_dropwithreset(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, int32_t * ti_locked, int32_t rstreason, int32_t tlen) +rack_do_dropwithreset(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, + int32_t rstreason, int32_t tlen) { - if (*ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - *ti_locked = TI_UNLOCKED; - } if (tp != NULL) { tcp_dropwithreset(m, th, tp, tlen, rstreason); INP_WUNLOCK(tp->t_inpcb); @@ -1528,7 +1522,7 @@ rack_do_dropwithreset(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, int32 * and valid. */ static void -rack_do_dropafterack(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, int32_t * ti_locked, int32_t thflags, int32_t tlen, int32_t * ret_val) +rack_do_dropafterack(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, int32_t thflags, int32_t tlen, int32_t * ret_val) { /* * Generate an ACK dropping incoming segment if it occupies sequence @@ -1550,14 +1544,10 @@ rack_do_dropafterack(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, int32_ (SEQ_GT(tp->snd_una, th->th_ack) || SEQ_GT(th->th_ack, tp->snd_max))) { *ret_val = 1; - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return; } else *ret_val = 0; - if (*ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - *ti_locked = TI_UNLOCKED; - } rack = (struct tcp_rack *)tp->t_fb_ptr; rack->r_wanted_output++; tp->t_flags |= TF_ACKNOW; @@ -1567,7 +1557,7 @@ rack_do_dropafterack(struct mbuf *m, struct tcpcb *tp, struct tcphdr *th, int32_ static int -rack_process_rst(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int32_t * ti_locked) +rack_process_rst(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp) { /* * RFC5961 Section 3.2 @@ -1586,9 +1576,6 @@ rack_process_rst(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tc (tp->rcv_wnd == 0 && tp->last_ack_sent == th->th_seq)) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - KASSERT(*ti_locked == TI_RLOCKED, - ("%s: TH_RST ti_locked %d, th %p tp %p", - __func__, *ti_locked, th, tp)); KASSERT(tp->t_state != TCPS_SYN_SENT, ("%s: TH_RST for TCPS_SYN_SENT th %p tp %p", __func__, th, tp)); @@ -1617,7 +1604,7 @@ rack_process_rst(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tc tp = tcp_close(tp); } dropped = 1; - rack_do_drop(m, tp, ti_locked); + rack_do_drop(m, tp); } else { TCPSTAT_INC(tcps_badrst); /* Send challenge ACK. */ @@ -1638,10 +1625,9 @@ rack_process_rst(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tc * and valid. */ static void -rack_challenge_ack(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t * ti_locked, int32_t * ret_val) +rack_challenge_ack(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t * ret_val) { - KASSERT(*ti_locked == TI_RLOCKED, - ("tcp_do_segment: TH_SYN ti_locked %d", *ti_locked)); + INP_INFO_RLOCK_ASSERT(&V_tcbinfo); TCPSTAT_INC(tcps_badsyn); @@ -1650,7 +1636,7 @@ rack_challenge_ack(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) { tp = tcp_drop(tp, ECONNRESET); *ret_val = 1; - rack_do_drop(m, tp, ti_locked); + rack_do_drop(m, tp); } else { /* Send challenge ACK. */ tcp_respond(tp, mtod(m, void *), th, m, tp->rcv_nxt, @@ -1658,7 +1644,7 @@ rack_challenge_ack(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t tp->last_ack_sent = tp->rcv_nxt; m = NULL; *ret_val = 0; - rack_do_drop(m, NULL, ti_locked); + rack_do_drop(m, NULL); } } @@ -1669,7 +1655,7 @@ rack_challenge_ack(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t * TCB is still valid and locked. */ static int -rack_ts_check(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t * ti_locked, int32_t tlen, int32_t thflags, int32_t * ret_val) +rack_ts_check(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t tlen, int32_t thflags, int32_t * ret_val) { /* Check to see if ts_recent is over 24 days old. */ @@ -1691,9 +1677,9 @@ rack_ts_check(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t * ti_ TCPSTAT_INC(tcps_pawsdrop); *ret_val = 0; if (tlen) { - rack_do_dropafterack(m, tp, th, ti_locked, thflags, tlen, ret_val); + rack_do_dropafterack(m, tp, th, thflags, tlen, ret_val); } else { - rack_do_drop(m, NULL, ti_locked); + rack_do_drop(m, NULL); } return (1); } @@ -1707,7 +1693,7 @@ rack_ts_check(struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t * ti_ * TCB is still valid and locked. */ static int -rack_drop_checks(struct tcpopt *to, struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t * tlenp, int32_t * ti_locked, int32_t * thf, int32_t * drop_hdrlen, int32_t * ret_val) +rack_drop_checks(struct tcpopt *to, struct mbuf *m, struct tcphdr *th, struct tcpcb *tp, int32_t * tlenp, int32_t * thf, int32_t * drop_hdrlen, int32_t * ret_val) { int32_t todrop; int32_t thflags; @@ -1779,7 +1765,7 @@ rack_drop_checks(struct tcpopt *to, struct mbuf *m, struct tcphdr *th, struct tc tp->t_flags |= TF_ACKNOW; TCPSTAT_INC(tcps_rcvwinprobe); } else { - rack_do_dropafterack(m, tp, th, ti_locked, thflags, tlen, ret_val); + rack_do_dropafterack(m, tp, th, thflags, tlen, ret_val); return (1); } } else @@ -4482,7 +4468,7 @@ out: static int rack_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, - int32_t * ti_locked, uint32_t tiwin, int32_t tlen, + uint32_t tiwin, int32_t tlen, int32_t * ofia, int32_t thflags, int32_t * ret_val) { int32_t ourfinisacked = 0; @@ -4494,7 +4480,7 @@ rack_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so, rack = (struct tcp_rack *)tp->t_fb_ptr; if (SEQ_GT(th->th_ack, tp->snd_max)) { - rack_do_dropafterack(m, tp, th, ti_locked, thflags, tlen, ret_val); + rack_do_dropafterack(m, tp, th, thflags, tlen, ret_val); return (1); } if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) { @@ -4642,7 +4628,7 @@ rack_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so, */ *ret_val = 1; tp = tcp_close(tp); - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_UNLIMITED, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, tlen); return (1); } } @@ -4660,7 +4646,7 @@ rack_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so, static int rack_process_data(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) + uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) { /* * Update window information. Don't look at window if no ACK: TAC's @@ -4883,28 +4869,16 @@ dodata: /* XXX */ case TCPS_FIN_WAIT_2: rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - KASSERT(*ti_locked == TI_RLOCKED, ("%s: dodata " - "TCP_FIN_WAIT_2 ti_locked: %d", __func__, - *ti_locked)); tcp_twstart(tp); - *ti_locked = TI_UNLOCKED; - INP_INFO_RUNLOCK(&V_tcbinfo); return (1); } } - if (*ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - *ti_locked = TI_UNLOCKED; - } /* * Return any desired output. */ if ((tp->t_flags & TF_ACKNOW) || (sbavail(&so->so_snd) > (tp->snd_max - tp->snd_una))) { rack->r_wanted_output++; } - KASSERT(*ti_locked == TI_UNLOCKED, ("%s: check_delack ti_locked %d", - __func__, *ti_locked)); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); return (0); } @@ -4917,7 +4891,7 @@ dodata: /* XXX */ static int rack_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t nxt_pkt) + uint32_t tiwin, int32_t nxt_pkt) { int32_t nsegs; int32_t newsize = 0; /* automatic sockbuf scaling */ @@ -4969,10 +4943,6 @@ rack_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, * This is a pure, in-sequence data packet with nothing on the * reassembly queue and we have enough buffer space to take it. */ - if (*ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - *ti_locked = TI_UNLOCKED; - } nsegs = max(1, m->m_pkthdr.lro_nsegs); @@ -5041,7 +5011,7 @@ rack_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, static int rack_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t nxt_pkt, uint32_t cts) + uint32_t tiwin, int32_t nxt_pkt, uint32_t cts) { int32_t acked; int32_t nsegs; @@ -5117,10 +5087,6 @@ rack_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, /* * This is a pure ack for outstanding data. */ - if (*ti_locked == TI_RLOCKED) { - INP_INFO_RUNLOCK(&V_tcbinfo); - *ti_locked = TI_UNLOCKED; - } TCPSTAT_INC(tcps_predack); /* @@ -5199,7 +5165,7 @@ rack_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, static int rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) + uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) { int32_t ret_val = 0; int32_t todrop; @@ -5220,22 +5186,22 @@ rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((thflags & TH_ACK) && (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return (1); } if ((thflags & (TH_ACK | TH_RST)) == (TH_ACK | TH_RST)) { TCP_PROBE5(connect__refused, NULL, tp, mtod(m, const char *), tp, th); tp = tcp_drop(tp, ECONNREFUSED); - rack_do_drop(m, tp, ti_locked); + rack_do_drop(m, tp); return (1); } if (thflags & TH_RST) { - rack_do_drop(m, tp, ti_locked); + rack_do_drop(m, tp); return (1); } if (!(thflags & TH_SYN)) { - rack_do_drop(m, tp, ti_locked); + rack_do_drop(m, tp); return (1); } tp->irs = th->th_seq; @@ -5323,8 +5289,6 @@ rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so, tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN); tcp_state_change(tp, TCPS_SYN_RECEIVED); } - KASSERT(*ti_locked == TI_RLOCKED, ("%s: trimthenstep6: " - "ti_locked %d", __func__, *ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(tp->t_inpcb); /* @@ -5349,7 +5313,7 @@ rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so, * of step 5, ack processing. Otherwise, goto step 6. */ if (thflags & TH_ACK) { - if (rack_process_ack(m, th, so, tp, to, ti_locked, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) + if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) return (ret_val); /* We may have changed to FIN_WAIT_1 above */ if (tp->t_state == TCPS_FIN_WAIT_1) { @@ -5381,7 +5345,7 @@ rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so, } } return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } /* @@ -5392,7 +5356,7 @@ rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so, static int rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) + uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) { int32_t ret_val = 0; int32_t ourfinisacked = 0; @@ -5402,7 +5366,7 @@ rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((thflags & TH_ACK) && (SEQ_LEQ(th->th_ack, tp->snd_una) || SEQ_GT(th->th_ack, tp->snd_max))) { - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return (1); } if (IS_FASTOPEN(tp->t_flags)) { @@ -5414,7 +5378,7 @@ rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, * FIN, or a RST. */ if ((thflags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) { - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return (1); } else if (thflags & TH_SYN) { /* non-initial SYN is ignored */ @@ -5424,22 +5388,22 @@ rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT) || (rack->r_ctl.rc_hpts_flags & PACE_TMR_TLP) || (rack->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) { - rack_do_drop(m, NULL, ti_locked); + rack_do_drop(m, NULL); return (0); } } else if (!(thflags & (TH_ACK | TH_FIN | TH_RST))) { - rack_do_drop(m, NULL, ti_locked); + rack_do_drop(m, NULL); return (0); } } if (thflags & TH_RST) - return (rack_process_rst(m, th, so, tp, ti_locked)); + return (rack_process_rst(m, th, so, tp)); /* * RFC5961 Section 4.2 Send challenge ACK for any SYN in * synchronized state. */ if (thflags & TH_SYN) { - rack_challenge_ack(m, th, tp, ti_locked, &ret_val); + rack_challenge_ack(m, th, tp, &ret_val); return (ret_val); } /* @@ -5448,7 +5412,7 @@ rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && TSTMP_LT(to->to_tsval, tp->ts_recent)) { - if (rack_ts_check(m, th, tp, ti_locked, tlen, thflags, &ret_val)) + if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val)) return (ret_val); } /* @@ -5459,10 +5423,10 @@ rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, * "LAND" DoS attack. */ if (SEQ_LT(th->th_seq, tp->irs)) { - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return (1); } - if (rack_drop_checks(to, m, th, tp, &tlen, ti_locked, &thflags, &drop_hdrlen, &ret_val)) { + if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { return (ret_val); } /* @@ -5497,7 +5461,7 @@ rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, cc_conn_init(tp); } return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } TCPSTAT_INC(tcps_connects); soisconnected(so); @@ -5546,7 +5510,7 @@ rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, (void)tcp_reass(tp, (struct tcphdr *)0, 0, (struct mbuf *)0); tp->snd_wl1 = th->th_seq - 1; - if (rack_process_ack(m, th, so, tp, to, ti_locked, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { + if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { return (ret_val); } if (tp->t_state == TCPS_FIN_WAIT_1) { @@ -5577,7 +5541,7 @@ rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, } } return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } /* @@ -5588,7 +5552,7 @@ rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, static int rack_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) + uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) { int32_t ret_val = 0; @@ -5615,12 +5579,12 @@ rack_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so, rack = (struct tcp_rack *)tp->t_fb_ptr; if (tlen == 0) { if (rack_fastack(m, th, so, tp, to, drop_hdrlen, tlen, - ti_locked, tiwin, nxt_pkt, rack->r_ctl.rc_rcvtime)) { + tiwin, nxt_pkt, rack->r_ctl.rc_rcvtime)) { return (0); } } else { if (rack_do_fastnewdata(m, th, so, tp, to, drop_hdrlen, tlen, - ti_locked, tiwin, nxt_pkt)) { + tiwin, nxt_pkt)) { return (0); } } @@ -5628,14 +5592,14 @@ rack_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so, rack_calc_rwin(so, tp); if (thflags & TH_RST) - return (rack_process_rst(m, th, so, tp, ti_locked)); + return (rack_process_rst(m, th, so, tp)); /* * RFC5961 Section 4.2 Send challenge ACK for any SYN in * synchronized state. */ if (thflags & TH_SYN) { - rack_challenge_ack(m, th, tp, ti_locked, &ret_val); + rack_challenge_ack(m, th, tp, &ret_val); return (ret_val); } /* @@ -5644,10 +5608,10 @@ rack_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && TSTMP_LT(to->to_tsval, tp->ts_recent)) { - if (rack_ts_check(m, th, tp, ti_locked, tlen, thflags, &ret_val)) + if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val)) return (ret_val); } - if (rack_drop_checks(to, m, th, tp, &tlen, ti_locked, &thflags, &drop_hdrlen, &ret_val)) { + if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { return (ret_val); } /* @@ -5680,32 +5644,32 @@ rack_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so, if (tp->t_flags & TF_NEEDSYN) { return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } else if (tp->t_flags & TF_ACKNOW) { - rack_do_dropafterack(m, tp, th, ti_locked, thflags, tlen, &ret_val); + rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); return (ret_val); } else { - rack_do_drop(m, NULL, ti_locked); + rack_do_drop(m, NULL); return (0); } } /* * Ack processing. */ - if (rack_process_ack(m, th, so, tp, to, ti_locked, tiwin, tlen, NULL, thflags, &ret_val)) { + if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) { return (ret_val); } if (sbavail(&so->so_snd)) { if (rack_progress_timeout_check(tp)) { tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT); - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return (1); } } /* State changes only happen in rack_process_data() */ return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } /* @@ -5716,19 +5680,19 @@ rack_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so, static int rack_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) + uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) { int32_t ret_val = 0; rack_calc_rwin(so, tp); if (thflags & TH_RST) - return (rack_process_rst(m, th, so, tp, ti_locked)); + return (rack_process_rst(m, th, so, tp)); /* * RFC5961 Section 4.2 Send challenge ACK for any SYN in * synchronized state. */ if (thflags & TH_SYN) { - rack_challenge_ack(m, th, tp, ti_locked, &ret_val); + rack_challenge_ack(m, th, tp, &ret_val); return (ret_val); } /* @@ -5737,10 +5701,10 @@ rack_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && TSTMP_LT(to->to_tsval, tp->ts_recent)) { - if (rack_ts_check(m, th, tp, ti_locked, tlen, thflags, &ret_val)) + if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val)) return (ret_val); } - if (rack_drop_checks(to, m, th, tp, &tlen, ti_locked, &thflags, &drop_hdrlen, &ret_val)) { + if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { return (ret_val); } /* @@ -5772,48 +5736,46 @@ rack_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((thflags & TH_ACK) == 0) { if (tp->t_flags & TF_NEEDSYN) { return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } else if (tp->t_flags & TF_ACKNOW) { - rack_do_dropafterack(m, tp, th, ti_locked, thflags, tlen, &ret_val); + rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); return (ret_val); } else { - rack_do_drop(m, NULL, ti_locked); + rack_do_drop(m, NULL); return (0); } } /* * Ack processing. */ - if (rack_process_ack(m, th, so, tp, to, ti_locked, tiwin, tlen, NULL, thflags, &ret_val)) { + if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) { return (ret_val); } if (sbavail(&so->so_snd)) { if (rack_progress_timeout_check(tp)) { tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT); - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return (1); } } return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } static int rack_check_data_after_close(struct mbuf *m, - struct tcpcb *tp, int32_t *ti_locked, int32_t *tlen, struct tcphdr *th, struct socket *so) + struct tcpcb *tp, int32_t *tlen, struct tcphdr *th, struct socket *so) { - struct tcp_rack *rack; + struct tcp_rack *rack; - KASSERT(*ti_locked == TI_RLOCKED, ("%s: SS_NOFDEREF && " - "CLOSE_WAIT && tlen ti_locked %d", __func__, *ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); rack = (struct tcp_rack *)tp->t_fb_ptr; if (rack->rc_allow_data_af_clo == 0) { close_now: tp = tcp_close(tp); TCPSTAT_INC(tcps_rcvafterclose); - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_UNLIMITED, (*tlen)); + rack_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, (*tlen)); return (1); } if (sbavail(&so->so_snd) == 0) @@ -5834,7 +5796,7 @@ rack_check_data_after_close(struct mbuf *m, static int rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) + uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) { int32_t ret_val = 0; int32_t ourfinisacked = 0; @@ -5842,13 +5804,13 @@ rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, rack_calc_rwin(so, tp); if (thflags & TH_RST) - return (rack_process_rst(m, th, so, tp, ti_locked)); + return (rack_process_rst(m, th, so, tp)); /* * RFC5961 Section 4.2 Send challenge ACK for any SYN in * synchronized state. */ if (thflags & TH_SYN) { - rack_challenge_ack(m, th, tp, ti_locked, &ret_val); + rack_challenge_ack(m, th, tp, &ret_val); return (ret_val); } /* @@ -5857,10 +5819,10 @@ rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && TSTMP_LT(to->to_tsval, tp->ts_recent)) { - if (rack_ts_check(m, th, tp, ti_locked, tlen, thflags, &ret_val)) + if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val)) return (ret_val); } - if (rack_drop_checks(to, m, th, tp, &tlen, ti_locked, &thflags, &drop_hdrlen, &ret_val)) { + if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { return (ret_val); } /* @@ -5868,7 +5830,7 @@ rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, * are gone, then RST the other end. */ if ((so->so_state & SS_NOFDREF) && tlen) { - if (rack_check_data_after_close(m, tp, ti_locked, &tlen, th, so)) + if (rack_check_data_after_close(m, tp, &tlen, th, so)) return (1); } /* @@ -5900,19 +5862,19 @@ rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((thflags & TH_ACK) == 0) { if (tp->t_flags & TF_NEEDSYN) { return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } else if (tp->t_flags & TF_ACKNOW) { - rack_do_dropafterack(m, tp, th, ti_locked, thflags, tlen, &ret_val); + rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); return (ret_val); } else { - rack_do_drop(m, NULL, ti_locked); + rack_do_drop(m, NULL); return (0); } } /* * Ack processing. */ - if (rack_process_ack(m, th, so, tp, to, ti_locked, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { + if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { return (ret_val); } if (ourfinisacked) { @@ -5937,12 +5899,12 @@ rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, if (sbavail(&so->so_snd)) { if (rack_progress_timeout_check(tp)) { tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT); - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return (1); } } return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } /* @@ -5953,7 +5915,7 @@ rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, static int rack_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) + uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) { int32_t ret_val = 0; int32_t ourfinisacked = 0; @@ -5961,13 +5923,13 @@ rack_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so, rack_calc_rwin(so, tp); if (thflags & TH_RST) - return (rack_process_rst(m, th, so, tp, ti_locked)); + return (rack_process_rst(m, th, so, tp)); /* * RFC5961 Section 4.2 Send challenge ACK for any SYN in * synchronized state. */ if (thflags & TH_SYN) { - rack_challenge_ack(m, th, tp, ti_locked, &ret_val); + rack_challenge_ack(m, th, tp, &ret_val); return (ret_val); } /* @@ -5976,10 +5938,10 @@ rack_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && TSTMP_LT(to->to_tsval, tp->ts_recent)) { - if (rack_ts_check(m, th, tp, ti_locked, tlen, thflags, &ret_val)) + if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val)) return (ret_val); } - if (rack_drop_checks(to, m, th, tp, &tlen, ti_locked, &thflags, &drop_hdrlen, &ret_val)) { + if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { return (ret_val); } /* @@ -5987,7 +5949,7 @@ rack_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so, * are gone, then RST the other end. */ if ((so->so_state & SS_NOFDREF) && tlen) { - if (rack_check_data_after_close(m, tp, ti_locked, &tlen, th, so)) + if (rack_check_data_after_close(m, tp, &tlen, th, so)) return (1); } /* @@ -6019,38 +5981,36 @@ rack_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((thflags & TH_ACK) == 0) { if (tp->t_flags & TF_NEEDSYN) { return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } else if (tp->t_flags & TF_ACKNOW) { - rack_do_dropafterack(m, tp, th, ti_locked, thflags, tlen, &ret_val); + rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); return (ret_val); } else { - rack_do_drop(m, NULL, ti_locked); + rack_do_drop(m, NULL); return (0); } } /* * Ack processing. */ - if (rack_process_ack(m, th, so, tp, to, ti_locked, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { + if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { return (ret_val); } if (ourfinisacked) { INP_INFO_RLOCK_ASSERT(&V_tcbinfo); tcp_twstart(tp); - INP_INFO_RUNLOCK(&V_tcbinfo); - *ti_locked = TI_UNLOCKED; m_freem(m); return (1); } if (sbavail(&so->so_snd)) { if (rack_progress_timeout_check(tp)) { tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT); - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return (1); } } return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } /* @@ -6061,7 +6021,7 @@ rack_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so, static int rack_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) + uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) { int32_t ret_val = 0; int32_t ourfinisacked = 0; @@ -6069,13 +6029,13 @@ rack_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so, rack_calc_rwin(so, tp); if (thflags & TH_RST) - return (rack_process_rst(m, th, so, tp, ti_locked)); + return (rack_process_rst(m, th, so, tp)); /* * RFC5961 Section 4.2 Send challenge ACK for any SYN in * synchronized state. */ if (thflags & TH_SYN) { - rack_challenge_ack(m, th, tp, ti_locked, &ret_val); + rack_challenge_ack(m, th, tp, &ret_val); return (ret_val); } /* @@ -6084,10 +6044,10 @@ rack_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && TSTMP_LT(to->to_tsval, tp->ts_recent)) { - if (rack_ts_check(m, th, tp, ti_locked, tlen, thflags, &ret_val)) + if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val)) return (ret_val); } - if (rack_drop_checks(to, m, th, tp, &tlen, ti_locked, &thflags, &drop_hdrlen, &ret_val)) { + if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { return (ret_val); } /* @@ -6095,7 +6055,7 @@ rack_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so, * are gone, then RST the other end. */ if ((so->so_state & SS_NOFDREF) && tlen) { - if (rack_check_data_after_close(m, tp, ti_locked, &tlen, th, so)) + if (rack_check_data_after_close(m, tp, &tlen, th, so)) return (1); } /* @@ -6127,36 +6087,37 @@ rack_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((thflags & TH_ACK) == 0) { if (tp->t_flags & TF_NEEDSYN) { return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } else if (tp->t_flags & TF_ACKNOW) { - rack_do_dropafterack(m, tp, th, ti_locked, thflags, tlen, &ret_val); + rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); return (ret_val); } else { - rack_do_drop(m, NULL, ti_locked); + rack_do_drop(m, NULL); return (0); } } /* * case TCPS_LAST_ACK: Ack processing. */ - if (rack_process_ack(m, th, so, tp, to, ti_locked, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { + if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { return (ret_val); } if (ourfinisacked) { + INP_INFO_RLOCK_ASSERT(&V_tcbinfo); tp = tcp_close(tp); - rack_do_drop(m, tp, ti_locked); + rack_do_drop(m, tp); return (1); } if (sbavail(&so->so_snd)) { if (rack_progress_timeout_check(tp)) { tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT); - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return (1); } } return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } @@ -6168,7 +6129,7 @@ rack_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so, static int rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, - int32_t * ti_locked, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) + uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) { int32_t ret_val = 0; int32_t ourfinisacked = 0; @@ -6177,13 +6138,13 @@ rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so, /* Reset receive buffer auto scaling when not in bulk receive mode. */ if (thflags & TH_RST) - return (rack_process_rst(m, th, so, tp, ti_locked)); + return (rack_process_rst(m, th, so, tp)); /* * RFC5961 Section 4.2 Send challenge ACK for any SYN in * synchronized state. */ if (thflags & TH_SYN) { - rack_challenge_ack(m, th, tp, ti_locked, &ret_val); + rack_challenge_ack(m, th, tp, &ret_val); return (ret_val); } /* @@ -6192,10 +6153,10 @@ rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && TSTMP_LT(to->to_tsval, tp->ts_recent)) { - if (rack_ts_check(m, th, tp, ti_locked, tlen, thflags, &ret_val)) + if (rack_ts_check(m, th, tp, tlen, thflags, &ret_val)) return (ret_val); } - if (rack_drop_checks(to, m, th, tp, &tlen, ti_locked, &thflags, &drop_hdrlen, &ret_val)) { + if (rack_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { return (ret_val); } /* @@ -6204,7 +6165,7 @@ rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((so->so_state & SS_NOFDREF) && tlen) { - if (rack_check_data_after_close(m, tp, ti_locked, &tlen, th, so)) + if (rack_check_data_after_close(m, tp, &tlen, th, so)) return (1); } /* @@ -6236,30 +6197,30 @@ rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((thflags & TH_ACK) == 0) { if (tp->t_flags & TF_NEEDSYN) { return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } else if (tp->t_flags & TF_ACKNOW) { - rack_do_dropafterack(m, tp, th, ti_locked, thflags, tlen, &ret_val); + rack_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); return (ret_val); } else { - rack_do_drop(m, NULL, ti_locked); + rack_do_drop(m, NULL); return (0); } } /* * Ack processing. */ - if (rack_process_ack(m, th, so, tp, to, ti_locked, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { + if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { return (ret_val); } if (sbavail(&so->so_snd)) { if (rack_progress_timeout_check(tp)) { tcp_set_inp_to_drop(tp->t_inpcb, ETIMEDOUT); - rack_do_dropwithreset(m, tp, th, ti_locked, BANDLIM_RST_OPENPORT, tlen); + rack_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); return (1); } } return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, - ti_locked, tiwin, thflags, nxt_pkt)); + tiwin, thflags, nxt_pkt)); } @@ -6533,7 +6494,7 @@ rack_timer_audit(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb) static void rack_hpts_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos, - int32_t ti_locked, int32_t nxt_pkt, struct timeval *tv) + int32_t nxt_pkt, struct timeval *tv) { int32_t thflags, retval, did_out = 0; int32_t way_out = 0; @@ -6558,19 +6519,7 @@ rack_hpts_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, */ if ((thflags & (TH_SYN | TH_FIN | TH_RST)) != 0 || tp->t_state != TCPS_ESTABLISHED) { - KASSERT(ti_locked == TI_RLOCKED, ("%s ti_locked %d for " - "SYN/FIN/RST/!EST", __func__, ti_locked)); INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - } else { -#ifdef INVARIANTS - if (ti_locked == TI_RLOCKED) { - INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - } else { - KASSERT(ti_locked == TI_UNLOCKED, ("%s: EST " - "ti_locked: %d", __func__, ti_locked)); - INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); - } -#endif } INP_WLOCK_ASSERT(tp->t_inpcb); KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", @@ -6717,7 +6666,7 @@ rack_hpts_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, if ((tp->t_flags & TF_SACK_PERMIT) == 0) { tcp_switch_back_to_default(tp); (*tp->t_fb->tfb_tcp_do_segment) (m, th, so, tp, drop_hdrlen, - tlen, iptos, ti_locked); + tlen, iptos); return; } /* Set the flag */ @@ -6740,7 +6689,7 @@ rack_hpts_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, rack_clear_rate_sample(rack); retval = (*rack->r_substate) (m, th, so, tp, &to, drop_hdrlen, - tlen, &ti_locked, tiwin, thflags, nxt_pkt); + tlen, tiwin, thflags, nxt_pkt); #ifdef INVARIANTS if ((retval == 0) && (tp->t_inpcb == NULL)) { @@ -6748,11 +6697,6 @@ rack_hpts_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, retval, tp, prev_state); } #endif - if (ti_locked != TI_UNLOCKED) { - INP_INFO_RLOCK_ASSERT(&V_tcbinfo); - INP_INFO_RUNLOCK(&V_tcbinfo); - ti_locked = TI_UNLOCKED; - } if (retval == 0) { /* * If retval is 1 the tcb is unlocked and most likely the tp @@ -6824,14 +6768,13 @@ rack_hpts_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, void rack_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, - struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos, - int32_t ti_locked) + struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos) { struct timeval tv; #ifdef RSS struct tcp_function_block *tfb; struct tcp_rack *rack; - struct inpcb *inp; + struct epoch_tracker et; rack = (struct tcp_rack *)tp->t_fb_ptr; if (rack->r_state == 0) { @@ -6839,21 +6782,19 @@ rack_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, * Initial input (ACK to SYN-ACK etc)lets go ahead and get * it processed */ - INP_INFO_RLOCK(); - ti_locked = TI_RLOCKED; + INP_INFO_RLOCK_ET(&V_tcbinfo, et); tcp_get_usecs(&tv); rack_hpts_do_segment(m, th, so, tp, drop_hdrlen, - tlen, iptos, ti_locked, 0, &tv); + tlen, iptos, 0, &tv); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); return; } - if (ti_locked == TI_RLOCKED) - INP_INFO_RUNLOCK(&V_tcbinfo); - tcp_queue_to_input(tp, m, th, tlen, drop_hdrlen, iptos, (uint8_t) ti_locked); + tcp_queue_to_input(tp, m, th, tlen, drop_hdrlen, iptos); INP_WUNLOCK(tp->t_inpcb); #else tcp_get_usecs(&tv); rack_hpts_do_segment(m, th, so, tp, drop_hdrlen, - tlen, iptos, ti_locked, 0, &tv); + tlen, iptos, 0, &tv); #endif } diff --git a/sys/netinet/tcp_stacks/tcp_rack.h b/sys/netinet/tcp_stacks/tcp_rack.h index 36bc1e6ae0b8..c7132ba81893 100644 --- a/sys/netinet/tcp_stacks/tcp_rack.h +++ b/sys/netinet/tcp_stacks/tcp_rack.h @@ -281,7 +281,7 @@ struct tcp_rack { TAILQ_ENTRY(tcp_rack) r_hpts; /* hptsi queue next Lock(b) */ int32_t(*r_substate) (struct mbuf *, struct tcphdr *, struct socket *, struct tcpcb *, struct tcpopt *, - int32_t, int32_t, int32_t *, uint32_t, int, int); /* Lock(a) */ + int32_t, int32_t, uint32_t, int, int); /* Lock(a) */ struct tcpcb *rc_tp; /* The tcpcb Lock(a) */ struct inpcb *rc_inp; /* The inpcb Lock(a) */ uint32_t rc_free_cnt; /* Number of free entries on the rc_free list diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c index 8d3b85bbee96..51105a8f0777 100644 --- a/sys/netinet/tcp_subr.c +++ b/sys/netinet/tcp_subr.c @@ -1914,10 +1914,11 @@ tcp_timer_discard(void *ptp) { struct inpcb *inp; struct tcpcb *tp; + struct epoch_tracker et; tp = (struct tcpcb *)ptp; CURVNET_SET(tp->t_vnet); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); inp = tp->t_inpcb; KASSERT(inp != NULL, ("%s: tp %p tp->t_inpcb == NULL", __func__, tp)); @@ -1937,13 +1938,13 @@ tcp_timer_discard(void *ptp) tp->t_inpcb = NULL; uma_zfree(V_tcpcb_zone, tp); if (in_pcbrele_wlocked(inp)) { - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); return; } } INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); } @@ -2107,6 +2108,7 @@ tcp_pcblist(SYSCTL_HANDLER_ARGS) struct inpcb *inp, **inp_list; inp_gen_t gencnt; struct xinpgen xig; + struct epoch_tracker et; /* * The process of preparing the TCB list is too time-consuming and @@ -2193,14 +2195,14 @@ tcp_pcblist(SYSCTL_HANDLER_ARGS) } else INP_RUNLOCK(inp); } - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); for (i = 0; i < n; i++) { inp = inp_list[i]; INP_RLOCK(inp); if (!in_pcbrele_rlocked(inp)) INP_RUNLOCK(inp); } - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); if (!error) { /* @@ -2339,6 +2341,7 @@ tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip) struct inpcb *(*notify)(struct inpcb *, int) = tcp_notify; struct icmp *icp; struct in_conninfo inc; + struct epoch_tracker et; tcp_seq icmp_tcp_seq; int mtu; @@ -2370,7 +2373,7 @@ tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip) icp = (struct icmp *)((caddr_t)ip - offsetof(struct icmp, icmp_ip)); th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); inp = in_pcblookup(&V_tcbinfo, faddr, th->th_dport, ip->ip_src, th->th_sport, INPLOOKUP_WLOCKPCB, NULL); if (inp != NULL && PRC_IS_REDIRECT(cmd)) { @@ -2435,7 +2438,7 @@ tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip) out: if (inp != NULL) INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); } #endif /* INET */ @@ -2453,6 +2456,7 @@ tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d) struct ip6ctlparam *ip6cp = NULL; const struct sockaddr_in6 *sa6_src = NULL; struct in_conninfo inc; + struct epoch_tracker et; struct tcp_ports { uint16_t th_sport; uint16_t th_dport; @@ -2514,7 +2518,7 @@ tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d) } bzero(&t_ports, sizeof(struct tcp_ports)); m_copydata(m, off, sizeof(struct tcp_ports), (caddr_t)&t_ports); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); inp = in6_pcblookup(&V_tcbinfo, &ip6->ip6_dst, t_ports.th_dport, &ip6->ip6_src, t_ports.th_sport, INPLOOKUP_WLOCKPCB, NULL); if (inp != NULL && PRC_IS_REDIRECT(cmd)) { @@ -2586,7 +2590,7 @@ tcp6_ctlinput(int cmd, struct sockaddr *sa, void *d) out: if (inp != NULL) INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); } #endif /* INET6 */ @@ -2925,6 +2929,7 @@ sysctl_drop(SYSCTL_HANDLER_ARGS) struct tcpcb *tp; struct tcptw *tw; struct sockaddr_in *fin, *lin; + struct epoch_tracker et; #ifdef INET6 struct sockaddr_in6 *fin6, *lin6; #endif @@ -2984,7 +2989,7 @@ sysctl_drop(SYSCTL_HANDLER_ARGS) default: return (EINVAL); } - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); switch (addrs[0].ss_family) { #ifdef INET6 case AF_INET6: @@ -3023,7 +3028,7 @@ sysctl_drop(SYSCTL_HANDLER_ARGS) INP_WUNLOCK(inp); } else error = ESRCH; - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); return (error); } diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c index 799cd37f0db7..d8237c721579 100644 --- a/sys/netinet/tcp_timer.c +++ b/sys/netinet/tcp_timer.c @@ -274,55 +274,10 @@ tcp_timer_delack(void *xtp) CURVNET_RESTORE(); } -/* - * When a timer wants to remove a TCB it must - * hold the INP_INFO_RLOCK(). The timer function - * should only have grabbed the INP_WLOCK() when - * it entered. To safely switch to holding both the - * INP_INFO_RLOCK() and the INP_WLOCK() we must first - * grab a reference on the inp, which will hold the inp - * so that it can't be removed. We then unlock the INP_WLOCK(), - * and grab the INP_INFO_RLOCK() lock. Once we have the INP_INFO_RLOCK() - * we proceed again to get the INP_WLOCK() (this preserves proper - * lock order). After acquiring the INP_WLOCK we must check if someone - * else deleted the pcb i.e. the inp_flags check. - * If so we return 1 otherwise we return 0. - * - * No matter what the tcp_inpinfo_lock_add() function - * returns the caller must afterwards call tcp_inpinfo_lock_del() - * to drop the locks and reference properly. - */ - -int -tcp_inpinfo_lock_add(struct inpcb *inp) -{ - in_pcbref(inp); - INP_WUNLOCK(inp); - INP_INFO_RLOCK(&V_tcbinfo); - INP_WLOCK(inp); - if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { - return(1); - } - return(0); - -} - void tcp_inpinfo_lock_del(struct inpcb *inp, struct tcpcb *tp) { - INP_INFO_RUNLOCK(&V_tcbinfo); - if (inp && (tp == NULL)) { - /* - * If tcp_close/drop() gets called and tp - * returns NULL, then the function dropped - * the inp lock, we hold a reference keeping - * this around, so we must re-aquire the - * INP_WLOCK() in order to proceed with - * our dropping the inp reference. - */ - INP_WLOCK(inp); - } - if (inp && in_pcbrele_wlocked(inp) == 0) + if (inp && tp != NULL) INP_WUNLOCK(inp); } @@ -331,6 +286,7 @@ tcp_timer_2msl(void *xtp) { struct tcpcb *tp = xtp; struct inpcb *inp; + struct epoch_tracker et; CURVNET_SET(tp->t_vnet); #ifdef TCPDEBUG int ostate; @@ -377,11 +333,13 @@ tcp_timer_2msl(void *xtp) tp->t_inpcb && tp->t_inpcb->inp_socket && (tp->t_inpcb->inp_socket->so_rcv.sb_state & SBS_CANTRCVMORE)) { TCPSTAT_INC(tcps_finwait2_drops); - if (tcp_inpinfo_lock_add(inp)) { + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { tcp_inpinfo_lock_del(inp, tp); goto out; } + INP_INFO_RLOCK_ET(&V_tcbinfo, et); tp = tcp_close(tp); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); tcp_inpinfo_lock_del(inp, tp); goto out; } else { @@ -389,15 +347,17 @@ tcp_timer_2msl(void *xtp) callout_reset(&tp->t_timers->tt_2msl, TP_KEEPINTVL(tp), tcp_timer_2msl, tp); } else { - if (tcp_inpinfo_lock_add(inp)) { + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { tcp_inpinfo_lock_del(inp, tp); goto out; } + INP_INFO_RLOCK_ET(&V_tcbinfo, et); tp = tcp_close(tp); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); tcp_inpinfo_lock_del(inp, tp); goto out; } - } + } #ifdef TCPDEBUG if (tp != NULL && (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)) @@ -418,6 +378,7 @@ tcp_timer_keep(void *xtp) struct tcpcb *tp = xtp; struct tcptemp *t_template; struct inpcb *inp; + struct epoch_tracker et; CURVNET_SET(tp->t_vnet); #ifdef TCPDEBUG int ostate; @@ -511,11 +472,11 @@ tcp_timer_keep(void *xtp) dropit: TCPSTAT_INC(tcps_keepdrops); - - if (tcp_inpinfo_lock_add(inp)) { + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { tcp_inpinfo_lock_del(inp, tp); goto out; } + INP_INFO_RLOCK_ET(&V_tcbinfo, et); tp = tcp_drop(tp, ETIMEDOUT); #ifdef TCPDEBUG @@ -524,8 +485,9 @@ dropit: PRU_SLOWTIMO); #endif TCP_PROBE2(debug__user, tp, PRU_SLOWTIMO); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); tcp_inpinfo_lock_del(inp, tp); -out: + out: CURVNET_RESTORE(); } @@ -534,6 +496,7 @@ tcp_timer_persist(void *xtp) { struct tcpcb *tp = xtp; struct inpcb *inp; + struct epoch_tracker et; CURVNET_SET(tp->t_vnet); #ifdef TCPDEBUG int ostate; @@ -573,11 +536,13 @@ tcp_timer_persist(void *xtp) (ticks - tp->t_rcvtime >= tcp_maxpersistidle || ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) { TCPSTAT_INC(tcps_persistdrop); - if (tcp_inpinfo_lock_add(inp)) { + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { tcp_inpinfo_lock_del(inp, tp); goto out; } + INP_INFO_RLOCK_ET(&V_tcbinfo, et); tp = tcp_drop(tp, ETIMEDOUT); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); tcp_inpinfo_lock_del(inp, tp); goto out; } @@ -588,11 +553,13 @@ tcp_timer_persist(void *xtp) if (tp->t_state > TCPS_CLOSE_WAIT && (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) { TCPSTAT_INC(tcps_persistdrop); - if (tcp_inpinfo_lock_add(inp)) { + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { tcp_inpinfo_lock_del(inp, tp); goto out; } + INP_INFO_RLOCK_ET(&V_tcbinfo, et); tp = tcp_drop(tp, ETIMEDOUT); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); tcp_inpinfo_lock_del(inp, tp); goto out; } @@ -618,6 +585,7 @@ tcp_timer_rexmt(void * xtp) CURVNET_SET(tp->t_vnet); int rexmt; struct inpcb *inp; + struct epoch_tracker et; #ifdef TCPDEBUG int ostate; @@ -654,11 +622,13 @@ tcp_timer_rexmt(void * xtp) if (++tp->t_rxtshift > TCP_MAXRXTSHIFT) { tp->t_rxtshift = TCP_MAXRXTSHIFT; TCPSTAT_INC(tcps_timeoutdrop); - if (tcp_inpinfo_lock_add(inp)) { + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { tcp_inpinfo_lock_del(inp, tp); goto out; } + INP_INFO_RLOCK_ET(&V_tcbinfo, et); tp = tcp_drop(tp, ETIMEDOUT); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); tcp_inpinfo_lock_del(inp, tp); goto out; } diff --git a/sys/netinet/tcp_timer.h b/sys/netinet/tcp_timer.h index 9250196200dd..a2ab6ca56b0d 100644 --- a/sys/netinet/tcp_timer.h +++ b/sys/netinet/tcp_timer.h @@ -214,7 +214,6 @@ VNET_DECLARE(int, tcp_pmtud_blackhole_mss); VNET_DECLARE(int, tcp_v6pmtud_blackhole_mss); #define V_tcp_v6pmtud_blackhole_mss VNET(tcp_v6pmtud_blackhole_mss) -int tcp_inpinfo_lock_add(struct inpcb *inp); void tcp_inpinfo_lock_del(struct inpcb *inp, struct tcpcb *tp); void tcp_timer_init(void); diff --git a/sys/netinet/tcp_timewait.c b/sys/netinet/tcp_timewait.c index 75754d516c76..3ce0ceb3ad68 100644 --- a/sys/netinet/tcp_timewait.c +++ b/sys/netinet/tcp_timewait.c @@ -206,11 +206,12 @@ void tcp_tw_destroy(void) { struct tcptw *tw; + struct epoch_tracker et; - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); while ((tw = TAILQ_FIRST(&V_twq_2msl)) != NULL) tcp_twclose(tw, 0); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); TW_LOCK_DESTROY(V_tw_lock); uma_zdestroy(V_tcptw_zone); @@ -674,6 +675,7 @@ tcp_tw_2msl_scan(int reuse) { struct tcptw *tw; struct inpcb *inp; + struct epoch_tracker et; #ifdef INVARIANTS if (reuse) { @@ -707,12 +709,12 @@ tcp_tw_2msl_scan(int reuse) in_pcbref(inp); TW_RUNLOCK(V_tw_lock); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); tw = intotw(inp); if (in_pcbrele_wlocked(inp)) { if (__predict_true(tw == NULL)) { - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); continue; } else { /* This should not happen as in TIMEWAIT @@ -731,7 +733,7 @@ tcp_tw_2msl_scan(int reuse) "|| inp last reference) && tw != " "NULL", __func__); #endif - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); break; } } @@ -739,12 +741,12 @@ tcp_tw_2msl_scan(int reuse) if (tw == NULL) { /* tcp_twclose() has already been called */ INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); continue; } tcp_twclose(tw, reuse); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); if (reuse) return tw; } diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c index 2d41ccc8aaa0..75312fcd5847 100644 --- a/sys/netinet/tcp_usrreq.c +++ b/sys/netinet/tcp_usrreq.c @@ -276,11 +276,12 @@ tcp_usr_detach(struct socket *so) { struct inpcb *inp; int rlock = 0; + struct epoch_tracker et; inp = sotoinpcb(so); KASSERT(inp != NULL, ("tcp_usr_detach: inp == NULL")); if (!INP_INFO_WLOCKED(&V_tcbinfo)) { - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); rlock = 1; } INP_WLOCK(inp); @@ -288,7 +289,7 @@ tcp_usr_detach(struct socket *so) ("tcp_usr_detach: inp_socket == NULL")); tcp_detach(so, inp); if (rlock) - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); } #ifdef INET @@ -668,10 +669,11 @@ tcp_usr_disconnect(struct socket *so) { struct inpcb *inp; struct tcpcb *tp = NULL; + struct epoch_tracker et; int error = 0; TCPDEBUG0; - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); inp = sotoinpcb(so); KASSERT(inp != NULL, ("tcp_usr_disconnect: inp == NULL")); INP_WLOCK(inp); @@ -688,7 +690,7 @@ out: TCPDEBUG2(PRU_DISCONNECT); TCP_PROBE2(debug__user, tp, PRU_DISCONNECT); INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); return (error); } @@ -747,6 +749,7 @@ tcp6_usr_accept(struct socket *so, struct sockaddr **nam) struct tcpcb *tp = NULL; struct in_addr addr; struct in6_addr addr6; + struct epoch_tracker et; in_port_t port = 0; int v4 = 0; TCPDEBUG0; @@ -756,7 +759,7 @@ tcp6_usr_accept(struct socket *so, struct sockaddr **nam) inp = sotoinpcb(so); KASSERT(inp != NULL, ("tcp6_usr_accept: inp == NULL")); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { error = ECONNABORTED; @@ -783,7 +786,7 @@ out: TCPDEBUG2(PRU_ACCEPT); TCP_PROBE2(debug__user, tp, PRU_ACCEPT); INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); if (error == 0) { if (v4) *nam = in6_v4mapsin6_sockaddr(port, &addr); @@ -803,9 +806,10 @@ tcp_usr_shutdown(struct socket *so) int error = 0; struct inpcb *inp; struct tcpcb *tp = NULL; + struct epoch_tracker et; TCPDEBUG0; - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); inp = sotoinpcb(so); KASSERT(inp != NULL, ("inp == NULL")); INP_WLOCK(inp); @@ -824,7 +828,7 @@ out: TCPDEBUG2(PRU_SHUTDOWN); TCP_PROBE2(debug__user, tp, PRU_SHUTDOWN); INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); return (error); } @@ -887,6 +891,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, int error = 0; struct inpcb *inp; struct tcpcb *tp = NULL; + struct epoch_tracker net_et; #ifdef INET6 int isipv6; #endif @@ -897,7 +902,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m, * this call. */ if (flags & PRUS_EOF) - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, net_et); inp = sotoinpcb(so); KASSERT(inp != NULL, ("tcp_usr_send: inp == NULL")); INP_WLOCK(inp); @@ -1040,7 +1045,7 @@ out: ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND)); INP_WUNLOCK(inp); if (flags & PRUS_EOF) - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, net_et); return (error); } @@ -1079,12 +1084,13 @@ tcp_usr_abort(struct socket *so) { struct inpcb *inp; struct tcpcb *tp = NULL; + struct epoch_tracker et; TCPDEBUG0; inp = sotoinpcb(so); KASSERT(inp != NULL, ("tcp_usr_abort: inp == NULL")); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); KASSERT(inp->inp_socket != NULL, ("tcp_usr_abort: inp_socket == NULL")); @@ -1110,7 +1116,7 @@ tcp_usr_abort(struct socket *so) } INP_WUNLOCK(inp); dropped: - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); } /* @@ -1121,12 +1127,13 @@ tcp_usr_close(struct socket *so) { struct inpcb *inp; struct tcpcb *tp = NULL; + struct epoch_tracker et; TCPDEBUG0; inp = sotoinpcb(so); KASSERT(inp != NULL, ("tcp_usr_close: inp == NULL")); - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); INP_WLOCK(inp); KASSERT(inp->inp_socket != NULL, ("tcp_usr_close: inp_socket == NULL")); @@ -1150,7 +1157,7 @@ tcp_usr_close(struct socket *so) inp->inp_flags |= INP_SOCKREF; } INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); } /* @@ -2043,6 +2050,7 @@ tcp_attach(struct socket *so) { struct tcpcb *tp; struct inpcb *inp; + struct epoch_tracker et; int error; if (so->so_snd.sb_hiwat == 0 || so->so_rcv.sb_hiwat == 0) { @@ -2052,10 +2060,10 @@ tcp_attach(struct socket *so) } so->so_rcv.sb_flags |= SB_AUTOSIZE; so->so_snd.sb_flags |= SB_AUTOSIZE; - INP_INFO_RLOCK(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); error = in_pcballoc(so, &V_tcbinfo); if (error) { - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); return (error); } inp = sotoinpcb(so); @@ -2073,12 +2081,12 @@ tcp_attach(struct socket *so) if (tp == NULL) { in_pcbdetach(inp); in_pcbfree(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); return (ENOBUFS); } tp->t_state = TCPS_CLOSED; INP_WUNLOCK(inp); - INP_INFO_RUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); TCPSTATES_INC(TCPS_CLOSED); return (0); } diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h index 07ddfac3bfa0..b89e90976506 100644 --- a/sys/netinet/tcp_var.h +++ b/sys/netinet/tcp_var.h @@ -264,12 +264,11 @@ struct tcp_function_block { int (*tfb_tcp_output_wtime)(struct tcpcb *, const struct timeval *); void (*tfb_tcp_do_segment)(struct mbuf *, struct tcphdr *, struct socket *, struct tcpcb *, - int, int, uint8_t, - int); + int, int, uint8_t); void (*tfb_tcp_hpts_do_segment)(struct mbuf *, struct tcphdr *, struct socket *, struct tcpcb *, int, int, uint8_t, - int, int, struct timeval *); + int, struct timeval *); int (*tfb_tcp_ctloutput)(struct socket *so, struct sockopt *sopt, struct inpcb *inp, struct tcpcb *tp); /* Optional memory allocation/free routine */ @@ -862,8 +861,7 @@ int tcp_input(struct mbuf **, int *, int); int tcp_autorcvbuf(struct mbuf *, struct tcphdr *, struct socket *, struct tcpcb *, int); void tcp_do_segment(struct mbuf *, struct tcphdr *, - struct socket *, struct tcpcb *, int, int, uint8_t, - int); + struct socket *, struct tcpcb *, int, int, uint8_t); int register_tcp_functions(struct tcp_function_block *blk, int wait); int register_tcp_functions_as_names(struct tcp_function_block *blk, diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c index e56ec384064c..49bebb2fc7f5 100644 --- a/sys/netinet/udp_usrreq.c +++ b/sys/netinet/udp_usrreq.c @@ -399,6 +399,7 @@ udp_input(struct mbuf **mp, int *offp, int proto) struct sockaddr_in udp_in[2]; struct mbuf *m; struct m_tag *fwd_tag; + struct epoch_tracker et; int cscov_partial, iphlen; m = *mp; @@ -529,7 +530,7 @@ udp_input(struct mbuf **mp, int *offp, int proto) struct inpcbhead *pcblist; struct ip_moptions *imo; - INP_INFO_RLOCK(pcbinfo); + INP_INFO_RLOCK_ET(pcbinfo, et); pcblist = udp_get_pcblist(proto); last = NULL; CK_LIST_FOREACH(inp, pcblist, inp_list) { @@ -625,14 +626,14 @@ udp_input(struct mbuf **mp, int *offp, int proto) UDPSTAT_INC(udps_noportbcast); if (inp) INP_RUNLOCK(inp); - INP_INFO_RUNLOCK(pcbinfo); + INP_INFO_RUNLOCK_ET(pcbinfo, et); goto badunlocked; } UDP_PROBE(receive, NULL, last, ip, last, uh); if (udp_append(last, ip, m, iphlen, udp_in) == 0) INP_RUNLOCK(last); inp_lost: - INP_INFO_RUNLOCK(pcbinfo); + INP_INFO_RUNLOCK_ET(pcbinfo, et); return (IPPROTO_DONE); } @@ -839,6 +840,7 @@ udp_pcblist(SYSCTL_HANDLER_ARGS) struct inpcb *inp, **inp_list; inp_gen_t gencnt; struct xinpgen xig; + struct epoch_tracker et; /* * The process of preparing the PCB list is too time-consuming and @@ -857,10 +859,10 @@ udp_pcblist(SYSCTL_HANDLER_ARGS) /* * OK, now we're committed to doing something. */ - INP_INFO_RLOCK(&V_udbinfo); + INP_INFO_RLOCK_ET(&V_udbinfo, et); gencnt = V_udbinfo.ipi_gencnt; n = V_udbinfo.ipi_count; - INP_INFO_RUNLOCK(&V_udbinfo); + INP_INFO_RUNLOCK_ET(&V_udbinfo, et); error = sysctl_wire_old_buffer(req, 2 * (sizeof xig) + n * sizeof(struct xinpcb)); @@ -879,7 +881,7 @@ udp_pcblist(SYSCTL_HANDLER_ARGS) if (inp_list == NULL) return (ENOMEM); - INP_INFO_RLOCK(&V_udbinfo); + INP_INFO_RLOCK_ET(&V_udbinfo, et); for (inp = CK_LIST_FIRST(V_udbinfo.ipi_listhead), i = 0; inp && i < n; inp = CK_LIST_NEXT(inp, inp_list)) { INP_WLOCK(inp); @@ -890,7 +892,7 @@ udp_pcblist(SYSCTL_HANDLER_ARGS) } INP_WUNLOCK(inp); } - INP_INFO_RUNLOCK(&V_udbinfo); + INP_INFO_RUNLOCK_ET(&V_udbinfo, et); n = i; error = 0; @@ -922,11 +924,11 @@ udp_pcblist(SYSCTL_HANDLER_ARGS) * that something happened while we were processing this * request, and it might be necessary to retry. */ - INP_INFO_RLOCK(&V_udbinfo); + INP_INFO_RLOCK_ET(&V_udbinfo, et); xig.xig_gen = V_udbinfo.ipi_gencnt; xig.xig_sogen = so_gencnt; xig.xig_count = V_udbinfo.ipi_count; - INP_INFO_RUNLOCK(&V_udbinfo); + INP_INFO_RUNLOCK_ET(&V_udbinfo, et); error = SYSCTL_OUT(req, &xig, sizeof xig); } free(inp_list, M_TEMP); @@ -1108,6 +1110,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, struct cmsghdr *cm; struct inpcbinfo *pcbinfo; struct sockaddr_in *sin, src; + struct epoch_tracker et; int cscov_partial = 0; int error = 0; int ipflags; @@ -1264,7 +1267,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, (inp->inp_laddr.s_addr == INADDR_ANY) || (inp->inp_lport == 0))) || (src.sin_family == AF_INET)) { - INP_HASH_RLOCK(pcbinfo); + INP_HASH_RLOCK_ET(pcbinfo, et); unlock_udbinfo = UH_RLOCKED; } else unlock_udbinfo = UH_UNLOCKED; @@ -1520,7 +1523,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr, if (unlock_udbinfo == UH_WLOCKED) INP_HASH_WUNLOCK(pcbinfo); else if (unlock_udbinfo == UH_RLOCKED) - INP_HASH_RUNLOCK(pcbinfo); + INP_HASH_RUNLOCK_ET(pcbinfo, et); UDP_PROBE(send, NULL, inp, &ui->ui_i, inp, &ui->ui_u); error = ip_output(m, inp->inp_options, (unlock_inp == UH_WLOCKED ? &inp->inp_route : NULL), ipflags, @@ -1540,7 +1543,7 @@ release: } else if (unlock_udbinfo == UH_RLOCKED) { KASSERT(unlock_inp == UH_RLOCKED, ("%s: shared udbinfo lock, excl inp lock", __func__)); - INP_HASH_RUNLOCK(pcbinfo); + INP_HASH_RUNLOCK_ET(pcbinfo, et); INP_RUNLOCK(inp); } else if (unlock_inp == UH_WLOCKED) INP_WUNLOCK(inp); diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c index 957467f15eb1..4b6111c143dc 100644 --- a/sys/netinet6/icmp6.c +++ b/sys/netinet6/icmp6.c @@ -1896,6 +1896,7 @@ icmp6_rip6_input(struct mbuf **mp, int off) struct inpcb *last = NULL; struct sockaddr_in6 fromsa; struct icmp6_hdr *icmp6; + struct epoch_tracker et; struct mbuf *opts = NULL; #ifndef PULLDOWN_TEST @@ -1922,7 +1923,7 @@ icmp6_rip6_input(struct mbuf **mp, int off) return (IPPROTO_DONE); } - INP_INFO_RLOCK(&V_ripcbinfo); + INP_INFO_RLOCK_ET(&V_ripcbinfo, et); CK_LIST_FOREACH(in6p, &V_ripcb, inp_list) { if ((in6p->inp_vflag & INP_IPV6) == 0) continue; @@ -2000,7 +2001,7 @@ icmp6_rip6_input(struct mbuf **mp, int off) } last = in6p; } - INP_INFO_RUNLOCK(&V_ripcbinfo); + INP_INFO_RUNLOCK_ET(&V_ripcbinfo, et); if (last != NULL) { if (last->inp_flags & INP_CONTROLOPTS) ip6_savecontrol(last, m, &opts); diff --git a/sys/netinet6/in6_gif.c b/sys/netinet6/in6_gif.c index cbcbfae0abc9..0293237ffcb7 100644 --- a/sys/netinet6/in6_gif.c +++ b/sys/netinet6/in6_gif.c @@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$"); #include <sys/syslog.h> #include <sys/sysctl.h> #include <sys/malloc.h> +#include <sys/proc.h> #include <net/ethernet.h> #include <net/if.h> @@ -241,7 +242,7 @@ in6_gif_output(struct ifnet *ifp, struct mbuf *m, int proto, uint8_t ecn) int len; /* prepend new IP header */ - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); len = sizeof(struct ip6_hdr); #ifndef __NO_STRICT_ALIGNMENT if (proto == IPPROTO_ETHERIP) @@ -283,7 +284,7 @@ in6_gif_input(struct mbuf *m, int off, int proto, void *arg) struct ip6_hdr *ip6; uint8_t ecn; - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); if (sc == NULL) { m_freem(m); IP6STAT_INC(ip6s_nogif); @@ -312,7 +313,7 @@ in6_gif_lookup(const struct mbuf *m, int off, int proto, void **arg) if (V_ipv6_hashtbl == NULL) return (0); - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); /* * NOTE: it is safe to iterate without any locking here, because softc * can be reclaimed only when we are not within net_epoch_preempt diff --git a/sys/netinet6/ip6_gre.c b/sys/netinet6/ip6_gre.c index bb786c9416e8..48aa8cb03ac2 100644 --- a/sys/netinet6/ip6_gre.c +++ b/sys/netinet6/ip6_gre.c @@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/sysctl.h> #include <sys/malloc.h> +#include <sys/proc.h> #include <net/if.h> #include <net/if_var.h> @@ -110,7 +111,7 @@ in6_gre_lookup(const struct mbuf *m, int off, int proto, void **arg) if (V_ipv6_hashtbl == NULL) return (0); - MPASS(in_epoch()); + MPASS(in_epoch(net_epoch_preempt)); ip6 = mtod(m, const struct ip6_hdr *); CK_LIST_FOREACH(sc, &GRE_HASH(&ip6->ip6_dst, &ip6->ip6_src), chain) { /* diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c index 791aba11333a..192d1b320e3c 100644 --- a/sys/netinet6/raw_ip6.c +++ b/sys/netinet6/raw_ip6.c @@ -165,6 +165,7 @@ rip6_input(struct mbuf **mp, int *offp, int proto) struct inpcb *last = NULL; struct mbuf *opts = NULL; struct sockaddr_in6 fromsa; + struct epoch_tracker et; RIP6STAT_INC(rip6s_ipackets); @@ -172,7 +173,7 @@ rip6_input(struct mbuf **mp, int *offp, int proto) ifp = m->m_pkthdr.rcvif; - INP_INFO_RLOCK(&V_ripcbinfo); + INP_INFO_RLOCK_ET(&V_ripcbinfo, et); CK_LIST_FOREACH(in6p, &V_ripcb, inp_list) { /* XXX inp locking */ if ((in6p->inp_vflag & INP_IPV6) == 0) @@ -291,7 +292,7 @@ rip6_input(struct mbuf **mp, int *offp, int proto) } last = in6p; } - INP_INFO_RUNLOCK(&V_ripcbinfo); + INP_INFO_RUNLOCK_ET(&V_ripcbinfo, et); #if defined(IPSEC) || defined(IPSEC_SUPPORT) /* * Check AH/ESP integrity. diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c index 280add6e79fe..10d7c25764d1 100644 --- a/sys/netinet6/udp6_usrreq.c +++ b/sys/netinet6/udp6_usrreq.c @@ -214,6 +214,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto) int off = *offp; int cscov_partial; int plen, ulen; + struct epoch_tracker et; struct sockaddr_in6 fromsa[2]; struct m_tag *fwd_tag; uint16_t uh_sum; @@ -300,7 +301,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto) struct inpcbhead *pcblist; struct ip6_moptions *imo; - INP_INFO_RLOCK(pcbinfo); + INP_INFO_RLOCK_ET(pcbinfo, et); /* * In the event that laddr should be set to the link-local * address (this happens in RIPng), the multicast address @@ -426,7 +427,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto) INP_RUNLOCK(last); } else INP_RUNLOCK(last); - INP_INFO_RUNLOCK(pcbinfo); + INP_INFO_RUNLOCK_ET(pcbinfo, et); inp_lost: return (IPPROTO_DONE); } @@ -508,7 +509,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto) return (IPPROTO_DONE); badheadlocked: - INP_INFO_RUNLOCK(pcbinfo); + INP_INFO_RUNLOCK_ET(pcbinfo, et); badunlocked: if (m) m_freem(m); diff --git a/sys/sys/epoch.h b/sys/sys/epoch.h index aac133f1c027..095aab57f9d8 100644 --- a/sys/sys/epoch.h +++ b/sys/sys/epoch.h @@ -31,10 +31,9 @@ #define _SYS_EPOCH_H_ #ifdef _KERNEL #include <sys/lock.h> -#include <sys/proc.h> +#include <sys/pcpu.h> #endif -struct thread; struct epoch; typedef struct epoch *epoch_t; @@ -46,48 +45,49 @@ extern epoch_t global_epoch_preempt; struct epoch_context { void *data[2]; -} __aligned(sizeof(void *)); +} __aligned(sizeof(void *)); typedef struct epoch_context *epoch_context_t; + +struct epoch_tracker { + void *datap[3]; +#ifdef INVARIANTS + int datai[5]; +#else + int datai[1]; +#endif +} __aligned(sizeof(void *)); + +typedef struct epoch_tracker *epoch_tracker_t; + epoch_t epoch_alloc(int flags); void epoch_free(epoch_t epoch); -void epoch_enter(epoch_t epoch); -void epoch_enter_preempt_internal(epoch_t epoch, struct thread *td); -void epoch_exit(epoch_t epoch); -void epoch_exit_preempt_internal(epoch_t epoch, struct thread *td); void epoch_wait(epoch_t epoch); void epoch_wait_preempt(epoch_t epoch); void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t)); -int in_epoch(void); - +int in_epoch(epoch_t epoch); +int in_epoch_verbose(epoch_t epoch, int dump_onfail); #ifdef _KERNEL DPCPU_DECLARE(int, epoch_cb_count); DPCPU_DECLARE(struct grouptask, epoch_cb_task); +#define EPOCH_MAGIC0 0xFADECAFEF00DD00D +#define EPOCH_MAGIC1 0xBADDBABEDEEDFEED -static __inline void -epoch_enter_preempt(epoch_t epoch) -{ - struct thread *td; - int nesting __unused; +void epoch_enter_preempt_KBI(epoch_t epoch, epoch_tracker_t et); +void epoch_exit_preempt_KBI(epoch_t epoch, epoch_tracker_t et); +void epoch_enter_KBI(epoch_t epoch); +void epoch_exit_KBI(epoch_t epoch); - td = curthread; - nesting = td->td_epochnest++; -#ifndef INVARIANTS - if (nesting == 0) -#endif - epoch_enter_preempt_internal(epoch, td); -} -static __inline void -epoch_exit_preempt(epoch_t epoch) -{ - struct thread *td; +#if defined(KLD_MODULE) && !defined(KLD_TIED) +#define epoch_enter_preempt(e, t) epoch_enter_preempt_KBI((e), (t)) +#define epoch_exit_preempt(e, t) epoch_exit_preempt_KBI((e), (t)) +#define epoch_enter(e) epoch_enter_KBI((e)) +#define epoch_exit(e) epoch_exit_KBI((e)) +#else +#include <sys/epoch_private.h> +#endif /* KLD_MODULE */ - td = curthread; - MPASS(td->td_epochnest); - if (td->td_epochnest-- == 1) - epoch_exit_preempt_internal(epoch, td); -} -#endif /* _KERNEL */ +#endif /* _KERNEL */ #endif diff --git a/sys/sys/epoch_private.h b/sys/sys/epoch_private.h new file mode 100644 index 000000000000..b06078f5e2f3 --- /dev/null +++ b/sys/sys/epoch_private.h @@ -0,0 +1,203 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_EPOCH_PRIVATE_H_ +#define _SYS_EPOCH_PRIVATE_H_ +#ifndef _KERNEL +#error "no user serviceable parts" +#else +#include <ck_epoch.h> +#include <sys/kpilite.h> + +#include <sys/mutex.h> + +extern void epoch_adjust_prio(struct thread *td, u_char prio); +#ifndef _SYS_SYSTM_H_ +extern void critical_exit_preempt(void); +#endif + +#ifdef __amd64__ +#define EPOCH_ALIGN CACHE_LINE_SIZE*2 +#else +#define EPOCH_ALIGN CACHE_LINE_SIZE +#endif + +/* + * Standalone (_sa) routines for thread state manipulation + */ +static __inline void +critical_enter_sa(void *tdarg) +{ + struct thread_lite *td; + + td = tdarg; + td->td_critnest++; + __compiler_membar(); +} + +static __inline void +critical_exit_sa(void *tdarg) +{ + struct thread_lite *td; + + td = tdarg; + MPASS(td->td_critnest > 0); + __compiler_membar(); + td->td_critnest--; + __compiler_membar(); + if (__predict_false(td->td_owepreempt != 0)) + critical_exit_preempt(); +} + +typedef struct epoch_thread { +#ifdef INVARIANTS + uint64_t et_magic_pre; +#endif + TAILQ_ENTRY(epoch_thread) et_link; /* Epoch queue. */ + struct thread *et_td; /* pointer to thread in section */ + ck_epoch_section_t et_section; /* epoch section object */ +#ifdef INVARIANTS + uint64_t et_magic_post; +#endif +} *epoch_thread_t; +TAILQ_HEAD (epoch_tdlist, epoch_thread); + +typedef struct epoch_record { + ck_epoch_record_t er_record; + volatile struct epoch_tdlist er_tdlist; + volatile uint32_t er_gen; + uint32_t er_cpuid; +} __aligned(EPOCH_ALIGN) *epoch_record_t; + +struct epoch { + struct ck_epoch e_epoch __aligned(EPOCH_ALIGN); + struct epoch_record *e_pcpu_dom[MAXMEMDOM] __aligned(EPOCH_ALIGN); + int e_idx; + int e_flags; + struct epoch_record *e_pcpu[0]; +}; + +#define INIT_CHECK(epoch) \ + do { \ + if (__predict_false((epoch) == NULL)) \ + return; \ + } while (0) + +static __inline void +epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et) +{ + struct epoch_record *er; + struct epoch_thread *etd; + struct thread_lite *td; + MPASS(cold || epoch != NULL); + INIT_CHECK(epoch); + etd = (void *)et; +#ifdef INVARIANTS + MPASS(epoch->e_flags & EPOCH_PREEMPT); + etd->et_magic_pre = EPOCH_MAGIC0; + etd->et_magic_post = EPOCH_MAGIC1; +#endif + td = (struct thread_lite *)curthread; + etd->et_td = (void*)td; + td->td_epochnest++; + critical_enter_sa(td); + sched_pin_lite(td); + + td->td_pre_epoch_prio = td->td_priority; + er = epoch->e_pcpu[curcpu]; + TAILQ_INSERT_TAIL(&er->er_tdlist, etd, et_link); + ck_epoch_begin(&er->er_record, (ck_epoch_section_t *)&etd->et_section); + critical_exit_sa(td); +} + +static __inline void +epoch_enter(epoch_t epoch) +{ + ck_epoch_record_t *record; + struct thread_lite *td; + MPASS(cold || epoch != NULL); + INIT_CHECK(epoch); + td = (struct thread_lite *)curthread; + + td->td_epochnest++; + critical_enter_sa(td); + record = &epoch->e_pcpu[curcpu]->er_record; + ck_epoch_begin(record, NULL); +} + +static __inline void +epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et) +{ + struct epoch_record *er; + struct epoch_thread *etd; + struct thread_lite *td; + + INIT_CHECK(epoch); + td = (struct thread_lite *)curthread; + critical_enter_sa(td); + sched_unpin_lite(td); + MPASS(td->td_epochnest); + td->td_epochnest--; + er = epoch->e_pcpu[curcpu]; + MPASS(epoch->e_flags & EPOCH_PREEMPT); + etd = (void *)et; +#ifdef INVARIANTS + MPASS(etd != NULL); + MPASS(etd->et_td == (struct thread *)td); + MPASS(etd->et_magic_pre == EPOCH_MAGIC0); + MPASS(etd->et_magic_post == EPOCH_MAGIC1); + etd->et_magic_pre = 0; + etd->et_magic_post = 0; + etd->et_td = (void*)0xDEADBEEF; +#endif + ck_epoch_end(&er->er_record, + (ck_epoch_section_t *)&etd->et_section); + TAILQ_REMOVE(&er->er_tdlist, etd, et_link); + er->er_gen++; + if (__predict_false(td->td_pre_epoch_prio != td->td_priority)) + epoch_adjust_prio((struct thread *)td, td->td_pre_epoch_prio); + critical_exit_sa(td); +} + +static __inline void +epoch_exit(epoch_t epoch) +{ + ck_epoch_record_t *record; + struct thread_lite *td; + + INIT_CHECK(epoch); + td = (struct thread_lite *)curthread; + MPASS(td->td_epochnest); + td->td_epochnest--; + record = &epoch->e_pcpu[curcpu]->er_record; + ck_epoch_end(record, NULL); + critical_exit_sa(td); +} +#endif /* _KERNEL */ +#endif /* _SYS_EPOCH_PRIVATE_H_ */ diff --git a/sys/sys/pmckern.h b/sys/sys/pmckern.h index 69b72139d686..27e02af73cf2 100644 --- a/sys/sys/pmckern.h +++ b/sys/sys/pmckern.h @@ -201,11 +201,12 @@ extern struct pmc_domain_buffer_header *pmc_dom_hdrs[MAXMEMDOM]; /* Hook invocation; for use within the kernel */ #define PMC_CALL_HOOK(t, cmd, arg) \ -do { \ - epoch_enter_preempt(global_epoch_preempt); \ +do { \ + struct epoch_tracker et; \ + epoch_enter_preempt(global_epoch_preempt, &et); \ if (pmc_hook != NULL) \ (pmc_hook)((t), (cmd), (arg)); \ - epoch_exit_preempt(global_epoch_preempt); \ + epoch_exit_preempt(global_epoch_preempt, &et); \ } while (0) /* Hook invocation that needs an exclusive lock */ diff --git a/sys/sys/proc.h b/sys/sys/proc.h index 9f061d3c5a55..2d393435b35d 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -74,19 +74,6 @@ #include <machine/cpu.h> #endif - -/* - * A section object may be passed to every begin-end pair to allow for - * forward progress guarantees with-in prolonged active sections. - * - * We can't include ck_epoch.h so we define our own variant here and - * then CTASSERT that it's the same size in subr_epoch.c - */ -struct epoch_section { - unsigned int bucket; -}; -typedef struct epoch_section epoch_section_t; - /* * One structure allocated per session. * @@ -373,8 +360,6 @@ struct thread { int td_lastcpu; /* (t) Last cpu we were on. */ int td_oncpu; /* (t) Which cpu we are on. */ void *td_lkpi_task; /* LinuxKPI task struct pointer */ - TAILQ_ENTRY(thread) td_epochq; /* (t) Epoch queue. */ - epoch_section_t td_epoch_section; /* (t) epoch section object */ int td_pmcpend; }; |