/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _SYS_EPOCH_PRIVATE_H_
#define _SYS_EPOCH_PRIVATE_H_
#ifndef _KERNEL
#error "no user serviceable parts"
#else
#include <ck_epoch.h>
#include <sys/kpilite.h>
#include <sys/mutex.h>
extern void epoch_adjust_prio(struct thread *td, u_char prio);
#ifndef _SYS_SYSTM_H_
extern void critical_exit_preempt(void);
#endif
#ifdef __amd64__
#define EPOCH_ALIGN CACHE_LINE_SIZE*2
#else
#define EPOCH_ALIGN CACHE_LINE_SIZE
#endif
/*
* Standalone (_sa) routines for thread state manipulation
*/
static __inline void
critical_enter_sa(void *tdarg)
{
struct thread_lite *td;
td = tdarg;
td->td_critnest++;
__compiler_membar();
}
static __inline void
critical_exit_sa(void *tdarg)
{
struct thread_lite *td;
td = tdarg;
MPASS(td->td_critnest > 0);
__compiler_membar();
td->td_critnest--;
__compiler_membar();
if (__predict_false(td->td_owepreempt != 0))
critical_exit_preempt();
}
typedef struct epoch_thread {
#ifdef EPOCH_TRACKER_DEBUG
uint64_t et_magic_pre;
#endif
TAILQ_ENTRY(epoch_thread) et_link; /* Epoch queue. */
struct thread *et_td; /* pointer to thread in section */
ck_epoch_section_t et_section; /* epoch section object */
#ifdef EPOCH_TRACKER_DEBUG
uint64_t et_magic_post;
#endif
} *epoch_thread_t;
TAILQ_HEAD (epoch_tdlist, epoch_thread);
typedef struct epoch_record {
ck_epoch_record_t er_record;
volatile struct epoch_tdlist er_tdlist;
volatile uint32_t er_gen;
uint32_t er_cpuid;
} __aligned(EPOCH_ALIGN) *epoch_record_t;
struct epoch {
struct ck_epoch e_epoch __aligned(EPOCH_ALIGN);
epoch_record_t e_pcpu_record;
int e_idx;
int e_flags;
};
static epoch_record_t
epoch_currecord(epoch_t epoch)
{
return zpcpu_get_cpu(epoch->e_pcpu_record, curcpu);
}
#define INIT_CHECK(epoch) \
do { \
if (__predict_false((epoch) == NULL)) \
return; \
} while (0)
static __inline void
epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et)
{
struct epoch_record *er;
struct epoch_thread *etd;
struct thread_lite *td;
MPASS(cold || epoch != NULL);
INIT_CHECK(epoch);
etd = (void *)et;
MPASS(epoch->e_flags & EPOCH_PREEMPT);
#ifdef EPOCH_TRACKER_DEBUG
etd->et_magic_pre = EPOCH_MAGIC0;
etd->et_magic_post = EPOCH_MAGIC1;
#endif
td = (struct thread_lite *)curthread;
etd->et_td = (void*)td;
td->td_epochnest++;
critical_enter_sa(td);
sched_pin_lite(td);
td->td_pre_epoch_prio = td->td_priority;
er = epoch_currecord(epoch);
TAILQ_INSERT_TAIL(&er->er_tdlist, etd, et_link);
ck_epoch_begin(&er->er_record, (ck_epoch_section_t *)&etd->et_section);
critical_exit_sa(td);
}
static __inline void
epoch_enter(epoch_t epoch)
{
struct thread_lite *td;
epoch_record_t er;
MPASS(cold || epoch != NULL);
INIT_CHECK(epoch);
td = (struct thread_lite *)curthread;
td->td_epochnest++;
critical_enter_sa(td);
er = epoch_currecord(epoch);
ck_epoch_begin(&er->er_record, NULL);
}
static __inline void
epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et)
{
struct epoch_record *er;
struct epoch_thread *etd;
struct thread_lite *td;
INIT_CHECK(epoch);
td = (struct thread_lite *)curthread;
critical_enter_sa(td);
sched_unpin_lite(td);
MPASS(td->td_epochnest);
td->td_epochnest--;
er = epoch_currecord(epoch);
MPASS(epoch->e_flags & EPOCH_PREEMPT);
etd = (void *)et;
MPASS(etd != NULL);
MPASS(etd->et_td == (struct thread *)td);
#ifdef EPOCH_TRACKER_DEBUG
MPASS(etd->et_magic_pre == EPOCH_MAGIC0);
MPASS(etd->et_magic_post == EPOCH_MAGIC1);
etd->et_magic_pre = 0;
etd->et_magic_post = 0;
#endif
etd->et_td = (void*)0xDEADBEEF;
ck_epoch_end(&er->er_record,
(ck_epoch_section_t *)&etd->et_section);
TAILQ_REMOVE(&er->er_tdlist, etd, et_link);
er->er_gen++;
if (__predict_false(td->td_pre_epoch_prio != td->td_priority))
epoch_adjust_prio((struct thread *)td, td->td_pre_epoch_prio);
critical_exit_sa(td);
}
static __inline void
epoch_exit(epoch_t epoch)
{
struct thread_lite *td;
epoch_record_t er;
INIT_CHECK(epoch);
td = (struct thread_lite *)curthread;
MPASS(td->td_epochnest);
td->td_epochnest--;
er = epoch_currecord(epoch);
ck_epoch_end(&er->er_record, NULL);
critical_exit_sa(td);
}
#endif /* _KERNEL */
#endif /* _SYS_EPOCH_PRIVATE_H_ */
