diff options
Diffstat (limited to 'sys/compat/linuxkpi/common')
137 files changed, 25952 insertions, 0 deletions
diff --git a/sys/compat/linuxkpi/common/include/asm/atomic-long.h b/sys/compat/linuxkpi/common/include/asm/atomic-long.h new file mode 100644 index 000000000000..e2bea06c0953 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/atomic-long.h @@ -0,0 +1,134 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _ATOMIC_LONG_H_ +#define _ATOMIC_LONG_H_ + +#include <linux/compiler.h> +#include <sys/types.h> +#include <machine/atomic.h> + +#define ATOMIC_LONG_INIT(x) { .counter = (x) } + +typedef struct { + volatile long counter; +} atomic_long_t; + +#define atomic_long_add(i, v) atomic_long_add_return((i), (v)) +#define atomic_long_sub(i, v) atomic_long_add_return(-(i), (v)) +#define atomic_long_inc_return(v) atomic_long_add_return(1, (v)) +#define atomic_long_inc_not_zero(v) atomic_long_add_unless((v), 1, 0) + +static inline long +atomic_long_add_return(long i, atomic_long_t *v) +{ + return i + atomic_fetchadd_long(&v->counter, i); +} + +static inline void +atomic_long_set(atomic_long_t *v, long i) +{ + WRITE_ONCE(v->counter, i); +} + +static inline long +atomic_long_read(atomic_long_t *v) +{ + return READ_ONCE(v->counter); +} + +static inline long +atomic_long_inc(atomic_long_t *v) +{ + return atomic_fetchadd_long(&v->counter, 1) + 1; +} + +static inline long +atomic_long_dec(atomic_long_t *v) +{ + return atomic_fetchadd_long(&v->counter, -1) - 1; +} + +static inline long +atomic_long_xchg(atomic_long_t *v, long val) +{ + return atomic_swap_long(&v->counter, val); +} + +static inline long +atomic_long_cmpxchg(atomic_long_t *v, long old, long new) +{ + long ret = old; + + for (;;) { + if (atomic_fcmpset_long(&v->counter, &ret, new)) + break; + if (ret != old) + break; + } + return (ret); +} + +static inline int +atomic_long_add_unless(atomic_long_t *v, long a, long u) +{ + long c = atomic_long_read(v); + + for (;;) { + if (unlikely(c == u)) + break; + if (likely(atomic_fcmpset_long(&v->counter, &c, c + a))) + break; + } + return (c != u); +} + +static inline long +atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) +{ + long c = atomic_long_read(v); + + for (;;) { + if (unlikely(c == u)) + break; + if (likely(atomic_fcmpset_long(&v->counter, &c, c + a))) + break; + } + return (c); +} + +static inline long +atomic_long_dec_and_test(atomic_long_t *v) +{ + long i = atomic_long_add(-1, v); + return i == 0 ; +} + +#endif /* _ATOMIC_LONG_H_ */ diff --git a/sys/compat/linuxkpi/common/include/asm/atomic.h b/sys/compat/linuxkpi/common/include/asm/atomic.h new file mode 100644 index 000000000000..fff06f61d856 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/atomic.h @@ -0,0 +1,321 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2018 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ASM_ATOMIC_H_ +#define _ASM_ATOMIC_H_ + +#include <linux/compiler.h> +#include <sys/types.h> +#include <machine/atomic.h> + +#define ATOMIC_INIT(x) { .counter = (x) } + +typedef struct { + volatile int counter; +} atomic_t; + +/*------------------------------------------------------------------------* + * 32-bit atomic operations + *------------------------------------------------------------------------*/ + +#define atomic_add(i, v) atomic_add_return((i), (v)) +#define atomic_sub(i, v) atomic_sub_return((i), (v)) +#define atomic_inc_return(v) atomic_add_return(1, (v)) +#define atomic_add_negative(i, v) (atomic_add_return((i), (v)) < 0) +#define atomic_add_and_test(i, v) (atomic_add_return((i), (v)) == 0) +#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0) +#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) +#define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0) +#define atomic_dec_return(v) atomic_sub_return(1, (v)) +#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) + +static inline int +atomic_add_return(int i, atomic_t *v) +{ + return i + atomic_fetchadd_int(&v->counter, i); +} + +static inline int +atomic_sub_return(int i, atomic_t *v) +{ + return atomic_fetchadd_int(&v->counter, -i) - i; +} + +static inline void +atomic_set(atomic_t *v, int i) +{ + WRITE_ONCE(v->counter, i); +} + +static inline void +atomic_set_release(atomic_t *v, int i) +{ + atomic_store_rel_int(&v->counter, i); +} + +static inline void +atomic_set_mask(unsigned int mask, atomic_t *v) +{ + atomic_set_int(&v->counter, mask); +} + +static inline int +atomic_read(const atomic_t *v) +{ + return READ_ONCE(v->counter); +} + +static inline int +atomic_inc(atomic_t *v) +{ + return atomic_fetchadd_int(&v->counter, 1) + 1; +} + +static inline int +atomic_dec(atomic_t *v) +{ + return atomic_fetchadd_int(&v->counter, -1) - 1; +} + +static inline int +atomic_add_unless(atomic_t *v, int a, int u) +{ + int c = atomic_read(v); + + for (;;) { + if (unlikely(c == u)) + break; + if (likely(atomic_fcmpset_int(&v->counter, &c, c + a))) + break; + } + return (c != u); +} + +static inline int +atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + int c = atomic_read(v); + + for (;;) { + if (unlikely(c == u)) + break; + if (likely(atomic_fcmpset_int(&v->counter, &c, c + a))) + break; + } + return (c); +} + +static inline void +atomic_clear_mask(unsigned int mask, atomic_t *v) +{ + atomic_clear_int(&v->counter, mask); +} + +static inline int +atomic_xchg(atomic_t *v, int i) +{ + return (atomic_swap_int(&v->counter, i)); +} + +static inline int +atomic_cmpxchg(atomic_t *v, int old, int new) +{ + int ret = old; + + for (;;) { + if (atomic_fcmpset_int(&v->counter, &ret, new)) + break; + if (ret != old) + break; + } + return (ret); +} + +#if defined(__amd64__) || defined(__arm64__) || defined(__i386__) +#define LINUXKPI_ATOMIC_8(...) __VA_ARGS__ +#define LINUXKPI_ATOMIC_16(...) __VA_ARGS__ +#else +#define LINUXKPI_ATOMIC_8(...) +#define LINUXKPI_ATOMIC_16(...) +#endif + +#if !(defined(i386) || (defined(__mips__) && !(defined(__mips_n32) || \ + defined(__mips_n64))) || (defined(__powerpc__) && \ + !defined(__powerpc64__))) +#define LINUXKPI_ATOMIC_64(...) __VA_ARGS__ +#else +#define LINUXKPI_ATOMIC_64(...) +#endif + +#define cmpxchg(ptr, old, new) ({ \ + union { \ + __typeof(*(ptr)) val; \ + u8 u8[0]; \ + u16 u16[0]; \ + u32 u32[0]; \ + u64 u64[0]; \ + } __ret = { .val = (old) }, __new = { .val = (new) }; \ + \ + CTASSERT( \ + LINUXKPI_ATOMIC_8(sizeof(__ret.val) == 1 ||) \ + LINUXKPI_ATOMIC_16(sizeof(__ret.val) == 2 ||) \ + LINUXKPI_ATOMIC_64(sizeof(__ret.val) == 8 ||) \ + sizeof(__ret.val) == 4); \ + \ + switch (sizeof(__ret.val)) { \ + LINUXKPI_ATOMIC_8( \ + case 1: \ + while (!atomic_fcmpset_8((volatile u8 *)(ptr), \ + __ret.u8, __new.u8[0]) && __ret.val == (old)) \ + ; \ + break; \ + ) \ + LINUXKPI_ATOMIC_16( \ + case 2: \ + while (!atomic_fcmpset_16((volatile u16 *)(ptr), \ + __ret.u16, __new.u16[0]) && __ret.val == (old)) \ + ; \ + break; \ + ) \ + case 4: \ + while (!atomic_fcmpset_32((volatile u32 *)(ptr), \ + __ret.u32, __new.u32[0]) && __ret.val == (old)) \ + ; \ + break; \ + LINUXKPI_ATOMIC_64( \ + case 8: \ + while (!atomic_fcmpset_64((volatile u64 *)(ptr), \ + __ret.u64, __new.u64[0]) && __ret.val == (old)) \ + ; \ + break; \ + ) \ + } \ + __ret.val; \ +}) + +#define cmpxchg_relaxed(...) cmpxchg(__VA_ARGS__) + +#define xchg(ptr, new) ({ \ + union { \ + __typeof(*(ptr)) val; \ + u8 u8[0]; \ + u16 u16[0]; \ + u32 u32[0]; \ + u64 u64[0]; \ + } __ret, __new = { .val = (new) }; \ + \ + CTASSERT( \ + LINUXKPI_ATOMIC_8(sizeof(__ret.val) == 1 ||) \ + LINUXKPI_ATOMIC_16(sizeof(__ret.val) == 2 ||) \ + LINUXKPI_ATOMIC_64(sizeof(__ret.val) == 8 ||) \ + sizeof(__ret.val) == 4); \ + \ + switch (sizeof(__ret.val)) { \ + LINUXKPI_ATOMIC_8( \ + case 1: \ + __ret.val = READ_ONCE(*ptr); \ + while (!atomic_fcmpset_8((volatile u8 *)(ptr), \ + __ret.u8, __new.u8[0])) \ + ; \ + break; \ + ) \ + LINUXKPI_ATOMIC_16( \ + case 2: \ + __ret.val = READ_ONCE(*ptr); \ + while (!atomic_fcmpset_16((volatile u16 *)(ptr), \ + __ret.u16, __new.u16[0])) \ + ; \ + break; \ + ) \ + case 4: \ + __ret.u32[0] = atomic_swap_32((volatile u32 *)(ptr), \ + __new.u32[0]); \ + break; \ + LINUXKPI_ATOMIC_64( \ + case 8: \ + __ret.u64[0] = atomic_swap_64((volatile u64 *)(ptr), \ + __new.u64[0]); \ + break; \ + ) \ + } \ + __ret.val; \ +}) + +static inline int +atomic_dec_if_positive(atomic_t *v) +{ + int retval; + int old; + + old = atomic_read(v); + for (;;) { + retval = old - 1; + if (unlikely(retval < 0)) + break; + if (likely(atomic_fcmpset_int(&v->counter, &old, retval))) + break; + } + return (retval); +} + +#define LINUX_ATOMIC_OP(op, c_op) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + int c, old; \ + \ + c = v->counter; \ + while ((old = atomic_cmpxchg(v, c, c c_op i)) != c) \ + c = old; \ +} + +#define LINUX_ATOMIC_FETCH_OP(op, c_op) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int c, old; \ + \ + c = v->counter; \ + while ((old = atomic_cmpxchg(v, c, c c_op i)) != c) \ + c = old; \ + \ + return (c); \ +} + +LINUX_ATOMIC_OP(or, |) +LINUX_ATOMIC_OP(and, &) +LINUX_ATOMIC_OP(andnot, &~) +LINUX_ATOMIC_OP(xor, ^) + +LINUX_ATOMIC_FETCH_OP(or, |) +LINUX_ATOMIC_FETCH_OP(and, &) +LINUX_ATOMIC_FETCH_OP(andnot, &~) +LINUX_ATOMIC_FETCH_OP(xor, ^) + +#endif /* _ASM_ATOMIC_H_ */ diff --git a/sys/compat/linuxkpi/common/include/asm/atomic64.h b/sys/compat/linuxkpi/common/include/asm/atomic64.h new file mode 100644 index 000000000000..c60117b5cb8c --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/atomic64.h @@ -0,0 +1,149 @@ +/*- + * Copyright (c) 2016-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _ASM_ATOMIC64_H_ +#define _ASM_ATOMIC64_H_ + +#include <linux/compiler.h> +#include <sys/types.h> +#include <machine/atomic.h> + +typedef struct { + volatile int64_t counter; +} atomic64_t; + +#define ATOMIC64_INIT(x) { .counter = (x) } + +/*------------------------------------------------------------------------* + * 64-bit atomic operations + *------------------------------------------------------------------------*/ + +#define atomic64_add(i, v) atomic64_add_return((i), (v)) +#define atomic64_sub(i, v) atomic64_sub_return((i), (v)) +#define atomic64_inc_return(v) atomic64_add_return(1, (v)) +#define atomic64_add_negative(i, v) (atomic64_add_return((i), (v)) < 0) +#define atomic64_add_and_test(i, v) (atomic64_add_return((i), (v)) == 0) +#define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0) +#define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0) +#define atomic64_inc_and_test(v) (atomic64_add_return(1, (v)) == 0) +#define atomic64_dec_return(v) atomic64_sub_return(1, (v)) +#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) + +static inline int64_t +atomic64_add_return(int64_t i, atomic64_t *v) +{ + return i + atomic_fetchadd_64(&v->counter, i); +} + +static inline int64_t +atomic64_sub_return(int64_t i, atomic64_t *v) +{ + return atomic_fetchadd_64(&v->counter, -i) - i; +} + +static inline void +atomic64_set(atomic64_t *v, int64_t i) +{ + atomic_store_rel_64(&v->counter, i); +} + +static inline int64_t +atomic64_read(atomic64_t *v) +{ + return READ_ONCE(v->counter); +} + +static inline int64_t +atomic64_inc(atomic64_t *v) +{ + return atomic_fetchadd_64(&v->counter, 1) + 1; +} + +static inline int64_t +atomic64_dec(atomic64_t *v) +{ + return atomic_fetchadd_64(&v->counter, -1) - 1; +} + +static inline int64_t +atomic64_add_unless(atomic64_t *v, int64_t a, int64_t u) +{ + int64_t c = atomic64_read(v); + + for (;;) { + if (unlikely(c == u)) + break; + if (likely(atomic_fcmpset_64(&v->counter, &c, c + a))) + break; + } + return (c != u); +} + +static inline int64_t +atomic64_fetch_add_unless(atomic64_t *v, int64_t a, int64_t u) +{ + int64_t c = atomic64_read(v); + + for (;;) { + if (unlikely(c == u)) + break; + if (likely(atomic_fcmpset_64(&v->counter, &c, c + a))) + break; + } + return (c); +} + +static inline int64_t +atomic64_xchg(atomic64_t *v, int64_t i) +{ +#if !((defined(__mips__) && !(defined(__mips_n32) || defined(__mips_n64))) || \ + (defined(__powerpc__) && !defined(__powerpc64__))) + return (atomic_swap_64(&v->counter, i)); +#else + int64_t ret = atomic64_read(v); + + while (!atomic_fcmpset_64(&v->counter, &ret, i)) + ; + return (ret); +#endif +} + +static inline int64_t +atomic64_cmpxchg(atomic64_t *v, int64_t old, int64_t new) +{ + int64_t ret = old; + + for (;;) { + if (atomic_fcmpset_64(&v->counter, &ret, new)) + break; + if (ret != old) + break; + } + return (ret); +} + +#endif /* _ASM_ATOMIC64_H_ */ diff --git a/sys/compat/linuxkpi/common/include/asm/byteorder.h b/sys/compat/linuxkpi/common/include/asm/byteorder.h new file mode 100644 index 000000000000..0a6c2b039c66 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/byteorder.h @@ -0,0 +1,94 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _ASM_BYTEORDER_H_ +#define _ASM_BYTEORDER_H_ + +#include <sys/types.h> +#include <sys/endian.h> +#include <asm/types.h> + +#if BYTE_ORDER == LITTLE_ENDIAN +#define __LITTLE_ENDIAN +#else +#define __BIG_ENDIAN +#endif + +#define cpu_to_le64(x) htole64(x) +#define le64_to_cpu(x) le64toh(x) +#define cpu_to_le32(x) htole32(x) +#define le32_to_cpu(x) le32toh(x) +#define cpu_to_le16(x) htole16(x) +#define le16_to_cpu(x) le16toh(x) +#define cpu_to_be64(x) htobe64(x) +#define be64_to_cpu(x) be64toh(x) +#define cpu_to_be32(x) htobe32(x) +#define be32_to_cpu(x) be32toh(x) +#define cpu_to_be16(x) htobe16(x) +#define be16_to_cpu(x) be16toh(x) +#define __be16_to_cpu(x) be16toh(x) + +#define cpu_to_le64p(x) htole64(*((const uint64_t *)(x))) +#define le64_to_cpup(x) le64toh(*((const uint64_t *)(x))) +#define cpu_to_le32p(x) htole32(*((const uint32_t *)(x))) +#define le32_to_cpup(x) le32toh(*((const uint32_t *)(x))) +#define cpu_to_le16p(x) htole16(*((const uint16_t *)(x))) +#define le16_to_cpup(x) le16toh(*((const uint16_t *)(x))) +#define cpu_to_be64p(x) htobe64(*((const uint64_t *)(x))) +#define be64_to_cpup(x) be64toh(*((const uint64_t *)(x))) +#define cpu_to_be32p(x) htobe32(*((const uint32_t *)(x))) +#define be32_to_cpup(x) be32toh(*((const uint32_t *)(x))) +#define cpu_to_be16p(x) htobe16(*((const uint16_t *)(x))) +#define be16_to_cpup(x) be16toh(*((const uint16_t *)(x))) + +#define cpu_to_le64s(x) do { *((uint64_t *)(x)) = cpu_to_le64p((x)); } while (0) +#define le64_to_cpus(x) do { *((uint64_t *)(x)) = le64_to_cpup((x)); } while (0) +#define cpu_to_le32s(x) do { *((uint32_t *)(x)) = cpu_to_le32p((x)); } while (0) +#define le32_to_cpus(x) do { *((uint32_t *)(x)) = le32_to_cpup((x)); } while (0) +#define cpu_to_le16s(x) do { *((uint16_t *)(x)) = cpu_to_le16p((x)); } while (0) +#define le16_to_cpus(x) do { *((uint16_t *)(x)) = le16_to_cpup((x)); } while (0) +#define cpu_to_be64s(x) do { *((uint64_t *)(x)) = cpu_to_be64p((x)); } while (0) +#define be64_to_cpus(x) do { *((uint64_t *)(x)) = be64_to_cpup((x)); } while (0) +#define cpu_to_be32s(x) do { *((uint32_t *)(x)) = cpu_to_be32p((x)); } while (0) +#define be32_to_cpus(x) do { *((uint32_t *)(x)) = be32_to_cpup((x)); } while (0) +#define cpu_to_be16s(x) do { *((uint16_t *)(x)) = cpu_to_be16p((x)); } while (0) +#define be16_to_cpus(x) do { *((uint16_t *)(x)) = be16_to_cpup((x)); } while (0) + +#define swab16(x) bswap16(x) +#define swab32(x) bswap32(x) +#define swab64(x) bswap64(x) + +static inline void +be16_add_cpu(uint16_t *var, uint16_t val) +{ + *var = cpu_to_be16(be16_to_cpu(*var) + val); +} + +#endif /* _ASM_BYTEORDER_H_ */ diff --git a/sys/compat/linuxkpi/common/include/asm/fcntl.h b/sys/compat/linuxkpi/common/include/asm/fcntl.h new file mode 100644 index 000000000000..f24624e6eeed --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/fcntl.h @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _ASM_FCNTL_H_ +#define _ASM_FCNTL_H_ + +#include <sys/fcntl.h> + +#endif /* _ASM_FCNTL_H_ */ diff --git a/sys/compat/linuxkpi/common/include/asm/io.h b/sys/compat/linuxkpi/common/include/asm/io.h new file mode 100644 index 000000000000..24d350f4743e --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/io.h @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _ASM_IO_H_ +#define _ASM_IO_H_ + +#include <linux/io.h> + +#endif /* _ASM_IO_H_ */ diff --git a/sys/compat/linuxkpi/common/include/asm/msr.h b/sys/compat/linuxkpi/common/include/asm/msr.h new file mode 100644 index 000000000000..1697c20767a5 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/msr.h @@ -0,0 +1,37 @@ +/*- + * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ASM_MSR_H_ +#define _ASM_MSR_H_ + +#include <machine/cpufunc.h> + +#define rdmsrl(msr, val) ((val) = rdmsr(msr)) +#define rdmsrl_safe(msr, val) rdmsr_safe(msr, val) + +#endif /* _ASM_MSR_H_ */ diff --git a/sys/compat/linuxkpi/common/include/asm/pgtable.h b/sys/compat/linuxkpi/common/include/asm/pgtable.h new file mode 100644 index 000000000000..1df103060368 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/pgtable.h @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _ASM_PGTABLE_H_ +#define _ASM_PGTABLE_H_ + +#include <linux/page.h> + +typedef unsigned long pteval_t; +typedef unsigned long pmdval_t; +typedef unsigned long pudval_t; +typedef unsigned long pgdval_t; +typedef unsigned long pgprotval_t; +typedef struct page *pgtable_t; + +#endif /* _ASM_PGTABLE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/asm/smp.h b/sys/compat/linuxkpi/common/include/asm/smp.h new file mode 100644 index 000000000000..dd25b02115a8 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/smp.h @@ -0,0 +1,48 @@ +/*- + * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ASM_SMP_H_ +#define _ASM_SMP_H_ + +#if defined(__i386__) || defined(__amd64__) + +#define wbinvd_on_all_cpus() linux_wbinvd_on_all_cpus() + +int linux_wbinvd_on_all_cpus(void); + +#endif + +#define get_cpu() ({ \ + critical_enter(); \ + PCPU_GET(cpuid); \ +}) + +#define put_cpu() \ + critical_exit() + +#endif /* _ASM_SMP_H_ */ diff --git a/sys/compat/linuxkpi/common/include/asm/types.h b/sys/compat/linuxkpi/common/include/asm/types.h new file mode 100644 index 000000000000..63e1ac6aca8c --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/types.h @@ -0,0 +1,64 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _ASM_TYPES_H_ +#define _ASM_TYPES_H_ + +#ifdef _KERNEL + +#include <sys/types.h> + +typedef uint8_t u8; +typedef uint8_t __u8; +typedef uint16_t u16; +typedef uint16_t __u16; +typedef uint32_t u32; +typedef uint32_t __u32; +typedef uint64_t u64; +typedef uint64_t __u64; + +typedef int8_t s8; +typedef int8_t __s8; +typedef int16_t s16; +typedef int16_t __s16; +typedef int32_t s32; +typedef int32_t __s32; +typedef int64_t s64; +typedef int64_t __s64; + +/* DMA addresses come in generic and 64-bit flavours. */ +typedef vm_paddr_t dma_addr_t; +typedef vm_paddr_t dma64_addr_t; + +typedef unsigned short umode_t; + +#endif /* _KERNEL */ + +#endif /* _ASM_TYPES_H_ */ diff --git a/sys/compat/linuxkpi/common/include/asm/uaccess.h b/sys/compat/linuxkpi/common/include/asm/uaccess.h new file mode 100644 index 000000000000..0ae20b09dc8c --- /dev/null +++ b/sys/compat/linuxkpi/common/include/asm/uaccess.h @@ -0,0 +1,72 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _ASM_UACCESS_H_ +#define _ASM_UACCESS_H_ + +#include <linux/uaccess.h> + +static inline long +copy_to_user(void *to, const void *from, unsigned long n) +{ + if (linux_copyout(from, to, n) != 0) + return n; + return 0; +} +#define __copy_to_user(...) copy_to_user(__VA_ARGS__) + +static inline long +copy_from_user(void *to, const void *from, unsigned long n) +{ + if (linux_copyin(from, to, n) != 0) + return n; + return 0; +} +#define __copy_from_user(...) copy_from_user(__VA_ARGS__) +#define __copy_in_user(...) copy_from_user(__VA_ARGS__) + +#if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 50000 +#define user_access_begin(ptr, len) access_ok(ptr, len) +#else +#define user_access_begin() do { } while (0) +#endif +#define user_access_end() do { } while (0) + +#define unsafe_get_user(x, ptr, err) do { \ + if (unlikely(__get_user(x, ptr))) \ + goto err; \ +} while (0) + +#define unsafe_put_user(x, ptr, err) do { \ + if (unlikely(__put_user(x, ptr))) \ + goto err; \ +} while (0) + +#endif /* _ASM_UACCESS_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/atomic.h b/sys/compat/linuxkpi/common/include/linux/atomic.h new file mode 100644 index 000000000000..f8aa5c9276e7 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/atomic.h @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_ATOMIC_H_ +#define _LINUX_ATOMIC_H_ + +#include <asm/atomic.h> +#include <asm/atomic64.h> +#include <asm/atomic-long.h> + +#endif /* _LINUX_ATOMIC_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/bitmap.h b/sys/compat/linuxkpi/common/include/linux/bitmap.h new file mode 100644 index 000000000000..4de94333c99f --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/bitmap.h @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_BITMAP_H_ +#define _LINUX_BITMAP_H_ + +#include <linux/bitops.h> +#include <linux/slab.h> + +static inline void +bitmap_zero(unsigned long *addr, const unsigned int size) +{ + memset(addr, 0, BITS_TO_LONGS(size) * sizeof(long)); +} + +static inline void +bitmap_fill(unsigned long *addr, const unsigned int size) +{ + const unsigned int tail = size & (BITS_PER_LONG - 1); + + memset(addr, 0xff, BIT_WORD(size) * sizeof(long)); + + if (tail) + addr[BIT_WORD(size)] = BITMAP_LAST_WORD_MASK(tail); +} + +static inline int +bitmap_full(unsigned long *addr, const unsigned int size) +{ + const unsigned int end = BIT_WORD(size); + const unsigned int tail = size & (BITS_PER_LONG - 1); + unsigned int i; + + for (i = 0; i != end; i++) { + if (addr[i] != ~0UL) + return (0); + } + + if (tail) { + const unsigned long mask = BITMAP_LAST_WORD_MASK(tail); + + if ((addr[end] & mask) != mask) + return (0); + } + return (1); +} + +static inline int +bitmap_empty(unsigned long *addr, const unsigned int size) +{ + const unsigned int end = BIT_WORD(size); + const unsigned int tail = size & (BITS_PER_LONG - 1); + unsigned int i; + + for (i = 0; i != end; i++) { + if (addr[i] != 0) + return (0); + } + + if (tail) { + const unsigned long mask = BITMAP_LAST_WORD_MASK(tail); + + if ((addr[end] & mask) != 0) + return (0); + } + return (1); +} + +static inline void +bitmap_set(unsigned long *map, unsigned int start, int nr) +{ + const unsigned int size = start + nr; + int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG); + unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start); + + map += BIT_WORD(start); + + while (nr - bits_to_set >= 0) { + *map |= mask_to_set; + nr -= bits_to_set; + bits_to_set = BITS_PER_LONG; + mask_to_set = ~0UL; + map++; + } + + if (nr) { + mask_to_set &= BITMAP_LAST_WORD_MASK(size); + *map |= mask_to_set; + } +} + +static inline void +bitmap_clear(unsigned long *map, unsigned int start, int nr) +{ + const unsigned int size = start + nr; + int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); + unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); + + map += BIT_WORD(start); + + while (nr - bits_to_clear >= 0) { + *map &= ~mask_to_clear; + nr -= bits_to_clear; + bits_to_clear = BITS_PER_LONG; + mask_to_clear = ~0UL; + map++; + } + + if (nr) { + mask_to_clear &= BITMAP_LAST_WORD_MASK(size); + *map &= ~mask_to_clear; + } +} + +static inline unsigned int +bitmap_find_next_zero_area_off(const unsigned long *map, + const unsigned int size, unsigned int start, + unsigned int nr, unsigned int align_mask, + unsigned int align_offset) +{ + unsigned int index; + unsigned int end; + unsigned int i; + +retry: + index = find_next_zero_bit(map, size, start); + + index = (((index + align_offset) + align_mask) & ~align_mask) - align_offset; + + end = index + nr; + if (end > size) + return (end); + + i = find_next_bit(map, end, index); + if (i < end) { + start = i + 1; + goto retry; + } + return (index); +} + +static inline unsigned int +bitmap_find_next_zero_area(const unsigned long *map, + const unsigned int size, unsigned int start, + unsigned int nr, unsigned int align_mask) +{ + return (bitmap_find_next_zero_area_off(map, size, + start, nr, align_mask, 0)); +} + +static inline int +bitmap_find_free_region(unsigned long *bitmap, int bits, int order) +{ + int pos; + int end; + + for (pos = 0; (end = pos + (1 << order)) <= bits; pos = end) { + if (!linux_reg_op(bitmap, pos, order, REG_OP_ISFREE)) + continue; + linux_reg_op(bitmap, pos, order, REG_OP_ALLOC); + return (pos); + } + return (-ENOMEM); +} + +static inline int +bitmap_allocate_region(unsigned long *bitmap, int pos, int order) +{ + if (!linux_reg_op(bitmap, pos, order, REG_OP_ISFREE)) + return (-EBUSY); + linux_reg_op(bitmap, pos, order, REG_OP_ALLOC); + return (0); +} + +static inline void +bitmap_release_region(unsigned long *bitmap, int pos, int order) +{ + linux_reg_op(bitmap, pos, order, REG_OP_RELEASE); +} + +static inline unsigned int +bitmap_weight(unsigned long *addr, const unsigned int size) +{ + const unsigned int end = BIT_WORD(size); + const unsigned int tail = size & (BITS_PER_LONG - 1); + unsigned int retval = 0; + unsigned int i; + + for (i = 0; i != end; i++) + retval += hweight_long(addr[i]); + + if (tail) { + const unsigned long mask = BITMAP_LAST_WORD_MASK(tail); + + retval += hweight_long(addr[end] & mask); + } + return (retval); +} + +static inline int +bitmap_equal(const unsigned long *pa, + const unsigned long *pb, unsigned size) +{ + const unsigned int end = BIT_WORD(size); + const unsigned int tail = size & (BITS_PER_LONG - 1); + unsigned int i; + + for (i = 0; i != end; i++) { + if (pa[i] != pb[i]) + return (0); + } + + if (tail) { + const unsigned long mask = BITMAP_LAST_WORD_MASK(tail); + + if ((pa[end] ^ pb[end]) & mask) + return (0); + } + return (1); +} + +static inline int +bitmap_subset(const unsigned long *pa, + const unsigned long *pb, unsigned size) +{ + const unsigned end = BIT_WORD(size); + const unsigned tail = size & (BITS_PER_LONG - 1); + unsigned i; + + for (i = 0; i != end; i++) { + if (pa[i] & ~pb[i]) + return (0); + } + + if (tail) { + const unsigned long mask = BITMAP_LAST_WORD_MASK(tail); + + if (pa[end] & ~pb[end] & mask) + return (0); + } + return (1); +} + +static inline void +bitmap_complement(unsigned long *dst, const unsigned long *src, + const unsigned int size) +{ + const unsigned int end = BITS_TO_LONGS(size); + unsigned int i; + + for (i = 0; i != end; i++) + dst[i] = ~src[i]; +} + +static inline void +bitmap_copy(unsigned long *dst, const unsigned long *src, + const unsigned int size) +{ + const unsigned int end = BITS_TO_LONGS(size); + unsigned int i; + + for (i = 0; i != end; i++) + dst[i] = src[i]; +} + +static inline void +bitmap_or(unsigned long *dst, const unsigned long *src1, + const unsigned long *src2, const unsigned int size) +{ + const unsigned int end = BITS_TO_LONGS(size); + unsigned int i; + + for (i = 0; i != end; i++) + dst[i] = src1[i] | src2[i]; +} + +static inline void +bitmap_and(unsigned long *dst, const unsigned long *src1, + const unsigned long *src2, const unsigned int size) +{ + const unsigned int end = BITS_TO_LONGS(size); + unsigned int i; + + for (i = 0; i != end; i++) + dst[i] = src1[i] & src2[i]; +} + +static inline void +bitmap_andnot(unsigned long *dst, const unsigned long *src1, + const unsigned long *src2, const unsigned int size) +{ + const unsigned int end = BITS_TO_LONGS(size); + unsigned int i; + + for (i = 0; i != end; i++) + dst[i] = src1[i] & ~src2[i]; +} + +static inline void +bitmap_xor(unsigned long *dst, const unsigned long *src1, + const unsigned long *src2, const unsigned int size) +{ + const unsigned int end = BITS_TO_LONGS(size); + unsigned int i; + + for (i = 0; i != end; i++) + dst[i] = src1[i] ^ src2[i]; +} + +static inline unsigned long * +bitmap_alloc(unsigned int size, gfp_t flags) +{ + return (kmalloc_array(BITS_TO_LONGS(size), + sizeof(unsigned long), flags)); +} + +static inline unsigned long * +bitmap_zalloc(unsigned int size, gfp_t flags) +{ + return (bitmap_alloc(size, flags | __GFP_ZERO)); +} + +static inline void +bitmap_free(const unsigned long *bitmap) +{ + kfree(bitmap); +} + +#endif /* _LINUX_BITMAP_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/bitops.h b/sys/compat/linuxkpi/common/include/linux/bitops.h new file mode 100644 index 000000000000..1bcbd681203c --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/bitops.h @@ -0,0 +1,410 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_BITOPS_H_ +#define _LINUX_BITOPS_H_ + +#include <sys/param.h> +#include <sys/types.h> +#include <sys/systm.h> +#include <sys/errno.h> +#include <sys/libkern.h> + +#define BIT(nr) (1UL << (nr)) +#define BIT_ULL(nr) (1ULL << (nr)) +#ifdef __LP64__ +#define BITS_PER_LONG 64 +#else +#define BITS_PER_LONG 32 +#endif + +#define BITS_PER_LONG_LONG 64 + +#define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) % BITS_PER_LONG)) +#define BITMAP_LAST_WORD_MASK(n) (~0UL >> (BITS_PER_LONG - (n))) +#define BITS_TO_LONGS(n) howmany((n), BITS_PER_LONG) +#define BIT_MASK(nr) (1UL << ((nr) & (BITS_PER_LONG - 1))) +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) +#define GENMASK(h, l) (((~0UL) >> (BITS_PER_LONG - (h) - 1)) & ((~0UL) << (l))) +#define GENMASK_ULL(h, l) (((~0ULL) >> (BITS_PER_LONG_LONG - (h) - 1)) & ((~0ULL) << (l))) +#define BITS_PER_BYTE 8 +#define BITS_PER_TYPE(t) (sizeof(t) * BITS_PER_BYTE) + +#define hweight8(x) bitcount((uint8_t)(x)) +#define hweight16(x) bitcount16(x) +#define hweight32(x) bitcount32(x) +#define hweight64(x) bitcount64(x) +#define hweight_long(x) bitcountl(x) + +static inline int +__ffs(int mask) +{ + return (ffs(mask) - 1); +} + +static inline int +__fls(int mask) +{ + return (fls(mask) - 1); +} + +static inline int +__ffsl(long mask) +{ + return (ffsl(mask) - 1); +} + +static inline int +__flsl(long mask) +{ + return (flsl(mask) - 1); +} + +static inline int +fls64(uint64_t mask) +{ + return (flsll(mask)); +} + +static inline uint32_t +ror32(uint32_t word, unsigned int shift) +{ + return ((word >> shift) | (word << (32 - shift))); +} + +#define ffz(mask) __ffs(~(mask)) + +static inline int get_count_order(unsigned int count) +{ + int order; + + order = fls(count) - 1; + if (count & (count - 1)) + order++; + return order; +} + +static inline unsigned long +find_first_bit(const unsigned long *addr, unsigned long size) +{ + long mask; + int bit; + + for (bit = 0; size >= BITS_PER_LONG; + size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { + if (*addr == 0) + continue; + return (bit + __ffsl(*addr)); + } + if (size) { + mask = (*addr) & BITMAP_LAST_WORD_MASK(size); + if (mask) + bit += __ffsl(mask); + else + bit += size; + } + return (bit); +} + +static inline unsigned long +find_first_zero_bit(const unsigned long *addr, unsigned long size) +{ + long mask; + int bit; + + for (bit = 0; size >= BITS_PER_LONG; + size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { + if (~(*addr) == 0) + continue; + return (bit + __ffsl(~(*addr))); + } + if (size) { + mask = ~(*addr) & BITMAP_LAST_WORD_MASK(size); + if (mask) + bit += __ffsl(mask); + else + bit += size; + } + return (bit); +} + +static inline unsigned long +find_last_bit(const unsigned long *addr, unsigned long size) +{ + long mask; + int offs; + int bit; + int pos; + + pos = size / BITS_PER_LONG; + offs = size % BITS_PER_LONG; + bit = BITS_PER_LONG * pos; + addr += pos; + if (offs) { + mask = (*addr) & BITMAP_LAST_WORD_MASK(offs); + if (mask) + return (bit + __flsl(mask)); + } + while (pos--) { + addr--; + bit -= BITS_PER_LONG; + if (*addr) + return (bit + __flsl(*addr)); + } + return (size); +} + +static inline unsigned long +find_next_bit(const unsigned long *addr, unsigned long size, unsigned long offset) +{ + long mask; + int offs; + int bit; + int pos; + + if (offset >= size) + return (size); + pos = offset / BITS_PER_LONG; + offs = offset % BITS_PER_LONG; + bit = BITS_PER_LONG * pos; + addr += pos; + if (offs) { + mask = (*addr) & ~BITMAP_LAST_WORD_MASK(offs); + if (mask) + return (bit + __ffsl(mask)); + if (size - bit <= BITS_PER_LONG) + return (size); + bit += BITS_PER_LONG; + addr++; + } + for (size -= bit; size >= BITS_PER_LONG; + size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { + if (*addr == 0) + continue; + return (bit + __ffsl(*addr)); + } + if (size) { + mask = (*addr) & BITMAP_LAST_WORD_MASK(size); + if (mask) + bit += __ffsl(mask); + else + bit += size; + } + return (bit); +} + +static inline unsigned long +find_next_zero_bit(const unsigned long *addr, unsigned long size, + unsigned long offset) +{ + long mask; + int offs; + int bit; + int pos; + + if (offset >= size) + return (size); + pos = offset / BITS_PER_LONG; + offs = offset % BITS_PER_LONG; + bit = BITS_PER_LONG * pos; + addr += pos; + if (offs) { + mask = ~(*addr) & ~BITMAP_LAST_WORD_MASK(offs); + if (mask) + return (bit + __ffsl(mask)); + if (size - bit <= BITS_PER_LONG) + return (size); + bit += BITS_PER_LONG; + addr++; + } + for (size -= bit; size >= BITS_PER_LONG; + size -= BITS_PER_LONG, bit += BITS_PER_LONG, addr++) { + if (~(*addr) == 0) + continue; + return (bit + __ffsl(~(*addr))); + } + if (size) { + mask = ~(*addr) & BITMAP_LAST_WORD_MASK(size); + if (mask) + bit += __ffsl(mask); + else + bit += size; + } + return (bit); +} + +#define __set_bit(i, a) \ + atomic_set_long(&((volatile unsigned long *)(a))[BIT_WORD(i)], BIT_MASK(i)) + +#define set_bit(i, a) \ + atomic_set_long(&((volatile unsigned long *)(a))[BIT_WORD(i)], BIT_MASK(i)) + +#define __clear_bit(i, a) \ + atomic_clear_long(&((volatile unsigned long *)(a))[BIT_WORD(i)], BIT_MASK(i)) + +#define clear_bit(i, a) \ + atomic_clear_long(&((volatile unsigned long *)(a))[BIT_WORD(i)], BIT_MASK(i)) + +#define clear_bit_unlock(i, a) \ + atomic_clear_rel_long(&((volatile unsigned long *)(a))[BIT_WORD(i)], BIT_MASK(i)) + +#define test_bit(i, a) \ + !!(READ_ONCE(((volatile const unsigned long *)(a))[BIT_WORD(i)]) & BIT_MASK(i)) + +static inline int +test_and_clear_bit(long bit, volatile unsigned long *var) +{ + long val; + + var += BIT_WORD(bit); + bit %= BITS_PER_LONG; + bit = (1UL << bit); + + val = *var; + while (!atomic_fcmpset_long(var, &val, val & ~bit)) + ; + return !!(val & bit); +} + +static inline int +__test_and_clear_bit(long bit, volatile unsigned long *var) +{ + long val; + + var += BIT_WORD(bit); + bit %= BITS_PER_LONG; + bit = (1UL << bit); + + val = *var; + *var &= ~bit; + + return !!(val & bit); +} + +static inline int +test_and_set_bit(long bit, volatile unsigned long *var) +{ + long val; + + var += BIT_WORD(bit); + bit %= BITS_PER_LONG; + bit = (1UL << bit); + + val = *var; + while (!atomic_fcmpset_long(var, &val, val | bit)) + ; + return !!(val & bit); +} + +static inline int +__test_and_set_bit(long bit, volatile unsigned long *var) +{ + long val; + + var += BIT_WORD(bit); + bit %= BITS_PER_LONG; + bit = (1UL << bit); + + val = *var; + *var |= bit; + + return !!(val & bit); +} + +enum { + REG_OP_ISFREE, + REG_OP_ALLOC, + REG_OP_RELEASE, +}; + +static inline int +linux_reg_op(unsigned long *bitmap, int pos, int order, int reg_op) +{ + int nbits_reg; + int index; + int offset; + int nlongs_reg; + int nbitsinlong; + unsigned long mask; + int i; + int ret = 0; + + nbits_reg = 1 << order; + index = pos / BITS_PER_LONG; + offset = pos - (index * BITS_PER_LONG); + nlongs_reg = BITS_TO_LONGS(nbits_reg); + nbitsinlong = min(nbits_reg, BITS_PER_LONG); + + mask = (1UL << (nbitsinlong - 1)); + mask += mask - 1; + mask <<= offset; + + switch (reg_op) { + case REG_OP_ISFREE: + for (i = 0; i < nlongs_reg; i++) { + if (bitmap[index + i] & mask) + goto done; + } + ret = 1; + break; + + case REG_OP_ALLOC: + for (i = 0; i < nlongs_reg; i++) + bitmap[index + i] |= mask; + break; + + case REG_OP_RELEASE: + for (i = 0; i < nlongs_reg; i++) + bitmap[index + i] &= ~mask; + break; + } +done: + return ret; +} + +#define for_each_set_bit(bit, addr, size) \ + for ((bit) = find_first_bit((addr), (size)); \ + (bit) < (size); \ + (bit) = find_next_bit((addr), (size), (bit) + 1)) + +#define for_each_clear_bit(bit, addr, size) \ + for ((bit) = find_first_zero_bit((addr), (size)); \ + (bit) < (size); \ + (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) + +static inline uint64_t +sign_extend64(uint64_t value, int index) +{ + uint8_t shift = 63 - index; + + return ((int64_t)(value << shift) >> shift); +} + +#endif /* _LINUX_BITOPS_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/bottom_half.h b/sys/compat/linuxkpi/common/include/linux/bottom_half.h new file mode 100644 index 000000000000..9f8dc02f2798 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/bottom_half.h @@ -0,0 +1,34 @@ +/*- + * Copyright (c) 2017 Hans Petter Selasky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_BOTTOM_HALF_H_ +#define _LINUX_BOTTOM_HALF_H_ + +extern void local_bh_enable(void); +extern void local_bh_disable(void); + +#endif /* _LINUX_BOTTOM_HALF_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/cache.h b/sys/compat/linuxkpi/common/include/linux/cache.h new file mode 100644 index 000000000000..a269e55eb90b --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/cache.h @@ -0,0 +1,37 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_CACHE_H_ +#define _LINUX_CACHE_H_ + +#define cache_line_size() CACHE_LINE_SIZE +#define L1_CACHE_BYTES CACHE_LINE_SIZE + +#endif /* _LINUX_CACHE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/cdev.h b/sys/compat/linuxkpi/common/include/linux/cdev.h new file mode 100644 index 000000000000..2a472da91866 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/cdev.h @@ -0,0 +1,152 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_CDEV_H_ +#define _LINUX_CDEV_H_ + +#include <linux/kobject.h> +#include <linux/sysfs.h> +#include <linux/kdev_t.h> +#include <linux/list.h> + +#include <asm/atomic-long.h> + +struct file_operations; +struct inode; +struct module; + +extern struct cdevsw linuxcdevsw; +extern const struct kobj_type linux_cdev_ktype; +extern const struct kobj_type linux_cdev_static_ktype; + +struct linux_cdev { + struct kobject kobj; + struct module *owner; + struct cdev *cdev; + dev_t dev; + const struct file_operations *ops; + u_int refs; + u_int siref; +}; + +static inline void +cdev_init(struct linux_cdev *cdev, const struct file_operations *ops) +{ + + kobject_init(&cdev->kobj, &linux_cdev_static_ktype); + cdev->ops = ops; + cdev->refs = 1; +} + +static inline struct linux_cdev * +cdev_alloc(void) +{ + struct linux_cdev *cdev; + + cdev = kzalloc(sizeof(struct linux_cdev), M_WAITOK); + kobject_init(&cdev->kobj, &linux_cdev_ktype); + cdev->refs = 1; + return (cdev); +} + +static inline void +cdev_put(struct linux_cdev *p) +{ + kobject_put(&p->kobj); +} + +static inline int +cdev_add(struct linux_cdev *cdev, dev_t dev, unsigned count) +{ + struct make_dev_args args; + int error; + + if (count != 1) + return (-EINVAL); + + cdev->dev = dev; + + /* Setup arguments for make_dev_s() */ + make_dev_args_init(&args); + args.mda_devsw = &linuxcdevsw; + args.mda_uid = 0; + args.mda_gid = 0; + args.mda_mode = 0700; + args.mda_si_drv1 = cdev; + + error = make_dev_s(&args, &cdev->cdev, "%s", + kobject_name(&cdev->kobj)); + if (error) + return (-error); + + kobject_get(cdev->kobj.parent); + return (0); +} + +static inline int +cdev_add_ext(struct linux_cdev *cdev, dev_t dev, uid_t uid, gid_t gid, int mode) +{ + struct make_dev_args args; + int error; + + cdev->dev = dev; + + /* Setup arguments for make_dev_s() */ + make_dev_args_init(&args); + args.mda_devsw = &linuxcdevsw; + args.mda_uid = uid; + args.mda_gid = gid; + args.mda_mode = mode; + args.mda_si_drv1 = cdev; + + error = make_dev_s(&args, &cdev->cdev, "%s/%d", + kobject_name(&cdev->kobj), MINOR(dev)); + if (error) + return (-error); + + kobject_get(cdev->kobj.parent); + return (0); +} + +void linux_destroy_dev(struct linux_cdev *); + +static inline void +cdev_del(struct linux_cdev *cdev) +{ + + linux_destroy_dev(cdev); + kobject_put(&cdev->kobj); +} + +struct linux_cdev *linux_find_cdev(const char *name, unsigned major, unsigned minor); + +#define cdev linux_cdev + +#endif /* _LINUX_CDEV_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/clocksource.h b/sys/compat/linuxkpi/common/include/linux/clocksource.h new file mode 100644 index 000000000000..f775aa6bb329 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/clocksource.h @@ -0,0 +1,38 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_CLOCKSOURCE_H +#define _LINUX_CLOCKSOURCE_H + +#include <asm/types.h> + +#define CLOCKSOURCE_MASK(x) ((u64)(-1ULL >> ((-(x)) & 63))) + +#endif /* _LINUX_CLOCKSOURCE_H */ diff --git a/sys/compat/linuxkpi/common/include/linux/compat.h b/sys/compat/linuxkpi/common/include/linux/compat.h new file mode 100644 index 000000000000..e9b2e4dfd1a7 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/compat.h @@ -0,0 +1,60 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_COMPAT_H_ +#define _LINUX_COMPAT_H_ + +#include <sys/param.h> +#include <sys/proc.h> +#include <sys/malloc.h> + +struct thread; +struct task_struct; + +extern int linux_alloc_current(struct thread *, int flags); +extern void linux_free_current(struct task_struct *); + + +static inline void +linux_set_current(struct thread *td) +{ + if (__predict_false(td->td_lkpi_task == NULL)) + lkpi_alloc_current(td, M_WAITOK); +} + +static inline int +linux_set_current_flags(struct thread *td, int flags) +{ + if (__predict_false(td->td_lkpi_task == NULL)) + return (lkpi_alloc_current(td, flags)); + return (0); +} + +#endif /* _LINUX_COMPAT_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/compiler.h b/sys/compat/linuxkpi/common/include/linux/compiler.h new file mode 100644 index 000000000000..1177674aa68f --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/compiler.h @@ -0,0 +1,117 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_COMPILER_H_ +#define _LINUX_COMPILER_H_ + +#include <sys/cdefs.h> + +#define __user +#define __kernel +#define __safe +#define __force +#define __nocast +#define __iomem +#define __chk_user_ptr(x) ((void)0) +#define __chk_io_ptr(x) ((void)0) +#define __builtin_warning(x, y...) (1) +#define __acquires(x) +#define __releases(x) +#define __acquire(x) do { } while (0) +#define __release(x) do { } while (0) +#define __cond_lock(x,c) (c) +#define __bitwise +#define __devinitdata +#define __deprecated +#define __init +#define __initconst +#define __devinit +#define __devexit +#define __exit +#define __rcu +#define __percpu +#define __weak __weak_symbol +#define __malloc +#define ___stringify(...) #__VA_ARGS__ +#define __stringify(...) ___stringify(__VA_ARGS__) +#define __attribute_const__ __attribute__((__const__)) +#undef __always_inline +#define __always_inline inline +#define noinline __noinline +#define ____cacheline_aligned __aligned(CACHE_LINE_SIZE) + +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#define typeof(x) __typeof(x) + +#define uninitialized_var(x) x = x +#define __maybe_unused __unused +#define __always_unused __unused +#define __must_check __result_use_check + +#define __printf(a,b) __printflike(a,b) + +#define barrier() __asm__ __volatile__("": : :"memory") + +#if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 50000 +/* Moved from drm_os_freebsd.h */ +#define lower_32_bits(n) ((u32)(n)) +#define upper_32_bits(n) ((u32)(((n) >> 16) >> 16)) +#endif + +#define ___PASTE(a,b) a##b +#define __PASTE(a,b) ___PASTE(a,b) + +#define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x)) + +#define WRITE_ONCE(x,v) do { \ + barrier(); \ + ACCESS_ONCE(x) = (v); \ + barrier(); \ +} while (0) + +#define READ_ONCE(x) ({ \ + __typeof(x) __var = ({ \ + barrier(); \ + ACCESS_ONCE(x); \ + }); \ + barrier(); \ + __var; \ +}) + +#define lockless_dereference(p) READ_ONCE(p) + +#define _AT(T,X) ((T)(X)) + +#define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b)) +#define __must_be_array(a) __same_type(a, &(a)[0]) + +#endif /* _LINUX_COMPILER_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/completion.h b/sys/compat/linuxkpi/common/include/linux/completion.h new file mode 100644 index 000000000000..09d499309e5a --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/completion.h @@ -0,0 +1,69 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_COMPLETION_H_ +#define _LINUX_COMPLETION_H_ + +#include <linux/errno.h> + +struct completion { + unsigned int done; +}; + +#define INIT_COMPLETION(c) \ + ((c).done = 0) +#define init_completion(c) \ + do { (c)->done = 0; } while (0) +#define reinit_completion(c) \ + do { (c)->done = 0; } while (0) +#define complete(c) \ + linux_complete_common((c), 0) +#define complete_all(c) \ + linux_complete_common((c), 1) +#define wait_for_completion(c) \ + linux_wait_for_common((c), 0) +#define wait_for_completion_interruptible(c) \ + linux_wait_for_common((c), 1) +#define wait_for_completion_timeout(c, timeout) \ + linux_wait_for_timeout_common((c), (timeout), 0) +#define wait_for_completion_interruptible_timeout(c, timeout) \ + linux_wait_for_timeout_common((c), (timeout), 1) +#define try_wait_for_completion(c) \ + linux_try_wait_for_completion(c) +#define completion_done(c) \ + linux_completion_done(c) + +extern void linux_complete_common(struct completion *, int); +extern int linux_wait_for_common(struct completion *, int); +extern int linux_wait_for_timeout_common(struct completion *, int, int); +extern int linux_try_wait_for_completion(struct completion *); +extern int linux_completion_done(struct completion *); + +#endif /* _LINUX_COMPLETION_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/dcache.h b/sys/compat/linuxkpi/common/include/linux/dcache.h new file mode 100644 index 000000000000..1bafa3dbd1fe --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/dcache.h @@ -0,0 +1,46 @@ +/*- + * Copyright (c) 2017 Limelight Networks, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __LINUX_DCACHE_H +#define __LINUX_DCACHE_H + +struct vnode; +struct pfs_node; + +struct dentry { + struct vnode *d_inode; + struct pfs_node *d_pfs_node; /* FreeBSD specific field */ +}; + +static inline struct vnode * +d_inode(const struct dentry *dentry) +{ + return (dentry->d_inode); +} + +#endif /* __LINUX_DCACHE_H */ diff --git a/sys/compat/linuxkpi/common/include/linux/debugfs.h b/sys/compat/linuxkpi/common/include/linux/debugfs.h new file mode 100644 index 000000000000..406016d7e2c8 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/debugfs.h @@ -0,0 +1,51 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2016-2018, Matthew Macy <mmacy@freebsd.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_DEBUGFS_H_ +#define _LINUX_DEBUGFS_H_ + +#include <linux/fs.h> +#include <linux/seq_file.h> + +#include <linux/types.h> + +void debugfs_remove(struct dentry *dentry); + +struct dentry *debugfs_create_file(const char *name, umode_t mode, + struct dentry *parent, void *data, + const struct file_operations *fops); + +struct dentry *debugfs_create_dir(const char *name, struct dentry *parent); + +struct dentry *debugfs_create_symlink(const char *name, struct dentry *parent, + const char *dest); + +void debugfs_remove_recursive(struct dentry *dentry); + +#endif diff --git a/sys/compat/linuxkpi/common/include/linux/delay.h b/sys/compat/linuxkpi/common/include/linux/delay.h new file mode 100644 index 000000000000..860d36368a8e --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/delay.h @@ -0,0 +1,76 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2014 François Tigeot + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_DELAY_H_ +#define _LINUX_DELAY_H_ + +#include <linux/jiffies.h> +#include <sys/systm.h> + +static inline void +linux_msleep(unsigned int ms) +{ + /* guard against invalid values */ + if (ms == 0) + ms = 1; + pause_sbt("lnxsleep", mstosbt(ms), 0, C_HARDCLOCK); +} + +#undef msleep +#define msleep(ms) linux_msleep(ms) + +#undef msleep_interruptible +#define msleep_interruptible(ms) linux_msleep_interruptible(ms) + +#define udelay(t) DELAY(t) + +static inline void +mdelay(unsigned long msecs) +{ + while (msecs--) + DELAY(1000); +} + +static inline void +ndelay(unsigned long x) +{ + DELAY(howmany(x, 1000)); +} + +static inline void +usleep_range(unsigned long min, unsigned long max) +{ + DELAY(min); +} + +extern unsigned int linux_msleep_interruptible(unsigned int ms); + +#endif /* _LINUX_DELAY_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/device.h b/sys/compat/linuxkpi/common/include/linux/device.h new file mode 100644 index 000000000000..5361236baea8 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/device.h @@ -0,0 +1,562 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_DEVICE_H_ +#define _LINUX_DEVICE_H_ + +#include <linux/err.h> +#include <linux/types.h> +#include <linux/kobject.h> +#include <linux/sysfs.h> +#include <linux/list.h> +#include <linux/compiler.h> +#include <linux/types.h> +#include <linux/module.h> +#include <linux/workqueue.h> +#include <linux/kdev_t.h> +#include <asm/atomic.h> + +#include <sys/bus.h> + +struct device; +struct fwnode_handle; + +struct class { + const char *name; + struct module *owner; + struct kobject kobj; + devclass_t bsdclass; + const struct dev_pm_ops *pm; + const struct attribute_group **dev_groups; + void (*class_release)(struct class *class); + void (*dev_release)(struct device *dev); + char * (*devnode)(struct device *dev, umode_t *mode); +}; + +struct dev_pm_ops { + int (*prepare)(struct device *dev); + int (*suspend)(struct device *dev); + int (*suspend_late)(struct device *dev); + int (*resume)(struct device *dev); + int (*resume_early)(struct device *dev); + int (*freeze)(struct device *dev); + int (*freeze_late)(struct device *dev); + int (*thaw)(struct device *dev); + int (*thaw_early)(struct device *dev); + int (*poweroff)(struct device *dev); + int (*poweroff_late)(struct device *dev); + int (*restore)(struct device *dev); + int (*restore_early)(struct device *dev); + int (*runtime_suspend)(struct device *dev); + int (*runtime_resume)(struct device *dev); + int (*runtime_idle)(struct device *dev); +}; + +struct device_driver { + const char *name; + const struct dev_pm_ops *pm; +}; + +struct device_type { + const char *name; +}; + +struct device { + struct device *parent; + struct list_head irqents; + device_t bsddev; + /* + * The following flag is used to determine if the LinuxKPI is + * responsible for detaching the BSD device or not. If the + * LinuxKPI got the BSD device using devclass_get_device(), it + * must not try to detach or delete it, because it's already + * done somewhere else. + */ + bool bsddev_attached_here; + struct device_driver *driver; + struct device_type *type; + dev_t devt; + struct class *class; + void (*release)(struct device *dev); + struct kobject kobj; + void *dma_priv; + void *driver_data; + unsigned int irq; +#define LINUX_IRQ_INVALID 65535 + unsigned int irq_start; + unsigned int irq_end; + const struct attribute_group **groups; + struct fwnode_handle *fwnode; + + spinlock_t devres_lock; + struct list_head devres_head; +}; + +extern struct device linux_root_device; +extern struct kobject linux_class_root; +extern const struct kobj_type linux_dev_ktype; +extern const struct kobj_type linux_class_ktype; + +struct class_attribute { + struct attribute attr; + ssize_t (*show)(struct class *, struct class_attribute *, char *); + ssize_t (*store)(struct class *, struct class_attribute *, const char *, size_t); + const void *(*namespace)(struct class *, const struct class_attribute *); +}; + +#define CLASS_ATTR(_name, _mode, _show, _store) \ + struct class_attribute class_attr_##_name = \ + { { #_name, NULL, _mode }, _show, _store } + +struct device_attribute { + struct attribute attr; + ssize_t (*show)(struct device *, + struct device_attribute *, char *); + ssize_t (*store)(struct device *, + struct device_attribute *, const char *, + size_t); +}; + +#define DEVICE_ATTR(_name, _mode, _show, _store) \ + struct device_attribute dev_attr_##_name = \ + __ATTR(_name, _mode, _show, _store) +#define DEVICE_ATTR_RO(_name) \ + struct device_attribute dev_attr_##_name = __ATTR_RO(_name) +#define DEVICE_ATTR_WO(_name) \ + struct device_attribute dev_attr_##_name = __ATTR_WO(_name) +#define DEVICE_ATTR_RW(_name) \ + struct device_attribute dev_attr_##_name = __ATTR_RW(_name) + +/* Simple class attribute that is just a static string */ +struct class_attribute_string { + struct class_attribute attr; + char *str; +}; + +static inline ssize_t +show_class_attr_string(struct class *class, + struct class_attribute *attr, char *buf) +{ + struct class_attribute_string *cs; + cs = container_of(attr, struct class_attribute_string, attr); + return snprintf(buf, PAGE_SIZE, "%s\n", cs->str); +} + +/* Currently read-only only */ +#define _CLASS_ATTR_STRING(_name, _mode, _str) \ + { __ATTR(_name, _mode, show_class_attr_string, NULL), _str } +#define CLASS_ATTR_STRING(_name, _mode, _str) \ + struct class_attribute_string class_attr_##_name = \ + _CLASS_ATTR_STRING(_name, _mode, _str) + +#define dev_err(dev, fmt, ...) device_printf((dev)->bsddev, fmt, ##__VA_ARGS__) +#define dev_warn(dev, fmt, ...) device_printf((dev)->bsddev, fmt, ##__VA_ARGS__) +#define dev_info(dev, fmt, ...) device_printf((dev)->bsddev, fmt, ##__VA_ARGS__) +#define dev_notice(dev, fmt, ...) device_printf((dev)->bsddev, fmt, ##__VA_ARGS__) +#define dev_dbg(dev, fmt, ...) do { } while (0) +#define dev_printk(lvl, dev, fmt, ...) \ + device_printf((dev)->bsddev, fmt, ##__VA_ARGS__) + +#define dev_err_once(dev, ...) do { \ + static bool __dev_err_once; \ + if (!__dev_err_once) { \ + __dev_err_once = 1; \ + dev_err(dev, __VA_ARGS__); \ + } \ +} while (0) + +#define dev_err_ratelimited(dev, ...) do { \ + static linux_ratelimit_t __ratelimited; \ + if (linux_ratelimited(&__ratelimited)) \ + dev_err(dev, __VA_ARGS__); \ +} while (0) + +#define dev_warn_ratelimited(dev, ...) do { \ + static linux_ratelimit_t __ratelimited; \ + if (linux_ratelimited(&__ratelimited)) \ + dev_warn(dev, __VA_ARGS__); \ +} while (0) + +static inline void * +dev_get_drvdata(const struct device *dev) +{ + + return dev->driver_data; +} + +static inline void +dev_set_drvdata(struct device *dev, void *data) +{ + + dev->driver_data = data; +} + +static inline struct device * +get_device(struct device *dev) +{ + + if (dev) + kobject_get(&dev->kobj); + + return (dev); +} + +static inline char * +dev_name(const struct device *dev) +{ + + return kobject_name(&dev->kobj); +} + +#define dev_set_name(_dev, _fmt, ...) \ + kobject_set_name(&(_dev)->kobj, (_fmt), ##__VA_ARGS__) + +static inline void +put_device(struct device *dev) +{ + + if (dev) + kobject_put(&dev->kobj); +} + +static inline int +class_register(struct class *class) +{ + + class->bsdclass = devclass_create(class->name); + kobject_init(&class->kobj, &linux_class_ktype); + kobject_set_name(&class->kobj, class->name); + kobject_add(&class->kobj, &linux_class_root, class->name); + + return (0); +} + +static inline void +class_unregister(struct class *class) +{ + + kobject_put(&class->kobj); +} + +static inline struct device *kobj_to_dev(struct kobject *kobj) +{ + return container_of(kobj, struct device, kobj); +} + +/* + * Devices are registered and created for exporting to sysfs. Create + * implies register and register assumes the device fields have been + * setup appropriately before being called. + */ +static inline void +device_initialize(struct device *dev) +{ + device_t bsddev = NULL; + int unit = -1; + + if (dev->devt) { + unit = MINOR(dev->devt); + bsddev = devclass_get_device(dev->class->bsdclass, unit); + dev->bsddev_attached_here = false; + } else if (dev->parent == NULL) { + bsddev = devclass_get_device(dev->class->bsdclass, 0); + dev->bsddev_attached_here = false; + } else { + dev->bsddev_attached_here = true; + } + + if (bsddev == NULL && dev->parent != NULL) { + bsddev = device_add_child(dev->parent->bsddev, + dev->class->kobj.name, unit); + } + + if (bsddev != NULL) + device_set_softc(bsddev, dev); + + dev->bsddev = bsddev; + MPASS(dev->bsddev != NULL); + kobject_init(&dev->kobj, &linux_dev_ktype); + + spin_lock_init(&dev->devres_lock); + INIT_LIST_HEAD(&dev->devres_head); +} + +static inline int +device_add(struct device *dev) +{ + if (dev->bsddev != NULL) { + if (dev->devt == 0) + dev->devt = makedev(0, device_get_unit(dev->bsddev)); + } + kobject_add(&dev->kobj, &dev->class->kobj, dev_name(dev)); + + if (dev->groups) + return (sysfs_create_groups(&dev->kobj, dev->groups)); + + return (0); +} + +static inline void +device_create_release(struct device *dev) +{ + kfree(dev); +} + +static inline struct device * +device_create_groups_vargs(struct class *class, struct device *parent, + dev_t devt, void *drvdata, const struct attribute_group **groups, + const char *fmt, va_list args) +{ + struct device *dev = NULL; + int retval = -ENODEV; + + if (class == NULL || IS_ERR(class)) + goto error; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) { + retval = -ENOMEM; + goto error; + } + + dev->devt = devt; + dev->class = class; + dev->parent = parent; + dev->groups = groups; + dev->release = device_create_release; + /* device_initialize() needs the class and parent to be set */ + device_initialize(dev); + dev_set_drvdata(dev, drvdata); + + retval = kobject_set_name_vargs(&dev->kobj, fmt, args); + if (retval) + goto error; + + retval = device_add(dev); + if (retval) + goto error; + + return dev; + +error: + put_device(dev); + return ERR_PTR(retval); +} + +static inline struct device * +device_create_with_groups(struct class *class, + struct device *parent, dev_t devt, void *drvdata, + const struct attribute_group **groups, const char *fmt, ...) +{ + va_list vargs; + struct device *dev; + + va_start(vargs, fmt); + dev = device_create_groups_vargs(class, parent, devt, drvdata, + groups, fmt, vargs); + va_end(vargs); + return dev; +} + +static inline bool +device_is_registered(struct device *dev) +{ + + return (dev->bsddev != NULL); +} + +static inline int +device_register(struct device *dev) +{ + device_t bsddev = NULL; + int unit = -1; + + if (device_is_registered(dev)) + goto done; + + if (dev->devt) { + unit = MINOR(dev->devt); + bsddev = devclass_get_device(dev->class->bsdclass, unit); + dev->bsddev_attached_here = false; + } else if (dev->parent == NULL) { + bsddev = devclass_get_device(dev->class->bsdclass, 0); + dev->bsddev_attached_here = false; + } else { + dev->bsddev_attached_here = true; + } + if (bsddev == NULL && dev->parent != NULL) { + bsddev = device_add_child(dev->parent->bsddev, + dev->class->kobj.name, unit); + } + if (bsddev != NULL) { + if (dev->devt == 0) + dev->devt = makedev(0, device_get_unit(bsddev)); + device_set_softc(bsddev, dev); + } + dev->bsddev = bsddev; +done: + kobject_init(&dev->kobj, &linux_dev_ktype); + kobject_add(&dev->kobj, &dev->class->kobj, dev_name(dev)); + + sysfs_create_groups(&dev->kobj, dev->class->dev_groups); + + return (0); +} + +static inline void +device_unregister(struct device *dev) +{ + device_t bsddev; + + sysfs_remove_groups(&dev->kobj, dev->class->dev_groups); + + bsddev = dev->bsddev; + dev->bsddev = NULL; + + if (bsddev != NULL && dev->bsddev_attached_here) { + mtx_lock(&Giant); + device_delete_child(device_get_parent(bsddev), bsddev); + mtx_unlock(&Giant); + } + put_device(dev); +} + +static inline void +device_del(struct device *dev) +{ + device_t bsddev; + + bsddev = dev->bsddev; + dev->bsddev = NULL; + + if (bsddev != NULL && dev->bsddev_attached_here) { + mtx_lock(&Giant); + device_delete_child(device_get_parent(bsddev), bsddev); + mtx_unlock(&Giant); + } +} + +struct device *device_create(struct class *class, struct device *parent, + dev_t devt, void *drvdata, const char *fmt, ...); + +static inline void +device_destroy(struct class *class, dev_t devt) +{ + device_t bsddev; + int unit; + + unit = MINOR(devt); + bsddev = devclass_get_device(class->bsdclass, unit); + if (bsddev != NULL) + device_unregister(device_get_softc(bsddev)); +} + +#define dev_pm_set_driver_flags(dev, flags) do { \ +} while (0) + +static inline void +linux_class_kfree(struct class *class) +{ + + kfree(class); +} + +static inline struct class * +class_create(struct module *owner, const char *name) +{ + struct class *class; + int error; + + class = kzalloc(sizeof(*class), M_WAITOK); + class->owner = owner; + class->name = name; + class->class_release = linux_class_kfree; + error = class_register(class); + if (error) { + kfree(class); + return (NULL); + } + + return (class); +} + +static inline void +class_destroy(struct class *class) +{ + + if (class == NULL) + return; + class_unregister(class); +} + +static inline int +device_create_file(struct device *dev, const struct device_attribute *attr) +{ + + if (dev) + return sysfs_create_file(&dev->kobj, &attr->attr); + return -EINVAL; +} + +static inline void +device_remove_file(struct device *dev, const struct device_attribute *attr) +{ + + if (dev) + sysfs_remove_file(&dev->kobj, &attr->attr); +} + +static inline int +class_create_file(struct class *class, const struct class_attribute *attr) +{ + + if (class) + return sysfs_create_file(&class->kobj, &attr->attr); + return -EINVAL; +} + +static inline void +class_remove_file(struct class *class, const struct class_attribute *attr) +{ + + if (class) + sysfs_remove_file(&class->kobj, &attr->attr); +} + +static inline int +dev_to_node(struct device *dev) +{ + return -1; +} + +char *kvasprintf(gfp_t, const char *, va_list); +char *kasprintf(gfp_t, const char *, ...); + +#endif /* _LINUX_DEVICE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/dma-attrs.h b/sys/compat/linuxkpi/common/include/linux/dma-attrs.h new file mode 100644 index 000000000000..5cb75e06cd38 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/dma-attrs.h @@ -0,0 +1,57 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_DMA_ATTR_H_ +#define _LINUX_DMA_ATTR_H_ + +#define DMA_ATTR_WRITE_BARRIER (1 << 0) +#define DMA_ATTR_WEAK_ORDERING (1 << 1) +#define DMA_ATTR_WRITE_COMBINE (1 << 2) +#define DMA_ATTR_NON_CONSISTENT (1 << 3) +#define DMA_ATTR_NO_KERNEL_MAPPING (1 << 4) +#define DMA_ATTR_SKIP_CPU_SYNC (1 << 5) +#define DMA_ATTR_FORCE_CONTIGUOUS (1 << 6) +#define DMA_ATTR_ALLOC_SINGLE_PAGES (1 << 7) +#define DMA_ATTR_NO_WARN (1 << 8) +#define DMA_ATTR_PRIVILEGED (1 << 9) + +struct dma_attrs { + unsigned long flags; +}; + +#define DEFINE_DMA_ATTRS(x) struct dma_attrs x = { } + +static inline void +init_dma_attrs(struct dma_attrs *attrs) +{ + attrs->flags = 0; +} + +#endif /* _LINUX_DMA_ATTR_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h new file mode 100644 index 000000000000..291bd4a8e7ce --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h @@ -0,0 +1,295 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_DMA_MAPPING_H_ +#define _LINUX_DMA_MAPPING_H_ + +#include <linux/types.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/dma-attrs.h> +#include <linux/scatterlist.h> +#include <linux/mm.h> +#include <linux/page.h> +#include <linux/sizes.h> + +#include <sys/systm.h> +#include <sys/malloc.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/pmap.h> + +#include <machine/bus.h> + +enum dma_data_direction { + DMA_BIDIRECTIONAL = 0, + DMA_TO_DEVICE = 1, + DMA_FROM_DEVICE = 2, + DMA_NONE = 3, +}; + +struct dma_map_ops { + void* (*alloc_coherent)(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp); + void (*free_coherent)(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle); + dma_addr_t (*map_page)(struct device *dev, struct page *page, + unsigned long offset, size_t size, enum dma_data_direction dir, + struct dma_attrs *attrs); + void (*unmap_page)(struct device *dev, dma_addr_t dma_handle, + size_t size, enum dma_data_direction dir, struct dma_attrs *attrs); + int (*map_sg)(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, struct dma_attrs *attrs); + void (*unmap_sg)(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, struct dma_attrs *attrs); + void (*sync_single_for_cpu)(struct device *dev, dma_addr_t dma_handle, + size_t size, enum dma_data_direction dir); + void (*sync_single_for_device)(struct device *dev, + dma_addr_t dma_handle, size_t size, enum dma_data_direction dir); + void (*sync_single_range_for_cpu)(struct device *dev, + dma_addr_t dma_handle, unsigned long offset, size_t size, + enum dma_data_direction dir); + void (*sync_single_range_for_device)(struct device *dev, + dma_addr_t dma_handle, unsigned long offset, size_t size, + enum dma_data_direction dir); + void (*sync_sg_for_cpu)(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir); + void (*sync_sg_for_device)(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir); + int (*mapping_error)(struct device *dev, dma_addr_t dma_addr); + int (*dma_supported)(struct device *dev, u64 mask); + int is_phys; +}; + +#define DMA_BIT_MASK(n) ((2ULL << ((n) - 1)) - 1ULL) + +int linux_dma_tag_init(struct device *dev, u64 mask); +void *linux_dma_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag); +dma_addr_t linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len); +void linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t size); +int linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, struct dma_attrs *attrs); +void linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, struct dma_attrs *attrs); + +static inline int +dma_supported(struct device *dev, u64 mask) +{ + + /* XXX busdma takes care of this elsewhere. */ + return (1); +} + +static inline int +dma_set_mask(struct device *dev, u64 dma_mask) +{ + + if (!dev->dma_priv || !dma_supported(dev, dma_mask)) + return -EIO; + + return (linux_dma_tag_init(dev, dma_mask)); +} + +static inline int +dma_set_coherent_mask(struct device *dev, u64 mask) +{ + + if (!dma_supported(dev, mask)) + return -EIO; + /* XXX Currently we don't support a separate coherent mask. */ + return 0; +} + +static inline int +dma_set_mask_and_coherent(struct device *dev, u64 mask) +{ + int r; + + r = dma_set_mask(dev, mask); + if (r == 0) + dma_set_coherent_mask(dev, mask); + return (r); +} + +static inline void * +dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t flag) +{ + return (linux_dma_alloc_coherent(dev, size, dma_handle, flag)); +} + +static inline void * +dma_zalloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t flag) +{ + + return (dma_alloc_coherent(dev, size, dma_handle, flag | __GFP_ZERO)); +} + +static inline void +dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_addr) +{ + + linux_dma_unmap(dev, dma_addr, size); + kmem_free((vm_offset_t)cpu_addr, size); +} + +static inline dma_addr_t +dma_map_single_attrs(struct device *dev, void *ptr, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + + return (linux_dma_map_phys(dev, vtophys(ptr), size)); +} + +static inline void +dma_unmap_single_attrs(struct device *dev, dma_addr_t dma_addr, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + + linux_dma_unmap(dev, dma_addr, size); +} + +static inline dma_addr_t +dma_map_page_attrs(struct device *dev, struct page *page, size_t offset, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + + return (linux_dma_map_phys(dev, VM_PAGE_TO_PHYS(page) + offset, size)); +} + +static inline int +dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + + return (linux_dma_map_sg_attrs(dev, sgl, nents, dir, attrs)); +} + +static inline void +dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + + linux_dma_unmap_sg_attrs(dev, sg, nents, dir, attrs); +} + +static inline dma_addr_t +dma_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, enum dma_data_direction direction) +{ + + return (linux_dma_map_phys(dev, VM_PAGE_TO_PHYS(page) + offset, size)); +} + +static inline void +dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, + enum dma_data_direction direction) +{ + + linux_dma_unmap(dev, dma_address, size); +} + +static inline void +dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, + enum dma_data_direction direction) +{ +} + +static inline void +dma_sync_single(struct device *dev, dma_addr_t addr, size_t size, + enum dma_data_direction dir) +{ + dma_sync_single_for_cpu(dev, addr, size, dir); +} + +static inline void +dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, + size_t size, enum dma_data_direction direction) +{ +} + +static inline void +dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, + enum dma_data_direction direction) +{ +} + +static inline void +dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, + enum dma_data_direction direction) +{ +} + +static inline void +dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle, + unsigned long offset, size_t size, int direction) +{ +} + +static inline void +dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle, + unsigned long offset, size_t size, int direction) +{ +} + +static inline int +dma_mapping_error(struct device *dev, dma_addr_t dma_addr) +{ + + return (dma_addr == 0); +} + +static inline unsigned int dma_set_max_seg_size(struct device *dev, + unsigned int size) +{ + return (0); +} + + +#define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, NULL) +#define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, NULL) +#define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, NULL) +#define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, NULL) + +#define DEFINE_DMA_UNMAP_ADDR(name) dma_addr_t name +#define DEFINE_DMA_UNMAP_LEN(name) __u32 name +#define dma_unmap_addr(p, name) ((p)->name) +#define dma_unmap_addr_set(p, name, v) (((p)->name) = (v)) +#define dma_unmap_len(p, name) ((p)->name) +#define dma_unmap_len_set(p, name, v) (((p)->name) = (v)) + +extern int uma_align_cache; +#define dma_get_cache_alignment() uma_align_cache + +#endif /* _LINUX_DMA_MAPPING_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/dmapool.h b/sys/compat/linuxkpi/common/include/linux/dmapool.h new file mode 100644 index 000000000000..80d74f153747 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/dmapool.h @@ -0,0 +1,86 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_DMAPOOL_H_ +#define _LINUX_DMAPOOL_H_ + +#include <linux/types.h> +#include <linux/io.h> +#include <linux/scatterlist.h> +#include <linux/device.h> +#include <linux/slab.h> + +struct dma_pool; +struct dma_pool *linux_dma_pool_create(char *name, struct device *dev, + size_t size, size_t align, size_t boundary); +void linux_dma_pool_destroy(struct dma_pool *pool); +void *linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, + dma_addr_t *handle); +void linux_dma_pool_free(struct dma_pool *pool, void *vaddr, + dma_addr_t dma_addr); + +static inline struct dma_pool * +dma_pool_create(char *name, struct device *dev, size_t size, + size_t align, size_t boundary) +{ + + return (linux_dma_pool_create(name, dev, size, align, boundary)); +} + +static inline void +dma_pool_destroy(struct dma_pool *pool) +{ + + linux_dma_pool_destroy(pool); +} + +static inline void * +dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, dma_addr_t *handle) +{ + + return (linux_dma_pool_alloc(pool, mem_flags, handle)); +} + +static inline void * +dma_pool_zalloc(struct dma_pool *pool, gfp_t mem_flags, dma_addr_t *handle) +{ + + return (dma_pool_alloc(pool, mem_flags | __GFP_ZERO, handle)); +} + +static inline void +dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr) +{ + + linux_dma_pool_free(pool, vaddr, dma_addr); +} + + +#endif /* _LINUX_DMAPOOL_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/err.h b/sys/compat/linuxkpi/common/include/linux/err.h new file mode 100644 index 000000000000..5f71ebf59b5b --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/err.h @@ -0,0 +1,83 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_ERR_H_ +#define _LINUX_ERR_H_ + +#include <sys/types.h> + +#include <linux/compiler.h> + +#define MAX_ERRNO 4095 + +#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO) + +static inline void * +ERR_PTR(long error) +{ + return (void *)error; +} + +static inline long +PTR_ERR(const void *ptr) +{ + return (long)ptr; +} + +static inline long +IS_ERR(const void *ptr) +{ + return IS_ERR_VALUE((unsigned long)ptr); +} + +static inline long +IS_ERR_OR_NULL(const void *ptr) +{ + return !ptr || IS_ERR_VALUE((unsigned long)ptr); +} + +static inline void * +ERR_CAST(const void *ptr) +{ + return __DECONST(void *, ptr); +} + +static inline int +PTR_ERR_OR_ZERO(const void *ptr) +{ + if (IS_ERR(ptr)) + return PTR_ERR(ptr); + else + return 0; +} + +#define PTR_RET(p) PTR_ERR_OR_ZERO(p) + +#endif /* _LINUX_ERR_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/errno.h b/sys/compat/linuxkpi/common/include/linux/errno.h new file mode 100644 index 000000000000..e824480ab640 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/errno.h @@ -0,0 +1,62 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_ERRNO_H_ +#define _LINUX_ERRNO_H_ + +#include <sys/errno.h> + +#define EBADRQC 56 /* Bad request code */ + +#define ECHRNG EDOM +#define ETIME ETIMEDOUT +#define ECOMM ESTALE +#define ENODATA ECONNREFUSED +#define ENOIOCTLCMD ENOIOCTL +/* Use same value as Linux, because BSD's ERESTART is negative */ +#define ERESTARTSYS 512 +#define ENOTSUPP EOPNOTSUPP +#define ENONET EHOSTDOWN + +#define ERESTARTNOINTR 513 +#define ERESTARTNOHAND 514 +#define ERESTART_RESTARTBLOCK 516 +#define EPROBE_DEFER 517 +#define EOPENSTALE 518 +#define EBADHANDLE 521 +#define ENOTSYNC 522 +#define EBADCOOKIE 523 +#define ETOOSMALL 525 +#define ESERVERFAULT 526 +#define EBADTYPE 527 +#define EJUKEBOX 528 +#define EIOCBQUEUED 529 + +#endif /* _LINUX_ERRNO_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/etherdevice.h b/sys/compat/linuxkpi/common/include/linux/etherdevice.h new file mode 100644 index 000000000000..392f395a5feb --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/etherdevice.h @@ -0,0 +1,117 @@ +/*- + * Copyright (c) 2015-2016 Mellanox Technologies, Ltd. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_ETHERDEVICE +#define _LINUX_ETHERDEVICE + +#include <linux/types.h> + +#include <sys/random.h> +#include <sys/libkern.h> + +#define ETH_MODULE_SFF_8079 1 +#define ETH_MODULE_SFF_8079_LEN 256 +#define ETH_MODULE_SFF_8472 2 +#define ETH_MODULE_SFF_8472_LEN 512 +#define ETH_MODULE_SFF_8636 3 +#define ETH_MODULE_SFF_8636_LEN 256 +#define ETH_MODULE_SFF_8436 4 +#define ETH_MODULE_SFF_8436_LEN 256 + +struct ethtool_eeprom { + u32 offset; + u32 len; +}; + +struct ethtool_modinfo { + u32 type; + u32 eeprom_len; +}; + +static inline bool +is_zero_ether_addr(const u8 * addr) +{ + return ((addr[0] + addr[1] + addr[2] + addr[3] + addr[4] + addr[5]) == 0x00); +} + +static inline bool +is_multicast_ether_addr(const u8 * addr) +{ + return (0x01 & addr[0]); +} + +static inline bool +is_broadcast_ether_addr(const u8 * addr) +{ + return ((addr[0] + addr[1] + addr[2] + addr[3] + addr[4] + addr[5]) == (6 * 0xff)); +} + +static inline bool +is_valid_ether_addr(const u8 * addr) +{ + return !is_multicast_ether_addr(addr) && !is_zero_ether_addr(addr); +} + +static inline void +ether_addr_copy(u8 * dst, const u8 * src) +{ + memcpy(dst, src, 6); +} + +static inline bool +ether_addr_equal(const u8 *pa, const u8 *pb) +{ + return (memcmp(pa, pb, 6) == 0); +} + +static inline bool +ether_addr_equal_64bits(const u8 *pa, const u8 *pb) +{ + return (memcmp(pa, pb, 6) == 0); +} + +static inline void +eth_broadcast_addr(u8 *pa) +{ + memset(pa, 0xff, 6); +} + +static inline void +eth_zero_addr(u8 *pa) +{ + memset(pa, 0, 6); +} + +static inline void +random_ether_addr(u8 * dst) +{ + arc4random_buf(dst, 6); + + dst[0] &= 0xfe; + dst[0] |= 0x02; +} + +#endif /* _LINUX_ETHERDEVICE */ diff --git a/sys/compat/linuxkpi/common/include/linux/export.h b/sys/compat/linuxkpi/common/include/linux/export.h new file mode 100644 index 000000000000..00d51114f774 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/export.h @@ -0,0 +1,33 @@ +/*- + * Copyright (c) 2018 Johannes Lundberg <johalun0@gmail.com> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_EXPORT_H +#define _LINUX_EXPORT_H + +#define EXPORT_SYMBOL(name) +#define EXPORT_SYMBOL_GPL(name) + +#endif diff --git a/sys/compat/linuxkpi/common/include/linux/file.h b/sys/compat/linuxkpi/common/include/linux/file.h new file mode 100644 index 000000000000..21b254211cfd --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/file.h @@ -0,0 +1,190 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_FILE_H_ +#define _LINUX_FILE_H_ + +#include <sys/param.h> +#include <sys/file.h> +#include <sys/filedesc.h> +#include <sys/refcount.h> +#include <sys/capsicum.h> +#include <sys/proc.h> + +#include <linux/fs.h> +#include <linux/slab.h> + +struct linux_file; + +#undef file + +extern struct fileops linuxfileops; + +static inline struct linux_file * +linux_fget(unsigned int fd) +{ + struct file *file; + + /* lookup file pointer by file descriptor index */ + if (fget_unlocked(curthread->td_proc->p_fd, fd, + &cap_no_rights, &file) != 0) + return (NULL); + + /* check if file handle really belongs to us */ + if (file->f_data == NULL || + file->f_ops != &linuxfileops) { + fdrop(file, curthread); + return (NULL); + } + return ((struct linux_file *)file->f_data); +} + +extern void linux_file_free(struct linux_file *filp); + +static inline void +fput(struct linux_file *filp) +{ + if (refcount_release(filp->_file == NULL ? + &filp->f_count : &filp->_file->f_count)) { + linux_file_free(filp); + } +} + +static inline unsigned int +file_count(struct linux_file *filp) +{ + return (filp->_file == NULL ? + filp->f_count : filp->_file->f_count); +} + +static inline void +put_unused_fd(unsigned int fd) +{ + struct file *file; + + if (fget_unlocked(curthread->td_proc->p_fd, fd, + &cap_no_rights, &file) != 0) { + return; + } + /* + * NOTE: We should only get here when the "fd" has not been + * installed, so no need to free the associated Linux file + * structure. + */ + fdclose(curthread, file, fd); + + /* drop extra reference */ + fdrop(file, curthread); +} + +static inline void +fd_install(unsigned int fd, struct linux_file *filp) +{ + struct file *file; + + if (fget_unlocked(curthread->td_proc->p_fd, fd, + &cap_no_rights, &file) != 0) { + filp->_file = NULL; + } else { + filp->_file = file; + finit(file, filp->f_mode, DTYPE_DEV, filp, &linuxfileops); + + /* transfer reference count from "filp" to "file" */ + while (refcount_release(&filp->f_count) == 0) + refcount_acquire(&file->f_count); + } + + /* drop the extra reference */ + fput(filp); +} + +static inline int +get_unused_fd(void) +{ + struct file *file; + int error; + int fd; + + error = falloc(curthread, &file, &fd, 0); + if (error) + return -error; + /* drop the extra reference */ + fdrop(file, curthread); + return fd; +} + +static inline int +get_unused_fd_flags(int flags) +{ + struct file *file; + int error; + int fd; + + error = falloc(curthread, &file, &fd, flags); + if (error) + return -error; + /* drop the extra reference */ + fdrop(file, curthread); + return fd; +} + +extern struct linux_file *linux_file_alloc(void); + +static inline struct linux_file * +alloc_file(int mode, const struct file_operations *fops) +{ + struct linux_file *filp; + + filp = linux_file_alloc(); + filp->f_op = fops; + filp->f_mode = mode; + + return (filp); +} + +struct fd { + struct linux_file *linux_file; +}; + +static inline void fdput(struct fd fd) +{ + fput(fd.linux_file); +} + +static inline struct fd fdget(unsigned int fd) +{ + struct linux_file *f = linux_fget(fd); + return (struct fd){f}; +} + +#define file linux_file +#define fget(...) linux_fget(__VA_ARGS__) + +#endif /* _LINUX_FILE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/fs.h b/sys/compat/linuxkpi/common/include/linux/fs.h new file mode 100644 index 000000000000..f400fc7cbfda --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/fs.h @@ -0,0 +1,305 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2018 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_FS_H_ +#define _LINUX_FS_H_ + +#include <sys/cdefs.h> +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/conf.h> +#include <sys/vnode.h> +#include <sys/file.h> +#include <sys/filedesc.h> +#include <linux/types.h> +#include <linux/wait.h> +#include <linux/semaphore.h> +#include <linux/spinlock.h> +#include <linux/dcache.h> + +struct module; +struct kiocb; +struct iovec; +struct dentry; +struct page; +struct file_lock; +struct pipe_inode_info; +struct vm_area_struct; +struct poll_table_struct; +struct files_struct; +struct pfs_node; +struct linux_cdev; + +#define inode vnode +#define i_cdev v_rdev +#define i_private v_data + +#define S_IRUGO (S_IRUSR | S_IRGRP | S_IROTH) +#define S_IWUGO (S_IWUSR | S_IWGRP | S_IWOTH) + + +typedef struct files_struct *fl_owner_t; + +struct file_operations; + +struct linux_file_wait_queue { + struct wait_queue wq; + struct wait_queue_head *wqh; + atomic_t state; +#define LINUX_FWQ_STATE_INIT 0 +#define LINUX_FWQ_STATE_NOT_READY 1 +#define LINUX_FWQ_STATE_QUEUED 2 +#define LINUX_FWQ_STATE_READY 3 +#define LINUX_FWQ_STATE_MAX 4 +}; + +struct linux_file { + struct file *_file; + const struct file_operations *f_op; + void *private_data; + int f_flags; + int f_mode; /* Just starting mode. */ + struct dentry *f_dentry; + struct dentry f_dentry_store; + struct selinfo f_selinfo; + struct sigio *f_sigio; + struct vnode *f_vnode; +#define f_inode f_vnode + volatile u_int f_count; + + /* anonymous shmem object */ + vm_object_t f_shmem; + + /* kqfilter support */ + int f_kqflags; +#define LINUX_KQ_FLAG_HAS_READ (1 << 0) +#define LINUX_KQ_FLAG_HAS_WRITE (1 << 1) +#define LINUX_KQ_FLAG_NEED_READ (1 << 2) +#define LINUX_KQ_FLAG_NEED_WRITE (1 << 3) + /* protects f_selinfo.si_note */ + spinlock_t f_kqlock; + struct linux_file_wait_queue f_wait_queue; + + /* pointer to associated character device, if any */ + struct linux_cdev *f_cdev; +}; + +#define file linux_file +#define fasync_struct sigio * + +#define fasync_helper(fd, filp, on, queue) \ +({ \ + if ((on)) \ + *(queue) = &(filp)->f_sigio; \ + else \ + *(queue) = NULL; \ + 0; \ +}) + +#define kill_fasync(queue, sig, pollstat) \ +do { \ + if (*(queue) != NULL) \ + pgsigio(*(queue), (sig), 0); \ +} while (0) + +typedef int (*filldir_t)(void *, const char *, int, off_t, u64, unsigned); + +struct file_operations { + struct module *owner; + ssize_t (*read)(struct linux_file *, char __user *, size_t, off_t *); + ssize_t (*write)(struct linux_file *, const char __user *, size_t, off_t *); + unsigned int (*poll) (struct linux_file *, struct poll_table_struct *); + long (*unlocked_ioctl)(struct linux_file *, unsigned int, unsigned long); + long (*compat_ioctl)(struct linux_file *, unsigned int, unsigned long); + int (*mmap)(struct linux_file *, struct vm_area_struct *); + int (*open)(struct inode *, struct file *); + int (*release)(struct inode *, struct linux_file *); + int (*fasync)(int, struct linux_file *, int); + +/* Although not supported in FreeBSD, to align with Linux code + * we are adding llseek() only when it is mapped to no_llseek which returns + * an illegal seek error + */ + off_t (*llseek)(struct linux_file *, off_t, int); +#if 0 + /* We do not support these methods. Don't permit them to compile. */ + loff_t (*llseek)(struct file *, loff_t, int); + ssize_t (*aio_read)(struct kiocb *, const struct iovec *, + unsigned long, loff_t); + ssize_t (*aio_write)(struct kiocb *, const struct iovec *, + unsigned long, loff_t); + int (*readdir)(struct file *, void *, filldir_t); + int (*ioctl)(struct inode *, struct file *, unsigned int, + unsigned long); + int (*flush)(struct file *, fl_owner_t id); + int (*fsync)(struct file *, struct dentry *, int datasync); + int (*aio_fsync)(struct kiocb *, int datasync); + int (*lock)(struct file *, int, struct file_lock *); + ssize_t (*sendpage)(struct file *, struct page *, int, size_t, + loff_t *, int); + unsigned long (*get_unmapped_area)(struct file *, unsigned long, + unsigned long, unsigned long, unsigned long); + int (*check_flags)(int); + int (*flock)(struct file *, int, struct file_lock *); + ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, + loff_t *, size_t, unsigned int); + ssize_t (*splice_read)(struct file *, loff_t *, + struct pipe_inode_info *, size_t, unsigned int); + int (*setlease)(struct file *, long, struct file_lock **); +#endif +}; +#define fops_get(fops) (fops) +#define replace_fops(f, fops) ((f)->f_op = (fops)) + +#define FMODE_READ FREAD +#define FMODE_WRITE FWRITE +#define FMODE_EXEC FEXEC + +int __register_chrdev(unsigned int major, unsigned int baseminor, + unsigned int count, const char *name, + const struct file_operations *fops); +int __register_chrdev_p(unsigned int major, unsigned int baseminor, + unsigned int count, const char *name, + const struct file_operations *fops, uid_t uid, + gid_t gid, int mode); +void __unregister_chrdev(unsigned int major, unsigned int baseminor, + unsigned int count, const char *name); + +static inline void +unregister_chrdev(unsigned int major, const char *name) +{ + + __unregister_chrdev(major, 0, 256, name); +} + +static inline int +register_chrdev(unsigned int major, const char *name, + const struct file_operations *fops) +{ + + return (__register_chrdev(major, 0, 256, name, fops)); +} + +static inline int +register_chrdev_p(unsigned int major, const char *name, + const struct file_operations *fops, uid_t uid, gid_t gid, int mode) +{ + + return (__register_chrdev_p(major, 0, 256, name, fops, uid, gid, mode)); +} + +static inline int +register_chrdev_region(dev_t dev, unsigned range, const char *name) +{ + + return 0; +} + +static inline void +unregister_chrdev_region(dev_t dev, unsigned range) +{ + + return; +} + +static inline int +alloc_chrdev_region(dev_t *dev, unsigned baseminor, unsigned count, + const char *name) +{ + + return 0; +} + +/* No current support for seek op in FreeBSD */ +static inline int +nonseekable_open(struct inode *inode, struct file *filp) +{ + return 0; +} + +extern unsigned int linux_iminor(struct inode *); +#define iminor(...) linux_iminor(__VA_ARGS__) + +static inline struct linux_file * +get_file(struct linux_file *f) +{ + + refcount_acquire(f->_file == NULL ? &f->f_count : &f->_file->f_count); + return (f); +} + +static inline struct inode * +igrab(struct inode *inode) +{ + int error; + + error = vget(inode, 0); + if (error) + return (NULL); + + return (inode); +} + +static inline void +iput(struct inode *inode) +{ + + vrele(inode); +} + +static inline loff_t +no_llseek(struct file *file, loff_t offset, int whence) +{ + + return (-ESPIPE); +} + +static inline loff_t +noop_llseek(struct linux_file *file, loff_t offset, int whence) +{ + + return (file->_file->f_offset); +} + +static inline struct vnode * +file_inode(const struct linux_file *file) +{ + + return (file->f_vnode); +} + +static inline int +call_mmap(struct linux_file *file, struct vm_area_struct *vma) +{ + + return (file->f_op->mmap(file, vma)); +} + +#endif /* _LINUX_FS_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/gfp.h b/sys/compat/linuxkpi/common/include/linux/gfp.h new file mode 100644 index 000000000000..1366e75ad26a --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/gfp.h @@ -0,0 +1,187 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_GFP_H_ +#define _LINUX_GFP_H_ + +#include <sys/cdefs.h> +#include <sys/types.h> +#include <sys/systm.h> +#include <sys/malloc.h> + +#include <linux/page.h> + +#include <vm/vm_param.h> +#include <vm/vm_object.h> +#include <vm/vm_extern.h> +#include <vm/vm_kern.h> + +#define __GFP_NOWARN 0 +#define __GFP_HIGHMEM 0 +#define __GFP_ZERO M_ZERO +#define __GFP_NORETRY 0 +#define __GFP_RECLAIM 0 +#define __GFP_RECLAIMABLE 0 +#define __GFP_RETRY_MAYFAIL 0 +#define __GFP_MOVABLE 0 +#define __GFP_COMP 0 +#define __GFP_KSWAPD_RECLAIM 0 + +#define __GFP_IO 0 +#define __GFP_NO_KSWAPD 0 +#define __GFP_KSWAPD_RECLAIM 0 +#define __GFP_WAIT M_WAITOK +#define __GFP_DMA32 (1U << 24) /* LinuxKPI only */ +#define __GFP_BITS_SHIFT 25 +#define __GFP_BITS_MASK ((1 << __GFP_BITS_SHIFT) - 1) +#define __GFP_NOFAIL M_WAITOK + +#define GFP_NOWAIT M_NOWAIT +#define GFP_ATOMIC (M_NOWAIT | M_USE_RESERVE) +#define GFP_KERNEL M_WAITOK +#define GFP_USER M_WAITOK +#define GFP_HIGHUSER M_WAITOK +#define GFP_HIGHUSER_MOVABLE M_WAITOK +#define GFP_IOFS M_NOWAIT +#define GFP_NOIO M_NOWAIT +#define GFP_DMA32 __GFP_DMA32 +#define GFP_TEMPORARY M_NOWAIT +#define GFP_NATIVE_MASK (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_ZERO) +#define GFP_TRANSHUGE 0 +#define GFP_TRANSHUGE_LIGHT 0 + +CTASSERT((__GFP_DMA32 & GFP_NATIVE_MASK) == 0); +CTASSERT((__GFP_BITS_MASK & GFP_NATIVE_MASK) == GFP_NATIVE_MASK); + +/* + * Resolve a page into a virtual address: + * + * NOTE: This function only works for pages allocated by the kernel. + */ +extern void *linux_page_address(struct page *); + +#define page_address(page) linux_page_address(page) + +/* + * Page management for unmapped pages: + */ +extern vm_page_t linux_alloc_pages(gfp_t flags, unsigned int order); +extern void linux_free_pages(vm_page_t page, unsigned int order); + +static inline struct page * +alloc_page(gfp_t flags) +{ + + return (linux_alloc_pages(flags, 0)); +} + +static inline struct page * +alloc_pages(gfp_t flags, unsigned int order) +{ + + return (linux_alloc_pages(flags, order)); +} + +static inline struct page * +alloc_pages_node(int node_id, gfp_t flags, unsigned int order) +{ + + return (linux_alloc_pages(flags, order)); +} + +static inline void +__free_pages(struct page *page, unsigned int order) +{ + + linux_free_pages(page, order); +} + +static inline void +__free_page(struct page *page) +{ + + linux_free_pages(page, 0); +} + +/* + * Page management for mapped pages: + */ +extern vm_offset_t linux_alloc_kmem(gfp_t flags, unsigned int order); +extern void linux_free_kmem(vm_offset_t, unsigned int order); + +static inline vm_offset_t +get_zeroed_page(gfp_t flags) +{ + + return (linux_alloc_kmem(flags | __GFP_ZERO, 0)); +} + +static inline vm_offset_t +__get_free_page(gfp_t flags) +{ + + return (linux_alloc_kmem(flags, 0)); +} + +static inline vm_offset_t +__get_free_pages(gfp_t flags, unsigned int order) +{ + + return (linux_alloc_kmem(flags, order)); +} + +static inline void +free_pages(uintptr_t addr, unsigned int order) +{ + if (addr == 0) + return; + + linux_free_kmem(addr, order); +} + +static inline void +free_page(uintptr_t addr) +{ + if (addr == 0) + return; + + linux_free_kmem(addr, 0); +} + +static inline bool +gfpflags_allow_blocking(const gfp_t gfp_flags) +{ + return ((gfp_flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK); +} + +#define SetPageReserved(page) do { } while (0) /* NOP */ +#define ClearPageReserved(page) do { } while (0) /* NOP */ + +#endif /* _LINUX_GFP_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/hardirq.h b/sys/compat/linuxkpi/common/include/linux/hardirq.h new file mode 100644 index 000000000000..d338e11c5a6f --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/hardirq.h @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_HARDIRQ_H_ +#define _LINUX_HARDIRQ_H_ + +#include <linux/types.h> +#include <linux/lockdep.h> + +#include <sys/param.h> +#include <sys/bus.h> +#include <sys/interrupt.h> + +#define synchronize_irq(irq) _intr_drain((irq)) + +#endif /* _LINUX_HARDIRQ_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/hrtimer.h b/sys/compat/linuxkpi/common/include/linux/hrtimer.h new file mode 100644 index 000000000000..a5ff7480a271 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/hrtimer.h @@ -0,0 +1,89 @@ +/*- + * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_HRTIMER_H_ +#define _LINUX_HRTIMER_H_ + +#include <sys/_callout.h> +#include <sys/_mutex.h> + +#include <linux/ktime.h> +#include <linux/timer.h> + +enum hrtimer_mode { + HRTIMER_MODE_REL, + HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL, +}; + +enum hrtimer_restart { + HRTIMER_RESTART, + HRTIMER_NORESTART, +}; + +struct hrtimer { + enum hrtimer_restart (*function)(struct hrtimer *); + struct mtx mtx; + struct callout callout; + s64 expires; /* relative time in nanoseconds */ + s64 precision; /* in nanoseconds */ +}; + +#define hrtimer_active(hrtimer) linux_hrtimer_active(hrtimer) +#define hrtimer_cancel(hrtimer) linux_hrtimer_cancel(hrtimer) + +#define hrtimer_init(hrtimer, clock, mode) do { \ + CTASSERT((clock) == CLOCK_MONOTONIC); \ + CTASSERT((mode) == HRTIMER_MODE_REL); \ + linux_hrtimer_init(hrtimer); \ +} while (0) + +#define hrtimer_set_expires(hrtimer, time) \ + linux_hrtimer_set_expires(hrtimer, time) + +#define hrtimer_start(hrtimer, time, mode) do { \ + CTASSERT((mode) == HRTIMER_MODE_REL); \ + linux_hrtimer_start(hrtimer, time); \ +} while (0) + +#define hrtimer_start_range_ns(hrtimer, time, prec, mode) do { \ + CTASSERT((mode) == HRTIMER_MODE_REL); \ + linux_hrtimer_start_range_ns(hrtimer, time, prec); \ +} while (0) + +#define hrtimer_forward_now(hrtimer, interval) do { \ + linux_hrtimer_forward_now(hrtimer, interval); \ +} while (0) + +bool linux_hrtimer_active(struct hrtimer *); +int linux_hrtimer_cancel(struct hrtimer *); +void linux_hrtimer_init(struct hrtimer *); +void linux_hrtimer_set_expires(struct hrtimer *, ktime_t); +void linux_hrtimer_start(struct hrtimer *, ktime_t); +void linux_hrtimer_start_range_ns(struct hrtimer *, ktime_t, int64_t); +void linux_hrtimer_forward_now(struct hrtimer *, ktime_t); + +#endif /* _LINUX_HRTIMER_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/idr.h b/sys/compat/linuxkpi/common/include/linux/idr.h new file mode 100644 index 000000000000..7387032fd989 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/idr.h @@ -0,0 +1,150 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_IDR_H_ +#define _LINUX_IDR_H_ + +#include <sys/param.h> +#include <sys/lock.h> +#include <sys/mutex.h> + +#include <linux/types.h> + +#define IDR_BITS 5 +#define IDR_SIZE (1 << IDR_BITS) +#define IDR_MASK (IDR_SIZE - 1) + +#define MAX_ID_SHIFT ((sizeof(int) * NBBY) - 1) +#define MAX_ID_BIT (1U << MAX_ID_SHIFT) +#define MAX_ID_MASK (MAX_ID_BIT - 1) +#define MAX_LEVEL (MAX_ID_SHIFT + IDR_BITS - 1) / IDR_BITS + +#define MAX_IDR_SHIFT (sizeof(int)*8 - 1) +#define MAX_IDR_BIT (1U << MAX_IDR_SHIFT) +#define MAX_IDR_MASK (MAX_IDR_BIT - 1) + +struct idr_layer { + unsigned long bitmap; + struct idr_layer *ary[IDR_SIZE]; +}; + +struct idr { + struct mtx lock; + struct idr_layer *top; + struct idr_layer *free; + int layers; + int next_cyclic_id; +}; + +/* NOTE: It is the applications responsibility to destroy the IDR */ +#define DEFINE_IDR(name) \ + struct idr name; \ + SYSINIT(name##_idr_sysinit, SI_SUB_DRIVERS, SI_ORDER_FIRST, \ + idr_init, &(name)) + +/* NOTE: It is the applications responsibility to destroy the IDA */ +#define DEFINE_IDA(name) \ + struct ida name; \ + SYSINIT(name##_ida_sysinit, SI_SUB_DRIVERS, SI_ORDER_FIRST, \ + ida_init, &(name)) + +void idr_preload(gfp_t gfp_mask); +void idr_preload_end(void); +void *idr_find(struct idr *idp, int id); +void *idr_get_next(struct idr *idp, int *nextid); +bool idr_is_empty(struct idr *idp); +int idr_pre_get(struct idr *idp, gfp_t gfp_mask); +int idr_get_new(struct idr *idp, void *ptr, int *id); +int idr_get_new_above(struct idr *idp, void *ptr, int starting_id, int *id); +void *idr_replace(struct idr *idp, void *ptr, int id); +void *idr_remove(struct idr *idp, int id); +void idr_remove_all(struct idr *idp); +void idr_destroy(struct idr *idp); +void idr_init(struct idr *idp); +int idr_alloc(struct idr *idp, void *ptr, int start, int end, gfp_t); +int idr_alloc_cyclic(struct idr *idp, void *ptr, int start, int end, gfp_t); +int idr_for_each(struct idr *idp, int (*fn)(int id, void *p, void *data), void *data); + +#define idr_for_each_entry(idp, entry, id) \ + for ((id) = 0; ((entry) = idr_get_next(idp, &(id))) != NULL; ++(id)) + +#define IDA_CHUNK_SIZE 128 /* 128 bytes per chunk */ +#define IDA_BITMAP_LONGS (IDA_CHUNK_SIZE / sizeof(long) - 1) +#define IDA_BITMAP_BITS (IDA_BITMAP_LONGS * sizeof(long) * 8) + +struct ida_bitmap { + long nr_busy; + unsigned long bitmap[IDA_BITMAP_LONGS]; +}; + +struct ida { + struct idr idr; + struct ida_bitmap *free_bitmap; +}; + +int ida_pre_get(struct ida *ida, gfp_t gfp_mask); +int ida_get_new_above(struct ida *ida, int starting_id, int *p_id); +void ida_remove(struct ida *ida, int id); +void ida_destroy(struct ida *ida); +void ida_init(struct ida *ida); + +int ida_simple_get(struct ida *ida, unsigned int start, unsigned int end, + gfp_t gfp_mask); +void ida_simple_remove(struct ida *ida, unsigned int id); + +static inline void +ida_free(struct ida *ida, int id) +{ + + ida_remove(ida, id); +} + +static inline int +ida_get_new(struct ida *ida, int *p_id) +{ + + return (ida_get_new_above(ida, 0, p_id)); +} + +static inline int +ida_alloc_max(struct ida *ida, unsigned int max, gfp_t gfp) +{ + + return (ida_simple_get(ida, 0, max, gfp)); +} + +static inline bool +ida_is_empty(struct ida *ida) +{ + + return (idr_is_empty(&ida->idr)); +} + +#endif /* _LINUX_IDR_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/if_arp.h b/sys/compat/linuxkpi/common/include/linux/if_arp.h new file mode 100644 index 000000000000..9235e2dff9cf --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/if_arp.h @@ -0,0 +1,37 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_IF_ARP_H_ +#define _LINUX_IF_ARP_H_ +#include <sys/param.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <net/if_arp.h> +#endif /* _LINUX_IF_ARP_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/if_ether.h b/sys/compat/linuxkpi/common/include/linux/if_ether.h new file mode 100644 index 000000000000..77e8f1192816 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/if_ether.h @@ -0,0 +1,55 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_IF_ETHER_H_ +#define _LINUX_IF_ETHER_H_ + +#include <linux/types.h> + +#include <net/ethernet.h> + +#define ETH_HLEN ETHER_HDR_LEN /* Total octets in header. */ +#ifndef ETH_ALEN +#define ETH_ALEN ETHER_ADDR_LEN +#endif +#define ETH_FCS_LEN 4 /* Octets in the FCS */ +#define VLAN_HLEN 4 /* The additional bytes (on top of the Ethernet header) + * that VLAN requires. */ +/* + * defined Ethernet Protocol ID's. + */ +#define ETH_P_IP ETHERTYPE_IP +#define ETH_P_IPV6 ETHERTYPE_IPV6 +#define ETH_P_MPLS_UC ETHERTYPE_MPLS +#define ETH_P_MPLS_MC ETHERTYPE_MPLS_MCAST +#define ETH_P_8021Q ETHERTYPE_VLAN +#define ETH_P_8021AD ETHERTYPE_QINQ + +#endif /* _LINUX_IF_ETHER_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/if_vlan.h b/sys/compat/linuxkpi/common/include/linux/if_vlan.h new file mode 100644 index 000000000000..4d8b8926a032 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/if_vlan.h @@ -0,0 +1,59 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_IF_VLAN_H_ +#define _LINUX_IF_VLAN_H_ + +#include <sys/types.h> +#include <sys/socket.h> +#include <net/if.h> +#include <net/ethernet.h> +#include <net/if_var.h> +#include <net/if_vlan_var.h> +#include <net/if_types.h> + +#define VLAN_N_VID 4096 + +static inline int +is_vlan_dev(struct ifnet *ifp) +{ + return (ifp->if_type == IFT_L2VLAN); +} + +static inline uint16_t +vlan_dev_vlan_id(struct ifnet *ifp) +{ + uint16_t vtag; + if (VLAN_TAG(ifp, &vtag) == 0) + return (vtag); + return (0); +} + +#endif /* _LINUX_IF_VLAN_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/in.h b/sys/compat/linuxkpi/common/include/linux/in.h new file mode 100644 index 000000000000..96d66fbae2f0 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/in.h @@ -0,0 +1,47 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_IN_H_ +#define _LINUX_IN_H_ + +#include "opt_inet.h" + +#include <sys/cdefs.h> +#include <sys/param.h> +#include <sys/systm.h> +#include <netinet/in.h> +#include <asm/byteorder.h> + +#define ipv4_is_zeronet(be) IN_ZERONET(ntohl(be)) +#define ipv4_is_loopback(be) IN_LOOPBACK(ntohl(be)) +#define ipv4_is_multicast(be) IN_MULTICAST(ntohl(be)) +#define ipv4_is_lbcast(be) ((be) == INADDR_BROADCAST) + +#endif /* _LINUX_IN_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/in6.h b/sys/compat/linuxkpi/common/include/linux/in6.h new file mode 100644 index 000000000000..c72bfb6f0b88 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/in6.h @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_IN6_H_ +#define _LINUX_IN6_H_ + +#include "opt_inet6.h" + +#endif /* _LINUX_IN6_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/inetdevice.h b/sys/compat/linuxkpi/common/include/linux/inetdevice.h new file mode 100644 index 000000000000..87c880392b71 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/inetdevice.h @@ -0,0 +1,93 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_INETDEVICE_H_ +#define _LINUX_INETDEVICE_H_ + +#include <linux/netdevice.h> + +static inline struct net_device * +ip_dev_find(struct vnet *vnet, uint32_t addr) +{ + struct sockaddr_in sin; + struct epoch_tracker et; + struct ifaddr *ifa; + struct ifnet *ifp; + + memset(&sin, 0, sizeof(sin)); + sin.sin_addr.s_addr = addr; + sin.sin_len = sizeof(sin); + sin.sin_family = AF_INET; + NET_EPOCH_ENTER(et); + CURVNET_SET_QUIET(vnet); + ifa = ifa_ifwithaddr((struct sockaddr *)&sin); + CURVNET_RESTORE(); + if (ifa) { + ifp = ifa->ifa_ifp; + if_ref(ifp); + } else { + ifp = NULL; + } + NET_EPOCH_EXIT(et); + return (ifp); +} + +static inline struct net_device * +ip6_dev_find(struct vnet *vnet, struct in6_addr addr, uint16_t scope_id) +{ + struct sockaddr_in6 sin6; + struct epoch_tracker et; + struct ifaddr *ifa; + struct ifnet *ifp; + + memset(&sin6, 0, sizeof(sin6)); + sin6.sin6_addr = addr; + sin6.sin6_len = sizeof(sin6); + sin6.sin6_family = AF_INET6; + if (IN6_IS_SCOPE_LINKLOCAL(&addr) || + IN6_IS_ADDR_MC_INTFACELOCAL(&addr)) { + /* embed the IPv6 scope ID */ + sin6.sin6_addr.s6_addr16[1] = htons(scope_id); + } + NET_EPOCH_ENTER(et); + CURVNET_SET_QUIET(vnet); + ifa = ifa_ifwithaddr((struct sockaddr *)&sin6); + CURVNET_RESTORE(); + if (ifa != NULL) { + ifp = ifa->ifa_ifp; + if_ref(ifp); + } else { + ifp = NULL; + } + NET_EPOCH_EXIT(et); + return (ifp); +} + +#endif /* _LINUX_INETDEVICE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/interrupt.h b/sys/compat/linuxkpi/common/include/linux/interrupt.h new file mode 100644 index 000000000000..ca26cd24ffda --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/interrupt.h @@ -0,0 +1,215 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_INTERRUPT_H_ +#define _LINUX_INTERRUPT_H_ + +#include <linux/device.h> +#include <linux/pci.h> +#include <linux/irqreturn.h> + +#include <sys/bus.h> +#include <sys/rman.h> + +typedef irqreturn_t (*irq_handler_t)(int, void *); + +#define IRQF_SHARED RF_SHAREABLE + +struct irq_ent { + struct list_head links; + struct device *dev; + struct resource *res; + void *arg; + irqreturn_t (*handler)(int, void *); + void *tag; + unsigned int irq; +}; + +static inline int +linux_irq_rid(struct device *dev, unsigned int irq) +{ + /* check for MSI- or MSIX- interrupt */ + if (irq >= dev->irq_start && irq < dev->irq_end) + return (irq - dev->irq_start + 1); + else + return (0); +} + +extern void linux_irq_handler(void *); + +static inline struct irq_ent * +linux_irq_ent(struct device *dev, unsigned int irq) +{ + struct irq_ent *irqe; + + list_for_each_entry(irqe, &dev->irqents, links) + if (irqe->irq == irq) + return (irqe); + + return (NULL); +} + +static inline int +request_irq(unsigned int irq, irq_handler_t handler, unsigned long flags, + const char *name, void *arg) +{ + struct resource *res; + struct irq_ent *irqe; + struct device *dev; + int error; + int rid; + + dev = linux_pci_find_irq_dev(irq); + if (dev == NULL) + return -ENXIO; + rid = linux_irq_rid(dev, irq); + res = bus_alloc_resource_any(dev->bsddev, SYS_RES_IRQ, &rid, + flags | RF_ACTIVE); + if (res == NULL) + return (-ENXIO); + irqe = kmalloc(sizeof(*irqe), GFP_KERNEL); + irqe->dev = dev; + irqe->res = res; + irqe->arg = arg; + irqe->handler = handler; + irqe->irq = irq; + error = bus_setup_intr(dev->bsddev, res, INTR_TYPE_NET | INTR_MPSAFE, + NULL, linux_irq_handler, irqe, &irqe->tag); + if (error) { + bus_release_resource(dev->bsddev, SYS_RES_IRQ, rid, irqe->res); + kfree(irqe); + return (-error); + } + list_add(&irqe->links, &dev->irqents); + + return 0; +} + +static inline int +enable_irq(unsigned int irq) +{ + struct irq_ent *irqe; + struct device *dev; + + dev = linux_pci_find_irq_dev(irq); + if (dev == NULL) + return -EINVAL; + irqe = linux_irq_ent(dev, irq); + if (irqe == NULL || irqe->tag != NULL) + return -EINVAL; + return -bus_setup_intr(dev->bsddev, irqe->res, INTR_TYPE_NET | INTR_MPSAFE, + NULL, linux_irq_handler, irqe, &irqe->tag); +} + +static inline void +disable_irq(unsigned int irq) +{ + struct irq_ent *irqe; + struct device *dev; + + dev = linux_pci_find_irq_dev(irq); + if (dev == NULL) + return; + irqe = linux_irq_ent(dev, irq); + if (irqe == NULL) + return; + if (irqe->tag != NULL) + bus_teardown_intr(dev->bsddev, irqe->res, irqe->tag); + irqe->tag = NULL; +} + +static inline int +bind_irq_to_cpu(unsigned int irq, int cpu_id) +{ + struct irq_ent *irqe; + struct device *dev; + + dev = linux_pci_find_irq_dev(irq); + if (dev == NULL) + return (-ENOENT); + + irqe = linux_irq_ent(dev, irq); + if (irqe == NULL) + return (-ENOENT); + + return (-bus_bind_intr(dev->bsddev, irqe->res, cpu_id)); +} + +static inline void +free_irq(unsigned int irq, void *device) +{ + struct irq_ent *irqe; + struct device *dev; + int rid; + + dev = linux_pci_find_irq_dev(irq); + if (dev == NULL) + return; + rid = linux_irq_rid(dev, irq); + irqe = linux_irq_ent(dev, irq); + if (irqe == NULL) + return; + if (irqe->tag != NULL) + bus_teardown_intr(dev->bsddev, irqe->res, irqe->tag); + bus_release_resource(dev->bsddev, SYS_RES_IRQ, rid, irqe->res); + list_del(&irqe->links); + kfree(irqe); +} + +/* + * LinuxKPI tasklet support + */ +typedef void tasklet_func_t(unsigned long); + +struct tasklet_struct { + TAILQ_ENTRY(tasklet_struct) entry; + tasklet_func_t *func; + /* Our "state" implementation is different. Avoid same name as Linux. */ + volatile u_int tasklet_state; + atomic_t count; + unsigned long data; +}; + +#define DECLARE_TASKLET(_name, _func, _data) \ +struct tasklet_struct _name = { .func = (_func), .data = (_data) } + +#define tasklet_hi_schedule(t) tasklet_schedule(t) + +extern void tasklet_schedule(struct tasklet_struct *); +extern void tasklet_kill(struct tasklet_struct *); +extern void tasklet_init(struct tasklet_struct *, tasklet_func_t *, + unsigned long data); +extern void tasklet_enable(struct tasklet_struct *); +extern void tasklet_disable(struct tasklet_struct *); +extern int tasklet_trylock(struct tasklet_struct *); +extern void tasklet_unlock(struct tasklet_struct *); +extern void tasklet_unlock_wait(struct tasklet_struct *ts); + +#endif /* _LINUX_INTERRUPT_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/io-mapping.h b/sys/compat/linuxkpi/common/include/linux/io-mapping.h new file mode 100644 index 000000000000..12643be9c6e6 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/io-mapping.h @@ -0,0 +1,117 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_IO_MAPPING_H_ +#define _LINUX_IO_MAPPING_H_ + +#include <sys/types.h> +#include <machine/vm.h> + +#include <linux/types.h> +#include <linux/io.h> +#include <linux/slab.h> + +struct io_mapping { + unsigned long base; + unsigned long size; + void *mem; + vm_memattr_t attr; +}; + +static inline struct io_mapping * +io_mapping_init_wc(struct io_mapping *mapping, resource_size_t base, + unsigned long size) +{ + + mapping->base = base; + mapping->size = size; +#ifdef VM_MEMATTR_WRITE_COMBINING + mapping->mem = ioremap_wc(base, size); + mapping->attr = VM_MEMATTR_WRITE_COMBINING; +#else + mapping->mem = ioremap_nocache(base, size); + mapping->attr = VM_MEMATTR_UNCACHEABLE; +#endif + return (mapping); +} + +static inline struct io_mapping * +io_mapping_create_wc(resource_size_t base, unsigned long size) +{ + struct io_mapping *mapping; + + mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); + if (mapping == NULL) + return (NULL); + return (io_mapping_init_wc(mapping, base, size)); +} + +static inline void +io_mapping_fini(struct io_mapping *mapping) +{ + + iounmap(mapping->mem); +} + +static inline void +io_mapping_free(struct io_mapping *mapping) +{ + + io_mapping_fini(mapping->mem); + kfree(mapping); +} + +static inline void * +io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset) +{ + + return ((char *)mapping->mem + offset); +} + +static inline void +io_mapping_unmap_atomic(void *vaddr) +{ +} + +static inline void * +io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset, + unsigned long size) +{ + + return ((char *)mapping->mem + offset); +} + +static inline void +io_mapping_unmap(void *vaddr) +{ +} + +#endif /* _LINUX_IO_MAPPING_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/io.h b/sys/compat/linuxkpi/common/include/linux/io.h new file mode 100644 index 000000000000..754bc4303ded --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/io.h @@ -0,0 +1,482 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_IO_H_ +#define _LINUX_IO_H_ + +#include <sys/endian.h> +#include <sys/types.h> + +#include <machine/vm.h> + +#include <linux/compiler.h> +#include <linux/types.h> + +/* + * XXX This is all x86 specific. It should be bus space access. + */ + + +/* rmb and wmb are declared in machine/atomic.h, so should be included first. */ +#ifndef __io_br +#define __io_br() __compiler_membar() +#endif + +#ifndef __io_ar +#ifdef rmb +#define __io_ar() rmb() +#else +#define __io_ar() __compiler_membar() +#endif +#endif + +#ifndef __io_bw +#ifdef wmb +#define __io_bw() wmb() +#else +#define __io_bw() __compiler_membar() +#endif +#endif + +#ifndef __io_aw +#define __io_aw() __compiler_membar() +#endif + +/* Access MMIO registers atomically without barriers and byte swapping. */ + +static inline uint8_t +__raw_readb(const volatile void *addr) +{ + return (*(const volatile uint8_t *)addr); +} +#define __raw_readb(addr) __raw_readb(addr) + +static inline void +__raw_writeb(uint8_t v, volatile void *addr) +{ + *(volatile uint8_t *)addr = v; +} +#define __raw_writeb(v, addr) __raw_writeb(v, addr) + +static inline uint16_t +__raw_readw(const volatile void *addr) +{ + return (*(const volatile uint16_t *)addr); +} +#define __raw_readw(addr) __raw_readw(addr) + +static inline void +__raw_writew(uint16_t v, volatile void *addr) +{ + *(volatile uint16_t *)addr = v; +} +#define __raw_writew(v, addr) __raw_writew(v, addr) + +static inline uint32_t +__raw_readl(const volatile void *addr) +{ + return (*(const volatile uint32_t *)addr); +} +#define __raw_readl(addr) __raw_readl(addr) + +static inline void +__raw_writel(uint32_t v, volatile void *addr) +{ + *(volatile uint32_t *)addr = v; +} +#define __raw_writel(v, addr) __raw_writel(v, addr) + +#ifdef __LP64__ +static inline uint64_t +__raw_readq(const volatile void *addr) +{ + return (*(const volatile uint64_t *)addr); +} +#define __raw_readq(addr) __raw_readq(addr) + +static inline void +__raw_writeq(uint64_t v, volatile void *addr) +{ + *(volatile uint64_t *)addr = v; +} +#define __raw_writeq(v, addr) __raw_writeq(v, addr) +#endif + +#define mmiowb() barrier() + +/* Access little-endian MMIO registers atomically with memory barriers. */ + +#undef readb +static inline uint8_t +readb(const volatile void *addr) +{ + uint8_t v; + + __io_br(); + v = *(const volatile uint8_t *)addr; + __io_ar(); + return (v); +} +#define readb(addr) readb(addr) + +#undef writeb +static inline void +writeb(uint8_t v, volatile void *addr) +{ + __io_bw(); + *(volatile uint8_t *)addr = v; + __io_aw(); +} +#define writeb(v, addr) writeb(v, addr) + +#undef readw +static inline uint16_t +readw(const volatile void *addr) +{ + uint16_t v; + + __io_br(); + v = le16toh(__raw_readw(addr)); + __io_ar(); + return (v); +} +#define readw(addr) readw(addr) + +#undef writew +static inline void +writew(uint16_t v, volatile void *addr) +{ + __io_bw(); + __raw_writew(htole16(v), addr); + __io_aw(); +} +#define writew(v, addr) writew(v, addr) + +#undef readl +static inline uint32_t +readl(const volatile void *addr) +{ + uint32_t v; + + __io_br(); + v = le32toh(__raw_readl(addr)); + __io_ar(); + return (v); +} +#define readl(addr) readl(addr) + +#undef writel +static inline void +writel(uint32_t v, volatile void *addr) +{ + __io_bw(); + __raw_writel(htole32(v), addr); + __io_aw(); +} +#define writel(v, addr) writel(v, addr) + +#undef readq +#undef writeq +#ifdef __LP64__ +static inline uint64_t +readq(const volatile void *addr) +{ + uint64_t v; + + __io_br(); + v = le64toh(__raw_readq(addr)); + __io_ar(); + return (v); +} +#define readq(addr) readq(addr) + +static inline void +writeq(uint64_t v, volatile void *addr) +{ + __io_bw(); + __raw_writeq(htole64(v), addr); + __io_aw(); +} +#define writeq(v, addr) writeq(v, addr) +#endif + +/* Access little-endian MMIO registers atomically without memory barriers. */ + +#undef readb_relaxed +static inline uint8_t +readb_relaxed(const volatile void *addr) +{ + return (__raw_readb(addr)); +} +#define readb_relaxed(addr) readb_relaxed(addr) + +#undef writeb_relaxed +static inline void +writeb_relaxed(uint8_t v, volatile void *addr) +{ + __raw_writeb(v, addr); +} +#define writeb_relaxed(v, addr) writeb_relaxed(v, addr) + +#undef readw_relaxed +static inline uint16_t +readw_relaxed(const volatile void *addr) +{ + return (le16toh(__raw_readw(addr))); +} +#define readw_relaxed(addr) readw_relaxed(addr) + +#undef writew_relaxed +static inline void +writew_relaxed(uint16_t v, volatile void *addr) +{ + __raw_writew(htole16(v), addr); +} +#define writew_relaxed(v, addr) writew_relaxed(v, addr) + +#undef readl_relaxed +static inline uint32_t +readl_relaxed(const volatile void *addr) +{ + return (le32toh(__raw_readl(addr))); +} +#define readl_relaxed(addr) readl_relaxed(addr) + +#undef writel_relaxed +static inline void +writel_relaxed(uint32_t v, volatile void *addr) +{ + __raw_writel(htole32(v), addr); +} +#define writel_relaxed(v, addr) writel_relaxed(v, addr) + +#undef readq_relaxed +#undef writeq_relaxed +#ifdef __LP64__ +static inline uint64_t +readq_relaxed(const volatile void *addr) +{ + return (le64toh(__raw_readq(addr))); +} +#define readq_relaxed(addr) readq_relaxed(addr) + +static inline void +writeq_relaxed(uint64_t v, volatile void *addr) +{ + __raw_writeq(htole64(v), addr); +} +#define writeq_relaxed(v, addr) writeq_relaxed(v, addr) +#endif + +/* XXX On Linux ioread and iowrite handle both MMIO and port IO. */ + +#undef ioread8 +static inline uint8_t +ioread8(const volatile void *addr) +{ + return (readb(addr)); +} +#define ioread8(addr) ioread8(addr) + +#undef ioread16 +static inline uint16_t +ioread16(const volatile void *addr) +{ + return (readw(addr)); +} +#define ioread16(addr) ioread16(addr) + +#undef ioread16be +static inline uint16_t +ioread16be(const volatile void *addr) +{ + uint16_t v; + + __io_br(); + v = (be16toh(__raw_readw(addr))); + __io_ar(); + + return (v); +} +#define ioread16be(addr) ioread16be(addr) + +#undef ioread32 +static inline uint32_t +ioread32(const volatile void *addr) +{ + return (readl(addr)); +} +#define ioread32(addr) ioread32(addr) + +#undef ioread32be +static inline uint32_t +ioread32be(const volatile void *addr) +{ + uint32_t v; + + __io_br(); + v = (be32toh(__raw_readl(addr))); + __io_ar(); + + return (v); +} +#define ioread32be(addr) ioread32be(addr) + +#undef iowrite8 +static inline void +iowrite8(uint8_t v, volatile void *addr) +{ + writeb(v, addr); +} +#define iowrite8(v, addr) iowrite8(v, addr) + +#undef iowrite16 +static inline void +iowrite16(uint16_t v, volatile void *addr) +{ + writew(v, addr); +} +#define iowrite16 iowrite16 + +#undef iowrite32 +static inline void +iowrite32(uint32_t v, volatile void *addr) +{ + writel(v, addr); +} +#define iowrite32(v, addr) iowrite32(v, addr) + +#undef iowrite32be +static inline void +iowrite32be(uint32_t v, volatile void *addr) +{ + __io_bw(); + __raw_writel(htobe32(v), addr); + __io_aw(); +} +#define iowrite32be(v, addr) iowrite32be(v, addr) + +#if defined(__i386__) || defined(__amd64__) +static inline void +_outb(u_char data, u_int port) +{ + __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); +} +#endif + +#if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) || defined(__aarch64__) +void *_ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr); +#else +#define _ioremap_attr(...) NULL +#endif + +#ifdef VM_MEMATTR_DEVICE +#define ioremap_nocache(addr, size) \ + _ioremap_attr((addr), (size), VM_MEMATTR_DEVICE) +#define ioremap_wt(addr, size) \ + _ioremap_attr((addr), (size), VM_MEMATTR_DEVICE) +#define ioremap(addr, size) \ + _ioremap_attr((addr), (size), VM_MEMATTR_DEVICE) +#else +#define ioremap_nocache(addr, size) \ + _ioremap_attr((addr), (size), VM_MEMATTR_UNCACHEABLE) +#define ioremap_wt(addr, size) \ + _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_THROUGH) +#define ioremap(addr, size) \ + _ioremap_attr((addr), (size), VM_MEMATTR_UNCACHEABLE) +#endif +#define ioremap_wc(addr, size) \ + _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_COMBINING) +#define ioremap_wb(addr, size) \ + _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_BACK) +void iounmap(void *addr); + +#define memset_io(a, b, c) memset((a), (b), (c)) +#define memcpy_fromio(a, b, c) memcpy((a), (b), (c)) +#define memcpy_toio(a, b, c) memcpy((a), (b), (c)) + +static inline void +__iowrite32_copy(void *to, void *from, size_t count) +{ + uint32_t *src; + uint32_t *dst; + int i; + + for (i = 0, src = from, dst = to; i < count; i++, src++, dst++) + __raw_writel(*src, dst); +} + +static inline void +__iowrite64_copy(void *to, void *from, size_t count) +{ +#ifdef __LP64__ + uint64_t *src; + uint64_t *dst; + int i; + + for (i = 0, src = from, dst = to; i < count; i++, src++, dst++) + __raw_writeq(*src, dst); +#else + __iowrite32_copy(to, from, count * 2); +#endif +} + +enum { + MEMREMAP_WB = 1 << 0, + MEMREMAP_WT = 1 << 1, + MEMREMAP_WC = 1 << 2, +}; + +static inline void * +memremap(resource_size_t offset, size_t size, unsigned long flags) +{ + void *addr = NULL; + + if ((flags & MEMREMAP_WB) && + (addr = ioremap_wb(offset, size)) != NULL) + goto done; + if ((flags & MEMREMAP_WT) && + (addr = ioremap_wt(offset, size)) != NULL) + goto done; + if ((flags & MEMREMAP_WC) && + (addr = ioremap_wc(offset, size)) != NULL) + goto done; +done: + return (addr); +} + +static inline void +memunmap(void *addr) +{ + /* XXX May need to check if this is RAM */ + iounmap(addr); +} + +#endif /* _LINUX_IO_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/ioctl.h b/sys/compat/linuxkpi/common/include/linux/ioctl.h new file mode 100644 index 000000000000..fdedf9c4a179 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/ioctl.h @@ -0,0 +1,38 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_IOCTL_H_ +#define _LINUX_IOCTL_H_ + +#include <sys/ioccom.h> + +#define _IOC_SIZE(cmd) IOCPARM_LEN(cmd) + +#endif /* _LINUX_IOCTL_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/irq_work.h b/sys/compat/linuxkpi/common/include/linux/irq_work.h new file mode 100644 index 000000000000..b44e78230b0d --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/irq_work.h @@ -0,0 +1,52 @@ +/*- + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Emmanuel Vadot under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __LINUX_IRQ_WORK_H__ +#define __LINUX_IRQ_WORK_H__ + +#include <linux/workqueue.h> + +struct irq_work { + struct work_struct work; +}; + +static inline void +init_irq_work(struct irq_work *irqw, void (*func)(struct irq_work *)) +{ + INIT_WORK(&irqw->work, (work_func_t)func); +} + +static inline void +irq_work_queue(struct irq_work *irqw) +{ + schedule_work(&irqw->work); +} + +#endif /* __LINUX_IRQ_WORK_H__ */ diff --git a/sys/compat/linuxkpi/common/include/linux/irqreturn.h b/sys/compat/linuxkpi/common/include/linux/irqreturn.h new file mode 100644 index 000000000000..780fccad017c --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/irqreturn.h @@ -0,0 +1,40 @@ +/*- + * Copyright (c) 2017 Limelight Networks, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_IRQRETURN_H +#define _LINUX_IRQRETURN_H + +typedef enum irqreturn { + IRQ_NONE = 0, + IRQ_HANDLED = 1, + IRQ_WAKE_THREAD = 2 +} irqreturn_t; + +#define IRQ_RETVAL(x) ((x) ? IRQ_HANDLED : IRQ_NONE) + +#endif /* _LINUX_IRQRETURN_H */ diff --git a/sys/compat/linuxkpi/common/include/linux/jhash.h b/sys/compat/linuxkpi/common/include/linux/jhash.h new file mode 100644 index 000000000000..c3904e276778 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/jhash.h @@ -0,0 +1,147 @@ +#ifndef _LINUX_JHASH_H_ +#define _LINUX_JHASH_H_ + +#include <asm/types.h> + +/* jhash.h: Jenkins hash support. + * + * Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net) + * + * http://burtleburtle.net/bob/hash/ + * + * These are the credits from Bob's sources: + * + * lookup2.c, by Bob Jenkins, December 1996, Public Domain. + * hash(), hash2(), hash3, and mix() are externally useful functions. + * Routines to test the hash are included if SELF_TEST is defined. + * You can use this free for any purpose. It has no warranty. + * + * Copyright (C) 2003 David S. Miller (davem@redhat.com) + * + * I've modified Bob's hash to be useful in the Linux kernel, and + * any bugs present are surely my fault. -DaveM + * + * $FreeBSD$ + */ + +/* NOTE: Arguments are modified. */ +#define __jhash_mix(a, b, c) \ +{ \ + a -= b; a -= c; a ^= (c>>13); \ + b -= c; b -= a; b ^= (a<<8); \ + c -= a; c -= b; c ^= (b>>13); \ + a -= b; a -= c; a ^= (c>>12); \ + b -= c; b -= a; b ^= (a<<16); \ + c -= a; c -= b; c ^= (b>>5); \ + a -= b; a -= c; a ^= (c>>3); \ + b -= c; b -= a; b ^= (a<<10); \ + c -= a; c -= b; c ^= (b>>15); \ +} + +/* The golden ration: an arbitrary value */ +#define JHASH_GOLDEN_RATIO 0x9e3779b9 + +/* The most generic version, hashes an arbitrary sequence + * of bytes. No alignment or length assumptions are made about + * the input key. + */ +static inline u32 jhash(const void *key, u32 length, u32 initval) +{ + u32 a, b, c, len; + const u8 *k = key; + + len = length; + a = b = JHASH_GOLDEN_RATIO; + c = initval; + + while (len >= 12) { + a += (k[0] +((u32)k[1]<<8) +((u32)k[2]<<16) +((u32)k[3]<<24)); + b += (k[4] +((u32)k[5]<<8) +((u32)k[6]<<16) +((u32)k[7]<<24)); + c += (k[8] +((u32)k[9]<<8) +((u32)k[10]<<16)+((u32)k[11]<<24)); + + __jhash_mix(a,b,c); + + k += 12; + len -= 12; + } + + c += length; + switch (len) { + case 11: c += ((u32)k[10]<<24); + case 10: c += ((u32)k[9]<<16); + case 9 : c += ((u32)k[8]<<8); + case 8 : b += ((u32)k[7]<<24); + case 7 : b += ((u32)k[6]<<16); + case 6 : b += ((u32)k[5]<<8); + case 5 : b += k[4]; + case 4 : a += ((u32)k[3]<<24); + case 3 : a += ((u32)k[2]<<16); + case 2 : a += ((u32)k[1]<<8); + case 1 : a += k[0]; + }; + + __jhash_mix(a,b,c); + + return c; +} + +/* A special optimized version that handles 1 or more of u32s. + * The length parameter here is the number of u32s in the key. + */ +static inline u32 jhash2(const u32 *k, u32 length, u32 initval) +{ + u32 a, b, c, len; + + a = b = JHASH_GOLDEN_RATIO; + c = initval; + len = length; + + while (len >= 3) { + a += k[0]; + b += k[1]; + c += k[2]; + __jhash_mix(a, b, c); + k += 3; len -= 3; + } + + c += length * 4; + + switch (len) { + case 2 : b += k[1]; + case 1 : a += k[0]; + }; + + __jhash_mix(a,b,c); + + return c; +} + + +/* A special ultra-optimized versions that knows they are hashing exactly + * 3, 2 or 1 word(s). + * + * NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally + * done at the end is not done here. + */ +static inline u32 jhash_3words(u32 a, u32 b, u32 c, u32 initval) +{ + a += JHASH_GOLDEN_RATIO; + b += JHASH_GOLDEN_RATIO; + c += initval; + + __jhash_mix(a, b, c); + + return c; +} + +static inline u32 jhash_2words(u32 a, u32 b, u32 initval) +{ + return jhash_3words(a, b, 0, initval); +} + +static inline u32 jhash_1word(u32 a, u32 initval) +{ + return jhash_3words(a, 0, 0, initval); +} + +#endif /* _LINUX_JHASH_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/jiffies.h b/sys/compat/linuxkpi/common/include/linux/jiffies.h new file mode 100644 index 000000000000..ed2c5f774d23 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/jiffies.h @@ -0,0 +1,166 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_JIFFIES_H_ +#define _LINUX_JIFFIES_H_ + +#include <linux/types.h> +#include <linux/time.h> + +#include <sys/time.h> +#include <sys/kernel.h> +#include <sys/limits.h> + +#define jiffies ticks +#define jiffies_64 ticks +#define jiffies_to_msecs(x) (((int64_t)(int)(x)) * 1000 / hz) + +#define MAX_JIFFY_OFFSET ((INT_MAX >> 1) - 1) + +#define time_after(a, b) ((int)((b) - (a)) < 0) +#define time_after32(a, b) ((int32_t)((uint32_t)(b) - (uint32_t)(a)) < 0) +#define time_before(a, b) time_after(b,a) +#define time_before32(a, b) time_after32(b, a) +#define time_after_eq(a, b) ((int)((a) - (b)) >= 0) +#define time_before_eq(a, b) time_after_eq(b, a) +#define time_in_range(a,b,c) \ + (time_after_eq(a,b) && time_before_eq(a,c)) +#define time_is_after_eq_jiffies(a) time_after_eq(a, jiffies) + +#define HZ hz + +extern uint64_t lkpi_nsec2hz_rem; +extern uint64_t lkpi_nsec2hz_div; +extern uint64_t lkpi_nsec2hz_max; + +extern uint64_t lkpi_usec2hz_rem; +extern uint64_t lkpi_usec2hz_div; +extern uint64_t lkpi_usec2hz_max; + +extern uint64_t lkpi_msec2hz_rem; +extern uint64_t lkpi_msec2hz_div; +extern uint64_t lkpi_msec2hz_max; + +static inline int +timespec_to_jiffies(const struct timespec *ts) +{ + u64 result; + + result = ((u64)hz * ts->tv_sec) + + (((u64)hz * ts->tv_nsec + NSEC_PER_SEC - 1) / NSEC_PER_SEC); + if (result > MAX_JIFFY_OFFSET) + result = MAX_JIFFY_OFFSET; + + return ((int)result); +} + +static inline int +msecs_to_jiffies(uint64_t msec) +{ + uint64_t result; + + if (msec > lkpi_msec2hz_max) + msec = lkpi_msec2hz_max; + result = howmany(msec * lkpi_msec2hz_rem, lkpi_msec2hz_div); + if (result > MAX_JIFFY_OFFSET) + result = MAX_JIFFY_OFFSET; + + return ((int)result); +} + +static inline int +usecs_to_jiffies(uint64_t usec) +{ + uint64_t result; + + if (usec > lkpi_usec2hz_max) + usec = lkpi_usec2hz_max; + result = howmany(usec * lkpi_usec2hz_rem, lkpi_usec2hz_div); + if (result > MAX_JIFFY_OFFSET) + result = MAX_JIFFY_OFFSET; + + return ((int)result); +} + +static inline uint64_t +nsecs_to_jiffies64(uint64_t nsec) +{ + + if (nsec > lkpi_nsec2hz_max) + nsec = lkpi_nsec2hz_max; + return (howmany(nsec * lkpi_nsec2hz_rem, lkpi_nsec2hz_div)); +} + +static inline unsigned long +nsecs_to_jiffies(uint64_t nsec) +{ + + if (sizeof(unsigned long) >= sizeof(uint64_t)) { + if (nsec > lkpi_nsec2hz_max) + nsec = lkpi_nsec2hz_max; + } else { + if (nsec > (lkpi_nsec2hz_max >> 32)) + nsec = (lkpi_nsec2hz_max >> 32); + } + return (howmany(nsec * lkpi_nsec2hz_rem, lkpi_nsec2hz_div)); +} + +static inline uint64_t +jiffies_to_nsecs(int j) +{ + + return ((1000000000ULL / hz) * (uint64_t)(unsigned int)j); +} + +static inline uint64_t +jiffies_to_usecs(int j) +{ + + return ((1000000ULL / hz) * (uint64_t)(unsigned int)j); +} + +static inline uint64_t +get_jiffies_64(void) +{ + + return ((uint64_t)(unsigned int)ticks); +} + +static inline int +linux_timer_jiffies_until(int expires) +{ + int delta = expires - jiffies; + /* guard against already expired values */ + if (delta < 1) + delta = 1; + return (delta); +} + +#endif /* _LINUX_JIFFIES_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/kdev_t.h b/sys/compat/linuxkpi/common/include/linux/kdev_t.h new file mode 100644 index 000000000000..9477ba739e31 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/kdev_t.h @@ -0,0 +1,46 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_KDEV_T_H_ +#define _LINUX_KDEV_T_H_ + +#include <sys/types.h> + +#define MAJOR(dev) major(dev) +#define MINOR(dev) minor(dev) +#define MKDEV(ma, mi) makedev(ma, mi) + +static inline uint16_t +old_encode_dev(dev_t dev) +{ + return ((MAJOR(dev) << 8) | MINOR(dev)); +} + +#endif /* _LINUX_KDEV_T_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/kernel.h b/sys/compat/linuxkpi/common/include/linux/kernel.h new file mode 100644 index 000000000000..098f0493f659 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/kernel.h @@ -0,0 +1,599 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * Copyright (c) 2014-2015 François Tigeot + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_KERNEL_H_ +#define _LINUX_KERNEL_H_ + +#include <sys/cdefs.h> +#include <sys/types.h> +#include <sys/systm.h> +#include <sys/param.h> +#include <sys/libkern.h> +#include <sys/stat.h> +#include <sys/smp.h> +#include <sys/stddef.h> +#include <sys/syslog.h> +#include <sys/time.h> + +#include <linux/bitops.h> +#include <linux/compiler.h> +#include <linux/errno.h> +#include <linux/sched.h> +#include <linux/types.h> +#include <linux/jiffies.h> +#include <linux/log2.h> + +#include <asm/byteorder.h> +#include <asm/uaccess.h> + +#include <machine/stdarg.h> + +#define KERN_CONT "" +#define KERN_EMERG "<0>" +#define KERN_ALERT "<1>" +#define KERN_CRIT "<2>" +#define KERN_ERR "<3>" +#define KERN_WARNING "<4>" +#define KERN_NOTICE "<5>" +#define KERN_INFO "<6>" +#define KERN_DEBUG "<7>" + +#define U8_MAX ((u8)~0U) +#define S8_MAX ((s8)(U8_MAX >> 1)) +#define S8_MIN ((s8)(-S8_MAX - 1)) +#define U16_MAX ((u16)~0U) +#define S16_MAX ((s16)(U16_MAX >> 1)) +#define S16_MIN ((s16)(-S16_MAX - 1)) +#define U32_MAX ((u32)~0U) +#define S32_MAX ((s32)(U32_MAX >> 1)) +#define S32_MIN ((s32)(-S32_MAX - 1)) +#define U64_MAX ((u64)~0ULL) +#define S64_MAX ((s64)(U64_MAX >> 1)) +#define S64_MIN ((s64)(-S64_MAX - 1)) + +#define S8_C(x) x +#define U8_C(x) x ## U +#define S16_C(x) x +#define U16_C(x) x ## U +#define S32_C(x) x +#define U32_C(x) x ## U +#define S64_C(x) x ## LL +#define U64_C(x) x ## ULL + +#define BUILD_BUG() do { CTASSERT(0); } while (0) +#define BUILD_BUG_ON(x) CTASSERT(!(x)) +#define BUILD_BUG_ON_MSG(x, msg) BUILD_BUG_ON(x) +#define BUILD_BUG_ON_NOT_POWER_OF_2(x) BUILD_BUG_ON(!powerof2(x)) +#define BUILD_BUG_ON_INVALID(expr) while (0) { (void)(expr); } + +extern const volatile int lkpi_build_bug_on_zero; +#define BUILD_BUG_ON_ZERO(x) ((x) ? lkpi_build_bug_on_zero : 0) + +#define BUG() panic("BUG at %s:%d", __FILE__, __LINE__) +#define BUG_ON(cond) do { \ + if (cond) { \ + panic("BUG ON %s failed at %s:%d", \ + __stringify(cond), __FILE__, __LINE__); \ + } \ +} while (0) + +#define WARN_ON(cond) ({ \ + bool __ret = (cond); \ + if (__ret) { \ + printf("WARNING %s failed at %s:%d\n", \ + __stringify(cond), __FILE__, __LINE__); \ + linux_dump_stack(); \ + } \ + unlikely(__ret); \ +}) + +#define WARN_ON_SMP(cond) WARN_ON(cond) + +#define WARN_ON_ONCE(cond) ({ \ + static bool __warn_on_once; \ + bool __ret = (cond); \ + if (__ret && !__warn_on_once) { \ + __warn_on_once = 1; \ + printf("WARNING %s failed at %s:%d\n", \ + __stringify(cond), __FILE__, __LINE__); \ + linux_dump_stack(); \ + } \ + unlikely(__ret); \ +}) + +#define oops_in_progress SCHEDULER_STOPPED() + +#undef ALIGN +#define ALIGN(x, y) roundup2((x), (y)) +#undef PTR_ALIGN +#define PTR_ALIGN(p, a) ((__typeof(p))ALIGN((uintptr_t)(p), (a))) +#if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 50000 +/* Moved from linuxkpi_gplv2 */ +#define IS_ALIGNED(x, a) (((x) & ((__typeof(x))(a) - 1)) == 0) +#endif +#define DIV_ROUND_UP(x, n) howmany(x, n) +#define __KERNEL_DIV_ROUND_UP(x, n) howmany(x, n) +#define DIV_ROUND_UP_ULL(x, n) DIV_ROUND_UP((unsigned long long)(x), (n)) +#define DIV_ROUND_DOWN_ULL(x, n) (((unsigned long long)(x) / (n)) * (n)) +#define FIELD_SIZEOF(t, f) sizeof(((t *)0)->f) + +#define printk(...) printf(__VA_ARGS__) +#define vprintk(f, a) vprintf(f, a) + +#define asm __asm + +extern void linux_dump_stack(void); +#define dump_stack() linux_dump_stack() + +struct va_format { + const char *fmt; + va_list *va; +}; + +static inline int +vscnprintf(char *buf, size_t size, const char *fmt, va_list args) +{ + ssize_t ssize = size; + int i; + + i = vsnprintf(buf, size, fmt, args); + + return ((i >= ssize) ? (ssize - 1) : i); +} + +static inline int +scnprintf(char *buf, size_t size, const char *fmt, ...) +{ + va_list args; + int i; + + va_start(args, fmt); + i = vscnprintf(buf, size, fmt, args); + va_end(args); + + return (i); +} + +/* + * The "pr_debug()" and "pr_devel()" macros should produce zero code + * unless DEBUG is defined: + */ +#ifdef DEBUG +extern int linuxkpi_debug; +#define pr_debug(fmt, ...) \ + do { \ + if (linuxkpi_debug) \ + log(LOG_DEBUG, fmt, ##__VA_ARGS__); \ + } while (0) +#define pr_devel(fmt, ...) \ + log(LOG_DEBUG, pr_fmt(fmt), ##__VA_ARGS__) +#else +#define pr_debug(fmt, ...) \ + ({ if (0) log(LOG_DEBUG, fmt, ##__VA_ARGS__); 0; }) +#define pr_devel(fmt, ...) \ + ({ if (0) log(LOG_DEBUG, pr_fmt(fmt), ##__VA_ARGS__); 0; }) +#endif + +#ifndef pr_fmt +#define pr_fmt(fmt) fmt +#endif + +/* + * Print a one-time message (analogous to WARN_ONCE() et al): + */ +#define printk_once(...) do { \ + static bool __print_once; \ + \ + if (!__print_once) { \ + __print_once = true; \ + printk(__VA_ARGS__); \ + } \ +} while (0) + +/* + * Log a one-time message (analogous to WARN_ONCE() et al): + */ +#define log_once(level,...) do { \ + static bool __log_once; \ + \ + if (unlikely(!__log_once)) { \ + __log_once = true; \ + log(level, __VA_ARGS__); \ + } \ +} while (0) + +#define pr_emerg(fmt, ...) \ + log(LOG_EMERG, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_alert(fmt, ...) \ + log(LOG_ALERT, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_crit(fmt, ...) \ + log(LOG_CRIT, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_err(fmt, ...) \ + log(LOG_ERR, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_warning(fmt, ...) \ + log(LOG_WARNING, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_warn(...) \ + pr_warning(__VA_ARGS__) +#define pr_warn_once(fmt, ...) \ + log_once(LOG_WARNING, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_notice(fmt, ...) \ + log(LOG_NOTICE, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_info(fmt, ...) \ + log(LOG_INFO, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_info_once(fmt, ...) \ + log_once(LOG_INFO, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_cont(fmt, ...) \ + printk(KERN_CONT fmt, ##__VA_ARGS__) +#define pr_warn_ratelimited(...) do { \ + static linux_ratelimit_t __ratelimited; \ + if (linux_ratelimited(&__ratelimited)) \ + pr_warning(__VA_ARGS__); \ +} while (0) + +#ifndef WARN +#define WARN(condition, ...) ({ \ + bool __ret_warn_on = (condition); \ + if (unlikely(__ret_warn_on)) \ + pr_warning(__VA_ARGS__); \ + unlikely(__ret_warn_on); \ +}) +#endif + +#ifndef WARN_ONCE +#define WARN_ONCE(condition, ...) ({ \ + bool __ret_warn_on = (condition); \ + if (unlikely(__ret_warn_on)) \ + pr_warn_once(__VA_ARGS__); \ + unlikely(__ret_warn_on); \ +}) +#endif + +#define container_of(ptr, type, member) \ +({ \ + const __typeof(((type *)0)->member) *__p = (ptr); \ + (type *)((uintptr_t)__p - offsetof(type, member)); \ +}) + +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + +#define u64_to_user_ptr(val) ((void *)(uintptr_t)(val)) + +static inline unsigned long long +simple_strtoull(const char *cp, char **endp, unsigned int base) +{ + return (strtouq(cp, endp, base)); +} + +static inline long long +simple_strtoll(const char *cp, char **endp, unsigned int base) +{ + return (strtoq(cp, endp, base)); +} + +static inline unsigned long +simple_strtoul(const char *cp, char **endp, unsigned int base) +{ + return (strtoul(cp, endp, base)); +} + +static inline long +simple_strtol(const char *cp, char **endp, unsigned int base) +{ + return (strtol(cp, endp, base)); +} + +static inline int +kstrtoul(const char *cp, unsigned int base, unsigned long *res) +{ + char *end; + + *res = strtoul(cp, &end, base); + + /* skip newline character, if any */ + if (*end == '\n') + end++; + if (*cp == 0 || *end != 0) + return (-EINVAL); + return (0); +} + +static inline int +kstrtol(const char *cp, unsigned int base, long *res) +{ + char *end; + + *res = strtol(cp, &end, base); + + /* skip newline character, if any */ + if (*end == '\n') + end++; + if (*cp == 0 || *end != 0) + return (-EINVAL); + return (0); +} + +static inline int +kstrtoint(const char *cp, unsigned int base, int *res) +{ + char *end; + long temp; + + *res = temp = strtol(cp, &end, base); + + /* skip newline character, if any */ + if (*end == '\n') + end++; + if (*cp == 0 || *end != 0) + return (-EINVAL); + if (temp != (int)temp) + return (-ERANGE); + return (0); +} + +static inline int +kstrtouint(const char *cp, unsigned int base, unsigned int *res) +{ + char *end; + unsigned long temp; + + *res = temp = strtoul(cp, &end, base); + + /* skip newline character, if any */ + if (*end == '\n') + end++; + if (*cp == 0 || *end != 0) + return (-EINVAL); + if (temp != (unsigned int)temp) + return (-ERANGE); + return (0); +} + +static inline int +kstrtou16(const char *cp, unsigned int base, u16 *res) +{ + char *end; + unsigned long temp; + + *res = temp = strtoul(cp, &end, base); + + /* skip newline character, if any */ + if (*end == '\n') + end++; + if (*cp == 0 || *end != 0) + return (-EINVAL); + if (temp != (u16)temp) + return (-ERANGE); + return (0); +} + +static inline int +kstrtou32(const char *cp, unsigned int base, u32 *res) +{ + char *end; + unsigned long temp; + + *res = temp = strtoul(cp, &end, base); + + /* skip newline character, if any */ + if (*end == '\n') + end++; + if (*cp == 0 || *end != 0) + return (-EINVAL); + if (temp != (u32)temp) + return (-ERANGE); + return (0); +} + +static inline int +kstrtou64(const char *cp, unsigned int base, u64 *res) +{ + char *end; + + *res = strtouq(cp, &end, base); + + /* skip newline character, if any */ + if (*end == '\n') + end++; + if (*cp == 0 || *end != 0) + return (-EINVAL); + return (0); +} + +static inline int +kstrtobool(const char *s, bool *res) +{ + int len; + + if (s == NULL || (len = strlen(s)) == 0 || res == NULL) + return (-EINVAL); + + /* skip newline character, if any */ + if (s[len - 1] == '\n') + len--; + + if (len == 1 && strchr("yY1", s[0]) != NULL) + *res = true; + else if (len == 1 && strchr("nN0", s[0]) != NULL) + *res = false; + else if (strncasecmp("on", s, len) == 0) + *res = true; + else if (strncasecmp("off", s, len) == 0) + *res = false; + else + return (-EINVAL); + + return (0); +} + +static inline int +kstrtobool_from_user(const char __user *s, size_t count, bool *res) +{ + char buf[8] = {}; + + if (count > (sizeof(buf) - 1)) + count = (sizeof(buf) - 1); + + if (copy_from_user(buf, s, count)) + return (-EFAULT); + + return (kstrtobool(buf, res)); +} + +#define min(x, y) ((x) < (y) ? (x) : (y)) +#define max(x, y) ((x) > (y) ? (x) : (y)) + +#define min3(a, b, c) min(a, min(b,c)) +#define max3(a, b, c) max(a, max(b,c)) + +#define min_t(type, x, y) ({ \ + type __min1 = (x); \ + type __min2 = (y); \ + __min1 < __min2 ? __min1 : __min2; }) + +#define max_t(type, x, y) ({ \ + type __max1 = (x); \ + type __max2 = (y); \ + __max1 > __max2 ? __max1 : __max2; }) + +#define offsetofend(t, m) \ + (offsetof(t, m) + sizeof((((t *)0)->m))) + +#define clamp_t(type, _x, min, max) min_t(type, max_t(type, _x, min), max) +#define clamp(x, lo, hi) min( max(x,lo), hi) +#define clamp_val(val, lo, hi) clamp_t(typeof(val), val, lo, hi) + +/* + * This looks more complex than it should be. But we need to + * get the type for the ~ right in round_down (it needs to be + * as wide as the result!), and we want to evaluate the macro + * arguments just once each. + */ +#define __round_mask(x, y) ((__typeof__(x))((y)-1)) +#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1) +#define round_down(x, y) ((x) & ~__round_mask(x, y)) + +#define smp_processor_id() PCPU_GET(cpuid) +#define num_possible_cpus() mp_ncpus +#define num_online_cpus() mp_ncpus + +#if defined(__i386__) || defined(__amd64__) +extern bool linux_cpu_has_clflush; +#define cpu_has_clflush linux_cpu_has_clflush +#endif + +typedef struct pm_message { + int event; +} pm_message_t; + +/* Swap values of a and b */ +#define swap(a, b) do { \ + typeof(a) _swap_tmp = a; \ + a = b; \ + b = _swap_tmp; \ +} while (0) + +#define DIV_ROUND_CLOSEST(x, divisor) (((x) + ((divisor) / 2)) / (divisor)) + +#define DIV_ROUND_CLOSEST_ULL(x, divisor) ({ \ + __typeof(divisor) __d = (divisor); \ + unsigned long long __ret = (x) + (__d) / 2; \ + __ret /= __d; \ + __ret; \ +}) + +static inline uintmax_t +mult_frac(uintmax_t x, uintmax_t multiplier, uintmax_t divisor) +{ + uintmax_t q = (x / divisor); + uintmax_t r = (x % divisor); + + return ((q * multiplier) + ((r * multiplier) / divisor)); +} + +static inline int64_t +abs64(int64_t x) +{ + return (x < 0 ? -x : x); +} + +typedef struct linux_ratelimit { + struct timeval lasttime; + int counter; +} linux_ratelimit_t; + +static inline bool +linux_ratelimited(linux_ratelimit_t *rl) +{ + return (ppsratecheck(&rl->lasttime, &rl->counter, 1)); +} + +#define struct_size(ptr, field, num) ({ \ + const size_t __size = offsetof(__typeof(*(ptr)), field); \ + const size_t __max = (SIZE_MAX - __size) / sizeof((ptr)->field[0]); \ + ((num) > __max) ? SIZE_MAX : (__size + sizeof((ptr)->field[0]) * (num)); \ +}) + +#define __is_constexpr(x) \ + __builtin_constant_p(x) + +/* + * The is_signed() macro below returns true if the passed data type is + * signed. Else false is returned. + */ +#define is_signed(datatype) (((datatype)-1 / (datatype)2) == (datatype)0) + +/* + * The type_max() macro below returns the maxium positive value the + * passed data type can hold. + */ +#define type_max(datatype) ( \ + (sizeof(datatype) >= 8) ? (is_signed(datatype) ? INT64_MAX : UINT64_MAX) : \ + (sizeof(datatype) >= 4) ? (is_signed(datatype) ? INT32_MAX : UINT32_MAX) : \ + (sizeof(datatype) >= 2) ? (is_signed(datatype) ? INT16_MAX : UINT16_MAX) : \ + (is_signed(datatype) ? INT8_MAX : UINT8_MAX) \ +) + +/* + * The type_min() macro below returns the minimum value the passed + * data type can hold. For unsigned types the minimum value is always + * zero. For signed types it may vary. + */ +#define type_min(datatype) ( \ + (sizeof(datatype) >= 8) ? (is_signed(datatype) ? INT64_MIN : 0) : \ + (sizeof(datatype) >= 4) ? (is_signed(datatype) ? INT32_MIN : 0) : \ + (sizeof(datatype) >= 2) ? (is_signed(datatype) ? INT16_MIN : 0) : \ + (is_signed(datatype) ? INT8_MIN : 0) \ +) + +#define TAINT_WARN 0 +#define test_taint(x) (0) + +#endif /* _LINUX_KERNEL_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/kmod.h b/sys/compat/linuxkpi/common/include/linux/kmod.h new file mode 100644 index 000000000000..3fd42fab584c --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/kmod.h @@ -0,0 +1,52 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_KMOD_H_ +#define _LINUX_KMOD_H_ + +#include <sys/types.h> +#include <sys/syscallsubr.h> +#include <sys/refcount.h> +#include <sys/sbuf.h> +#include <machine/stdarg.h> +#include <sys/proc.h> + +#define request_module(...) \ +({\ + char modname[128]; \ + int fileid; \ + snprintf(modname, sizeof(modname), __VA_ARGS__); \ + kern_kldload(curthread, modname, &fileid); \ +}) + +#define request_module_nowait request_module + + +#endif /* _LINUX_KMOD_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/kobject.h b/sys/compat/linuxkpi/common/include/linux/kobject.h new file mode 100644 index 000000000000..bd9e1c4ec6f5 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/kobject.h @@ -0,0 +1,154 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_KOBJECT_H_ +#define _LINUX_KOBJECT_H_ + +#include <machine/stdarg.h> + +#include <linux/kernel.h> +#include <linux/kref.h> +#include <linux/list.h> +#include <linux/slab.h> + +struct kobject; +struct sysctl_oid; + +struct kobj_type { + void (*release)(struct kobject *kobj); + const struct sysfs_ops *sysfs_ops; + struct attribute **default_attrs; +}; + +extern const struct kobj_type linux_kfree_type; + +struct kobject { + struct kobject *parent; + char *name; + struct kref kref; + const struct kobj_type *ktype; + struct list_head entry; + struct sysctl_oid *oidp; +}; + +extern struct kobject *mm_kobj; + +struct attribute { + const char *name; + struct module *owner; + mode_t mode; +}; + +struct kobj_attribute { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr, + char *buf); + ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count); +}; + +static inline void +kobject_init(struct kobject *kobj, const struct kobj_type *ktype) +{ + + kref_init(&kobj->kref); + INIT_LIST_HEAD(&kobj->entry); + kobj->ktype = ktype; + kobj->oidp = NULL; +} + +void linux_kobject_release(struct kref *kref); + +static inline void +kobject_put(struct kobject *kobj) +{ + + if (kobj) + kref_put(&kobj->kref, linux_kobject_release); +} + +static inline struct kobject * +kobject_get(struct kobject *kobj) +{ + + if (kobj) + kref_get(&kobj->kref); + return kobj; +} + +int kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list); +int kobject_add(struct kobject *kobj, struct kobject *parent, + const char *fmt, ...); + +static inline struct kobject * +kobject_create(void) +{ + struct kobject *kobj; + + kobj = kzalloc(sizeof(*kobj), GFP_KERNEL); + if (kobj == NULL) + return (NULL); + kobject_init(kobj, &linux_kfree_type); + + return (kobj); +} + +static inline struct kobject * +kobject_create_and_add(const char *name, struct kobject *parent) +{ + struct kobject *kobj; + + kobj = kobject_create(); + if (kobj == NULL) + return (NULL); + if (kobject_add(kobj, parent, "%s", name) == 0) + return (kobj); + kobject_put(kobj); + + return (NULL); +} + +static inline void +kobject_del(struct kobject *kobj __unused) +{ +} + +static inline char * +kobject_name(const struct kobject *kobj) +{ + + return kobj->name; +} + +int kobject_set_name(struct kobject *kobj, const char *fmt, ...); +int kobject_init_and_add(struct kobject *kobj, const struct kobj_type *ktype, + struct kobject *parent, const char *fmt, ...); + +#endif /* _LINUX_KOBJECT_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/kref.h b/sys/compat/linuxkpi/common/include/linux/kref.h new file mode 100644 index 000000000000..cfc5c0231e23 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/kref.h @@ -0,0 +1,132 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * Copyright (c) 2013 François Tigeot + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_KREF_H_ +#define _LINUX_KREF_H_ + +#include <sys/types.h> +#include <sys/refcount.h> + +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/mutex.h> +#include <linux/refcount.h> + +#include <asm/atomic.h> + +struct kref { + atomic_t refcount; +}; + +static inline void +kref_init(struct kref *kref) +{ + + refcount_init(&kref->refcount.counter, 1); +} + +static inline unsigned int +kref_read(const struct kref *kref) +{ + + return (atomic_read(&kref->refcount)); +} + +static inline void +kref_get(struct kref *kref) +{ + + refcount_acquire(&kref->refcount.counter); +} + +static inline int +kref_put(struct kref *kref, void (*rel)(struct kref *kref)) +{ + + if (refcount_release(&kref->refcount.counter)) { + rel(kref); + return 1; + } + return 0; +} + +static inline int +kref_put_lock(struct kref *kref, void (*rel)(struct kref *kref), + spinlock_t *lock) +{ + + if (refcount_release(&kref->refcount.counter)) { + spin_lock(lock); + rel(kref); + return (1); + } + return (0); +} + + +static inline int +kref_sub(struct kref *kref, unsigned int count, + void (*rel)(struct kref *kref)) +{ + + while (count--) { + if (refcount_release(&kref->refcount.counter)) { + rel(kref); + return 1; + } + } + return 0; +} + +static inline int __must_check +kref_get_unless_zero(struct kref *kref) +{ + + return atomic_add_unless(&kref->refcount, 1, 0); +} + +static inline int kref_put_mutex(struct kref *kref, + void (*release)(struct kref *kref), struct mutex *lock) +{ + WARN_ON(release == NULL); + if (unlikely(!atomic_add_unless(&kref->refcount, -1, 1))) { + mutex_lock(lock); + if (unlikely(!atomic_dec_and_test(&kref->refcount))) { + mutex_unlock(lock); + return 0; + } + release(kref); + return 1; + } + return 0; +} + +#endif /* _LINUX_KREF_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/kthread.h b/sys/compat/linuxkpi/common/include/linux/kthread.h new file mode 100644 index 000000000000..3afd21dc9356 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/kthread.h @@ -0,0 +1,73 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_KTHREAD_H_ +#define _LINUX_KTHREAD_H_ + +#include <linux/sched.h> + +#include <sys/unistd.h> +#include <sys/kthread.h> + +#define kthread_run(fn, data, fmt, ...) ({ \ + struct task_struct *__task; \ + struct thread *__td; \ + \ + if (kthread_add(linux_kthread_fn, NULL, NULL, &__td, \ + RFSTOPPED, 0, fmt, ## __VA_ARGS__)) \ + __task = NULL; \ + else \ + __task = linux_kthread_setup_and_run(__td, fn, data); \ + __task; \ +}) + +int linux_kthread_stop(struct task_struct *); +bool linux_kthread_should_stop_task(struct task_struct *); +bool linux_kthread_should_stop(void); +int linux_kthread_park(struct task_struct *); +void linux_kthread_parkme(void); +bool linux_kthread_should_park(void); +void linux_kthread_unpark(struct task_struct *); +void linux_kthread_fn(void *); +struct task_struct *linux_kthread_setup_and_run(struct thread *, + linux_task_fn_t *, void *arg); +int linux_in_atomic(void); + +#define kthread_stop(task) linux_kthread_stop(task) +#define kthread_should_stop() linux_kthread_should_stop() +#define kthread_should_stop_task(task) linux_kthread_should_stop_task(task) +#define kthread_park(task) linux_kthread_park(task) +#define kthread_parkme() linux_kthread_parkme() +#define kthread_should_park() linux_kthread_should_park() +#define kthread_unpark(task) linux_kthread_unpark(task) + +#define in_atomic() linux_in_atomic() + +#endif /* _LINUX_KTHREAD_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/ktime.h b/sys/compat/linuxkpi/common/include/linux/ktime.h new file mode 100644 index 000000000000..e480de3181a9 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/ktime.h @@ -0,0 +1,259 @@ +/*- + * Copyright (c) 2018 Limelight Networks, Inc. + * Copyright (c) 2014-2018 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_KTIME_H +#define _LINUX_KTIME_H + +#include <linux/types.h> +#include <linux/time.h> +#include <linux/jiffies.h> + +/* time values in nanoseconds */ +typedef s64 ktime_t; + +#define KTIME_MAX ((s64)~((u64)1 << 63)) +#define KTIME_SEC_MAX (KTIME_MAX / NSEC_PER_SEC) + +static inline int64_t +ktime_to_ns(ktime_t kt) +{ + return (kt); +} + +static inline ktime_t +ns_to_ktime(uint64_t nsec) +{ + return (nsec); +} + +static inline int64_t +ktime_divns(const ktime_t kt, int64_t div) +{ + return (kt / div); +} + +static inline int64_t +ktime_to_us(ktime_t kt) +{ + return (ktime_divns(kt, NSEC_PER_USEC)); +} + +static inline int64_t +ktime_to_ms(ktime_t kt) +{ + return (ktime_divns(kt, NSEC_PER_MSEC)); +} + +static inline struct timeval +ktime_to_timeval(ktime_t kt) +{ + return (ns_to_timeval(kt)); +} + +static inline ktime_t +ktime_add_ns(ktime_t kt, int64_t ns) +{ + return (kt + ns); +} + +static inline ktime_t +ktime_add_ms(ktime_t kt, int64_t ms) +{ + + return (ktime_add_ns(kt, ms * NSEC_PER_MSEC)); +} + +static inline ktime_t +ktime_add_us(ktime_t kt, int64_t us) +{ + + return (ktime_add_ns(kt, us * NSEC_PER_USEC)); +} + +static inline ktime_t +ktime_sub_ns(ktime_t kt, int64_t ns) +{ + return (kt - ns); +} + +static inline ktime_t +ktime_set(const long secs, const unsigned long nsecs) +{ + ktime_t retval = {(s64) secs * NSEC_PER_SEC + (s64) nsecs}; + + return (retval); +} + +static inline ktime_t +ktime_sub(ktime_t lhs, ktime_t rhs) +{ + return (lhs - rhs); +} + +static inline int64_t +ktime_us_delta(ktime_t later, ktime_t earlier) +{ + ktime_t diff = ktime_sub(later, earlier); + + return (ktime_to_us(diff)); +} + +static inline int64_t +ktime_ms_delta(ktime_t later, ktime_t earlier) +{ + ktime_t diff = ktime_sub(later, earlier); + + return (ktime_to_ms(diff)); +} + +static inline ktime_t +ktime_add(ktime_t lhs, ktime_t rhs) +{ + return (lhs + rhs); +} + +static inline int +ktime_compare(const ktime_t cmp1, const ktime_t cmp2) +{ + + if (cmp1 > cmp2) + return (1); + else if (cmp1 < cmp2) + return (-1); + else + return (0); +} + +static inline bool +ktime_after(const ktime_t cmp1, const ktime_t cmp2) +{ + + return (ktime_compare(cmp1, cmp2) > 0); +} + +static inline bool +ktime_before(const ktime_t cmp1, const ktime_t cmp2) +{ + + return (ktime_compare(cmp1, cmp2) < 0); +} + +static inline ktime_t +timespec_to_ktime(struct timespec ts) +{ + return (ktime_set(ts.tv_sec, ts.tv_nsec)); +} + +static inline ktime_t +timeval_to_ktime(struct timeval tv) +{ + return (ktime_set(tv.tv_sec, tv.tv_usec * NSEC_PER_USEC)); +} + +static inline int64_t +timespec64_to_ns(struct timespec64 *ts) +{ + return (timespec_to_ns(ts)); +} + +#define ktime_to_timespec(kt) ns_to_timespec(kt) +#define ktime_to_timespec64(kt) ns_to_timespec(kt) +#define ktime_to_timeval(kt) ns_to_timeval(kt) +#define ktime_to_ns(kt) (kt) +#define ktime_get_ts(ts) getnanouptime(ts) +#define ktime_get_ts64(ts) getnanouptime(ts) +#define ktime_get_raw_ts64(ts) getnanouptime(ts) +#define getrawmonotonic64(ts) getnanouptime(ts) + +static inline int64_t +ktime_get_ns(void) +{ + struct timespec ts; + + ktime_get_ts(&ts); + + return (ktime_to_ns(timespec_to_ktime(ts))); +} + +static inline ktime_t +ktime_get(void) +{ + struct timespec ts; + + ktime_get_ts(&ts); + return (timespec_to_ktime(ts)); +} + +static inline ktime_t +ktime_get_boottime(void) +{ + struct timespec ts; + + nanouptime(&ts); + return (timespec_to_ktime(ts)); +} + +static inline ktime_t +ktime_get_real(void) +{ + struct timespec ts; + + nanotime(&ts); + return (timespec_to_ktime(ts)); +} + +static inline ktime_t +ktime_get_real_seconds(void) +{ + struct timespec ts; + + nanotime(&ts); + return (ts.tv_sec); +} + +static inline ktime_t +ktime_get_raw(void) +{ + struct timespec ts; + + nanouptime(&ts); + return (timespec_to_ktime(ts)); +} + +static inline u64 +ktime_get_raw_ns(void) +{ + struct timespec ts; + + nanouptime(&ts); + return (ktime_to_ns(timespec_to_ktime(ts))); +} + +#endif /* _LINUX_KTIME_H */ diff --git a/sys/compat/linuxkpi/common/include/linux/list.h b/sys/compat/linuxkpi/common/include/linux/list.h new file mode 100644 index 000000000000..c16007eab143 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/list.h @@ -0,0 +1,498 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_LIST_H_ +#define _LINUX_LIST_H_ + +/* + * Since LIST_HEAD conflicts with the linux definition we must include any + * FreeBSD header which requires it here so it is resolved with the correct + * definition prior to the undef. + */ +#include <linux/types.h> + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/queue.h> +#include <sys/cpuset.h> +#include <sys/jail.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/proc.h> +#include <sys/vnode.h> +#include <sys/conf.h> +#include <sys/socket.h> +#include <sys/mbuf.h> + +#include <net/bpf.h> +#include <net/if.h> +#include <net/if_var.h> +#include <net/if_types.h> +#include <net/if_media.h> +#include <net/vnet.h> + +#include <netinet/in.h> +#include <netinet/in_pcb.h> +#include <netinet/in_var.h> +#include <netinet/tcp_lro.h> + +#include <netinet6/in6_var.h> +#include <netinet6/nd6.h> + +#include <vm/vm.h> +#include <vm/vm_object.h> +#include <vm/pmap.h> + +#ifndef prefetch +#define prefetch(x) +#endif + +#define LINUX_LIST_HEAD_INIT(name) { &(name), &(name) } + +#define LINUX_LIST_HEAD(name) \ + struct list_head name = LINUX_LIST_HEAD_INIT(name) + +#ifndef LIST_HEAD_DEF +#define LIST_HEAD_DEF +struct list_head { + struct list_head *next; + struct list_head *prev; +}; +#endif + +static inline void +INIT_LIST_HEAD(struct list_head *list) +{ + + list->next = list->prev = list; +} + +static inline int +list_empty(const struct list_head *head) +{ + + return (head->next == head); +} + +static inline int +list_empty_careful(const struct list_head *head) +{ + struct list_head *next = head->next; + + return ((next == head) && (next == head->prev)); +} + +static inline void +__list_del(struct list_head *prev, struct list_head *next) +{ + next->prev = prev; + WRITE_ONCE(prev->next, next); +} + +static inline void +__list_del_entry(struct list_head *entry) +{ + + __list_del(entry->prev, entry->next); +} + +static inline void +list_del(struct list_head *entry) +{ + + __list_del(entry->prev, entry->next); +} + +static inline void +list_replace(struct list_head *old, struct list_head *new) +{ + new->next = old->next; + new->next->prev = new; + new->prev = old->prev; + new->prev->next = new; +} + +static inline void +list_replace_init(struct list_head *old, struct list_head *new) +{ + list_replace(old, new); + INIT_LIST_HEAD(old); +} + +static inline void +linux_list_add(struct list_head *new, struct list_head *prev, + struct list_head *next) +{ + + next->prev = new; + new->next = next; + new->prev = prev; + prev->next = new; +} + +static inline void +list_del_init(struct list_head *entry) +{ + + list_del(entry); + INIT_LIST_HEAD(entry); +} + +#define list_entry(ptr, type, field) container_of(ptr, type, field) + +#define list_first_entry(ptr, type, member) \ + list_entry((ptr)->next, type, member) + +#define list_last_entry(ptr, type, member) \ + list_entry((ptr)->prev, type, member) + +#define list_first_entry_or_null(ptr, type, member) \ + (!list_empty(ptr) ? list_first_entry(ptr, type, member) : NULL) + +#define list_next_entry(ptr, member) \ + list_entry(((ptr)->member.next), typeof(*(ptr)), member) + +#define list_safe_reset_next(ptr, n, member) \ + (n) = list_next_entry(ptr, member) + +#define list_prev_entry(ptr, member) \ + list_entry(((ptr)->member.prev), typeof(*(ptr)), member) + +#define list_for_each(p, head) \ + for (p = (head)->next; p != (head); p = (p)->next) + +#define list_for_each_safe(p, n, head) \ + for (p = (head)->next, n = (p)->next; p != (head); p = n, n = (p)->next) + +#define list_for_each_entry(p, h, field) \ + for (p = list_entry((h)->next, typeof(*p), field); &(p)->field != (h); \ + p = list_entry((p)->field.next, typeof(*p), field)) + +#define list_for_each_entry_safe(p, n, h, field) \ + for (p = list_entry((h)->next, typeof(*p), field), \ + n = list_entry((p)->field.next, typeof(*p), field); &(p)->field != (h);\ + p = n, n = list_entry(n->field.next, typeof(*n), field)) + +#define list_for_each_entry_from(p, h, field) \ + for ( ; &(p)->field != (h); \ + p = list_entry((p)->field.next, typeof(*p), field)) + +#define list_for_each_entry_continue(p, h, field) \ + for (p = list_next_entry((p), field); &(p)->field != (h); \ + p = list_next_entry((p), field)) + +#define list_for_each_entry_safe_from(pos, n, head, member) \ + for (n = list_entry((pos)->member.next, typeof(*pos), member); \ + &(pos)->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + +#define list_for_each_entry_reverse(p, h, field) \ + for (p = list_entry((h)->prev, typeof(*p), field); &(p)->field != (h); \ + p = list_entry((p)->field.prev, typeof(*p), field)) + +#define list_for_each_entry_safe_reverse(p, n, h, field) \ + for (p = list_entry((h)->prev, typeof(*p), field), \ + n = list_entry((p)->field.prev, typeof(*p), field); &(p)->field != (h); \ + p = n, n = list_entry(n->field.prev, typeof(*n), field)) + +#define list_for_each_entry_continue_reverse(p, h, field) \ + for (p = list_entry((p)->field.prev, typeof(*p), field); &(p)->field != (h); \ + p = list_entry((p)->field.prev, typeof(*p), field)) + +#define list_for_each_prev(p, h) for (p = (h)->prev; p != (h); p = (p)->prev) + +#define list_for_each_entry_from_reverse(p, h, field) \ + for (; &p->field != (h); \ + p = list_prev_entry(p, field)) + +static inline void +list_add(struct list_head *new, struct list_head *head) +{ + + linux_list_add(new, head, head->next); +} + +static inline void +list_add_tail(struct list_head *new, struct list_head *head) +{ + + linux_list_add(new, head->prev, head); +} + +static inline void +list_move(struct list_head *list, struct list_head *head) +{ + + list_del(list); + list_add(list, head); +} + +static inline void +list_move_tail(struct list_head *entry, struct list_head *head) +{ + + list_del(entry); + list_add_tail(entry, head); +} + +static inline void +list_bulk_move_tail(struct list_head *head, struct list_head *first, + struct list_head *last) +{ + first->prev->next = last->next; + last->next->prev = first->prev; + head->prev->next = first; + first->prev = head->prev; + last->next = head; + head->prev = last; +} + +static inline void +linux_list_splice(const struct list_head *list, struct list_head *prev, + struct list_head *next) +{ + struct list_head *first; + struct list_head *last; + + if (list_empty(list)) + return; + first = list->next; + last = list->prev; + first->prev = prev; + prev->next = first; + last->next = next; + next->prev = last; +} + +static inline void +list_splice(const struct list_head *list, struct list_head *head) +{ + + linux_list_splice(list, head, head->next); +} + +static inline void +list_splice_tail(struct list_head *list, struct list_head *head) +{ + + linux_list_splice(list, head->prev, head); +} + +static inline void +list_splice_init(struct list_head *list, struct list_head *head) +{ + + linux_list_splice(list, head, head->next); + INIT_LIST_HEAD(list); +} + +static inline void +list_splice_tail_init(struct list_head *list, struct list_head *head) +{ + + linux_list_splice(list, head->prev, head); + INIT_LIST_HEAD(list); +} + +#undef LIST_HEAD +#define LIST_HEAD(name) struct list_head name = { &(name), &(name) } + + +struct hlist_head { + struct hlist_node *first; +}; + +struct hlist_node { + struct hlist_node *next, **pprev; +}; + +#define HLIST_HEAD_INIT { } +#define HLIST_HEAD(name) struct hlist_head name = HLIST_HEAD_INIT +#define INIT_HLIST_HEAD(head) (head)->first = NULL +#define INIT_HLIST_NODE(node) \ +do { \ + (node)->next = NULL; \ + (node)->pprev = NULL; \ +} while (0) + +static inline int +hlist_unhashed(const struct hlist_node *h) +{ + + return !h->pprev; +} + +static inline int +hlist_empty(const struct hlist_head *h) +{ + + return !READ_ONCE(h->first); +} + +static inline void +hlist_del(struct hlist_node *n) +{ + + WRITE_ONCE(*(n->pprev), n->next); + if (n->next != NULL) + n->next->pprev = n->pprev; +} + +static inline void +hlist_del_init(struct hlist_node *n) +{ + + if (hlist_unhashed(n)) + return; + hlist_del(n); + INIT_HLIST_NODE(n); +} + +static inline void +hlist_add_head(struct hlist_node *n, struct hlist_head *h) +{ + + n->next = h->first; + if (h->first != NULL) + h->first->pprev = &n->next; + WRITE_ONCE(h->first, n); + n->pprev = &h->first; +} + +static inline void +hlist_add_before(struct hlist_node *n, struct hlist_node *next) +{ + + n->pprev = next->pprev; + n->next = next; + next->pprev = &n->next; + WRITE_ONCE(*(n->pprev), n); +} + +static inline void +hlist_add_behind(struct hlist_node *n, struct hlist_node *prev) +{ + + n->next = prev->next; + WRITE_ONCE(prev->next, n); + n->pprev = &prev->next; + + if (n->next != NULL) + n->next->pprev = &n->next; +} + +static inline void +hlist_move_list(struct hlist_head *old, struct hlist_head *new) +{ + + new->first = old->first; + if (new->first) + new->first->pprev = &new->first; + old->first = NULL; +} + +static inline int list_is_singular(const struct list_head *head) +{ + return !list_empty(head) && (head->next == head->prev); +} + +static inline void __list_cut_position(struct list_head *list, + struct list_head *head, struct list_head *entry) +{ + struct list_head *new_first = entry->next; + list->next = head->next; + list->next->prev = list; + list->prev = entry; + entry->next = list; + head->next = new_first; + new_first->prev = head; +} + +static inline void list_cut_position(struct list_head *list, + struct list_head *head, struct list_head *entry) +{ + if (list_empty(head)) + return; + if (list_is_singular(head) && + (head->next != entry && head != entry)) + return; + if (entry == head) + INIT_LIST_HEAD(list); + else + __list_cut_position(list, head, entry); +} + +static inline int list_is_first(const struct list_head *list, + const struct list_head *head) +{ + + return (list->prev == head); +} + +static inline int list_is_last(const struct list_head *list, + const struct list_head *head) +{ + return list->next == head; +} + +#define hlist_entry(ptr, type, field) container_of(ptr, type, field) + +#define hlist_for_each(p, head) \ + for (p = (head)->first; p; p = (p)->next) + +#define hlist_for_each_safe(p, n, head) \ + for (p = (head)->first; p && ({ n = (p)->next; 1; }); p = n) + +#define hlist_entry_safe(ptr, type, member) \ + ((ptr) ? hlist_entry(ptr, type, member) : NULL) + +#define hlist_for_each_entry(pos, head, member) \ + for (pos = hlist_entry_safe((head)->first, typeof(*(pos)), member);\ + pos; \ + pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member)) + +#define hlist_for_each_entry_continue(pos, member) \ + for (pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member); \ + (pos); \ + pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member)) + +#define hlist_for_each_entry_from(pos, member) \ + for (; (pos); \ + pos = hlist_entry_safe((pos)->member.next, typeof(*(pos)), member)) + +#define hlist_for_each_entry_safe(pos, n, head, member) \ + for (pos = hlist_entry_safe((head)->first, typeof(*(pos)), member); \ + (pos) && ({ n = (pos)->member.next; 1; }); \ + pos = hlist_entry_safe(n, typeof(*(pos)), member)) + +extern void list_sort(void *priv, struct list_head *head, int (*cmp)(void *priv, + struct list_head *a, struct list_head *b)); + +#endif /* _LINUX_LIST_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/lockdep.h b/sys/compat/linuxkpi/common/include/linux/lockdep.h new file mode 100644 index 000000000000..d2b3d4485dde --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/lockdep.h @@ -0,0 +1,87 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_LOCKDEP_H_ +#define _LINUX_LOCKDEP_H_ + +#include <sys/lock.h> + +struct lock_class_key { +}; + +#define lockdep_set_class(lock, key) +#define lockdep_set_subclass(lock, sub) +#define lockdep_set_class_and_name(lock, key, name) +#define lockdep_set_current_reclaim_state(g) do { } while (0) +#define lockdep_clear_current_reclaim_state() do { } while (0) + +#ifdef INVARIANTS +#define lockdep_assert_held(m) do { \ + struct lock_object *__lock = (struct lock_object *)(m); \ + LOCK_CLASS(__lock)->lc_assert(__lock, LA_LOCKED); \ +} while (0) + +#define lockdep_assert_held_once(m) do { \ + struct lock_object *__lock = (struct lock_object *)(m); \ + LOCK_CLASS(__lock)->lc_assert(__lock, LA_LOCKED | LA_NOTRECURSED); \ +} while (0) + +static __inline bool +lockdep_is_held(void *__m) +{ + struct lock_object *__lock; + struct thread *__td; + + __lock = __m; + return (LOCK_CLASS(__lock)->lc_owner(__lock, &__td) != 0); +} +#define lockdep_is_held_type(_m, _t) lockdep_is_held(_m) + +#else +#define lockdep_assert_held(m) do { } while (0) + +#define lockdep_assert_held_once(m) do { } while (0) + +#define lockdep_is_held(m) 1 +#define lockdep_is_held_type(_m, _t) 1 +#endif + +#define might_lock(m) do { } while (0) +#define might_lock_read(m) do { } while (0) + +#define lock_acquire(...) do { } while (0) +#define lock_release(...) do { } while (0) +#define lock_acquire_shared_recursive(...) do { } while (0) + +#define mutex_acquire(...) do { } while (0) +#define mutex_release(...) do { } while (0) + +#endif /* _LINUX_LOCKDEP_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/log2.h b/sys/compat/linuxkpi/common/include/linux/log2.h new file mode 100644 index 000000000000..d79eedf38176 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/log2.h @@ -0,0 +1,131 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2015 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_LOG2_H_ +#define _LINUX_LOG2_H_ + +#include <linux/types.h> + +#include <sys/libkern.h> + +static inline unsigned long +roundup_pow_of_two(unsigned long x) +{ + return (1UL << flsl(x - 1)); +} + +static inline int +is_power_of_2(unsigned long n) +{ + return (n == roundup_pow_of_two(n)); +} + +static inline unsigned long +rounddown_pow_of_two(unsigned long x) +{ + return (1UL << (flsl(x) - 1)); +} + +#define ilog2(n) \ +( \ + __builtin_constant_p(n) ? ( \ + (n) < 1 ? -1 : \ + (n) & (1ULL << 63) ? 63 : \ + (n) & (1ULL << 62) ? 62 : \ + (n) & (1ULL << 61) ? 61 : \ + (n) & (1ULL << 60) ? 60 : \ + (n) & (1ULL << 59) ? 59 : \ + (n) & (1ULL << 58) ? 58 : \ + (n) & (1ULL << 57) ? 57 : \ + (n) & (1ULL << 56) ? 56 : \ + (n) & (1ULL << 55) ? 55 : \ + (n) & (1ULL << 54) ? 54 : \ + (n) & (1ULL << 53) ? 53 : \ + (n) & (1ULL << 52) ? 52 : \ + (n) & (1ULL << 51) ? 51 : \ + (n) & (1ULL << 50) ? 50 : \ + (n) & (1ULL << 49) ? 49 : \ + (n) & (1ULL << 48) ? 48 : \ + (n) & (1ULL << 47) ? 47 : \ + (n) & (1ULL << 46) ? 46 : \ + (n) & (1ULL << 45) ? 45 : \ + (n) & (1ULL << 44) ? 44 : \ + (n) & (1ULL << 43) ? 43 : \ + (n) & (1ULL << 42) ? 42 : \ + (n) & (1ULL << 41) ? 41 : \ + (n) & (1ULL << 40) ? 40 : \ + (n) & (1ULL << 39) ? 39 : \ + (n) & (1ULL << 38) ? 38 : \ + (n) & (1ULL << 37) ? 37 : \ + (n) & (1ULL << 36) ? 36 : \ + (n) & (1ULL << 35) ? 35 : \ + (n) & (1ULL << 34) ? 34 : \ + (n) & (1ULL << 33) ? 33 : \ + (n) & (1ULL << 32) ? 32 : \ + (n) & (1ULL << 31) ? 31 : \ + (n) & (1ULL << 30) ? 30 : \ + (n) & (1ULL << 29) ? 29 : \ + (n) & (1ULL << 28) ? 28 : \ + (n) & (1ULL << 27) ? 27 : \ + (n) & (1ULL << 26) ? 26 : \ + (n) & (1ULL << 25) ? 25 : \ + (n) & (1ULL << 24) ? 24 : \ + (n) & (1ULL << 23) ? 23 : \ + (n) & (1ULL << 22) ? 22 : \ + (n) & (1ULL << 21) ? 21 : \ + (n) & (1ULL << 20) ? 20 : \ + (n) & (1ULL << 19) ? 19 : \ + (n) & (1ULL << 18) ? 18 : \ + (n) & (1ULL << 17) ? 17 : \ + (n) & (1ULL << 16) ? 16 : \ + (n) & (1ULL << 15) ? 15 : \ + (n) & (1ULL << 14) ? 14 : \ + (n) & (1ULL << 13) ? 13 : \ + (n) & (1ULL << 12) ? 12 : \ + (n) & (1ULL << 11) ? 11 : \ + (n) & (1ULL << 10) ? 10 : \ + (n) & (1ULL << 9) ? 9 : \ + (n) & (1ULL << 8) ? 8 : \ + (n) & (1ULL << 7) ? 7 : \ + (n) & (1ULL << 6) ? 6 : \ + (n) & (1ULL << 5) ? 5 : \ + (n) & (1ULL << 4) ? 4 : \ + (n) & (1ULL << 3) ? 3 : \ + (n) & (1ULL << 2) ? 2 : \ + (n) & (1ULL << 1) ? 1 : \ + (n) & (1ULL << 0) ? 0 : \ + -1) : \ + (sizeof(n) <= 4) ? \ + fls((u32)(n)) - 1 : flsll((u64)(n)) - 1 \ +) + +#define order_base_2(x) ilog2(roundup_pow_of_two(x)) + +#endif /* _LINUX_LOG2_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/math64.h b/sys/compat/linuxkpi/common/include/linux/math64.h new file mode 100644 index 000000000000..a3af81c92c9e --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/math64.h @@ -0,0 +1,103 @@ +/*- + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2014-2015 Mellanox Technologies, Ltd. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_MATH64_H +#define _LINUX_MATH64_H + +#include <sys/stdint.h> + +#define do_div(n, base) ({ \ + uint32_t __base = (base); \ + uint32_t __rem; \ + __rem = ((uint64_t)(n)) % __base; \ + (n) = ((uint64_t)(n)) / __base; \ + __rem; \ +}) + +static inline uint64_t +div64_u64_rem(uint64_t dividend, uint64_t divisor, uint64_t *remainder) +{ + + *remainder = dividend % divisor; + return (dividend / divisor); +} + +static inline int64_t +div64_s64(int64_t dividend, int64_t divisor) +{ + + return (dividend / divisor); +} + +static inline uint64_t +div64_u64(uint64_t dividend, uint64_t divisor) +{ + + return (dividend / divisor); +} + +static inline uint64_t +div_u64_rem(uint64_t dividend, uint32_t divisor, uint32_t *remainder) +{ + + *remainder = dividend % divisor; + return (dividend / divisor); +} + +static inline int64_t +div_s64(int64_t dividend, int32_t divisor) +{ + + return (dividend / divisor); +} + +static inline uint64_t +div_u64(uint64_t dividend, uint32_t divisor) +{ + + return (dividend / divisor); +} + +static inline uint64_t +mul_u32_u32(uint32_t a, uint32_t b) +{ + + return ((uint64_t)a * b); +} + +static inline uint64_t +div64_u64_round_up(uint64_t dividend, uint64_t divisor) +{ + return ((dividend + divisor - 1) / divisor); +} + +#define DIV64_U64_ROUND_UP(...) \ + div64_u64_round_up(__VA_ARGS__) + +#endif /* _LINUX_MATH64_H */ diff --git a/sys/compat/linuxkpi/common/include/linux/miscdevice.h b/sys/compat/linuxkpi/common/include/linux/miscdevice.h new file mode 100644 index 000000000000..26a4a2c049c1 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/miscdevice.h @@ -0,0 +1,76 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_MISCDEVICE_H_ +#define _LINUX_MISCDEVICE_H_ + +#define MISC_DYNAMIC_MINOR -1 + +#include <linux/device.h> +#include <linux/cdev.h> + +struct miscdevice { + const char *name; + struct device *this_device; + const struct file_operations *fops; + struct cdev *cdev; + int minor; + const char *nodename; + umode_t mode; +}; + +extern struct class linux_class_misc; + +static inline int +misc_register(struct miscdevice *misc) +{ + misc->this_device = device_create(&linux_class_misc, + &linux_root_device, 0, misc, misc->name); + misc->cdev = cdev_alloc(); + if (misc->cdev == NULL) + return -ENOMEM; + misc->cdev->owner = THIS_MODULE; + misc->cdev->ops = misc->fops; + kobject_set_name(&misc->cdev->kobj, misc->name); + if (cdev_add(misc->cdev, misc->this_device->devt, 1)) + return -EINVAL; + return (0); +} + +static inline int +misc_deregister(struct miscdevice *misc) +{ + device_destroy(&linux_class_misc, misc->this_device->devt); + cdev_del(misc->cdev); + + return (0); +} + +#endif /* _LINUX_MISCDEVICE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/mm.h b/sys/compat/linuxkpi/common/include/linux/mm.h new file mode 100644 index 000000000000..f6f53afbc8a9 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/mm.h @@ -0,0 +1,266 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot + * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_MM_H_ +#define _LINUX_MM_H_ + +#include <linux/spinlock.h> +#include <linux/gfp.h> +#include <linux/kernel.h> +#include <linux/mm_types.h> +#include <linux/pfn.h> +#include <linux/list.h> + +#include <asm/pgtable.h> + +#define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE) + +/* + * Make sure our LinuxKPI defined virtual memory flags don't conflict + * with the ones defined by FreeBSD: + */ +CTASSERT((VM_PROT_ALL & -(1 << 8)) == 0); + +#define VM_READ VM_PROT_READ +#define VM_WRITE VM_PROT_WRITE +#define VM_EXEC VM_PROT_EXECUTE + +#define VM_PFNINTERNAL (1 << 8) /* FreeBSD private flag to vm_insert_pfn() */ +#define VM_MIXEDMAP (1 << 9) +#define VM_NORESERVE (1 << 10) +#define VM_PFNMAP (1 << 11) +#define VM_IO (1 << 12) +#define VM_MAYWRITE (1 << 13) +#define VM_DONTCOPY (1 << 14) +#define VM_DONTEXPAND (1 << 15) +#define VM_DONTDUMP (1 << 16) + +#define VMA_MAX_PREFAULT_RECORD 1 + +#define FOLL_WRITE (1 << 0) +#define FOLL_FORCE (1 << 1) + +#define VM_FAULT_OOM (1 << 0) +#define VM_FAULT_SIGBUS (1 << 1) +#define VM_FAULT_MAJOR (1 << 2) +#define VM_FAULT_WRITE (1 << 3) +#define VM_FAULT_HWPOISON (1 << 4) +#define VM_FAULT_HWPOISON_LARGE (1 << 5) +#define VM_FAULT_SIGSEGV (1 << 6) +#define VM_FAULT_NOPAGE (1 << 7) +#define VM_FAULT_LOCKED (1 << 8) +#define VM_FAULT_RETRY (1 << 9) +#define VM_FAULT_FALLBACK (1 << 10) + +#define FAULT_FLAG_WRITE (1 << 0) +#define FAULT_FLAG_MKWRITE (1 << 1) +#define FAULT_FLAG_ALLOW_RETRY (1 << 2) +#define FAULT_FLAG_RETRY_NOWAIT (1 << 3) +#define FAULT_FLAG_KILLABLE (1 << 4) +#define FAULT_FLAG_TRIED (1 << 5) +#define FAULT_FLAG_USER (1 << 6) +#define FAULT_FLAG_REMOTE (1 << 7) +#define FAULT_FLAG_INSTRUCTION (1 << 8) + +typedef int (*pte_fn_t)(linux_pte_t *, pgtable_t, unsigned long addr, void *data); + +struct vm_area_struct { + vm_offset_t vm_start; + vm_offset_t vm_end; + vm_offset_t vm_pgoff; + pgprot_t vm_page_prot; + unsigned long vm_flags; + struct mm_struct *vm_mm; + void *vm_private_data; + const struct vm_operations_struct *vm_ops; + struct linux_file *vm_file; + + /* internal operation */ + vm_paddr_t vm_pfn; /* PFN for memory map */ + vm_size_t vm_len; /* length for memory map */ + vm_pindex_t vm_pfn_first; + int vm_pfn_count; + int *vm_pfn_pcount; + vm_object_t vm_obj; + vm_map_t vm_cached_map; + TAILQ_ENTRY(vm_area_struct) vm_entry; +}; + +struct vm_fault { + unsigned int flags; + pgoff_t pgoff; + union { + /* user-space address */ + void *virtual_address; /* < 4.11 */ + unsigned long address; /* >= 4.11 */ + }; + struct page *page; + struct vm_area_struct *vma; +}; + +struct vm_operations_struct { + void (*open) (struct vm_area_struct *); + void (*close) (struct vm_area_struct *); + int (*fault) (struct vm_area_struct *, struct vm_fault *); + int (*access) (struct vm_area_struct *, unsigned long, void *, int, int); +}; + +struct sysinfo { + uint64_t totalram; + uint64_t totalhigh; + uint32_t mem_unit; +}; + +/* + * Compute log2 of the power of two rounded up count of pages + * needed for size bytes. + */ +static inline int +get_order(unsigned long size) +{ + int order; + + size = (size - 1) >> PAGE_SHIFT; + order = 0; + while (size) { + order++; + size >>= 1; + } + return (order); +} + +static inline void * +lowmem_page_address(struct page *page) +{ + return (page_address(page)); +} + +/* + * This only works via memory map operations. + */ +static inline int +io_remap_pfn_range(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned long size, + vm_memattr_t prot) +{ + vma->vm_page_prot = prot; + vma->vm_pfn = pfn; + vma->vm_len = size; + + return (0); +} + +static inline int +apply_to_page_range(struct mm_struct *mm, unsigned long address, + unsigned long size, pte_fn_t fn, void *data) +{ + return (-ENOTSUP); +} + +int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, + unsigned long size); + +static inline int +remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned long size, pgprot_t prot) +{ + return (-ENOTSUP); +} + +static inline unsigned long +vma_pages(struct vm_area_struct *vma) +{ + return ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); +} + +#define offset_in_page(off) ((off) & (PAGE_SIZE - 1)) + +static inline void +set_page_dirty(struct vm_page *page) +{ + vm_page_dirty(page); +} + +static inline void +mark_page_accessed(struct vm_page *page) +{ + vm_page_reference(page); +} + +static inline void +get_page(struct vm_page *page) +{ + vm_page_wire(page); +} + +extern long +get_user_pages(unsigned long start, unsigned long nr_pages, + int gup_flags, struct page **, + struct vm_area_struct **); + +extern int +__get_user_pages_fast(unsigned long start, int nr_pages, int write, + struct page **); + +extern long +get_user_pages_remote(struct task_struct *, struct mm_struct *, + unsigned long start, unsigned long nr_pages, + int gup_flags, struct page **, + struct vm_area_struct **); + +static inline void +put_page(struct vm_page *page) +{ + vm_page_unwire(page, PQ_ACTIVE); +} + +#define copy_highpage(to, from) pmap_copy_page(from, to) + +static inline pgprot_t +vm_get_page_prot(unsigned long vm_flags) +{ + return (vm_flags & VM_PROT_ALL); +} + +static inline vm_page_t +vmalloc_to_page(const void *addr) +{ + vm_paddr_t paddr; + + paddr = pmap_kextract((vm_offset_t)addr); + return (PHYS_TO_VM_PAGE(paddr)); +} + +extern int is_vmalloc_addr(const void *addr); +void si_meminfo(struct sysinfo *si); + +#endif /* _LINUX_MM_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/mm_types.h b/sys/compat/linuxkpi/common/include/linux/mm_types.h new file mode 100644 index 000000000000..4c9eddf500d6 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/mm_types.h @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_MM_TYPES_H_ +#define _LINUX_MM_TYPES_H_ + +#include <linux/types.h> +#include <linux/page.h> +#include <linux/rwsem.h> + +#include <asm/atomic.h> + +typedef int vm_fault_t; + +struct vm_area_struct; +struct task_struct; + +struct mm_struct { + struct vm_area_struct *mmap; + atomic_t mm_count; + atomic_t mm_users; + size_t pinned_vm; + struct rw_semaphore mmap_sem; +}; + +extern void linux_mm_dtor(struct mm_struct *mm); + +static inline void +mmdrop(struct mm_struct *mm) +{ + if (__predict_false(atomic_dec_and_test(&mm->mm_count))) + linux_mm_dtor(mm); +} + +static inline bool +mmget_not_zero(struct mm_struct *mm) +{ + return (atomic_inc_not_zero(&mm->mm_users)); +} + +static inline void +mmput(struct mm_struct *mm) +{ + if (__predict_false(atomic_dec_and_test(&mm->mm_users))) + mmdrop(mm); +} + +static inline void +mmgrab(struct mm_struct *mm) +{ + atomic_inc(&mm->mm_count); +} + +extern struct mm_struct *linux_get_task_mm(struct task_struct *); +#define get_task_mm(task) linux_get_task_mm(task) + +#endif /* _LINUX_MM_TYPES_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/module.h b/sys/compat/linuxkpi/common/include/linux/module.h new file mode 100644 index 000000000000..a5a8dd540dd6 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/module.h @@ -0,0 +1,105 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_MODULE_H_ +#define _LINUX_MODULE_H_ + +#include <sys/cdefs.h> +#include <sys/types.h> +#include <sys/module.h> + +#include <linux/list.h> +#include <linux/compiler.h> +#include <linux/kmod.h> +#include <linux/kobject.h> +#include <linux/sysfs.h> +#include <linux/moduleparam.h> +#include <linux/slab.h> +#include <linux/export.h> + +#define MODULE_AUTHOR(name) +#define MODULE_DESCRIPTION(name) +#define MODULE_LICENSE(name) +#define MODULE_INFO(tag, info) +#define MODULE_FIRMWARE(firmware) + +#define THIS_MODULE ((struct module *)0) + +#define __MODULE_STRING(x) __stringify(x) + +/* OFED pre-module initialization */ +#define SI_SUB_OFED_PREINIT (SI_SUB_ROOT_CONF - 2) +/* OFED default module initialization */ +#define SI_SUB_OFED_MODINIT (SI_SUB_ROOT_CONF - 1) + +#include <sys/linker.h> + +static inline void +_module_run(void *arg) +{ + void (*fn)(void); +#ifdef OFED_DEBUG_INIT + char name[1024]; + caddr_t pc; + long offset; + + pc = (caddr_t)arg; + if (linker_search_symbol_name(pc, name, sizeof(name), &offset) != 0) + printf("Running ??? (%p)\n", pc); + else + printf("Running %s (%p)\n", name, pc); +#endif + fn = arg; + fn(); +} + +#define module_init(fn) \ + SYSINIT(fn, SI_SUB_OFED_MODINIT, SI_ORDER_FIRST, _module_run, (fn)) + +#define module_exit(fn) \ + SYSUNINIT(fn, SI_SUB_OFED_MODINIT, SI_ORDER_SECOND, _module_run, (fn)) + +/* + * The following two macros are a workaround for not having a module + * load and unload order resolver: + */ +#define module_init_order(fn, order) \ + SYSINIT(fn, SI_SUB_OFED_MODINIT, (order), _module_run, (fn)) + +#define module_exit_order(fn, order) \ + SYSUNINIT(fn, SI_SUB_OFED_MODINIT, (order), _module_run, (fn)) + +#define module_get(module) +#define module_put(module) +#define try_module_get(module) 1 + +#define postcore_initcall(fn) module_init(fn) + +#endif /* _LINUX_MODULE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/moduleparam.h b/sys/compat/linuxkpi/common/include/linux/moduleparam.h new file mode 100644 index 000000000000..3a16c20fd92e --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/moduleparam.h @@ -0,0 +1,133 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_MODULEPARAM_H_ +#define _LINUX_MODULEPARAM_H_ + +#include <sys/types.h> +#include <sys/sysctl.h> + +#include <linux/types.h> + +#ifndef LINUXKPI_PARAM_PARENT +#define LINUXKPI_PARAM_PARENT _compat_linuxkpi +#endif + +#ifndef LINUXKPI_PARAM_PREFIX +#define LINUXKPI_PARAM_PREFIX /* empty prefix is the default */ +#endif + +#ifndef LINUXKPI_PARAM_PERM +#define LINUXKPI_PARAM_PERM(perm) (((perm) & 0222) ? CTLFLAG_RWTUN : CTLFLAG_RDTUN) +#endif + +#define LINUXKPI_PARAM_CONCAT_SUB(a,b,c,d) a##b##c##d +#define LINUXKPI_PARAM_CONCAT(...) LINUXKPI_PARAM_CONCAT_SUB(__VA_ARGS__) +#define LINUXKPI_PARAM_PASS(...) __VA_ARGS__ +#define LINUXKPI_PARAM_DESC(name) LINUXKPI_PARAM_CONCAT(linuxkpi_,LINUXKPI_PARAM_PREFIX,name,_desc) +#define LINUXKPI_PARAM_NAME(name) LINUXKPI_PARAM_CONCAT(LINUXKPI_PARAM_PREFIX,name,,) + +#define LINUXKPI_PARAM_bool(name, var, perm) \ + extern const char LINUXKPI_PARAM_DESC(name)[]; \ + LINUXKPI_PARAM_PASS(SYSCTL_BOOL(LINUXKPI_PARAM_PARENT, OID_AUTO,\ + LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), &(var), 0, \ + LINUXKPI_PARAM_DESC(name))) + +#define LINUXKPI_PARAM_byte(name, var, perm) \ + extern const char LINUXKPI_PARAM_DESC(name)[]; \ + LINUXKPI_PARAM_PASS(SYSCTL_U8(LINUXKPI_PARAM_PARENT, OID_AUTO, \ + LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), &(var), 0, \ + LINUXKPI_PARAM_DESC(name))) + +#define LINUXKPI_PARAM_short(name, var, perm) \ + extern const char LINUXKPI_PARAM_DESC(name)[]; \ + LINUXKPI_PARAM_PASS(SYSCTL_S16(LINUXKPI_PARAM_PARENT, OID_AUTO, \ + LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), &(var), 0, \ + LINUXKPI_PARAM_DESC(name))) + +#define LINUXKPI_PARAM_ushort(name, var, perm) \ + extern const char LINUXKPI_PARAM_DESC(name)[]; \ + LINUXKPI_PARAM_PASS(SYSCTL_U16(LINUXKPI_PARAM_PARENT, OID_AUTO, \ + LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), &(var), 0, \ + LINUXKPI_PARAM_DESC(name))) + +#define LINUXKPI_PARAM_int(name, var, perm) \ + extern const char LINUXKPI_PARAM_DESC(name)[]; \ + LINUXKPI_PARAM_PASS(SYSCTL_INT(LINUXKPI_PARAM_PARENT, OID_AUTO, \ + LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), &(var), 0,\ + LINUXKPI_PARAM_DESC(name))) + +#define LINUXKPI_PARAM_uint(name, var, perm) \ + extern const char LINUXKPI_PARAM_DESC(name)[]; \ + LINUXKPI_PARAM_PASS(SYSCTL_UINT(LINUXKPI_PARAM_PARENT, OID_AUTO, \ + LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), &(var), 0, \ + LINUXKPI_PARAM_DESC(name))) + +#define LINUXKPI_PARAM_long(name, var, perm) \ + extern const char LINUXKPI_PARAM_DESC(name)[]; \ + LINUXKPI_PARAM_PASS(SYSCTL_LONG(LINUXKPI_PARAM_PARENT, OID_AUTO, \ + LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), &(var), 0, \ + LINUXKPI_PARAM_DESC(name))) + +#define LINUXKPI_PARAM_ulong(name, var, perm) \ + extern const char LINUXKPI_PARAM_DESC(name)[]; \ + LINUXKPI_PARAM_PASS(SYSCTL_ULONG(LINUXKPI_PARAM_PARENT, OID_AUTO, \ + LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), &(var), 0, \ + LINUXKPI_PARAM_DESC(name))) + +#define module_param_string(name, str, len, perm) \ + extern const char LINUXKPI_PARAM_DESC(name)[]; \ + LINUXKPI_PARAM_PASS(SYSCTL_STRING(LINUXKPI_PARAM_PARENT, OID_AUTO, \ + LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), (str), (len), \ + LINUXKPI_PARAM_DESC(name))) + +#define module_param_named(name, var, type, mode) \ + LINUXKPI_PARAM_##type(name, var, mode) + +#define module_param(var, type, mode) \ + LINUXKPI_PARAM_##type(var, var, mode) + +#define module_param_named_unsafe(name, var, type, mode) \ + LINUXKPI_PARAM_##type(name, var, mode) + +#define module_param_unsafe(var, type, mode) \ + LINUXKPI_PARAM_##type(var, var, mode) + +#define module_param_array(var, type, addr_argc, mode) + +#define MODULE_PARM_DESC(name, desc) \ + const char LINUXKPI_PARAM_DESC(name)[] = { desc } + +#define kernel_param_lock(...) do {} while (0) +#define kernel_param_unlock(...) do {} while (0) + +SYSCTL_DECL(_compat_linuxkpi); + +#endif /* _LINUX_MODULEPARAM_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/mutex.h b/sys/compat/linuxkpi/common/include/linux/mutex.h new file mode 100644 index 000000000000..bdf0b4dbdb2d --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/mutex.h @@ -0,0 +1,176 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_MUTEX_H_ +#define _LINUX_MUTEX_H_ + +#include <sys/param.h> +#include <sys/proc.h> +#include <sys/lock.h> +#include <sys/sx.h> + +#include <linux/spinlock.h> +#include <asm/atomic.h> + +typedef struct mutex { + struct sx sx; +} mutex_t; + +/* + * By defining CONFIG_NO_MUTEX_SKIP LinuxKPI mutexes and asserts will + * not be skipped during panic(). + */ +#ifdef CONFIG_NO_MUTEX_SKIP +#define MUTEX_SKIP(void) 0 +#else +#define MUTEX_SKIP(void) unlikely(SCHEDULER_STOPPED() || kdb_active) +#endif + +#define mutex_lock(_m) do { \ + if (MUTEX_SKIP()) \ + break; \ + sx_xlock(&(_m)->sx); \ +} while (0) + +#define mutex_lock_nested(_m, _s) mutex_lock(_m) +#define mutex_lock_nest_lock(_m, _s) mutex_lock(_m) + +#define mutex_lock_interruptible(_m) ({ \ + MUTEX_SKIP() ? 0 : \ + linux_mutex_lock_interruptible(_m); \ +}) + +#define mutex_lock_interruptible_nested(m, c) mutex_lock_interruptible(m) + +/* + * Reuse the interruptable method since the SX + * lock handles both signals and interrupts: + */ +#define mutex_lock_killable(_m) ({ \ + MUTEX_SKIP() ? 0 : \ + linux_mutex_lock_interruptible(_m); \ +}) + +#define mutex_lock_killable_nested(_m, _sub) \ + mutex_lock_killable(_m) + +#define mutex_unlock(_m) do { \ + if (MUTEX_SKIP()) \ + break; \ + sx_xunlock(&(_m)->sx); \ +} while (0) + +#define mutex_trylock(_m) ({ \ + MUTEX_SKIP() ? 1 : \ + !!sx_try_xlock(&(_m)->sx); \ +}) + +enum mutex_trylock_recursive_enum { + MUTEX_TRYLOCK_FAILED = 0, + MUTEX_TRYLOCK_SUCCESS = 1, + MUTEX_TRYLOCK_RECURSIVE = 2, +}; + +static inline __must_check enum mutex_trylock_recursive_enum +mutex_trylock_recursive(struct mutex *lock) +{ + if (unlikely(sx_xholder(&lock->sx) == curthread)) + return (MUTEX_TRYLOCK_RECURSIVE); + + return (mutex_trylock(lock)); +} + +#define mutex_init(_m) \ + linux_mutex_init(_m, mutex_name(#_m), SX_NOWITNESS) + +#define __mutex_init(_m, _n, _l) \ + linux_mutex_init(_m, _n, SX_NOWITNESS) + +#define mutex_init_witness(_m) \ + linux_mutex_init(_m, mutex_name(#_m), SX_DUPOK) + +#define mutex_destroy(_m) \ + linux_mutex_destroy(_m) + +static inline bool +mutex_is_locked(mutex_t *m) +{ + return ((struct thread *)SX_OWNER(m->sx.sx_lock) != NULL); +} + +static inline bool +mutex_is_owned(mutex_t *m) +{ + return (sx_xlocked(&m->sx)); +} + +static inline int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *m) +{ + if (atomic_dec_and_test(cnt)) { + mutex_lock(m); + return (1); + } + + return (0); +} + +#ifdef WITNESS_ALL +/* NOTE: the maximum WITNESS name is 64 chars */ +#define __mutex_name(name, file, line) \ + (((const char *){file ":" #line "-" name}) + \ + (sizeof(file) > 16 ? sizeof(file) - 16 : 0)) +#else +#define __mutex_name(name, file, line) name +#endif +#define _mutex_name(...) __mutex_name(__VA_ARGS__) +#define mutex_name(name) _mutex_name(name, __FILE__, __LINE__) + +#define DEFINE_MUTEX(lock) \ + mutex_t lock; \ + SX_SYSINIT_FLAGS(lock, &(lock).sx, mutex_name(#lock), SX_DUPOK) + +static inline void +linux_mutex_init(mutex_t *m, const char *name, int flags) +{ + memset(m, 0, sizeof(*m)); + sx_init_flags(&m->sx, name, flags); +} + +static inline void +linux_mutex_destroy(mutex_t *m) +{ + if (mutex_is_owned(m)) + mutex_unlock(m); + sx_destroy(&m->sx); +} + +extern int linux_mutex_lock_interruptible(mutex_t *m); + +#endif /* _LINUX_MUTEX_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/net.h b/sys/compat/linuxkpi/common/include/linux/net.h new file mode 100644 index 000000000000..282a45d2db32 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/net.h @@ -0,0 +1,79 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_NET_H_ +#define _LINUX_NET_H_ + +#include <sys/types.h> +#include <sys/malloc.h> +#include <sys/proc.h> +#include <sys/protosw.h> +#include <sys/socket.h> +#include <sys/socketvar.h> +#include <sys/errno.h> + +static inline int +sock_create_kern(int family, int type, int proto, struct socket **res) +{ + return -socreate(family, res, type, proto, curthread->td_ucred, + curthread); +} + +static inline int +sock_getname(struct socket *so, struct sockaddr *addr, int *sockaddr_len, + int peer) +{ + struct sockaddr *nam; + int error; + + nam = NULL; + if (peer) { + if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) + return (-ENOTCONN); + + error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, &nam); + } else + error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, &nam); + if (error) + return (-error); + *addr = *nam; + *sockaddr_len = addr->sa_len; + + free(nam, M_SONAME); + return (0); +} + +static inline void +sock_release(struct socket *so) +{ + soclose(so); +} + +#endif /* _LINUX_NET_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/net_dim.h b/sys/compat/linuxkpi/common/include/linux/net_dim.h new file mode 100644 index 000000000000..f1b7e06b820e --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/net_dim.h @@ -0,0 +1,410 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0 + * + * Copyright (c) 2016, Mellanox Technologies. All rights reserved. + * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +/* This file implements Dynamic Interrupt Moderation, DIM */ + +#ifndef NET_DIM_H +#define NET_DIM_H + +#include <asm/types.h> + +#include <linux/workqueue.h> +#include <linux/ktime.h> + +struct net_dim_cq_moder { + u16 usec; + u16 pkts; + u8 cq_period_mode; +}; + +struct net_dim_sample { + ktime_t time; + u32 pkt_ctr; + u32 byte_ctr; + u16 event_ctr; +}; + +struct net_dim_stats { + int ppms; /* packets per msec */ + int bpms; /* bytes per msec */ + int epms; /* events per msec */ +}; + +struct net_dim { /* Adaptive Moderation */ + u8 state; + struct net_dim_stats prev_stats; + struct net_dim_sample start_sample; + struct work_struct work; + u16 event_ctr; + u8 profile_ix; + u8 mode; + u8 tune_state; + u8 steps_right; + u8 steps_left; + u8 tired; +}; + +enum { + NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, + NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, + NET_DIM_CQ_PERIOD_NUM_MODES = 0x2, + NET_DIM_CQ_PERIOD_MODE_DISABLED = 0xFF, +}; + +/* Adaptive moderation logic */ +enum { + NET_DIM_START_MEASURE, + NET_DIM_MEASURE_IN_PROGRESS, + NET_DIM_APPLY_NEW_PROFILE, +}; + +enum { + NET_DIM_PARKING_ON_TOP, + NET_DIM_PARKING_TIRED, + NET_DIM_GOING_RIGHT, + NET_DIM_GOING_LEFT, +}; + +enum { + NET_DIM_STATS_WORSE, + NET_DIM_STATS_SAME, + NET_DIM_STATS_BETTER, +}; + +enum { + NET_DIM_STEPPED, + NET_DIM_TOO_TIRED, + NET_DIM_ON_EDGE, +}; + +#define NET_DIM_PARAMS_NUM_PROFILES 5 +/* Adaptive moderation profiles */ +#define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 +#define NET_DIM_DEF_PROFILE_CQE 1 +#define NET_DIM_DEF_PROFILE_EQE 1 + +/* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */ +#define NET_DIM_EQE_PROFILES { \ + {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ +} + +#define NET_DIM_CQE_PROFILES { \ + {2, 256}, \ + {8, 128}, \ + {16, 64}, \ + {32, 64}, \ + {64, 64} \ +} + +static const struct net_dim_cq_moder + net_dim_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { + NET_DIM_EQE_PROFILES, + NET_DIM_CQE_PROFILES, +}; + +static inline struct net_dim_cq_moder +net_dim_get_profile(u8 cq_period_mode, + int ix) +{ + struct net_dim_cq_moder cq_moder; + + cq_moder = net_dim_profile[cq_period_mode][ix]; + cq_moder.cq_period_mode = cq_period_mode; + return cq_moder; +} + +static inline struct net_dim_cq_moder +net_dim_get_def_profile(u8 rx_cq_period_mode) +{ + int default_profile_ix; + + if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE) + default_profile_ix = NET_DIM_DEF_PROFILE_CQE; + else /* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */ + default_profile_ix = NET_DIM_DEF_PROFILE_EQE; + + return net_dim_get_profile(rx_cq_period_mode, default_profile_ix); +} + +static inline bool +net_dim_on_top(struct net_dim *dim) +{ + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + case NET_DIM_PARKING_TIRED: + return true; + case NET_DIM_GOING_RIGHT: + return (dim->steps_left > 1) && (dim->steps_right == 1); + default: /* NET_DIM_GOING_LEFT */ + return (dim->steps_right > 1) && (dim->steps_left == 1); + } +} + +static inline void +net_dim_turn(struct net_dim *dim) +{ + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + case NET_DIM_PARKING_TIRED: + break; + case NET_DIM_GOING_RIGHT: + dim->tune_state = NET_DIM_GOING_LEFT; + dim->steps_left = 0; + break; + case NET_DIM_GOING_LEFT: + dim->tune_state = NET_DIM_GOING_RIGHT; + dim->steps_right = 0; + break; + } +} + +static inline int +net_dim_step(struct net_dim *dim) +{ + if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) + return NET_DIM_TOO_TIRED; + + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + case NET_DIM_PARKING_TIRED: + break; + case NET_DIM_GOING_RIGHT: + if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1)) + return NET_DIM_ON_EDGE; + dim->profile_ix++; + dim->steps_right++; + break; + case NET_DIM_GOING_LEFT: + if (dim->profile_ix == 0) + return NET_DIM_ON_EDGE; + dim->profile_ix--; + dim->steps_left++; + break; + } + + dim->tired++; + return NET_DIM_STEPPED; +} + +static inline void +net_dim_park_on_top(struct net_dim *dim) +{ + dim->steps_right = 0; + dim->steps_left = 0; + dim->tired = 0; + dim->tune_state = NET_DIM_PARKING_ON_TOP; +} + +static inline void +net_dim_park_tired(struct net_dim *dim) +{ + dim->steps_right = 0; + dim->steps_left = 0; + dim->tune_state = NET_DIM_PARKING_TIRED; +} + +static inline void +net_dim_exit_parking(struct net_dim *dim) +{ + dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT : + NET_DIM_GOING_RIGHT; + net_dim_step(dim); +} + +#define IS_SIGNIFICANT_DIFF(val, ref) \ + (((100UL * abs((val) - (ref))) / (ref)) > 10) /* more than 10% + * difference */ + +static inline int +net_dim_stats_compare(struct net_dim_stats *curr, + struct net_dim_stats *prev) +{ + if (!prev->bpms) + return curr->bpms ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_SAME; + + if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) + return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_WORSE; + + if (!prev->ppms) + return curr->ppms ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_SAME; + + if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) + return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_WORSE; + + if (!prev->epms) + return NET_DIM_STATS_SAME; + + if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) + return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_WORSE; + + return NET_DIM_STATS_SAME; +} + +static inline bool +net_dim_decision(struct net_dim_stats *curr_stats, + struct net_dim *dim) +{ + int prev_state = dim->tune_state; + int prev_ix = dim->profile_ix; + int stats_res; + int step_res; + + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); + if (stats_res != NET_DIM_STATS_SAME) + net_dim_exit_parking(dim); + break; + + case NET_DIM_PARKING_TIRED: + dim->tired--; + if (!dim->tired) + net_dim_exit_parking(dim); + break; + + case NET_DIM_GOING_RIGHT: + case NET_DIM_GOING_LEFT: + stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); + if (stats_res != NET_DIM_STATS_BETTER) + net_dim_turn(dim); + + if (net_dim_on_top(dim)) { + net_dim_park_on_top(dim); + break; + } + step_res = net_dim_step(dim); + switch (step_res) { + case NET_DIM_ON_EDGE: + net_dim_park_on_top(dim); + break; + case NET_DIM_TOO_TIRED: + net_dim_park_tired(dim); + break; + } + + break; + } + + if ((prev_state != NET_DIM_PARKING_ON_TOP) || + (dim->tune_state != NET_DIM_PARKING_ON_TOP)) + dim->prev_stats = *curr_stats; + + return dim->profile_ix != prev_ix; +} + +static inline void +net_dim_sample(u16 event_ctr, + u64 packets, + u64 bytes, + struct net_dim_sample *s) +{ + s->time = ktime_get(); + s->pkt_ctr = packets; + s->byte_ctr = bytes; + s->event_ctr = event_ctr; +} + +#define NET_DIM_NEVENTS 64 +#define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1)) + +static inline void +net_dim_calc_stats(struct net_dim_sample *start, + struct net_dim_sample *end, + struct net_dim_stats *curr_stats) +{ + /* u32 holds up to 71 minutes, should be enough */ + u32 delta_us = ktime_us_delta(end->time, start->time); + u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr); + u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr, + start->byte_ctr); + + if (!delta_us) + return; + + curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); + curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); + curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC, + delta_us); +} + +static inline void +net_dim(struct net_dim *dim, + u64 packets, u64 bytes) +{ + struct net_dim_stats curr_stats; + struct net_dim_sample end_sample; + u16 nevents; + + dim->event_ctr++; + + switch (dim->state) { + case NET_DIM_MEASURE_IN_PROGRESS: + nevents = BIT_GAP(BITS_PER_TYPE(u16), + dim->event_ctr, + dim->start_sample.event_ctr); + if (nevents < NET_DIM_NEVENTS) + break; + net_dim_sample(dim->event_ctr, packets, bytes, &end_sample); + net_dim_calc_stats(&dim->start_sample, &end_sample, + &curr_stats); + if (net_dim_decision(&curr_stats, dim)) { + dim->state = NET_DIM_APPLY_NEW_PROFILE; + schedule_work(&dim->work); + break; + } + /* FALLTHROUGH */ + case NET_DIM_START_MEASURE: + net_dim_sample(dim->event_ctr, packets, bytes, &dim->start_sample); + dim->state = NET_DIM_MEASURE_IN_PROGRESS; + break; + case NET_DIM_APPLY_NEW_PROFILE: + break; + default: + break; + } +} + +#endif /* NET_DIM_H */ diff --git a/sys/compat/linuxkpi/common/include/linux/netdevice.h b/sys/compat/linuxkpi/common/include/linux/netdevice.h new file mode 100644 index 000000000000..336215b9f7c5 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/netdevice.h @@ -0,0 +1,143 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2019 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_NETDEVICE_H_ +#define _LINUX_NETDEVICE_H_ + +#include <linux/types.h> + +#include <sys/socket.h> + +#include <net/if_types.h> +#include <net/if.h> +#include <net/if_var.h> +#include <net/if_dl.h> + +#include <linux/list.h> +#include <linux/completion.h> +#include <linux/device.h> +#include <linux/workqueue.h> +#include <linux/net.h> +#include <linux/notifier.h> + +#ifdef VIMAGE +#define init_net *vnet0 +#else +#define init_net *((struct vnet *)0) +#endif + +#define MAX_ADDR_LEN 20 + +#define net_device ifnet + +static inline struct ifnet * +dev_get_by_index(struct vnet *vnet, int if_index) +{ + struct epoch_tracker et; + struct ifnet *retval; + + NET_EPOCH_ENTER(et); + CURVNET_SET(vnet); + retval = ifnet_byindex_ref(if_index); + CURVNET_RESTORE(); + NET_EPOCH_EXIT(et); + + return (retval); +} + +#define dev_hold(d) if_ref(d) +#define dev_put(d) if_rele(d) +#define dev_net(d) ((d)->if_vnet) + +#define net_eq(a,b) ((a) == (b)) + +#define netif_running(dev) !!((dev)->if_drv_flags & IFF_DRV_RUNNING) +#define netif_oper_up(dev) !!((dev)->if_flags & IFF_UP) +#define netif_carrier_ok(dev) ((dev)->if_link_state == LINK_STATE_UP) + +static inline void * +netdev_priv(const struct net_device *dev) +{ + return (dev->if_softc); +} + +static inline struct net_device * +netdev_notifier_info_to_dev(void *ifp) +{ + return (ifp); +} + +int register_netdevice_notifier(struct notifier_block *); +int register_inetaddr_notifier(struct notifier_block *); +int unregister_netdevice_notifier(struct notifier_block *); +int unregister_inetaddr_notifier(struct notifier_block *); + +#define rtnl_lock() +#define rtnl_unlock() + +static inline int +dev_mc_delete(struct net_device *dev, void *addr, int alen, int all) +{ + struct sockaddr_dl sdl; + + if (alen > sizeof(sdl.sdl_data)) + return (-EINVAL); + memset(&sdl, 0, sizeof(sdl)); + sdl.sdl_len = sizeof(sdl); + sdl.sdl_family = AF_LINK; + sdl.sdl_alen = alen; + memcpy(&sdl.sdl_data, addr, alen); + + return -if_delmulti(dev, (struct sockaddr *)&sdl); +} + +static inline int +dev_mc_del(struct net_device *dev, void *addr) +{ + return (dev_mc_delete(dev, addr, 6, 0)); +} + +static inline int +dev_mc_add(struct net_device *dev, void *addr, int alen, int newonly) +{ + struct sockaddr_dl sdl; + + if (alen > sizeof(sdl.sdl_data)) + return (-EINVAL); + memset(&sdl, 0, sizeof(sdl)); + sdl.sdl_len = sizeof(sdl); + sdl.sdl_family = AF_LINK; + sdl.sdl_alen = alen; + memcpy(&sdl.sdl_data, addr, alen); + + return -if_addmulti(dev, (struct sockaddr *)&sdl, NULL); +} + +#endif /* _LINUX_NETDEVICE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/notifier.h b/sys/compat/linuxkpi/common/include/linux/notifier.h new file mode 100644 index 000000000000..ae292439bf54 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/notifier.h @@ -0,0 +1,57 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_NOTIFIER_H_ +#define _LINUX_NOTIFIER_H_ + +#include <sys/types.h> +#include <sys/eventhandler.h> + +#define NOTIFY_DONE 0 + +enum { + NETDEV_CHANGE, + NETDEV_UP, + NETDEV_DOWN, + NETDEV_REGISTER, + NETDEV_UNREGISTER, + NETDEV_CHANGEADDR, + NETDEV_CHANGEIFADDR, + LINUX_NOTIFY_TAGS /* must be last */ +}; + +struct notifier_block { + int (*notifier_call) (struct notifier_block *, unsigned long, void *); + struct notifier_block *next; + int priority; + eventhandler_tag tags[LINUX_NOTIFY_TAGS]; +}; + +#endif /* _LINUX_NOTIFIER_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/overflow.h b/sys/compat/linuxkpi/common/include/linux/overflow.h new file mode 100644 index 000000000000..6f53f0b59384 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/overflow.h @@ -0,0 +1,65 @@ +/*- + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Emmanuel Vadot under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __LINUX_OVERFLOW_H__ +#define __LINUX_OVERFLOW_H__ + +#include <sys/stdint.h> +#include <sys/types.h> + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#if __has_builtin(__builtin_add_overflow) +#define check_add_overflow(a, b, c) \ + __builtin_add_overflow(a, b, c) +#else +#error "Compiler does not support __builtin_add_overflow" +#endif + +#if __has_builtin(__builtin_mul_overflow) +#define check_mul_overflow(a, b, c) \ + __builtin_mul_overflow(a, b, c) + +static inline size_t +array_size(size_t x, size_t y) +{ + size_t retval; + + if (__builtin_mul_overflow(x, y, &retval)) + retval = SIZE_MAX; + return (retval); +} +#else +#error "Compiler does not support __builtin_mul_overflow" +#endif + +#endif /* __LINUX_OVERFLOW_H__ */ diff --git a/sys/compat/linuxkpi/common/include/linux/page.h b/sys/compat/linuxkpi/common/include/linux/page.h new file mode 100644 index 000000000000..c2dbab769c2a --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/page.h @@ -0,0 +1,96 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_PAGE_H_ +#define _LINUX_PAGE_H_ + +#include <linux/types.h> + +#include <sys/param.h> +#include <sys/vmmeter.h> + +#include <machine/atomic.h> +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/pmap.h> + +typedef unsigned long linux_pte_t; +typedef unsigned long linux_pmd_t; +typedef unsigned long linux_pgd_t; +typedef unsigned long pgprot_t; + +#define page vm_page + +#define LINUXKPI_PROT_VALID (1 << 3) +#define LINUXKPI_CACHE_MODE_SHIFT 4 + +CTASSERT((VM_PROT_ALL & -LINUXKPI_PROT_VALID) == 0); + +static inline pgprot_t +cachemode2protval(vm_memattr_t attr) +{ + return ((attr << LINUXKPI_CACHE_MODE_SHIFT) | LINUXKPI_PROT_VALID); +} + +static inline vm_memattr_t +pgprot2cachemode(pgprot_t prot) +{ + if (prot & LINUXKPI_PROT_VALID) + return (prot >> LINUXKPI_CACHE_MODE_SHIFT); + else + return (VM_MEMATTR_DEFAULT); +} + +#define virt_to_page(x) PHYS_TO_VM_PAGE(vtophys(x)) +#define page_to_pfn(pp) (VM_PAGE_TO_PHYS(pp) >> PAGE_SHIFT) +#define pfn_to_page(pfn) (PHYS_TO_VM_PAGE((pfn) << PAGE_SHIFT)) +#define nth_page(page,n) pfn_to_page(page_to_pfn(page) + (n)) + +#define clear_page(page) memset(page, 0, PAGE_SIZE) +#define pgprot_noncached(prot) \ + (((prot) & VM_PROT_ALL) | cachemode2protval(VM_MEMATTR_UNCACHEABLE)) +#define pgprot_writecombine(prot) \ + (((prot) & VM_PROT_ALL) | cachemode2protval(VM_MEMATTR_WRITE_COMBINING)) + +#undef PAGE_MASK +#define PAGE_MASK (~(PAGE_SIZE-1)) +/* + * Modifying PAGE_MASK in the above way breaks trunc_page, round_page, + * and btoc macros. Therefore, redefine them in a way that makes sense + * so the LinuxKPI consumers don't get totally broken behavior. + */ +#undef btoc +#define btoc(x) (((vm_offset_t)(x) + PAGE_SIZE - 1) >> PAGE_SHIFT) +#undef round_page +#define round_page(x) ((((uintptr_t)(x)) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) +#undef trunc_page +#define trunc_page(x) ((uintptr_t)(x) & ~(PAGE_SIZE - 1)) + +#endif /* _LINUX_PAGE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/pci.h b/sys/compat/linuxkpi/common/include/linux/pci.h new file mode 100644 index 000000000000..2c23f2a42a12 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/pci.h @@ -0,0 +1,1065 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_PCI_H_ +#define _LINUX_PCI_H_ + +#define CONFIG_PCI_MSI + +#include <linux/types.h> + +#include <sys/param.h> +#include <sys/bus.h> +#include <sys/nv.h> +#include <sys/pciio.h> +#include <sys/rman.h> +#include <sys/bus.h> +#include <dev/pci/pcivar.h> +#include <dev/pci/pcireg.h> +#include <dev/pci/pci_private.h> + +#include <machine/resource.h> + +#include <linux/list.h> +#include <linux/dmapool.h> +#include <linux/dma-mapping.h> +#include <linux/compiler.h> +#include <linux/errno.h> +#include <asm/atomic.h> +#include <linux/device.h> + +struct pci_device_id { + uint32_t vendor; + uint32_t device; + uint32_t subvendor; + uint32_t subdevice; + uint32_t class; + uint32_t class_mask; + uintptr_t driver_data; +}; + +#define MODULE_DEVICE_TABLE(bus, table) + +#define PCI_BASE_CLASS_DISPLAY 0x03 +#define PCI_CLASS_DISPLAY_VGA 0x0300 +#define PCI_CLASS_DISPLAY_OTHER 0x0380 +#define PCI_BASE_CLASS_BRIDGE 0x06 +#define PCI_CLASS_BRIDGE_ISA 0x0601 + +#define PCI_ANY_ID -1U +#define PCI_VENDOR_ID_APPLE 0x106b +#define PCI_VENDOR_ID_ASUSTEK 0x1043 +#define PCI_VENDOR_ID_ATI 0x1002 +#define PCI_VENDOR_ID_DELL 0x1028 +#define PCI_VENDOR_ID_HP 0x103c +#define PCI_VENDOR_ID_IBM 0x1014 +#define PCI_VENDOR_ID_INTEL 0x8086 +#define PCI_VENDOR_ID_MELLANOX 0x15b3 +#define PCI_VENDOR_ID_REDHAT_QUMRANET 0x1af4 +#define PCI_VENDOR_ID_SERVERWORKS 0x1166 +#define PCI_VENDOR_ID_SONY 0x104d +#define PCI_VENDOR_ID_TOPSPIN 0x1867 +#define PCI_VENDOR_ID_VIA 0x1106 +#define PCI_SUBVENDOR_ID_REDHAT_QUMRANET 0x1af4 +#define PCI_DEVICE_ID_ATI_RADEON_QY 0x5159 +#define PCI_DEVICE_ID_MELLANOX_TAVOR 0x5a44 +#define PCI_DEVICE_ID_MELLANOX_TAVOR_BRIDGE 0x5a46 +#define PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT 0x6278 +#define PCI_DEVICE_ID_MELLANOX_ARBEL 0x6282 +#define PCI_DEVICE_ID_MELLANOX_SINAI_OLD 0x5e8c +#define PCI_DEVICE_ID_MELLANOX_SINAI 0x6274 +#define PCI_SUBDEVICE_ID_QEMU 0x1100 + +#define PCI_DEVFN(slot, func) ((((slot) & 0x1f) << 3) | ((func) & 0x07)) +#define PCI_SLOT(devfn) (((devfn) >> 3) & 0x1f) +#define PCI_FUNC(devfn) ((devfn) & 0x07) +#define PCI_BUS_NUM(devfn) (((devfn) >> 8) & 0xff) + +#define PCI_VDEVICE(_vendor, _device) \ + .vendor = PCI_VENDOR_ID_##_vendor, .device = (_device), \ + .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID +#define PCI_DEVICE(_vendor, _device) \ + .vendor = (_vendor), .device = (_device), \ + .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID + +#define to_pci_dev(n) container_of(n, struct pci_dev, dev) + +#define PCI_VENDOR_ID PCIR_DEVVENDOR +#define PCI_COMMAND PCIR_COMMAND +#define PCI_EXP_DEVCTL PCIER_DEVICE_CTL /* Device Control */ +#define PCI_EXP_LNKCTL PCIER_LINK_CTL /* Link Control */ +#define PCI_EXP_FLAGS_TYPE PCIEM_FLAGS_TYPE /* Device/Port type */ +#define PCI_EXP_DEVCAP PCIER_DEVICE_CAP /* Device capabilities */ +#define PCI_EXP_DEVSTA PCIER_DEVICE_STA /* Device Status */ +#define PCI_EXP_LNKCAP PCIER_LINK_CAP /* Link Capabilities */ +#define PCI_EXP_LNKSTA PCIER_LINK_STA /* Link Status */ +#define PCI_EXP_SLTCAP PCIER_SLOT_CAP /* Slot Capabilities */ +#define PCI_EXP_SLTCTL PCIER_SLOT_CTL /* Slot Control */ +#define PCI_EXP_SLTSTA PCIER_SLOT_STA /* Slot Status */ +#define PCI_EXP_RTCTL PCIER_ROOT_CTL /* Root Control */ +#define PCI_EXP_RTCAP PCIER_ROOT_CAP /* Root Capabilities */ +#define PCI_EXP_RTSTA PCIER_ROOT_STA /* Root Status */ +#define PCI_EXP_DEVCAP2 PCIER_DEVICE_CAP2 /* Device Capabilities 2 */ +#define PCI_EXP_DEVCTL2 PCIER_DEVICE_CTL2 /* Device Control 2 */ +#define PCI_EXP_LNKCAP2 PCIER_LINK_CAP2 /* Link Capabilities 2 */ +#define PCI_EXP_LNKCTL2 PCIER_LINK_CTL2 /* Link Control 2 */ +#define PCI_EXP_LNKSTA2 PCIER_LINK_STA2 /* Link Status 2 */ +#define PCI_EXP_FLAGS PCIER_FLAGS /* Capabilities register */ +#define PCI_EXP_FLAGS_VERS PCIEM_FLAGS_VERSION /* Capability version */ +#define PCI_EXP_TYPE_ROOT_PORT PCIEM_TYPE_ROOT_PORT /* Root Port */ +#define PCI_EXP_TYPE_ENDPOINT PCIEM_TYPE_ENDPOINT /* Express Endpoint */ +#define PCI_EXP_TYPE_LEG_END PCIEM_TYPE_LEGACY_ENDPOINT /* Legacy Endpoint */ +#define PCI_EXP_TYPE_DOWNSTREAM PCIEM_TYPE_DOWNSTREAM_PORT /* Downstream Port */ +#define PCI_EXP_FLAGS_SLOT PCIEM_FLAGS_SLOT /* Slot implemented */ +#define PCI_EXP_TYPE_RC_EC PCIEM_TYPE_ROOT_EC /* Root Complex Event Collector */ +#define PCI_EXP_LNKCAP_SLS_2_5GB 0x01 /* Supported Link Speed 2.5GT/s */ +#define PCI_EXP_LNKCAP_SLS_5_0GB 0x02 /* Supported Link Speed 5.0GT/s */ +#define PCI_EXP_LNKCAP_SLS_8_0GB 0x04 /* Supported Link Speed 8.0GT/s */ +#define PCI_EXP_LNKCAP_SLS_16_0GB 0x08 /* Supported Link Speed 16.0GT/s */ +#define PCI_EXP_LNKCAP_MLW 0x03f0 /* Maximum Link Width */ +#define PCI_EXP_LNKCAP2_SLS_2_5GB 0x02 /* Supported Link Speed 2.5GT/s */ +#define PCI_EXP_LNKCAP2_SLS_5_0GB 0x04 /* Supported Link Speed 5.0GT/s */ +#define PCI_EXP_LNKCAP2_SLS_8_0GB 0x08 /* Supported Link Speed 8.0GT/s */ +#define PCI_EXP_LNKCAP2_SLS_16_0GB 0x10 /* Supported Link Speed 16.0GT/s */ + +#define PCI_EXP_LNKCTL_HAWD PCIEM_LINK_CTL_HAWD +#define PCI_EXP_LNKCAP_CLKPM 0x00040000 +#define PCI_EXP_DEVSTA_TRPND 0x0020 + +#define IORESOURCE_MEM (1 << SYS_RES_MEMORY) +#define IORESOURCE_IO (1 << SYS_RES_IOPORT) +#define IORESOURCE_IRQ (1 << SYS_RES_IRQ) + +enum pci_bus_speed { + PCI_SPEED_UNKNOWN = -1, + PCIE_SPEED_2_5GT, + PCIE_SPEED_5_0GT, + PCIE_SPEED_8_0GT, + PCIE_SPEED_16_0GT, +}; + +enum pcie_link_width { + PCIE_LNK_WIDTH_RESRV = 0x00, + PCIE_LNK_X1 = 0x01, + PCIE_LNK_X2 = 0x02, + PCIE_LNK_X4 = 0x04, + PCIE_LNK_X8 = 0x08, + PCIE_LNK_X12 = 0x0c, + PCIE_LNK_X16 = 0x10, + PCIE_LNK_X32 = 0x20, + PCIE_LNK_WIDTH_UNKNOWN = 0xff, +}; + +typedef int pci_power_t; + +#define PCI_D0 PCI_POWERSTATE_D0 +#define PCI_D1 PCI_POWERSTATE_D1 +#define PCI_D2 PCI_POWERSTATE_D2 +#define PCI_D3hot PCI_POWERSTATE_D3 +#define PCI_D3cold 4 + +#define PCI_POWER_ERROR PCI_POWERSTATE_UNKNOWN + +struct pci_dev; + +struct pci_driver { + struct list_head links; + char *name; + const struct pci_device_id *id_table; + int (*probe)(struct pci_dev *dev, const struct pci_device_id *id); + void (*remove)(struct pci_dev *dev); + int (*suspend) (struct pci_dev *dev, pm_message_t state); /* Device suspended */ + int (*resume) (struct pci_dev *dev); /* Device woken up */ + void (*shutdown) (struct pci_dev *dev); /* Device shutdown */ + driver_t bsddriver; + devclass_t bsdclass; + struct device_driver driver; + const struct pci_error_handlers *err_handler; + bool isdrm; + int (*bsd_iov_init)(device_t dev, uint16_t num_vfs, + const nvlist_t *pf_config); + void (*bsd_iov_uninit)(device_t dev); + int (*bsd_iov_add_vf)(device_t dev, uint16_t vfnum, + const nvlist_t *vf_config); +}; + +struct pci_bus { + struct pci_dev *self; + int domain; + int number; +}; + +extern struct list_head pci_drivers; +extern struct list_head pci_devices; +extern spinlock_t pci_lock; + +#define __devexit_p(x) x + +struct pci_mmio_region { + TAILQ_ENTRY(pci_mmio_region) next; + struct resource *res; + int rid; + int type; +}; + +struct pci_dev { + struct device dev; + struct list_head links; + struct pci_driver *pdrv; + struct pci_bus *bus; + uint16_t device; + uint16_t vendor; + uint16_t subsystem_vendor; + uint16_t subsystem_device; + unsigned int irq; + unsigned int devfn; + uint32_t class; + uint8_t revision; + bool msi_enabled; + + TAILQ_HEAD(, pci_mmio_region) mmio; +}; + +static inline struct resource_list_entry * +linux_pci_get_rle(struct pci_dev *pdev, int type, int rid) +{ + struct pci_devinfo *dinfo; + struct resource_list *rl; + + dinfo = device_get_ivars(pdev->dev.bsddev); + rl = &dinfo->resources; + return resource_list_find(rl, type, rid); +} + +static inline struct resource_list_entry * +linux_pci_get_bar(struct pci_dev *pdev, int bar) +{ + struct resource_list_entry *rle; + + bar = PCIR_BAR(bar); + if ((rle = linux_pci_get_rle(pdev, SYS_RES_MEMORY, bar)) == NULL) + rle = linux_pci_get_rle(pdev, SYS_RES_IOPORT, bar); + return (rle); +} + +static inline struct device * +linux_pci_find_irq_dev(unsigned int irq) +{ + struct pci_dev *pdev; + struct device *found; + + found = NULL; + spin_lock(&pci_lock); + list_for_each_entry(pdev, &pci_devices, links) { + if (irq == pdev->dev.irq || + (irq >= pdev->dev.irq_start && irq < pdev->dev.irq_end)) { + found = &pdev->dev; + break; + } + } + spin_unlock(&pci_lock); + return (found); +} + +static inline int +pci_resource_type(struct pci_dev *pdev, int bar) +{ + struct pci_map *pm; + + pm = pci_find_bar(pdev->dev.bsddev, PCIR_BAR(bar)); + if (!pm) + return (-1); + + if (PCI_BAR_IO(pm->pm_value)) + return (SYS_RES_IOPORT); + else + return (SYS_RES_MEMORY); +} + +/* + * All drivers just seem to want to inspect the type not flags. + */ +static inline int +pci_resource_flags(struct pci_dev *pdev, int bar) +{ + int type; + + type = pci_resource_type(pdev, bar); + if (type < 0) + return (0); + return (1 << type); +} + +static inline const char * +pci_name(struct pci_dev *d) +{ + + return device_get_desc(d->dev.bsddev); +} + +static inline void * +pci_get_drvdata(struct pci_dev *pdev) +{ + + return dev_get_drvdata(&pdev->dev); +} + +static inline void +pci_set_drvdata(struct pci_dev *pdev, void *data) +{ + + dev_set_drvdata(&pdev->dev, data); +} + +static inline int +pci_enable_device(struct pci_dev *pdev) +{ + + pci_enable_io(pdev->dev.bsddev, SYS_RES_IOPORT); + pci_enable_io(pdev->dev.bsddev, SYS_RES_MEMORY); + return (0); +} + +static inline void +pci_disable_device(struct pci_dev *pdev) +{ + + pci_disable_busmaster(pdev->dev.bsddev); +} + +static inline int +pci_set_master(struct pci_dev *pdev) +{ + + pci_enable_busmaster(pdev->dev.bsddev); + return (0); +} + +static inline int +pci_set_power_state(struct pci_dev *pdev, int state) +{ + + pci_set_powerstate(pdev->dev.bsddev, state); + return (0); +} + +static inline int +pci_clear_master(struct pci_dev *pdev) +{ + + pci_disable_busmaster(pdev->dev.bsddev); + return (0); +} + +static inline int +pci_request_region(struct pci_dev *pdev, int bar, const char *res_name) +{ + int rid; + int type; + + type = pci_resource_type(pdev, bar); + if (type < 0) + return (-ENODEV); + rid = PCIR_BAR(bar); + if (bus_alloc_resource_any(pdev->dev.bsddev, type, &rid, + RF_ACTIVE) == NULL) + return (-EINVAL); + return (0); +} + +static inline void +pci_release_region(struct pci_dev *pdev, int bar) +{ + struct resource_list_entry *rle; + + if ((rle = linux_pci_get_bar(pdev, bar)) == NULL) + return; + bus_release_resource(pdev->dev.bsddev, rle->type, rle->rid, rle->res); +} + +static inline void +pci_release_regions(struct pci_dev *pdev) +{ + int i; + + for (i = 0; i <= PCIR_MAX_BAR_0; i++) + pci_release_region(pdev, i); +} + +static inline int +pci_request_regions(struct pci_dev *pdev, const char *res_name) +{ + int error; + int i; + + for (i = 0; i <= PCIR_MAX_BAR_0; i++) { + error = pci_request_region(pdev, i, res_name); + if (error && error != -ENODEV) { + pci_release_regions(pdev); + return (error); + } + } + return (0); +} + +static inline void +pci_disable_msix(struct pci_dev *pdev) +{ + + pci_release_msi(pdev->dev.bsddev); + + /* + * The MSIX IRQ numbers associated with this PCI device are no + * longer valid and might be re-assigned. Make sure + * linux_pci_find_irq_dev() does no longer see them by + * resetting their references to zero: + */ + pdev->dev.irq_start = 0; + pdev->dev.irq_end = 0; +} + +#define pci_disable_msi(pdev) \ + linux_pci_disable_msi(pdev) + +static inline void +linux_pci_disable_msi(struct pci_dev *pdev) +{ + + pci_release_msi(pdev->dev.bsddev); + + pdev->dev.irq_start = 0; + pdev->dev.irq_end = 0; + pdev->irq = pdev->dev.irq; + pdev->msi_enabled = false; +} + +unsigned long pci_resource_start(struct pci_dev *pdev, int bar); +unsigned long pci_resource_len(struct pci_dev *pdev, int bar); + +static inline bus_addr_t +pci_bus_address(struct pci_dev *pdev, int bar) +{ + + return (pci_resource_start(pdev, bar)); +} + +#define PCI_CAP_ID_EXP PCIY_EXPRESS +#define PCI_CAP_ID_PCIX PCIY_PCIX +#define PCI_CAP_ID_AGP PCIY_AGP +#define PCI_CAP_ID_PM PCIY_PMG + +#define PCI_EXP_DEVCTL PCIER_DEVICE_CTL +#define PCI_EXP_DEVCTL_PAYLOAD PCIEM_CTL_MAX_PAYLOAD +#define PCI_EXP_DEVCTL_READRQ PCIEM_CTL_MAX_READ_REQUEST +#define PCI_EXP_LNKCTL PCIER_LINK_CTL +#define PCI_EXP_LNKSTA PCIER_LINK_STA + +static inline int +pci_find_capability(struct pci_dev *pdev, int capid) +{ + int reg; + + if (pci_find_cap(pdev->dev.bsddev, capid, ®)) + return (0); + return (reg); +} + +static inline int pci_pcie_cap(struct pci_dev *dev) +{ + return pci_find_capability(dev, PCI_CAP_ID_EXP); +} + + +static inline int +pci_read_config_byte(struct pci_dev *pdev, int where, u8 *val) +{ + + *val = (u8)pci_read_config(pdev->dev.bsddev, where, 1); + return (0); +} + +static inline int +pci_read_config_word(struct pci_dev *pdev, int where, u16 *val) +{ + + *val = (u16)pci_read_config(pdev->dev.bsddev, where, 2); + return (0); +} + +static inline int +pci_read_config_dword(struct pci_dev *pdev, int where, u32 *val) +{ + + *val = (u32)pci_read_config(pdev->dev.bsddev, where, 4); + return (0); +} + +static inline int +pci_write_config_byte(struct pci_dev *pdev, int where, u8 val) +{ + + pci_write_config(pdev->dev.bsddev, where, val, 1); + return (0); +} + +static inline int +pci_write_config_word(struct pci_dev *pdev, int where, u16 val) +{ + + pci_write_config(pdev->dev.bsddev, where, val, 2); + return (0); +} + +static inline int +pci_write_config_dword(struct pci_dev *pdev, int where, u32 val) +{ + + pci_write_config(pdev->dev.bsddev, where, val, 4); + return (0); +} + +int linux_pci_register_driver(struct pci_driver *pdrv); +int linux_pci_register_drm_driver(struct pci_driver *pdrv); +void linux_pci_unregister_driver(struct pci_driver *pdrv); +void linux_pci_unregister_drm_driver(struct pci_driver *pdrv); + +#define pci_register_driver(pdrv) linux_pci_register_driver(pdrv) +#define pci_unregister_driver(pdrv) linux_pci_unregister_driver(pdrv) + +struct msix_entry { + int entry; + int vector; +}; + +/* + * Enable msix, positive errors indicate actual number of available + * vectors. Negative errors are failures. + * + * NB: define added to prevent this definition of pci_enable_msix from + * clashing with the native FreeBSD version. + */ +#define pci_enable_msix(...) \ + linux_pci_enable_msix(__VA_ARGS__) + +static inline int +pci_enable_msix(struct pci_dev *pdev, struct msix_entry *entries, int nreq) +{ + struct resource_list_entry *rle; + int error; + int avail; + int i; + + avail = pci_msix_count(pdev->dev.bsddev); + if (avail < nreq) { + if (avail == 0) + return -EINVAL; + return avail; + } + avail = nreq; + if ((error = -pci_alloc_msix(pdev->dev.bsddev, &avail)) != 0) + return error; + /* + * Handle case where "pci_alloc_msix()" may allocate less + * interrupts than available and return with no error: + */ + if (avail < nreq) { + pci_release_msi(pdev->dev.bsddev); + return avail; + } + rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1); + pdev->dev.irq_start = rle->start; + pdev->dev.irq_end = rle->start + avail; + for (i = 0; i < nreq; i++) + entries[i].vector = pdev->dev.irq_start + i; + return (0); +} + +#define pci_enable_msix_range(...) \ + linux_pci_enable_msix_range(__VA_ARGS__) + +static inline int +pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, + int minvec, int maxvec) +{ + int nvec = maxvec; + int rc; + + if (maxvec < minvec) + return (-ERANGE); + + do { + rc = pci_enable_msix(dev, entries, nvec); + if (rc < 0) { + return (rc); + } else if (rc > 0) { + if (rc < minvec) + return (-ENOSPC); + nvec = rc; + } + } while (rc); + return (nvec); +} + +#define pci_enable_msi(pdev) \ + linux_pci_enable_msi(pdev) + +static inline int +pci_enable_msi(struct pci_dev *pdev) +{ + struct resource_list_entry *rle; + int error; + int avail; + + avail = pci_msi_count(pdev->dev.bsddev); + if (avail < 1) + return -EINVAL; + + avail = 1; /* this function only enable one MSI IRQ */ + if ((error = -pci_alloc_msi(pdev->dev.bsddev, &avail)) != 0) + return error; + + rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1); + pdev->dev.irq_start = rle->start; + pdev->dev.irq_end = rle->start + avail; + pdev->irq = rle->start; + pdev->msi_enabled = true; + return (0); +} + +static inline int +pci_channel_offline(struct pci_dev *pdev) +{ + + return (pci_read_config(pdev->dev.bsddev, PCIR_VENDOR, 2) == PCIV_INVALID); +} + +static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) +{ + return -ENODEV; +} +static inline void pci_disable_sriov(struct pci_dev *dev) +{ +} + +static inline void * +pci_iomap(struct pci_dev *dev, int mmio_bar, int mmio_size __unused) +{ + struct pci_mmio_region *mmio; + + mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO); + mmio->rid = PCIR_BAR(mmio_bar); + mmio->type = pci_resource_type(dev, mmio_bar); + mmio->res = bus_alloc_resource_any(dev->dev.bsddev, mmio->type, + &mmio->rid, RF_ACTIVE); + if (mmio->res == NULL) { + free(mmio, M_DEVBUF); + return (NULL); + } + TAILQ_INSERT_TAIL(&dev->mmio, mmio, next); + + return ((void *)rman_get_bushandle(mmio->res)); +} + +static inline void +pci_iounmap(struct pci_dev *dev, void *res) +{ + struct pci_mmio_region *mmio, *p; + + TAILQ_FOREACH_SAFE(mmio, &dev->mmio, next, p) { + if (res != (void *)rman_get_bushandle(mmio->res)) + continue; + bus_release_resource(dev->dev.bsddev, + mmio->type, mmio->rid, mmio->res); + TAILQ_REMOVE(&dev->mmio, mmio, next); + free(mmio, M_DEVBUF); + return; + } +} + +#define DEFINE_PCI_DEVICE_TABLE(_table) \ + const struct pci_device_id _table[] __devinitdata + + +/* XXX This should not be necessary. */ +#define pcix_set_mmrbc(d, v) 0 +#define pcix_get_max_mmrbc(d) 0 +#define pcie_set_readrq(d, v) pci_set_max_read_req(&(d)->dev, (v)) + +#define PCI_DMA_BIDIRECTIONAL 0 +#define PCI_DMA_TODEVICE 1 +#define PCI_DMA_FROMDEVICE 2 +#define PCI_DMA_NONE 3 + +#define pci_pool dma_pool +#define pci_pool_destroy(...) dma_pool_destroy(__VA_ARGS__) +#define pci_pool_alloc(...) dma_pool_alloc(__VA_ARGS__) +#define pci_pool_free(...) dma_pool_free(__VA_ARGS__) +#define pci_pool_create(_name, _pdev, _size, _align, _alloc) \ + dma_pool_create(_name, &(_pdev)->dev, _size, _align, _alloc) +#define pci_free_consistent(_hwdev, _size, _vaddr, _dma_handle) \ + dma_free_coherent((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \ + _size, _vaddr, _dma_handle) +#define pci_map_sg(_hwdev, _sg, _nents, _dir) \ + dma_map_sg((_hwdev) == NULL ? NULL : &(_hwdev->dev), \ + _sg, _nents, (enum dma_data_direction)_dir) +#define pci_map_single(_hwdev, _ptr, _size, _dir) \ + dma_map_single((_hwdev) == NULL ? NULL : &(_hwdev->dev), \ + (_ptr), (_size), (enum dma_data_direction)_dir) +#define pci_unmap_single(_hwdev, _addr, _size, _dir) \ + dma_unmap_single((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \ + _addr, _size, (enum dma_data_direction)_dir) +#define pci_unmap_sg(_hwdev, _sg, _nents, _dir) \ + dma_unmap_sg((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \ + _sg, _nents, (enum dma_data_direction)_dir) +#define pci_map_page(_hwdev, _page, _offset, _size, _dir) \ + dma_map_page((_hwdev) == NULL ? NULL : &(_hwdev)->dev, _page,\ + _offset, _size, (enum dma_data_direction)_dir) +#define pci_unmap_page(_hwdev, _dma_address, _size, _dir) \ + dma_unmap_page((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \ + _dma_address, _size, (enum dma_data_direction)_dir) +#define pci_set_dma_mask(_pdev, mask) dma_set_mask(&(_pdev)->dev, (mask)) +#define pci_dma_mapping_error(_pdev, _dma_addr) \ + dma_mapping_error(&(_pdev)->dev, _dma_addr) +#define pci_set_consistent_dma_mask(_pdev, _mask) \ + dma_set_coherent_mask(&(_pdev)->dev, (_mask)) +#define DECLARE_PCI_UNMAP_ADDR(x) DEFINE_DMA_UNMAP_ADDR(x); +#define DECLARE_PCI_UNMAP_LEN(x) DEFINE_DMA_UNMAP_LEN(x); +#define pci_unmap_addr dma_unmap_addr +#define pci_unmap_addr_set dma_unmap_addr_set +#define pci_unmap_len dma_unmap_len +#define pci_unmap_len_set dma_unmap_len_set + +typedef unsigned int __bitwise pci_channel_state_t; +typedef unsigned int __bitwise pci_ers_result_t; + +enum pci_channel_state { + pci_channel_io_normal = 1, + pci_channel_io_frozen = 2, + pci_channel_io_perm_failure = 3, +}; + +enum pci_ers_result { + PCI_ERS_RESULT_NONE = 1, + PCI_ERS_RESULT_CAN_RECOVER = 2, + PCI_ERS_RESULT_NEED_RESET = 3, + PCI_ERS_RESULT_DISCONNECT = 4, + PCI_ERS_RESULT_RECOVERED = 5, +}; + + +/* PCI bus error event callbacks */ +struct pci_error_handlers { + pci_ers_result_t (*error_detected)(struct pci_dev *dev, + enum pci_channel_state error); + pci_ers_result_t (*mmio_enabled)(struct pci_dev *dev); + pci_ers_result_t (*link_reset)(struct pci_dev *dev); + pci_ers_result_t (*slot_reset)(struct pci_dev *dev); + void (*resume)(struct pci_dev *dev); +}; + +/* FreeBSD does not support SRIOV - yet */ +static inline struct pci_dev *pci_physfn(struct pci_dev *dev) +{ + return dev; +} + +static inline bool pci_is_pcie(struct pci_dev *dev) +{ + return !!pci_pcie_cap(dev); +} + +static inline u16 pcie_flags_reg(struct pci_dev *dev) +{ + int pos; + u16 reg16; + + pos = pci_find_capability(dev, PCI_CAP_ID_EXP); + if (!pos) + return 0; + + pci_read_config_word(dev, pos + PCI_EXP_FLAGS, ®16); + + return reg16; +} + + +static inline int pci_pcie_type(struct pci_dev *dev) +{ + return (pcie_flags_reg(dev) & PCI_EXP_FLAGS_TYPE) >> 4; +} + +static inline int pcie_cap_version(struct pci_dev *dev) +{ + return pcie_flags_reg(dev) & PCI_EXP_FLAGS_VERS; +} + +static inline bool pcie_cap_has_lnkctl(struct pci_dev *dev) +{ + int type = pci_pcie_type(dev); + + return pcie_cap_version(dev) > 1 || + type == PCI_EXP_TYPE_ROOT_PORT || + type == PCI_EXP_TYPE_ENDPOINT || + type == PCI_EXP_TYPE_LEG_END; +} + +static inline bool pcie_cap_has_devctl(const struct pci_dev *dev) +{ + return true; +} + +static inline bool pcie_cap_has_sltctl(struct pci_dev *dev) +{ + int type = pci_pcie_type(dev); + + return pcie_cap_version(dev) > 1 || type == PCI_EXP_TYPE_ROOT_PORT || + (type == PCI_EXP_TYPE_DOWNSTREAM && + pcie_flags_reg(dev) & PCI_EXP_FLAGS_SLOT); +} + +static inline bool pcie_cap_has_rtctl(struct pci_dev *dev) +{ + int type = pci_pcie_type(dev); + + return pcie_cap_version(dev) > 1 || type == PCI_EXP_TYPE_ROOT_PORT || + type == PCI_EXP_TYPE_RC_EC; +} + +static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos) +{ + if (!pci_is_pcie(dev)) + return false; + + switch (pos) { + case PCI_EXP_FLAGS_TYPE: + return true; + case PCI_EXP_DEVCAP: + case PCI_EXP_DEVCTL: + case PCI_EXP_DEVSTA: + return pcie_cap_has_devctl(dev); + case PCI_EXP_LNKCAP: + case PCI_EXP_LNKCTL: + case PCI_EXP_LNKSTA: + return pcie_cap_has_lnkctl(dev); + case PCI_EXP_SLTCAP: + case PCI_EXP_SLTCTL: + case PCI_EXP_SLTSTA: + return pcie_cap_has_sltctl(dev); + case PCI_EXP_RTCTL: + case PCI_EXP_RTCAP: + case PCI_EXP_RTSTA: + return pcie_cap_has_rtctl(dev); + case PCI_EXP_DEVCAP2: + case PCI_EXP_DEVCTL2: + case PCI_EXP_LNKCAP2: + case PCI_EXP_LNKCTL2: + case PCI_EXP_LNKSTA2: + return pcie_cap_version(dev) > 1; + default: + return false; + } +} + +static inline int +pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *dst) +{ + if (pos & 3) + return -EINVAL; + + if (!pcie_capability_reg_implemented(dev, pos)) + return -EINVAL; + + return pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, dst); +} + +static inline int +pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *dst) +{ + if (pos & 3) + return -EINVAL; + + if (!pcie_capability_reg_implemented(dev, pos)) + return -EINVAL; + + return pci_read_config_word(dev, pci_pcie_cap(dev) + pos, dst); +} + +static inline int +pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val) +{ + if (pos & 1) + return -EINVAL; + + if (!pcie_capability_reg_implemented(dev, pos)) + return 0; + + return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val); +} + +static inline int pcie_get_minimum_link(struct pci_dev *dev, + enum pci_bus_speed *speed, enum pcie_link_width *width) +{ + *speed = PCI_SPEED_UNKNOWN; + *width = PCIE_LNK_WIDTH_UNKNOWN; + return (0); +} + +static inline int +pci_num_vf(struct pci_dev *dev) +{ + return (0); +} + +static inline enum pci_bus_speed +pcie_get_speed_cap(struct pci_dev *dev) +{ + device_t root; + uint32_t lnkcap, lnkcap2; + int error, pos; + + root = device_get_parent(dev->dev.bsddev); + if (root == NULL) + return (PCI_SPEED_UNKNOWN); + root = device_get_parent(root); + if (root == NULL) + return (PCI_SPEED_UNKNOWN); + root = device_get_parent(root); + if (root == NULL) + return (PCI_SPEED_UNKNOWN); + + if (pci_get_vendor(root) == PCI_VENDOR_ID_VIA || + pci_get_vendor(root) == PCI_VENDOR_ID_SERVERWORKS) + return (PCI_SPEED_UNKNOWN); + + if ((error = pci_find_cap(root, PCIY_EXPRESS, &pos)) != 0) + return (PCI_SPEED_UNKNOWN); + + lnkcap2 = pci_read_config(root, pos + PCIER_LINK_CAP2, 4); + + if (lnkcap2) { /* PCIe r3.0-compliant */ + if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB) + return (PCIE_SPEED_2_5GT); + if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB) + return (PCIE_SPEED_5_0GT); + if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB) + return (PCIE_SPEED_8_0GT); + if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_16_0GB) + return (PCIE_SPEED_16_0GT); + } else { /* pre-r3.0 */ + lnkcap = pci_read_config(root, pos + PCIER_LINK_CAP, 4); + if (lnkcap & PCI_EXP_LNKCAP_SLS_2_5GB) + return (PCIE_SPEED_2_5GT); + if (lnkcap & PCI_EXP_LNKCAP_SLS_5_0GB) + return (PCIE_SPEED_5_0GT); + if (lnkcap & PCI_EXP_LNKCAP_SLS_8_0GB) + return (PCIE_SPEED_8_0GT); + if (lnkcap & PCI_EXP_LNKCAP_SLS_16_0GB) + return (PCIE_SPEED_16_0GT); + } + return (PCI_SPEED_UNKNOWN); +} + +static inline enum pcie_link_width +pcie_get_width_cap(struct pci_dev *dev) +{ + uint32_t lnkcap; + + pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); + if (lnkcap) + return ((lnkcap & PCI_EXP_LNKCAP_MLW) >> 4); + + return (PCIE_LNK_WIDTH_UNKNOWN); +} + +static inline int +pcie_get_mps(struct pci_dev *dev) +{ + return (pci_get_max_payload(dev->dev.bsddev)); +} + +static inline uint32_t +PCIE_SPEED2MBS_ENC(enum pci_bus_speed spd) +{ + + switch(spd) { + case PCIE_SPEED_16_0GT: + return (16000 * 128 / 130); + case PCIE_SPEED_8_0GT: + return (8000 * 128 / 130); + case PCIE_SPEED_5_0GT: + return (5000 * 8 / 10); + case PCIE_SPEED_2_5GT: + return (2500 * 8 / 10); + default: + return (0); + } +} + +static inline uint32_t +pcie_bandwidth_available(struct pci_dev *pdev, + struct pci_dev **limiting, + enum pci_bus_speed *speed, + enum pcie_link_width *width) +{ + enum pci_bus_speed nspeed = pcie_get_speed_cap(pdev); + enum pcie_link_width nwidth = pcie_get_width_cap(pdev); + + if (speed) + *speed = nspeed; + if (width) + *width = nwidth; + + return (nwidth * PCIE_SPEED2MBS_ENC(nspeed)); +} + +/* + * The following functions can be used to attach/detach the LinuxKPI's + * PCI device runtime. The pci_driver and pci_device_id pointer is + * allowed to be NULL. Other pointers must be all valid. + * The pci_dev structure should be zero-initialized before passed + * to the linux_pci_attach_device function. + */ +extern int linux_pci_attach_device(device_t, struct pci_driver *, + const struct pci_device_id *, struct pci_dev *); +extern int linux_pci_detach_device(struct pci_dev *); + +static inline int +pci_dev_present(const struct pci_device_id *cur) +{ + while (cur != NULL && (cur->vendor || cur->device)) { + if (pci_find_device(cur->vendor, cur->device) != NULL) { + return (1); + } + cur++; + } + return (0); +} + +#endif /* _LINUX_PCI_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/pfn.h b/sys/compat/linuxkpi/common/include/linux/pfn.h new file mode 100644 index 000000000000..162ca102c951 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/pfn.h @@ -0,0 +1,44 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_PFN_H_ +#define _LINUX_PFN_H_ + +#include <linux/types.h> + +typedef struct { + u64 val; +} pfn_t; + +#define PFN_ALIGN(x) (((unsigned long)(x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) +#define PFN_UP(x) (((x) + PAGE_SIZE - 1) >> PAGE_SHIFT) +#define PFN_DOWN(x) ((x) >> PAGE_SHIFT) +#define PFN_PHYS(x) ((phys_addr_t)(x) << PAGE_SHIFT) +#define PHYS_PFN(x) ((unsigned long)((x) >> PAGE_SHIFT)) + +#endif /* _LINUX_PFN_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/pfn_t.h b/sys/compat/linuxkpi/common/include/linux/pfn_t.h new file mode 100644 index 000000000000..bfa80b14ae94 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/pfn_t.h @@ -0,0 +1,56 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_PFN_T_H_ +#define _LINUX_PFN_T_H_ + +#include <linux/mm.h> + +CTASSERT(PAGE_SHIFT > 4); + +#define PFN_FLAGS_MASK (((u64)(PAGE_SIZE - 1)) << (64 - PAGE_SHIFT)) +#define PFN_SG_CHAIN (1ULL << (64 - 1)) +#define PFN_SG_LAST (1ULL << (64 - 2)) +#define PFN_DEV (1ULL << (64 - 3)) +#define PFN_MAP (1ULL << (64 - 4)) + +static inline pfn_t +__pfn_to_pfn_t(unsigned long pfn, u64 flags) +{ + pfn_t pfn_t = { pfn | (flags & PFN_FLAGS_MASK) }; + + return (pfn_t); +} + +static inline pfn_t +pfn_to_pfn_t(unsigned long pfn) +{ + return (__pfn_to_pfn_t (pfn, 0)); +} + +#endif /* _LINUX_PFN_T_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/pid.h b/sys/compat/linuxkpi/common/include/linux/pid.h new file mode 100644 index 000000000000..73d8f1f6edfe --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/pid.h @@ -0,0 +1,70 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_PID_H_ +#define _LINUX_PID_H_ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/proc.h> + +enum pid_type { + PIDTYPE_PID, + PIDTYPE_PGID, + PIDTYPE_SID, + PIDTYPE_MAX +}; + +#define pid_nr(n) (n) +#define pid_vnr(n) (n) +#define from_kuid_munged(a, uid) (uid) + +#define pid_task(pid, type) ({ \ + struct task_struct *__ts; \ + CTASSERT((type) == PIDTYPE_PID); \ + __ts = linux_pid_task(pid); \ + __ts; \ +}) + +#define get_pid_task(pid, type) ({ \ + struct task_struct *__ts; \ + CTASSERT((type) == PIDTYPE_PID); \ + __ts = linux_get_pid_task(pid); \ + __ts; \ +}) + +#define get_task_pid(task, type) ({ \ + CTASSERT((type) == PIDTYPE_PID); \ + (task)->task_thread->td_tid; \ +}) + +struct task_struct; +extern struct task_struct *linux_pid_task(pid_t); +extern struct task_struct *linux_get_pid_task(pid_t); + +#endif /* _LINUX_PID_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/poll.h b/sys/compat/linuxkpi/common/include/linux/poll.h new file mode 100644 index 000000000000..33501165df24 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/poll.h @@ -0,0 +1,48 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_POLL_H_ +#define _LINUX_POLL_H_ + +#include <sys/poll.h> +#include <sys/fcntl.h> + +#include <linux/wait.h> +#include <linux/file.h> + +typedef struct poll_table_struct { +} poll_table; + +extern void linux_poll_wait(struct linux_file *, wait_queue_head_t *, poll_table *); +#define poll_wait(...) linux_poll_wait(__VA_ARGS__) + +extern void linux_poll_wakeup(struct linux_file *); + +#endif /* _LINUX_POLL_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/preempt.h b/sys/compat/linuxkpi/common/include/linux/preempt.h new file mode 100644 index 000000000000..e19e7ee09a48 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/preempt.h @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_PREEMPT_H_ +#define _LINUX_PREEMPT_H_ + +#include <linux/hardirq.h> +#include <linux/list.h> + +#define in_interrupt() \ + (curthread->td_intr_nesting_level || curthread->td_critnest) + +#define in_task() (curthread->td_priority >= PI_SOFT) + +#define preempt_disable() critical_enter() +#define preempt_enable() critical_exit() + +#endif /* _LINUX_PREEMPT_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/printk.h b/sys/compat/linuxkpi/common/include/linux/printk.h new file mode 100644 index 000000000000..31802bdd2c99 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/printk.h @@ -0,0 +1,130 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_PRINTK_H_ +#define _LINUX_PRINTK_H_ + +#include <linux/kernel.h> + +/* GID printing macros */ +#define GID_PRINT_FMT "%.4x:%.4x:%.4x:%.4x:%.4x:%.4x:%.4x:%.4x" +#define GID_PRINT_ARGS(gid_raw) htons(((u16 *)gid_raw)[0]), htons(((u16 *)gid_raw)[1]),\ + htons(((u16 *)gid_raw)[2]), htons(((u16 *)gid_raw)[3]),\ + htons(((u16 *)gid_raw)[4]), htons(((u16 *)gid_raw)[5]),\ + htons(((u16 *)gid_raw)[6]), htons(((u16 *)gid_raw)[7]) + +enum { + DUMP_PREFIX_NONE, + DUMP_PREFIX_ADDRESS, + DUMP_PREFIX_OFFSET +}; + +static inline void +print_hex_dump(const char *level, const char *prefix_str, + const int prefix_type, const int rowsize, const int groupsize, + const void *buf, size_t len, const bool ascii) +{ + typedef const struct { long long value; } __packed *print_64p_t; + typedef const struct { uint32_t value; } __packed *print_32p_t; + typedef const struct { uint16_t value; } __packed *print_16p_t; + const void *buf_old = buf; + int row; + + while (len > 0) { + if (level != NULL) + printf("%s", level); + if (prefix_str != NULL) + printf("%s ", prefix_str); + + switch (prefix_type) { + case DUMP_PREFIX_ADDRESS: + printf("[%p] ", buf); + break; + case DUMP_PREFIX_OFFSET: + printf("[%p] ", (const char *)((const char *)buf - + (const char *)buf_old)); + break; + default: + break; + } + for (row = 0; row != rowsize; row++) { + if (groupsize == 8 && len > 7) { + printf("%016llx ", ((print_64p_t)buf)->value); + buf = (const uint8_t *)buf + 8; + len -= 8; + } else if (groupsize == 4 && len > 3) { + printf("%08x ", ((print_32p_t)buf)->value); + buf = (const uint8_t *)buf + 4; + len -= 4; + } else if (groupsize == 2 && len > 1) { + printf("%04x ", ((print_16p_t)buf)->value); + buf = (const uint8_t *)buf + 2; + len -= 2; + } else if (len > 0) { + printf("%02x ", *(const uint8_t *)buf); + buf = (const uint8_t *)buf + 1; + len--; + } else { + break; + } + } + printf("\n"); + } +} + +static inline void +print_hex_dump_bytes(const char *prefix_str, const int prefix_type, + const void *buf, size_t len) +{ + print_hex_dump(NULL, prefix_str, prefix_type, 16, 1, buf, len, 0); +} + +#define printk_ratelimit() ({ \ + static linux_ratelimit_t __ratelimited; \ + linux_ratelimited(&__ratelimited); \ +}) + +#define printk_ratelimited(...) ({ \ + bool __retval = printk_ratelimit(); \ + if (__retval) \ + printk(__VA_ARGS__); \ + __retval; \ +}) + +#define pr_err_ratelimited(fmt, ...) \ + printk_ratelimited(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) + +#define print_hex_dump_debug(...) \ + print_hex_dump(KERN_DEBUG, ##__VA_ARGS__) + +#define pr_info_ratelimited(fmt, ...) \ + printk_ratelimited(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__) + +#endif /* _LINUX_PRINTK_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/radix-tree.h b/sys/compat/linuxkpi/common/include/linux/radix-tree.h new file mode 100644 index 000000000000..1bef60c44c41 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/radix-tree.h @@ -0,0 +1,85 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2018 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_RADIX_TREE_H_ +#define _LINUX_RADIX_TREE_H_ + +#include <linux/types.h> + +#define RADIX_TREE_MAP_SHIFT 6 +#define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT) +#define RADIX_TREE_MAP_MASK (RADIX_TREE_MAP_SIZE - 1UL) +#define RADIX_TREE_MAX_HEIGHT \ + howmany(sizeof(long) * NBBY, RADIX_TREE_MAP_SHIFT) + +#define RADIX_TREE_ENTRY_MASK 3UL +#define RADIX_TREE_EXCEPTIONAL_ENTRY 2UL +#define RADIX_TREE_EXCEPTIONAL_SHIFT 2 + +struct radix_tree_node { + void *slots[RADIX_TREE_MAP_SIZE]; + int count; +}; + +struct radix_tree_root { + struct radix_tree_node *rnode; + gfp_t gfp_mask; + int height; +}; + +struct radix_tree_iter { + unsigned long index; +}; + +#define RADIX_TREE_INIT(mask) \ + { .rnode = NULL, .gfp_mask = mask, .height = 0 }; +#define INIT_RADIX_TREE(root, mask) \ + { (root)->rnode = NULL; (root)->gfp_mask = mask; (root)->height = 0; } +#define RADIX_TREE(name, mask) \ + struct radix_tree_root name = RADIX_TREE_INIT(mask) + +#define radix_tree_for_each_slot(slot, root, iter, start) \ + for ((iter)->index = (start); \ + radix_tree_iter_find(root, iter, &(slot)); (iter)->index++) + +static inline int +radix_tree_exception(void *arg) +{ + return ((uintptr_t)arg & RADIX_TREE_ENTRY_MASK); +} + +void *radix_tree_lookup(struct radix_tree_root *, unsigned long); +void *radix_tree_delete(struct radix_tree_root *, unsigned long); +int radix_tree_insert(struct radix_tree_root *, unsigned long, void *); +int radix_tree_store(struct radix_tree_root *, unsigned long, void **); +bool radix_tree_iter_find(struct radix_tree_root *, struct radix_tree_iter *, void ***); +void radix_tree_iter_delete(struct radix_tree_root *, struct radix_tree_iter *, void **); + +#endif /* _LINUX_RADIX_TREE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/random.h b/sys/compat/linuxkpi/common/include/linux/random.h new file mode 100644 index 000000000000..14ea88237e1b --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/random.h @@ -0,0 +1,71 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_RANDOM_H_ +#define _LINUX_RANDOM_H_ + +#include <sys/random.h> +#include <sys/libkern.h> + +#define get_random_u32() get_random_int() + +static inline void +get_random_bytes(void *buf, int nbytes) +{ + + arc4random_buf(buf, nbytes); +} + +static inline u_int +get_random_int(void) +{ + u_int val; + + get_random_bytes(&val, sizeof(val)); + return (val); +} + +static inline u_long +get_random_long(void) +{ + u_long val; + + get_random_bytes(&val, sizeof(val)); + return (val); +} + +static inline u32 +prandom_u32_max(u32 max) +{ + return (arc4random_uniform(max)); +} + +#endif /* _LINUX_RANDOM_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/rbtree.h b/sys/compat/linuxkpi/common/include/linux/rbtree.h new file mode 100644 index 000000000000..1a49b5d5cdd9 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/rbtree.h @@ -0,0 +1,98 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_RBTREE_H_ +#define _LINUX_RBTREE_H_ + +#include <sys/stddef.h> +#include <sys/tree.h> + +struct rb_node { + RB_ENTRY(rb_node) __entry; +}; +#define rb_left __entry.rbe_left +#define rb_right __entry.rbe_right + +/* + * We provide a false structure that has the same bit pattern as tree.h + * presents so it matches the member names expected by linux. + */ +struct rb_root { + struct rb_node *rb_node; +}; + +/* + * In linux all of the comparisons are done by the caller. + */ +int panic_cmp(struct rb_node *one, struct rb_node *two); + +RB_HEAD(linux_root, rb_node); +RB_PROTOTYPE(linux_root, rb_node, __entry, panic_cmp); + +#define rb_entry(ptr, type, member) container_of(ptr, type, member) + +#define RB_EMPTY_ROOT(root) RB_EMPTY((struct linux_root *)root) +#define RB_EMPTY_NODE(node) (RB_PARENT(node, __entry) == node) +#define RB_CLEAR_NODE(node) RB_SET_PARENT(node, node, __entry) + +#define rb_insert_color(node, root) \ + linux_root_RB_INSERT_COLOR((struct linux_root *)(root), (node)) +#define rb_erase(node, root) \ + linux_root_RB_REMOVE((struct linux_root *)(root), (node)) +#define rb_next(node) RB_NEXT(linux_root, NULL, (node)) +#define rb_prev(node) RB_PREV(linux_root, NULL, (node)) +#define rb_first(root) RB_MIN(linux_root, (struct linux_root *)(root)) +#define rb_last(root) RB_MAX(linux_root, (struct linux_root *)(root)) + +static inline void +rb_link_node(struct rb_node *node, struct rb_node *parent, + struct rb_node **rb_link) +{ + RB_SET(node, parent, __entry); + *rb_link = node; +} + +static inline void +rb_replace_node(struct rb_node *victim, struct rb_node *new, + struct rb_root *root) +{ + + RB_SWAP_CHILD((struct linux_root *)root, victim, new, __entry); + if (victim->rb_left) + RB_SET_PARENT(victim->rb_left, new, __entry); + if (victim->rb_right) + RB_SET_PARENT(victim->rb_right, new, __entry); + *new = *victim; +} + +#undef RB_ROOT +#define RB_ROOT (struct rb_root) { NULL } + +#endif /* _LINUX_RBTREE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/rculist.h b/sys/compat/linuxkpi/common/include/linux/rculist.h new file mode 100644 index 000000000000..0a4ad499c380 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/rculist.h @@ -0,0 +1,108 @@ +/*- + * Copyright (c) 2015 François Tigeot + * Copyright (c) 2016-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_RCULIST_H_ +#define _LINUX_RCULIST_H_ + +#include <linux/list.h> +#include <linux/rcupdate.h> + +#define list_entry_rcu(ptr, type, member) \ + container_of(READ_ONCE(ptr), type, member) + +#define list_next_rcu(head) (*((struct list_head **)(&(head)->next))) + +#define list_for_each_entry_rcu(pos, head, member) \ + for (pos = list_entry_rcu((head)->next, typeof(*(pos)), member); \ + &(pos)->member != (head); \ + pos = list_entry_rcu((pos)->member.next, typeof(*(pos)), member)) + +static inline void +list_add_rcu(struct list_head *new, struct list_head *prev) +{ + new->next = prev->next; + new->prev = prev; + rcu_assign_pointer(list_next_rcu(prev), new); + prev->prev = new; +} + +#define hlist_first_rcu(head) (*((struct hlist_node **)(&(head)->first))) +#define hlist_next_rcu(node) (*((struct hlist_node **)(&(node)->next))) +#define hlist_pprev_rcu(node) (*((struct hlist_node **)((node)->pprev))) + +static inline void +hlist_add_behind_rcu(struct hlist_node *n, struct hlist_node *prev) +{ + n->next = prev->next; + n->pprev = &prev->next; + rcu_assign_pointer(hlist_next_rcu(prev), n); + if (n->next) + n->next->pprev = &n->next; +} + +#define hlist_for_each_entry_rcu(pos, head, member) \ + for (pos = hlist_entry_safe (rcu_dereference_raw(hlist_first_rcu(head)),\ + typeof(*(pos)), member); \ + (pos); \ + pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \ + &(pos)->member)), typeof(*(pos)), member)) + +static inline void +hlist_del_rcu(struct hlist_node *n) +{ + struct hlist_node *next = n->next; + struct hlist_node **pprev = n->pprev; + + WRITE_ONCE(*pprev, next); + if (next) + next->pprev = pprev; +} + +static inline void +hlist_add_head_rcu(struct hlist_node *n, struct hlist_head *h) +{ + struct hlist_node *first = h->first; + + n->next = first; + n->pprev = &h->first; + rcu_assign_pointer(hlist_first_rcu(h), n); + if (first) + first->pprev = &n->next; +} + +static inline void +hlist_del_init_rcu(struct hlist_node *n) +{ + if (!hlist_unhashed(n)) { + hlist_del_rcu(n); + n->pprev = NULL; + } +} + +#endif /* _LINUX_RCULIST_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/rcupdate.h b/sys/compat/linuxkpi/common/include/linux/rcupdate.h new file mode 100644 index 000000000000..a43736e0285f --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/rcupdate.h @@ -0,0 +1,120 @@ +/*- + * Copyright (c) 2016-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_RCUPDATE_H_ +#define _LINUX_RCUPDATE_H_ + +#include <linux/compiler.h> +#include <linux/types.h> + +#include <machine/atomic.h> + +#define LINUX_KFREE_RCU_OFFSET_MAX 4096 /* exclusive */ + +/* BSD specific defines */ +#define RCU_TYPE_REGULAR 0 +#define RCU_TYPE_SLEEPABLE 1 +#define RCU_TYPE_MAX 2 + +#define RCU_INITIALIZER(v) \ + ((__typeof(*(v)) *)(v)) + +#define RCU_INIT_POINTER(p, v) do { \ + (p) = (v); \ +} while (0) + +#define call_rcu(ptr, func) do { \ + linux_call_rcu(RCU_TYPE_REGULAR, ptr, func); \ +} while (0) + +#define rcu_barrier(void) do { \ + linux_rcu_barrier(RCU_TYPE_REGULAR); \ +} while (0) + +#define rcu_read_lock(void) do { \ + linux_rcu_read_lock(RCU_TYPE_REGULAR); \ +} while (0) + +#define rcu_read_unlock(void) do { \ + linux_rcu_read_unlock(RCU_TYPE_REGULAR);\ +} while (0) + +#define synchronize_rcu(void) do { \ + linux_synchronize_rcu(RCU_TYPE_REGULAR); \ +} while (0) + +#define synchronize_rcu_expedited(void) do { \ + linux_synchronize_rcu(RCU_TYPE_REGULAR); \ +} while (0) + +#define kfree_rcu(ptr, rcu_head) do { \ + CTASSERT(offsetof(__typeof(*(ptr)), rcu_head) < \ + LINUX_KFREE_RCU_OFFSET_MAX); \ + call_rcu(&(ptr)->rcu_head, (rcu_callback_t)(uintptr_t) \ + offsetof(__typeof(*(ptr)), rcu_head)); \ +} while (0) + +#define rcu_access_pointer(p) \ + ((__typeof(*p) *)READ_ONCE(p)) + +#define rcu_dereference_protected(p, c) \ + ((__typeof(*p) *)READ_ONCE(p)) + +#define rcu_dereference(p) \ + rcu_dereference_protected(p, 0) + +#define rcu_dereference_raw(p) \ + ((__typeof(*p) *)READ_ONCE(p)) + +#define rcu_pointer_handoff(p) (p) + +#define rcu_assign_pointer(p, v) do { \ + atomic_store_rel_ptr((volatile uintptr_t *)&(p), \ + (uintptr_t)(v)); \ +} while (0) + +#define rcu_swap_protected(rcu, ptr, c) do { \ + typeof(ptr) p = rcu_dereference_protected(rcu, c); \ + rcu_assign_pointer(rcu, ptr); \ + (ptr) = p; \ +} while (0) + +/* prototypes */ + +extern void linux_call_rcu(unsigned type, struct rcu_head *ptr, rcu_callback_t func); +extern void linux_rcu_barrier(unsigned type); +extern void linux_rcu_read_lock(unsigned type); +extern void linux_rcu_read_unlock(unsigned type); +extern void linux_synchronize_rcu(unsigned type); + +/* Empty implementation for !DEBUG */ +#define init_rcu_head(...) +#define destroy_rcu_head(...) +#define init_rcu_head_on_stack(...) +#define destroy_rcu_head_on_stack(...) + +#endif /* _LINUX_RCUPDATE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/refcount.h b/sys/compat/linuxkpi/common/include/linux/refcount.h new file mode 100644 index 000000000000..02f9f32f7eb4 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/refcount.h @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Emmanuel Vadot under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_REFCOUNT_H +#define _LINUX_REFCOUNT_H + +#include <linux/atomic.h> + +struct refcount_linux { + atomic_t value; +}; +typedef struct refcount_linux refcount_t; + +static inline void +refcount_set(refcount_t *ref, unsigned int i) +{ + atomic_set(&ref->value, i); +} + +static inline void +refcount_inc(refcount_t *ref) +{ + atomic_inc(&ref->value); +} + +static inline bool +refcount_inc_not_zero(refcount_t *ref) +{ + return (atomic_inc_not_zero(&ref->value)); +} + +static inline void +refcount_dec(refcount_t *ref) +{ + atomic_dec(&ref->value); +} + +static inline unsigned int +refcount_read(refcount_t *ref) +{ + return atomic_read(&ref->value); +} + +static inline bool +refcount_dec_and_lock_irqsave(refcount_t *ref, spinlock_t *lock, + unsigned long *flags) +{ + if (atomic_dec_and_test(&ref->value) == true) { + spin_lock_irqsave(lock, flags); + return (true); + } + return (false); +} + +#endif /* __LINUX_REFCOUNT_H__ */ diff --git a/sys/compat/linuxkpi/common/include/linux/rwlock.h b/sys/compat/linuxkpi/common/include/linux/rwlock.h new file mode 100644 index 000000000000..621afb1e954b --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/rwlock.h @@ -0,0 +1,68 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_RWLOCK_H_ +#define _LINUX_RWLOCK_H_ + +#include <sys/types.h> +#include <sys/lock.h> +#include <sys/rwlock.h> +#include <sys/libkern.h> + +typedef struct { + struct rwlock rw; +} rwlock_t; + +#define read_lock(_l) rw_rlock(&(_l)->rw) +#define write_lock(_l) rw_wlock(&(_l)->rw) +#define read_unlock(_l) rw_runlock(&(_l)->rw) +#define write_unlock(_l) rw_wunlock(&(_l)->rw) +#define read_lock_irq(lock) read_lock((lock)) +#define read_unlock_irq(lock) read_unlock((lock)) +#define write_lock_irq(lock) write_lock((lock)) +#define write_unlock_irq(lock) write_unlock((lock)) +#define read_lock_irqsave(lock, flags) \ + do {(flags) = 0; read_lock(lock); } while (0) +#define write_lock_irqsave(lock, flags) \ + do {(flags) = 0; write_lock(lock); } while (0) +#define read_unlock_irqrestore(lock, flags) \ + do { read_unlock(lock); } while (0) +#define write_unlock_irqrestore(lock, flags) \ + do { write_unlock(lock); } while (0) + +static inline void +rwlock_init(rwlock_t *lock) +{ + + memset(&lock->rw, 0, sizeof(lock->rw)); + rw_init_flags(&lock->rw, "lnxrw", RW_NOWITNESS); +} + +#endif /* _LINUX_RWLOCK_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/rwsem.h b/sys/compat/linuxkpi/common/include/linux/rwsem.h new file mode 100644 index 000000000000..8850da91bd4d --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/rwsem.h @@ -0,0 +1,84 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_RWSEM_H_ +#define _LINUX_RWSEM_H_ + +#include <sys/param.h> +#include <sys/lock.h> +#include <sys/sx.h> +#include <sys/libkern.h> +#include <sys/kernel.h> + +struct rw_semaphore { + struct sx sx; +}; + +#define down_write(_rw) sx_xlock(&(_rw)->sx) +#define up_write(_rw) sx_xunlock(&(_rw)->sx) +#define down_read(_rw) sx_slock(&(_rw)->sx) +#define up_read(_rw) sx_sunlock(&(_rw)->sx) +#define down_read_trylock(_rw) !!sx_try_slock(&(_rw)->sx) +#define down_write_trylock(_rw) !!sx_try_xlock(&(_rw)->sx) +#define down_write_killable(_rw) linux_down_write_killable(_rw) +#define downgrade_write(_rw) sx_downgrade(&(_rw)->sx) +#define down_read_nested(_rw, _sc) down_read(_rw) +#define init_rwsem(_rw) linux_init_rwsem(_rw, rwsem_name("lnxrwsem")) + +#ifdef WITNESS_ALL +/* NOTE: the maximum WITNESS name is 64 chars */ +#define __rwsem_name(name, file, line) \ + (((const char *){file ":" #line "-" name}) + \ + (sizeof(file) > 16 ? sizeof(file) - 16 : 0)) +#else +#define __rwsem_name(name, file, line) name +#endif +#define _rwsem_name(...) __rwsem_name(__VA_ARGS__) +#define rwsem_name(name) _rwsem_name(name, __FILE__, __LINE__) + +#define DECLARE_RWSEM(name) \ +struct rw_semaphore name; \ +static void name##_rwsem_init(void *arg) \ +{ \ + linux_init_rwsem(&name, rwsem_name(#name)); \ +} \ +SYSINIT(name, SI_SUB_LOCK, SI_ORDER_SECOND, name##_rwsem_init, NULL) + +static inline void +linux_init_rwsem(struct rw_semaphore *rw, const char *name) +{ + + memset(rw, 0, sizeof(*rw)); + sx_init_flags(&rw->sx, name, SX_NOWITNESS); +} + +extern int linux_down_write_killable(struct rw_semaphore *); + +#endif /* _LINUX_RWSEM_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/scatterlist.h b/sys/compat/linuxkpi/common/include/linux/scatterlist.h new file mode 100644 index 000000000000..a23edfb0b4de --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/scatterlist.h @@ -0,0 +1,483 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_SCATTERLIST_H_ +#define _LINUX_SCATTERLIST_H_ + +#include <linux/page.h> +#include <linux/slab.h> +#include <linux/mm.h> + +struct bus_dmamap; +struct scatterlist { + unsigned long page_link; +#define SG_PAGE_LINK_CHAIN 0x1UL +#define SG_PAGE_LINK_LAST 0x2UL +#define SG_PAGE_LINK_MASK 0x3UL + unsigned int offset; + unsigned int length; + dma_addr_t dma_address; + struct bus_dmamap *dma_map; /* FreeBSD specific */ +}; + +CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0); + +struct sg_table { + struct scatterlist *sgl; + unsigned int nents; + unsigned int orig_nents; +}; + +struct sg_page_iter { + struct scatterlist *sg; + unsigned int sg_pgoffset; + unsigned int maxents; + struct { + unsigned int nents; + int pg_advance; + } internal; +}; + +struct sg_dma_page_iter { + struct sg_page_iter base; +}; + +#define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1)) + +#define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) + +#define SG_MAGIC 0x87654321UL +#define SG_CHAIN SG_PAGE_LINK_CHAIN +#define SG_END SG_PAGE_LINK_LAST + +#define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) +#define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) +#define sg_chain_ptr(sg) \ + ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK)) + +#define sg_dma_address(sg) (sg)->dma_address +#define sg_dma_len(sg) (sg)->length + +#define for_each_sg_page(sgl, iter, nents, pgoffset) \ + for (_sg_iter_init(sgl, iter, nents, pgoffset); \ + (iter)->sg; _sg_iter_next(iter)) +#define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \ + for_each_sg_page(sgl, &(iter)->base, nents, pgoffset) + +#define for_each_sg(sglist, sg, sgmax, iter) \ + for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg)) + +typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t); +typedef void (sg_free_fn) (struct scatterlist *, unsigned int); + +static inline void +sg_assign_page(struct scatterlist *sg, struct page *page) +{ + unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK; + + sg->page_link = page_link | (unsigned long)page; +} + +static inline void +sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, + unsigned int offset) +{ + sg_assign_page(sg, page); + sg->offset = offset; + sg->length = len; +} + +static inline struct page * +sg_page(struct scatterlist *sg) +{ + return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK)); +} + +static inline void +sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) +{ + sg_set_page(sg, virt_to_page(buf), buflen, + ((uintptr_t)buf) & (PAGE_SIZE - 1)); +} + +static inline struct scatterlist * +sg_next(struct scatterlist *sg) +{ + if (sg_is_last(sg)) + return (NULL); + sg++; + if (sg_is_chain(sg)) + sg = sg_chain_ptr(sg); + return (sg); +} + +static inline vm_paddr_t +sg_phys(struct scatterlist *sg) +{ + return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); +} + +static inline void * +sg_virt(struct scatterlist *sg) +{ + + return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); +} + +static inline void +sg_chain(struct scatterlist *prv, unsigned int prv_nents, + struct scatterlist *sgl) +{ + struct scatterlist *sg = &prv[prv_nents - 1]; + + sg->offset = 0; + sg->length = 0; + sg->page_link = ((unsigned long)sgl | + SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST; +} + +static inline void +sg_mark_end(struct scatterlist *sg) +{ + sg->page_link |= SG_PAGE_LINK_LAST; + sg->page_link &= ~SG_PAGE_LINK_CHAIN; +} + +static inline void +sg_init_table(struct scatterlist *sg, unsigned int nents) +{ + bzero(sg, sizeof(*sg) * nents); + sg_mark_end(&sg[nents - 1]); +} + +static struct scatterlist * +sg_kmalloc(unsigned int nents, gfp_t gfp_mask) +{ + if (nents == SG_MAX_SINGLE_ALLOC) { + return ((void *)__get_free_page(gfp_mask)); + } else + return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask)); +} + +static inline void +sg_kfree(struct scatterlist *sg, unsigned int nents) +{ + if (nents == SG_MAX_SINGLE_ALLOC) { + free_page((unsigned long)sg); + } else + kfree(sg); +} + +static inline void +__sg_free_table(struct sg_table *table, unsigned int max_ents, + bool skip_first_chunk, sg_free_fn * free_fn) +{ + struct scatterlist *sgl, *next; + + if (unlikely(!table->sgl)) + return; + + sgl = table->sgl; + while (table->orig_nents) { + unsigned int alloc_size = table->orig_nents; + unsigned int sg_size; + + if (alloc_size > max_ents) { + next = sg_chain_ptr(&sgl[max_ents - 1]); + alloc_size = max_ents; + sg_size = alloc_size - 1; + } else { + sg_size = alloc_size; + next = NULL; + } + + table->orig_nents -= sg_size; + if (skip_first_chunk) + skip_first_chunk = 0; + else + free_fn(sgl, alloc_size); + sgl = next; + } + + table->sgl = NULL; +} + +static inline void +sg_free_table(struct sg_table *table) +{ + __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); +} + +static inline int +__sg_alloc_table(struct sg_table *table, unsigned int nents, + unsigned int max_ents, struct scatterlist *first_chunk, + gfp_t gfp_mask, sg_alloc_fn *alloc_fn) +{ + struct scatterlist *sg, *prv; + unsigned int left; + + memset(table, 0, sizeof(*table)); + + if (nents == 0) + return (-EINVAL); + left = nents; + prv = NULL; + do { + unsigned int sg_size; + unsigned int alloc_size = left; + + if (alloc_size > max_ents) { + alloc_size = max_ents; + sg_size = alloc_size - 1; + } else + sg_size = alloc_size; + + left -= sg_size; + + if (first_chunk) { + sg = first_chunk; + first_chunk = NULL; + } else { + sg = alloc_fn(alloc_size, gfp_mask); + } + if (unlikely(!sg)) { + if (prv) + table->nents = ++table->orig_nents; + + return (-ENOMEM); + } + sg_init_table(sg, alloc_size); + table->nents = table->orig_nents += sg_size; + + if (prv) + sg_chain(prv, max_ents, sg); + else + table->sgl = sg; + + if (!left) + sg_mark_end(&sg[sg_size - 1]); + + prv = sg; + } while (left); + + return (0); +} + +static inline int +sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) +{ + int ret; + + ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, + NULL, gfp_mask, sg_kmalloc); + if (unlikely(ret)) + __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); + + return (ret); +} + +static inline int +__sg_alloc_table_from_pages(struct sg_table *sgt, + struct page **pages, unsigned int count, + unsigned long off, unsigned long size, + unsigned int max_segment, gfp_t gfp_mask) +{ + unsigned int i, segs, cur, len; + int rc; + struct scatterlist *s; + + if (__predict_false(!max_segment || offset_in_page(max_segment))) + return (-EINVAL); + + len = 0; + for (segs = i = 1; i < count; ++i) { + len += PAGE_SIZE; + if (len >= max_segment || + page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { + ++segs; + len = 0; + } + } + if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) + return (rc); + + cur = 0; + for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { + unsigned long seg_size; + unsigned int j; + + len = 0; + for (j = cur + 1; j < count; ++j) { + len += PAGE_SIZE; + if (len >= max_segment || page_to_pfn(pages[j]) != + page_to_pfn(pages[j - 1]) + 1) + break; + } + + seg_size = ((j - cur) << PAGE_SHIFT) - off; + sg_set_page(s, pages[cur], min(size, seg_size), off); + size -= seg_size; + off = 0; + cur = j; + } + return (0); +} + +static inline int +sg_alloc_table_from_pages(struct sg_table *sgt, + struct page **pages, unsigned int count, + unsigned long off, unsigned long size, + gfp_t gfp_mask) +{ + + return (__sg_alloc_table_from_pages(sgt, pages, count, off, size, + SCATTERLIST_MAX_SEGMENT, gfp_mask)); +} + +static inline int +sg_nents(struct scatterlist *sg) +{ + int nents; + + for (nents = 0; sg; sg = sg_next(sg)) + nents++; + return (nents); +} + +static inline void +__sg_page_iter_start(struct sg_page_iter *piter, + struct scatterlist *sglist, unsigned int nents, + unsigned long pgoffset) +{ + piter->internal.pg_advance = 0; + piter->internal.nents = nents; + + piter->sg = sglist; + piter->sg_pgoffset = pgoffset; +} + +static inline void +_sg_iter_next(struct sg_page_iter *iter) +{ + struct scatterlist *sg; + unsigned int pgcount; + + sg = iter->sg; + pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; + + ++iter->sg_pgoffset; + while (iter->sg_pgoffset >= pgcount) { + iter->sg_pgoffset -= pgcount; + sg = sg_next(sg); + --iter->maxents; + if (sg == NULL || iter->maxents == 0) + break; + pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT; + } + iter->sg = sg; +} + +static inline int +sg_page_count(struct scatterlist *sg) +{ + return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT); +} +#define sg_dma_page_count(sg) \ + sg_page_count(sg) + +static inline bool +__sg_page_iter_next(struct sg_page_iter *piter) +{ + unsigned int pgcount; + + if (piter->internal.nents == 0) + return (0); + if (piter->sg == NULL) + return (0); + + piter->sg_pgoffset += piter->internal.pg_advance; + piter->internal.pg_advance = 1; + + while (1) { + pgcount = sg_page_count(piter->sg); + if (likely(piter->sg_pgoffset < pgcount)) + break; + piter->sg_pgoffset -= pgcount; + piter->sg = sg_next(piter->sg); + if (--piter->internal.nents == 0) + return (0); + if (piter->sg == NULL) + return (0); + } + return (1); +} +#define __sg_page_iter_dma_next(itr) \ + __sg_page_iter_next(&(itr)->base) + +static inline void +_sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter, + unsigned int nents, unsigned long pgoffset) +{ + if (nents) { + iter->sg = sgl; + iter->sg_pgoffset = pgoffset - 1; + iter->maxents = nents; + _sg_iter_next(iter); + } else { + iter->sg = NULL; + iter->sg_pgoffset = 0; + iter->maxents = 0; + } +} + +/* + * sg_page_iter_dma_address() is implemented as a macro because it + * needs to accept two different and identical structure types. This + * allows both old and new code to co-exist. The compile time assert + * adds some safety, that the structure sizes match. + */ +#define sg_page_iter_dma_address(spi) ({ \ + struct sg_page_iter *__spi = (void *)(spi); \ + dma_addr_t __dma_address; \ + CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \ + __dma_address = __spi->sg->dma_address + \ + (__spi->sg_pgoffset << PAGE_SHIFT); \ + __dma_address; \ +}) + +static inline struct page * +sg_page_iter_page(struct sg_page_iter *piter) +{ + return (nth_page(sg_page(piter->sg), piter->sg_pgoffset)); +} + + +#endif /* _LINUX_SCATTERLIST_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/sched.h b/sys/compat/linuxkpi/common/include/linux/sched.h new file mode 100644 index 000000000000..da38d89eb639 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/sched.h @@ -0,0 +1,198 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2018 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_SCHED_H_ +#define _LINUX_SCHED_H_ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/proc.h> +#include <sys/sched.h> +#include <sys/sleepqueue.h> +#include <sys/time.h> + +#include <linux/bitmap.h> +#include <linux/compat.h> +#include <linux/completion.h> +#include <linux/mm_types.h> +#include <linux/pid.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/time.h> + +#include <asm/atomic.h> + +#define MAX_SCHEDULE_TIMEOUT INT_MAX + +#define TASK_RUNNING 0x0000 +#define TASK_INTERRUPTIBLE 0x0001 +#define TASK_UNINTERRUPTIBLE 0x0002 +#define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE) +#define TASK_WAKING 0x0100 +#define TASK_PARKED 0x0200 + +#define TASK_COMM_LEN (MAXCOMLEN + 1) + +struct work_struct; +struct task_struct { + struct thread *task_thread; + struct mm_struct *mm; + linux_task_fn_t *task_fn; + void *task_data; + int task_ret; + atomic_t usage; + atomic_t state; + atomic_t kthread_flags; + pid_t pid; /* BSD thread ID */ + const char *comm; + void *bsd_ioctl_data; + unsigned bsd_ioctl_len; + struct completion parked; + struct completion exited; +#define TS_RCU_TYPE_MAX 2 + TAILQ_ENTRY(task_struct) rcu_entry[TS_RCU_TYPE_MAX]; + int rcu_recurse[TS_RCU_TYPE_MAX]; + int bsd_interrupt_value; + struct work_struct *work; /* current work struct, if set */ + struct task_struct *group_leader; +}; + +#define current ({ \ + struct thread *__td = curthread; \ + linux_set_current(__td); \ + ((struct task_struct *)__td->td_lkpi_task); \ +}) + +#define task_pid_group_leader(task) (task)->task_thread->td_proc->p_pid +#define task_pid(task) ((task)->pid) +#define task_pid_nr(task) ((task)->pid) +#define task_pid_vnr(task) ((task)->pid) +#define get_pid(x) (x) +#define put_pid(x) do { } while (0) +#define current_euid() (curthread->td_ucred->cr_uid) +#define task_euid(task) ((task)->task_thread->td_ucred->cr_uid) + +#define get_task_state(task) atomic_read(&(task)->state) +#define set_task_state(task, x) atomic_set(&(task)->state, (x)) +#define __set_task_state(task, x) ((task)->state.counter = (x)) +#define set_current_state(x) set_task_state(current, x) +#define __set_current_state(x) __set_task_state(current, x) + +static inline void +get_task_struct(struct task_struct *task) +{ + atomic_inc(&task->usage); +} + +static inline void +put_task_struct(struct task_struct *task) +{ + if (atomic_dec_and_test(&task->usage)) + linux_free_current(task); +} + +#define cond_resched() do { if (!cold) sched_relinquish(curthread); } while (0) + +#define yield() kern_yield(PRI_UNCHANGED) +#define sched_yield() sched_relinquish(curthread) + +#define need_resched() (curthread->td_flags & TDF_NEEDRESCHED) + +bool linux_signal_pending(struct task_struct *task); +bool linux_fatal_signal_pending(struct task_struct *task); +bool linux_signal_pending_state(long state, struct task_struct *task); +void linux_send_sig(int signo, struct task_struct *task); + +#define signal_pending(task) linux_signal_pending(task) +#define fatal_signal_pending(task) linux_fatal_signal_pending(task) +#define signal_pending_state(state, task) \ + linux_signal_pending_state(state, task) +#define send_sig(signo, task, priv) do { \ + CTASSERT((priv) == 0); \ + linux_send_sig(signo, task); \ +} while (0) + +int linux_schedule_timeout(int timeout); + +static inline void +linux_schedule_save_interrupt_value(struct task_struct *task, int value) +{ + task->bsd_interrupt_value = value; +} + +bool linux_task_exiting(struct task_struct *task); + +#define current_exiting() \ + linux_task_exiting(current) + +static inline int +linux_schedule_get_interrupt_value(struct task_struct *task) +{ + int value = task->bsd_interrupt_value; + task->bsd_interrupt_value = 0; + return (value); +} + +#define schedule() \ + (void)linux_schedule_timeout(MAX_SCHEDULE_TIMEOUT) +#define schedule_timeout(timeout) \ + linux_schedule_timeout(timeout) +#define schedule_timeout_killable(timeout) \ + schedule_timeout_interruptible(timeout) +#define schedule_timeout_interruptible(timeout) ({ \ + set_current_state(TASK_INTERRUPTIBLE); \ + schedule_timeout(timeout); \ +}) +#define schedule_timeout_uninterruptible(timeout) ({ \ + set_current_state(TASK_UNINTERRUPTIBLE); \ + schedule_timeout(timeout); \ +}) + +#define io_schedule() schedule() +#define io_schedule_timeout(timeout) schedule_timeout(timeout) + +static inline uint64_t +local_clock(void) +{ + struct timespec ts; + + nanotime(&ts); + return ((uint64_t)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec); +} + +static inline const char * +get_task_comm(char *buf, struct task_struct *task) +{ + + buf[0] = 0; /* buffer is too small */ + return (task->comm); +} + +#endif /* _LINUX_SCHED_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/semaphore.h b/sys/compat/linuxkpi/common/include/linux/semaphore.h new file mode 100644 index 000000000000..59a35311a5cc --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/semaphore.h @@ -0,0 +1,70 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_SEMAPHORE_H_ +#define _LINUX_SEMAPHORE_H_ + +#include <sys/param.h> +#include <sys/lock.h> +#include <sys/sema.h> +#include <sys/libkern.h> + +/* + * XXX BSD semaphores are disused and slow. They also do not provide a + * sema_wait_sig method. This must be resolved eventually. + */ +struct semaphore { + struct sema sema; +}; + +#define down(_sem) sema_wait(&(_sem)->sema) +#define down_interruptible(_sem) sema_wait(&(_sem)->sema), 0 +#define down_trylock(_sem) !sema_trywait(&(_sem)->sema) +#define up(_sem) sema_post(&(_sem)->sema) + +static inline void +linux_sema_init(struct semaphore *sem, int val) +{ + + memset(&sem->sema, 0, sizeof(sem->sema)); + sema_init(&sem->sema, val, "lnxsema"); +} + +static inline void +init_MUTEX(struct semaphore *sem) +{ + + memset(&sem->sema, 0, sizeof(sem->sema)); + sema_init(&sem->sema, 1, "lnxsema"); +} + +#define sema_init linux_sema_init + +#endif /* _LINUX_SEMAPHORE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/seq_file.h b/sys/compat/linuxkpi/common/include/linux/seq_file.h new file mode 100644 index 000000000000..dab8020a0336 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/seq_file.h @@ -0,0 +1,88 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2016-2018, Matthew Macy <mmacy@freebsd.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_SEQ_FILE_H_ +#define _LINUX_SEQ_FILE_H_ + +#include <linux/types.h> +#include <linux/fs.h> +#include <sys/sbuf.h> + +#undef file +#define inode vnode + +#define DEFINE_SHOW_ATTRIBUTE(__name) \ +static int __name ## _open(struct inode *inode, struct linux_file *file) \ +{ \ + return single_open(file, __name ## _show, inode->i_private); \ +} \ + \ +static const struct file_operations __name ## _fops = { \ + .owner = THIS_MODULE, \ + .open = __name ## _open, \ + .read = seq_read, \ + .llseek = seq_lseek, \ + .release = single_release, \ +} + +struct seq_operations; + +struct seq_file { + struct sbuf *buf; + + const struct seq_operations *op; + const struct linux_file *file; + void *private; +}; + +struct seq_operations { + void * (*start) (struct seq_file *m, off_t *pos); + void (*stop) (struct seq_file *m, void *v); + void * (*next) (struct seq_file *m, void *v, off_t *pos); + int (*show) (struct seq_file *m, void *v); +}; + +ssize_t seq_read(struct linux_file *, char *, size_t, off_t *); +int seq_write(struct seq_file *seq, const void *data, size_t len); + +int seq_open(struct linux_file *f, const struct seq_operations *op); +int seq_release(struct inode *inode, struct linux_file *file); + +off_t seq_lseek(struct linux_file *file, off_t offset, int whence); +int single_open(struct linux_file *, int (*)(struct seq_file *, void *), void *); +int single_release(struct inode *, struct linux_file *); + +#define seq_printf(m, fmt, ...) sbuf_printf((m)->buf, (fmt), ##__VA_ARGS__) + +#define seq_puts(m, str) sbuf_printf((m)->buf, str) +#define seq_putc(m, str) sbuf_putc((m)->buf, str) + +#define file linux_file + +#endif /* _LINUX_SEQ_FILE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/shmem_fs.h b/sys/compat/linuxkpi/common/include/linux/shmem_fs.h new file mode 100644 index 000000000000..63aff012c6bb --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/shmem_fs.h @@ -0,0 +1,55 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2018 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_SHMEM_FS_H_ +#define _LINUX_SHMEM_FS_H_ + +/* Shared memory support */ +unsigned long linux_invalidate_mapping_pages(vm_object_t, pgoff_t, pgoff_t); +struct page *linux_shmem_read_mapping_page_gfp(vm_object_t, int, gfp_t); +struct linux_file *linux_shmem_file_setup(const char *, loff_t, unsigned long); +void linux_shmem_truncate_range(vm_object_t, loff_t, loff_t); + +#define invalidate_mapping_pages(...) \ + linux_invalidate_mapping_pages(__VA_ARGS__) + +#define shmem_read_mapping_page(...) \ + linux_shmem_read_mapping_page_gfp(__VA_ARGS__, 0) + +#define shmem_read_mapping_page_gfp(...) \ + linux_shmem_read_mapping_page_gfp(__VA_ARGS__) + +#define shmem_file_setup(...) \ + linux_shmem_file_setup(__VA_ARGS__) + +#define shmem_truncate_range(...) \ + linux_shmem_truncate_range(__VA_ARGS__) + +#endif /* _LINUX_SHMEM_FS_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/sizes.h b/sys/compat/linuxkpi/common/include/linux/sizes.h new file mode 100644 index 000000000000..a180cee5f022 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/sizes.h @@ -0,0 +1,51 @@ +/*- + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Emmanuel Vadot under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __LINUX_SIZES_H__ +#define __LINUX_SIZES_H__ + +#define SZ_1K (1024 * 1) +#define SZ_4K (1024 * 4) +#define SZ_8K (1024 * 8) +#define SZ_16K (1024 * 16) +#define SZ_32K (1024 * 32) +#define SZ_64K (1024 * 64) +#define SZ_128K (1024 * 128) +#define SZ_256K (1024 * 256) +#define SZ_512K (1024 * 512) + +#define SZ_1M (1024 * 1024 * 1) +#define SZ_2M (1024 * 1024 * 2) +#define SZ_8M (1024 * 1024 * 8) +#define SZ_16M (1024 * 1024 * 16) +#define SZ_32M (1024 * 1024 * 32) +#define SZ_64M (1024 * 1024 * 64) + +#endif diff --git a/sys/compat/linuxkpi/common/include/linux/slab.h b/sys/compat/linuxkpi/common/include/linux/slab.h new file mode 100644 index 000000000000..5ff7000f3658 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/slab.h @@ -0,0 +1,187 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_SLAB_H_ +#define _LINUX_SLAB_H_ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> +#include <sys/limits.h> +#include <vm/uma.h> + +#include <linux/types.h> +#include <linux/gfp.h> + +MALLOC_DECLARE(M_KMALLOC); + +#define kvmalloc(size, flags) kmalloc(size, flags) +#define kvzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO) +#define kvcalloc(n, size, flags) kvmalloc_array(n, size, (flags) | __GFP_ZERO) +#define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO) +#define kzalloc_node(size, flags, node) kmalloc(size, (flags) | __GFP_ZERO) +#define kfree_const(ptr) kfree(ptr) +#define vzalloc(size) __vmalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 0) +#define vfree(arg) kfree(arg) +#define kvfree(arg) kfree(arg) +#define vmalloc_node(size, node) __vmalloc(size, GFP_KERNEL, 0) +#define vmalloc_user(size) __vmalloc(size, GFP_KERNEL | __GFP_ZERO, 0) +#define vmalloc(size) __vmalloc(size, GFP_KERNEL, 0) +#define __kmalloc(...) kmalloc(__VA_ARGS__) +#define kmalloc_node(chunk, flags, n) kmalloc(chunk, flags) + +/* + * Prefix some functions with linux_ to avoid namespace conflict + * with the OpenSolaris code in the kernel. + */ +#define kmem_cache linux_kmem_cache +#define kmem_cache_create(...) linux_kmem_cache_create(__VA_ARGS__) +#define kmem_cache_alloc(...) linux_kmem_cache_alloc(__VA_ARGS__) +#define kmem_cache_free(...) linux_kmem_cache_free(__VA_ARGS__) +#define kmem_cache_destroy(...) linux_kmem_cache_destroy(__VA_ARGS__) + +#define KMEM_CACHE(__struct, flags) \ + linux_kmem_cache_create(#__struct, sizeof(struct __struct), \ + __alignof(struct __struct), (flags), NULL) + +typedef void linux_kmem_ctor_t (void *); + +struct linux_kmem_cache { + uma_zone_t cache_zone; + linux_kmem_ctor_t *cache_ctor; + unsigned cache_flags; + unsigned cache_size; +}; + +#define SLAB_HWCACHE_ALIGN (1 << 0) +#define SLAB_TYPESAFE_BY_RCU (1 << 1) +#define SLAB_RECLAIM_ACCOUNT (1 << 2) + +#define SLAB_DESTROY_BY_RCU \ + SLAB_TYPESAFE_BY_RCU + +#define ARCH_KMALLOC_MINALIGN \ + __alignof(unsigned long long) + +static inline gfp_t +linux_check_m_flags(gfp_t flags) +{ + const gfp_t m = M_NOWAIT | M_WAITOK; + + /* make sure either M_NOWAIT or M_WAITOK is set */ + if ((flags & m) == 0) + flags |= M_NOWAIT; + else if ((flags & m) == m) + flags &= ~M_WAITOK; + + /* mask away LinuxKPI specific flags */ + return (flags & GFP_NATIVE_MASK); +} + +static inline void * +kmalloc(size_t size, gfp_t flags) +{ + return (malloc(size, M_KMALLOC, linux_check_m_flags(flags))); +} + +static inline void * +kcalloc(size_t n, size_t size, gfp_t flags) +{ + flags |= __GFP_ZERO; + return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags))); +} + +static inline void * +__vmalloc(size_t size, gfp_t flags, int other) +{ + return (malloc(size, M_KMALLOC, linux_check_m_flags(flags))); +} + +static inline void * +vmalloc_32(size_t size) +{ + return (contigmalloc(size, M_KMALLOC, M_WAITOK, 0, UINT_MAX, 1, 1)); +} + +static inline void * +kmalloc_array(size_t n, size_t size, gfp_t flags) +{ + return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags))); +} + +static inline void * +kvmalloc_array(size_t n, size_t size, gfp_t flags) +{ + return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags))); +} + +static inline void * +krealloc(void *ptr, size_t size, gfp_t flags) +{ + return (realloc(ptr, size, M_KMALLOC, linux_check_m_flags(flags))); +} + +static inline void +kfree(const void *ptr) +{ + free(__DECONST(void *, ptr), M_KMALLOC); +} + +extern struct linux_kmem_cache *linux_kmem_cache_create(const char *name, + size_t size, size_t align, unsigned flags, linux_kmem_ctor_t *ctor); + +static inline void * +linux_kmem_cache_alloc(struct linux_kmem_cache *c, gfp_t flags) +{ + return (uma_zalloc_arg(c->cache_zone, c, + linux_check_m_flags(flags))); +} + +static inline void * +kmem_cache_zalloc(struct linux_kmem_cache *c, gfp_t flags) +{ + return (uma_zalloc_arg(c->cache_zone, c, + linux_check_m_flags(flags | M_ZERO))); +} + +extern void linux_kmem_cache_free_rcu(struct linux_kmem_cache *, void *); + +static inline void +linux_kmem_cache_free(struct linux_kmem_cache *c, void *m) +{ + if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU)) + linux_kmem_cache_free_rcu(c, m); + else + uma_zfree(c->cache_zone, m); +} + +extern void linux_kmem_cache_destroy(struct linux_kmem_cache *); + +#endif /* _LINUX_SLAB_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/smp.h b/sys/compat/linuxkpi/common/include/linux/smp.h new file mode 100644 index 000000000000..3f5684015547 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/smp.h @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_SMP_H_ +#define _LINUX_SMP_H_ + +#define on_each_cpu(cb, data, wait) ({ \ + CTASSERT(wait); \ + linux_on_each_cpu(cb, data); \ +}) + +extern int linux_on_each_cpu(void (*)(void *), void *); + +#endif /* _LINUX_SMP_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/socket.h b/sys/compat/linuxkpi/common/include/linux/socket.h new file mode 100644 index 000000000000..a9a952e4ffd8 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/socket.h @@ -0,0 +1,69 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_SOCKET_H_ +#define _LINUX_SOCKET_H_ + +#include <sys/socket.h> + +#ifdef notyet +static inline int +memcpy_toiovec(struct iovec *v, unsigned char *kdata, int len) +{ + struct uio uio; + int error; + + uio.uio_iov = v; + uio.uio_iovcnt = -1; + uio.uio_offset = 0; + uio.uio_resid = len; + uio.uio_segflag = UIO_USERSPACE; + uio.uio_rw = UIO_READ; + error = -uiomove(kdata, len, &uio); + return (error); +} + +static inline int +memcpy_fromiovec(unsigned char *kdata, struct iovec *iov, int len) +{ + struct uio uio; + int error; + + uio.uio_iov = v; + uio.uio_iovcnt = -1; + uio.uio_offset = 0; + uio.uio_resid = len; + uio.uio_segflag = UIO_USERSPACE; + uio.uio_rw = UIO_WRITE; + error = -uiomove(kdata, len, &uio); +} +#endif + +#endif /* _LINUX_SOCKET_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/spinlock.h b/sys/compat/linuxkpi/common/include/linux/spinlock.h new file mode 100644 index 000000000000..2309794b26ec --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/spinlock.h @@ -0,0 +1,163 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_SPINLOCK_H_ +#define _LINUX_SPINLOCK_H_ + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/kdb.h> + +#include <linux/compiler.h> +#include <linux/rwlock.h> +#include <linux/bottom_half.h> + +typedef struct { + struct mtx m; +} spinlock_t; + +/* + * By defining CONFIG_SPIN_SKIP LinuxKPI spinlocks and asserts will be + * skipped during panic(). By default it is disabled due to + * performance reasons. + */ +#ifdef CONFIG_SPIN_SKIP +#define SPIN_SKIP(void) unlikely(SCHEDULER_STOPPED() || kdb_active) +#else +#define SPIN_SKIP(void) 0 +#endif + +#define spin_lock(_l) do { \ + if (SPIN_SKIP()) \ + break; \ + mtx_lock(&(_l)->m); \ + local_bh_disable(); \ +} while (0) + +#define spin_lock_bh(_l) do { \ + spin_lock(_l); \ +} while (0) + +#define spin_lock_irq(_l) do { \ + spin_lock(_l); \ +} while (0) + +#define spin_unlock(_l) do { \ + if (SPIN_SKIP()) \ + break; \ + local_bh_enable(); \ + mtx_unlock(&(_l)->m); \ +} while (0) + +#define spin_unlock_bh(_l) do { \ + spin_unlock(_l); \ +} while (0) + +#define spin_unlock_irq(_l) do { \ + spin_unlock(_l); \ +} while (0) + +#define spin_trylock(_l) ({ \ + int __ret; \ + if (SPIN_SKIP()) { \ + __ret = 1; \ + } else { \ + __ret = mtx_trylock(&(_l)->m); \ + if (likely(__ret != 0)) \ + local_bh_disable(); \ + } \ + __ret; \ +}) + +#define spin_trylock_irq(_l) \ + spin_trylock(_l) + +#define spin_lock_nested(_l, _n) do { \ + if (SPIN_SKIP()) \ + break; \ + mtx_lock_flags(&(_l)->m, MTX_DUPOK); \ + local_bh_disable(); \ +} while (0) + +#define spin_lock_irqsave(_l, flags) do { \ + (flags) = 0; \ + spin_lock(_l); \ +} while (0) + +#define spin_lock_irqsave_nested(_l, flags, _n) do { \ + (flags) = 0; \ + spin_lock_nested(_l, _n); \ +} while (0) + +#define spin_unlock_irqrestore(_l, flags) do { \ + spin_unlock(_l); \ +} while (0) + +#ifdef WITNESS_ALL +/* NOTE: the maximum WITNESS name is 64 chars */ +#define __spin_lock_name(name, file, line) \ + (((const char *){file ":" #line "-" name}) + \ + (sizeof(file) > 16 ? sizeof(file) - 16 : 0)) +#else +#define __spin_lock_name(name, file, line) name +#endif +#define _spin_lock_name(...) __spin_lock_name(__VA_ARGS__) +#define spin_lock_name(name) _spin_lock_name(name, __FILE__, __LINE__) + +#define spin_lock_init(lock) linux_spin_lock_init(lock, spin_lock_name("lnxspin")) + +static inline void +linux_spin_lock_init(spinlock_t *lock, const char *name) +{ + + memset(lock, 0, sizeof(*lock)); + mtx_init(&lock->m, name, NULL, MTX_DEF | MTX_NOWITNESS); +} + +static inline void +spin_lock_destroy(spinlock_t *lock) +{ + + mtx_destroy(&lock->m); +} + +#define DEFINE_SPINLOCK(lock) \ + spinlock_t lock; \ + MTX_SYSINIT(lock, &(lock).m, spin_lock_name("lnxspin"), MTX_DEF) + +#define assert_spin_locked(_l) do { \ + if (SPIN_SKIP()) \ + break; \ + mtx_assert(&(_l)->m, MA_OWNED); \ +} while (0) + +#endif /* _LINUX_SPINLOCK_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/srcu.h b/sys/compat/linuxkpi/common/include/linux/srcu.h new file mode 100644 index 000000000000..1de7e8acedb8 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/srcu.h @@ -0,0 +1,56 @@ +/*- + * Copyright (c) 2015-2020 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_SRCU_H_ +#define _LINUX_SRCU_H_ + +#include <linux/compiler.h> + +struct srcu_struct { +}; + +#define srcu_dereference(p, srcu) \ + ((__typeof(*(p)) *)READ_ONCE(p)) + +#define DEFINE_STATIC_SRCU(name) \ + static struct srcu_struct name + +/* prototypes */ + +extern int srcu_read_lock(struct srcu_struct *); +extern void srcu_read_unlock(struct srcu_struct *, int index); +extern void synchronize_srcu(struct srcu_struct *); +extern void srcu_barrier(struct srcu_struct *); +extern int init_srcu_struct(struct srcu_struct *); +extern void cleanup_srcu_struct(struct srcu_struct *); + +#define synchronize_srcu_expedited(srcu) do { \ + synchronize_srcu(srcu); \ +} while (0) + +#endif /* _LINUX_SRCU_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/string.h b/sys/compat/linuxkpi/common/include/linux/string.h new file mode 100644 index 000000000000..39201e203162 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/string.h @@ -0,0 +1,170 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_STRING_H_ +#define _LINUX_STRING_H_ + +#include <sys/ctype.h> + +#include <linux/types.h> +#include <linux/gfp.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/err.h> + +#include <sys/libkern.h> + +#define strnicmp(...) strncasecmp(__VA_ARGS__) + +static inline int +match_string(const char *const *table, int n, const char *key) +{ + int i; + + for (i = 0; i != n && table[i] != NULL; i++) { + if (strcmp(table[i], key) == 0) + return (i); + } + return (-EINVAL); +} + +static inline void * +memdup_user(const void *ptr, size_t len) +{ + void *retval; + int error; + + retval = malloc(len, M_KMALLOC, M_WAITOK); + error = linux_copyin(ptr, retval, len); + if (error != 0) { + free(retval, M_KMALLOC); + return (ERR_PTR(error)); + } + return (retval); +} + +static inline void * +memdup_user_nul(const void *ptr, size_t len) +{ + char *retval; + int error; + + retval = malloc(len + 1, M_KMALLOC, M_WAITOK); + error = linux_copyin(ptr, retval, len); + if (error != 0) { + free(retval, M_KMALLOC); + return (ERR_PTR(error)); + } + retval[len] = '\0'; + return (retval); +} + +static inline void * +kmemdup(const void *src, size_t len, gfp_t gfp) +{ + void *dst; + + dst = kmalloc(len, gfp); + if (dst != NULL) + memcpy(dst, src, len); + return (dst); +} + +static inline char * +kstrdup(const char *string, gfp_t gfp) +{ + char *retval; + size_t len; + + if (string == NULL) + return (NULL); + len = strlen(string) + 1; + retval = kmalloc(len, gfp); + if (retval != NULL) + memcpy(retval, string, len); + return (retval); +} + +static inline char * +kstrndup(const char *string, size_t len, gfp_t gfp) +{ + char *retval; + + if (string == NULL) + return (NULL); + retval = kmalloc(len + 1, gfp); + if (retval != NULL) + strncpy(retval, string, len); + return (retval); +} + +static inline const char * +kstrdup_const(const char *src, gfp_t gfp) +{ + return (kmemdup(src, strlen(src) + 1, gfp)); +} + +static inline char * +skip_spaces(const char *str) +{ + while (isspace(*str)) + ++str; + return (__DECONST(char *, str)); +} + +static inline void * +memchr_inv(const void *start, int c, size_t length) +{ + const u8 *ptr; + const u8 *end; + u8 ch; + + ch = c; + ptr = start; + end = ptr + length; + + while (ptr != end) { + if (*ptr != ch) + return (__DECONST(void *, ptr)); + ptr++; + } + return (NULL); +} + +static inline size_t +str_has_prefix(const char *str, const char *prefix) +{ + size_t len; + + len = strlen(prefix); + return (strncmp(str, prefix, len) == 0 ? len : 0); +} + +#endif /* _LINUX_STRING_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/swap.h b/sys/compat/linuxkpi/common/include/linux/swap.h new file mode 100644 index 000000000000..33f8de4721a4 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/swap.h @@ -0,0 +1,52 @@ +/*- + * Copyright (c) 2018 Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_SWAP_H_ +#define _LINUX_SWAP_H_ + +#include <vm/swap_pager.h> +#include <vm/vm_pageout.h> + +static inline long +get_nr_swap_pages(void) +{ + int i, j; + + /* NB: This could be done cheaply by obtaining swap_total directly */ + swap_pager_status(&i, &j); + return i - j; +} + +static inline int +current_is_kswapd(void) +{ + + return (curproc == pageproc); +} + +#endif diff --git a/sys/compat/linuxkpi/common/include/linux/sysfs.h b/sys/compat/linuxkpi/common/include/linux/sysfs.h new file mode 100644 index 000000000000..59e30f152f6e --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/sysfs.h @@ -0,0 +1,300 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_SYSFS_H_ +#define _LINUX_SYSFS_H_ + +#include <sys/types.h> +#include <sys/sysctl.h> +#include <sys/errno.h> + +#include <linux/kobject.h> + +struct sysfs_ops { + ssize_t (*show)(struct kobject *, struct attribute *, char *); + ssize_t (*store)(struct kobject *, struct attribute *, const char *, + size_t); +}; + +struct attribute_group { + const char *name; + mode_t (*is_visible)(struct kobject *, + struct attribute *, int); + struct attribute **attrs; +}; + +#define __ATTR(_name, _mode, _show, _store) { \ + .attr = { .name = __stringify(_name), .mode = _mode }, \ + .show = _show, .store = _store, \ +} +#define __ATTR_RO(_name) __ATTR(_name, 0444, _name##_show, NULL) +#define __ATTR_WO(_name) __ATTR(_name, 0200, NULL, _name##_store) +#define __ATTR_RW(_name) __ATTR(_name, 0644, _name##_show, _name##_store) + +#define __ATTR_NULL { .attr = { .name = NULL } } + +#define ATTRIBUTE_GROUPS(_name) \ + static struct attribute_group _name##_group = { \ + .name = __stringify(_name), \ + .attrs = _name##_attrs, \ + }; \ + static const struct attribute_group *_name##_groups[] = { \ + &_name##_group, \ + NULL, \ + } + +/* + * Handle our generic '\0' terminated 'C' string. + * Two cases: + * a variable string: point arg1 at it, arg2 is max length. + * a constant string: point arg1 at it, arg2 is zero. + */ + +static inline int +sysctl_handle_attr(SYSCTL_HANDLER_ARGS) +{ + struct kobject *kobj; + struct attribute *attr; + const struct sysfs_ops *ops; + char *buf; + int error; + ssize_t len; + + kobj = arg1; + attr = (struct attribute *)(intptr_t)arg2; + if (kobj->ktype == NULL || kobj->ktype->sysfs_ops == NULL) + return (ENODEV); + buf = (char *)get_zeroed_page(GFP_KERNEL); + if (buf == NULL) + return (ENOMEM); + ops = kobj->ktype->sysfs_ops; + if (ops->show) { + len = ops->show(kobj, attr, buf); + /* + * It's valid to not have a 'show' so just return an + * empty string. + */ + if (len < 0) { + error = -len; + if (error != EIO) + goto out; + buf[0] = '\0'; + } else if (len) { + len--; + if (len >= PAGE_SIZE) + len = PAGE_SIZE - 1; + /* Trim trailing newline. */ + buf[len] = '\0'; + } + } + + /* Leave one trailing byte to append a newline. */ + error = sysctl_handle_string(oidp, buf, PAGE_SIZE - 1, req); + if (error != 0 || req->newptr == NULL || ops->store == NULL) + goto out; + len = strlcat(buf, "\n", PAGE_SIZE); + KASSERT(len < PAGE_SIZE, ("new attribute truncated")); + len = ops->store(kobj, attr, buf, len); + if (len < 0) + error = -len; +out: + free_page((unsigned long)buf); + + return (error); +} + +static inline int +sysfs_create_file(struct kobject *kobj, const struct attribute *attr) +{ + struct sysctl_oid *oid; + + oid = SYSCTL_ADD_OID(NULL, SYSCTL_CHILDREN(kobj->oidp), OID_AUTO, + attr->name, CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_MPSAFE, kobj, + (uintptr_t)attr, sysctl_handle_attr, "A", ""); + if (!oid) { + return (-ENOMEM); + } + + return (0); +} + +static inline void +sysfs_remove_file(struct kobject *kobj, const struct attribute *attr) +{ + + if (kobj->oidp) + sysctl_remove_name(kobj->oidp, attr->name, 1, 1); +} + +static inline int +sysfs_create_files(struct kobject *kobj, const struct attribute * const *attrs) +{ + int error = 0; + int i; + + for (i = 0; attrs[i] && !error; i++) + error = sysfs_create_file(kobj, attrs[i]); + while (error && --i >= 0) + sysfs_remove_file(kobj, attrs[i]); + + return (error); +} + +static inline void +sysfs_remove_files(struct kobject *kobj, const struct attribute * const *attrs) +{ + int i; + + for (i = 0; attrs[i]; i++) + sysfs_remove_file(kobj, attrs[i]); +} + +static inline int +sysfs_create_group(struct kobject *kobj, const struct attribute_group *grp) +{ + struct attribute **attr; + struct sysctl_oid *oidp; + + /* Don't create the group node if grp->name is undefined. */ + if (grp->name) + oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(kobj->oidp), + OID_AUTO, grp->name, CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, grp->name); + else + oidp = kobj->oidp; + for (attr = grp->attrs; *attr != NULL; attr++) { + SYSCTL_ADD_OID(NULL, SYSCTL_CHILDREN(oidp), OID_AUTO, + (*attr)->name, CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_MPSAFE, + kobj, (uintptr_t)*attr, sysctl_handle_attr, "A", ""); + } + + return (0); +} + +static inline void +sysfs_remove_group(struct kobject *kobj, const struct attribute_group *grp) +{ + + if (kobj->oidp) + sysctl_remove_name(kobj->oidp, grp->name, 1, 1); +} + +static inline int +sysfs_create_groups(struct kobject *kobj, const struct attribute_group **grps) +{ + int error = 0; + int i; + + if (grps == NULL) + goto done; + for (i = 0; grps[i] && !error; i++) + error = sysfs_create_group(kobj, grps[i]); + while (error && --i >= 0) + sysfs_remove_group(kobj, grps[i]); +done: + return (error); +} + +static inline void +sysfs_remove_groups(struct kobject *kobj, const struct attribute_group **grps) +{ + int i; + + if (grps == NULL) + return; + for (i = 0; grps[i]; i++) + sysfs_remove_group(kobj, grps[i]); +} + +static inline int +sysfs_merge_group(struct kobject *kobj, const struct attribute_group *grp) +{ + + /* Really expected behavior is to return failure if group exists. */ + return (sysfs_create_group(kobj, grp)); +} + +static inline void +sysfs_unmerge_group(struct kobject *kobj, const struct attribute_group *grp) +{ + struct attribute **attr; + struct sysctl_oid *oidp; + + SLIST_FOREACH(oidp, SYSCTL_CHILDREN(kobj->oidp), oid_link) { + if (strcmp(oidp->oid_name, grp->name) != 0) + continue; + for (attr = grp->attrs; *attr != NULL; attr++) { + sysctl_remove_name(oidp, (*attr)->name, 1, 1); + } + } +} + +static inline int +sysfs_create_dir(struct kobject *kobj) +{ + struct sysctl_oid *oid; + + oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(kobj->parent->oidp), + OID_AUTO, kobj->name, CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, kobj->name); + if (!oid) { + return (-ENOMEM); + } + kobj->oidp = oid; + + return (0); +} + +static inline void +sysfs_remove_dir(struct kobject *kobj) +{ + + if (kobj->oidp == NULL) + return; + sysctl_remove_oid(kobj->oidp, 1, 1); +} + +static inline bool +sysfs_streq(const char *s1, const char *s2) +{ + int l1, l2; + + l1 = strlen(s1); + l2 = strlen(s2); + + if (l1 != 0 && s1[l1-1] == '\n') + l1--; + if (l2 != 0 && s2[l2-1] == '\n') + l2--; + + return (l1 == l2 && strncmp(s1, s2, l1) == 0); +} + +#define sysfs_attr_init(attr) do {} while(0) + +#endif /* _LINUX_SYSFS_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/time.h b/sys/compat/linuxkpi/common/include/linux/time.h new file mode 100644 index 000000000000..1c07c69a67a6 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/time.h @@ -0,0 +1,135 @@ +/*- + * Copyright (c) 2014-2015 François Tigeot + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_TIME_H_ +#define _LINUX_TIME_H_ + +#define NSEC_PER_USEC 1000L +#define NSEC_PER_MSEC 1000000L +#define NSEC_PER_SEC 1000000000L + +#define USEC_PER_MSEC 1000L +#define USEC_PER_SEC 1000000L + +#define timespec64 timespec + +#include <sys/time.h> +#include <sys/stdint.h> + +static inline struct timeval +ns_to_timeval(const int64_t nsec) +{ + struct timeval tv; + long rem; + + if (nsec == 0) { + tv.tv_sec = 0; + tv.tv_usec = 0; + return (tv); + } + + tv.tv_sec = nsec / NSEC_PER_SEC; + rem = nsec % NSEC_PER_SEC; + if (rem < 0) { + tv.tv_sec--; + rem += NSEC_PER_SEC; + } + tv.tv_usec = rem / 1000; + return (tv); +} + +static inline int64_t +timeval_to_ns(const struct timeval *tv) +{ + return ((int64_t)tv->tv_sec * NSEC_PER_SEC) + + tv->tv_usec * NSEC_PER_USEC; +} + +#define getrawmonotonic(ts) nanouptime(ts) + +static inline struct timespec +timespec_sub(struct timespec lhs, struct timespec rhs) +{ + struct timespec ts; + + timespecsub(&lhs, &rhs, &ts); + + return ts; +} + +static inline void +set_normalized_timespec(struct timespec *ts, time_t sec, int64_t nsec) +{ + /* XXX: this doesn't actually normalize anything */ + ts->tv_sec = sec; + ts->tv_nsec = nsec; +} + +static inline int64_t +timespec_to_ns(const struct timespec *ts) +{ + return ((ts->tv_sec * NSEC_PER_SEC) + ts->tv_nsec); +} + +static inline struct timespec +ns_to_timespec(const int64_t nsec) +{ + struct timespec ts; + int32_t rem; + + if (nsec == 0) { + ts.tv_sec = 0; + ts.tv_nsec = 0; + return (ts); + } + + ts.tv_sec = nsec / NSEC_PER_SEC; + rem = nsec % NSEC_PER_SEC; + if (rem < 0) { + ts.tv_sec--; + rem += NSEC_PER_SEC; + } + ts.tv_nsec = rem; + return (ts); +} + +static inline int +timespec_valid(const struct timespec *ts) +{ + if (ts->tv_sec < 0 || ts->tv_sec > 100000000 || + ts->tv_nsec < 0 || ts->tv_nsec >= 1000000000) + return (0); + return (1); +} + +static inline unsigned long +get_seconds(void) +{ + return time_uptime; +} + +#endif /* _LINUX_TIME_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/timer.h b/sys/compat/linuxkpi/common/include/linux/timer.h new file mode 100644 index 000000000000..86048af31b2d --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/timer.h @@ -0,0 +1,94 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_TIMER_H_ +#define _LINUX_TIMER_H_ + +#include <linux/types.h> + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/callout.h> + +struct timer_list { + struct callout callout; + union { + void (*function) (unsigned long); /* < v4.15 */ + void (*function_415) (struct timer_list *); + }; + unsigned long data; + int expires; +}; + +extern unsigned long linux_timer_hz_mask; + +#define TIMER_IRQSAFE 0x0001 + +#define from_timer(var, arg, field) \ + container_of(arg, typeof(*(var)), field) + +#define timer_setup(timer, func, flags) do { \ + CTASSERT(((flags) & ~TIMER_IRQSAFE) == 0); \ + (timer)->function_415 = (func); \ + (timer)->data = (unsigned long)(timer); \ + callout_init(&(timer)->callout, 1); \ +} while (0) + +#define setup_timer(timer, func, dat) do { \ + (timer)->function = (func); \ + (timer)->data = (dat); \ + callout_init(&(timer)->callout, 1); \ +} while (0) + +#define __setup_timer(timer, func, dat, flags) do { \ + CTASSERT(((flags) & ~TIMER_IRQSAFE) == 0); \ + setup_timer(timer, func, dat); \ +} while (0) + +#define init_timer(timer) do { \ + (timer)->function = NULL; \ + (timer)->data = 0; \ + callout_init(&(timer)->callout, 1); \ +} while (0) + +extern int mod_timer(struct timer_list *, int); +extern void add_timer(struct timer_list *); +extern void add_timer_on(struct timer_list *, int cpu); +extern int del_timer(struct timer_list *); +extern int del_timer_sync(struct timer_list *); + +#define timer_pending(timer) callout_pending(&(timer)->callout) +#define round_jiffies(j) \ + ((int)(((j) + linux_timer_hz_mask) & ~linux_timer_hz_mask)) +#define round_jiffies_relative(j) round_jiffies(j) +#define round_jiffies_up(j) round_jiffies(j) +#define round_jiffies_up_relative(j) round_jiffies_up(j) + +#endif /* _LINUX_TIMER_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/types.h b/sys/compat/linuxkpi/common/include/linux/types.h new file mode 100644 index 000000000000..4d39e79e8a5d --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/types.h @@ -0,0 +1,87 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_TYPES_H_ +#define _LINUX_TYPES_H_ + +#include <sys/cdefs.h> +#include <sys/types.h> +#include <sys/param.h> +#include <sys/systm.h> +#include <linux/compiler.h> +#include <asm/types.h> + +#ifndef __bitwise__ +#ifdef __CHECKER__ +#define __bitwise__ __attribute__((bitwise)) +#else +#define __bitwise__ +#endif +#endif + +typedef uint16_t __le16; +typedef uint16_t __be16; +typedef uint32_t __le32; +typedef uint32_t __be32; +typedef uint64_t __le64; +typedef uint64_t __be64; + +typedef uint16_t __aligned_u16 __aligned(sizeof(uint16_t)); +typedef uint32_t __aligned_u32 __aligned(sizeof(uint32_t)); +typedef uint64_t __aligned_u64 __aligned(sizeof(uint64_t)); + +typedef unsigned short ushort; +typedef unsigned int uint; +typedef unsigned long ulong; +typedef unsigned gfp_t; +typedef off_t loff_t; +typedef vm_paddr_t resource_size_t; +typedef uint16_t __bitwise__ __sum16; +typedef unsigned long pgoff_t; +typedef unsigned __poll_t; + +typedef u64 phys_addr_t; + +typedef size_t __kernel_size_t; + +#define DECLARE_BITMAP(n, bits) \ + unsigned long n[howmany(bits, sizeof(long) * 8)] + +typedef unsigned long irq_hw_number_t; + +struct rcu_head { + void *raw[2]; +} __aligned(sizeof(void *)); + +typedef void (*rcu_callback_t)(struct rcu_head *head); +typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); +typedef int linux_task_fn_t(void *data); + +#endif /* _LINUX_TYPES_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/uaccess.h b/sys/compat/linuxkpi/common/include/linux/uaccess.h new file mode 100644 index 000000000000..c09c363a98a7 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/uaccess.h @@ -0,0 +1,96 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. + * Copyright (c) 2015 François Tigeot + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_UACCESS_H_ +#define _LINUX_UACCESS_H_ + +#include <sys/param.h> +#include <sys/lock.h> +#include <sys/proc.h> + +#include <vm/vm.h> +#include <vm/vm_extern.h> + +#include <linux/compiler.h> + +#define VERIFY_READ VM_PROT_READ +#define VERIFY_WRITE VM_PROT_WRITE + +#define __get_user(_x, _p) ({ \ + int __err; \ + __typeof(*(_p)) __x; \ + __err = linux_copyin((_p), &(__x), sizeof(*(_p))); \ + (_x) = __x; \ + __err; \ +}) + +#define __put_user(_x, _p) ({ \ + __typeof(*(_p)) __x = (_x); \ + linux_copyout(&(__x), (_p), sizeof(*(_p))); \ +}) +#define get_user(_x, _p) linux_copyin((_p), &(_x), sizeof(*(_p))) +#define put_user(_x, _p) __put_user(_x, _p) +#define clear_user(...) linux_clear_user(__VA_ARGS__) + +#if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 50000 +#define access_ok(a,b) linux_access_ok(a,b) +#else +#define access_ok(a,b,c) linux_access_ok(b,c) +#endif + +extern int linux_copyin(const void *uaddr, void *kaddr, size_t len); +extern int linux_copyout(const void *kaddr, void *uaddr, size_t len); +extern size_t linux_clear_user(void *uaddr, size_t len); +extern int linux_access_ok(const void *uaddr, size_t len); + +/* + * NOTE: Each pagefault_disable() call must have a corresponding + * pagefault_enable() call in the same scope. The former creates a new + * block and defines a temporary variable, and the latter uses the + * temporary variable and closes the block. Failure to balance the + * calls will result in a compile-time error. + */ +#define pagefault_disable(void) do { \ + int __saved_pflags = \ + vm_fault_disable_pagefaults() + +#define pagefault_enable(void) \ + vm_fault_enable_pagefaults(__saved_pflags); \ +} while (0) + +static inline bool +pagefault_disabled(void) +{ + return ((curthread->td_pflags & TDP_NOFAULTING) != 0); +} + +#endif /* _LINUX_UACCESS_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/usb.h b/sys/compat/linuxkpi/common/include/linux/usb.h new file mode 100644 index 000000000000..032c1e53a015 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/usb.h @@ -0,0 +1,318 @@ +/* $FreeBSD$ */ +/*- + * Copyright (c) 2007 Luigi Rizzo - Universita` di Pisa. All rights reserved. + * Copyright (c) 2007 Hans Petter Selasky. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _USB_COMPAT_LINUX_H +#define _USB_COMPAT_LINUX_H + +#include <sys/types.h> +#include <sys/param.h> +#include <sys/proc.h> +#include <sys/condvar.h> + +#include <dev/usb/usb.h> +#include <dev/usb/usbdi.h> +#include <dev/usb/usbdi_util.h> + +struct usb_device; +struct usb_interface; +struct usb_driver; +struct urb; + +typedef void *pm_message_t; +typedef void (usb_complete_t)(struct urb *); + +#define USB_MAX_FULL_SPEED_ISOC_FRAMES (60 * 1) +#define USB_MAX_HIGH_SPEED_ISOC_FRAMES (60 * 8) + +#define USB_DEVICE_ID_MATCH_DEVICE \ + (USB_DEVICE_ID_MATCH_VENDOR | USB_DEVICE_ID_MATCH_PRODUCT) + +#define USB_DEVICE(vend,prod) \ + .match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = (vend), \ + .idProduct = (prod) + +/* The "usb_driver" structure holds the Linux USB device driver + * callbacks, and a pointer to device ID's which this entry should + * match against. Usually this entry is exposed to the USB emulation + * layer using the "USB_DRIVER_EXPORT()" macro, which is defined + * below. + */ +struct usb_driver { + const char *name; + + int (*probe)(struct usb_interface *intf, + const struct usb_device_id *id); + + void (*disconnect)(struct usb_interface *intf); + + int (*ioctl)(struct usb_interface *intf, unsigned int code, void *buf); + + int (*suspend)(struct usb_interface *intf, pm_message_t message); + int (*resume)(struct usb_interface *intf); + + const struct usb_device_id *id_table; + + void (*shutdown)(struct usb_interface *intf); + + LIST_ENTRY(usb_driver) linux_driver_list; +}; + +#define USB_DRIVER_EXPORT(id,p_usb_drv) \ + SYSINIT(id,SI_SUB_KLD,SI_ORDER_FIRST,usb_linux_register,p_usb_drv); \ + SYSUNINIT(id,SI_SUB_KLD,SI_ORDER_ANY,usb_linux_deregister,p_usb_drv) + +#define USB_DT_ENDPOINT_SIZE 7 +#define USB_DT_ENDPOINT_AUDIO_SIZE 9 + +/* + * Endpoints + */ +#define USB_ENDPOINT_NUMBER_MASK 0x0f /* in bEndpointAddress */ +#define USB_ENDPOINT_DIR_MASK 0x80 + +#define USB_ENDPOINT_XFERTYPE_MASK 0x03 /* in bmAttributes */ +#define USB_ENDPOINT_XFER_CONTROL 0 +#define USB_ENDPOINT_XFER_ISOC 1 +#define USB_ENDPOINT_XFER_BULK 2 +#define USB_ENDPOINT_XFER_INT 3 +#define USB_ENDPOINT_MAX_ADJUSTABLE 0x80 + +/* CONTROL REQUEST SUPPORT */ + +/* + * Definition of direction mask for + * "bEndpointAddress" and "bmRequestType": + */ +#define USB_DIR_MASK 0x80 +#define USB_DIR_OUT 0x00 /* write to USB device */ +#define USB_DIR_IN 0x80 /* read from USB device */ + +/* + * Definition of type mask for + * "bmRequestType": + */ +#define USB_TYPE_MASK (0x03 << 5) +#define USB_TYPE_STANDARD (0x00 << 5) +#define USB_TYPE_CLASS (0x01 << 5) +#define USB_TYPE_VENDOR (0x02 << 5) +#define USB_TYPE_RESERVED (0x03 << 5) + +/* + * Definition of receiver mask for + * "bmRequestType": + */ +#define USB_RECIP_MASK 0x1f +#define USB_RECIP_DEVICE 0x00 +#define USB_RECIP_INTERFACE 0x01 +#define USB_RECIP_ENDPOINT 0x02 +#define USB_RECIP_OTHER 0x03 + +/* + * Definition of standard request values for + * "bRequest": + */ +#define USB_REQ_GET_STATUS 0x00 +#define USB_REQ_CLEAR_FEATURE 0x01 +#define USB_REQ_SET_FEATURE 0x03 +#define USB_REQ_SET_ADDRESS 0x05 +#define USB_REQ_GET_DESCRIPTOR 0x06 +#define USB_REQ_SET_DESCRIPTOR 0x07 +#define USB_REQ_GET_CONFIGURATION 0x08 +#define USB_REQ_SET_CONFIGURATION 0x09 +#define USB_REQ_GET_INTERFACE 0x0A +#define USB_REQ_SET_INTERFACE 0x0B +#define USB_REQ_SYNCH_FRAME 0x0C + +#define USB_REQ_SET_ENCRYPTION 0x0D /* Wireless USB */ +#define USB_REQ_GET_ENCRYPTION 0x0E +#define USB_REQ_SET_HANDSHAKE 0x0F +#define USB_REQ_GET_HANDSHAKE 0x10 +#define USB_REQ_SET_CONNECTION 0x11 +#define USB_REQ_SET_SECURITY_DATA 0x12 +#define USB_REQ_GET_SECURITY_DATA 0x13 +#define USB_REQ_SET_WUSB_DATA 0x14 +#define USB_REQ_LOOPBACK_DATA_WRITE 0x15 +#define USB_REQ_LOOPBACK_DATA_READ 0x16 +#define USB_REQ_SET_INTERFACE_DS 0x17 + +/* + * USB feature flags are written using USB_REQ_{CLEAR,SET}_FEATURE, and + * are read as a bit array returned by USB_REQ_GET_STATUS. (So there + * are at most sixteen features of each type.) + */ +#define USB_DEVICE_SELF_POWERED 0 /* (read only) */ +#define USB_DEVICE_REMOTE_WAKEUP 1 /* dev may initiate wakeup */ +#define USB_DEVICE_TEST_MODE 2 /* (wired high speed only) */ +#define USB_DEVICE_BATTERY 2 /* (wireless) */ +#define USB_DEVICE_B_HNP_ENABLE 3 /* (otg) dev may initiate HNP */ +#define USB_DEVICE_WUSB_DEVICE 3 /* (wireless) */ +#define USB_DEVICE_A_HNP_SUPPORT 4 /* (otg) RH port supports HNP */ +#define USB_DEVICE_A_ALT_HNP_SUPPORT 5 /* (otg) other RH port does */ +#define USB_DEVICE_DEBUG_MODE 6 /* (special devices only) */ + +#define USB_ENDPOINT_HALT 0 /* IN/OUT will STALL */ + +#define PIPE_ISOCHRONOUS 0x01 /* UE_ISOCHRONOUS */ +#define PIPE_INTERRUPT 0x03 /* UE_INTERRUPT */ +#define PIPE_CONTROL 0x00 /* UE_CONTROL */ +#define PIPE_BULK 0x02 /* UE_BULK */ + +/* Whenever Linux references an USB endpoint: + * a) to initialize "urb->endpoint" + * b) second argument passed to "usb_control_msg()" + * + * Then it uses one of the following macros. The "endpoint" argument + * is the physical endpoint value masked by 0xF. The "dev" argument + * is a pointer to "struct usb_device". + */ +#define usb_sndctrlpipe(dev,endpoint) \ + usb_find_host_endpoint(dev, PIPE_CONTROL, (endpoint) | USB_DIR_OUT) + +#define usb_rcvctrlpipe(dev,endpoint) \ + usb_find_host_endpoint(dev, PIPE_CONTROL, (endpoint) | USB_DIR_IN) + +#define usb_sndisocpipe(dev,endpoint) \ + usb_find_host_endpoint(dev, PIPE_ISOCHRONOUS, (endpoint) | USB_DIR_OUT) + +#define usb_rcvisocpipe(dev,endpoint) \ + usb_find_host_endpoint(dev, PIPE_ISOCHRONOUS, (endpoint) | USB_DIR_IN) + +#define usb_sndbulkpipe(dev,endpoint) \ + usb_find_host_endpoint(dev, PIPE_BULK, (endpoint) | USB_DIR_OUT) + +#define usb_rcvbulkpipe(dev,endpoint) \ + usb_find_host_endpoint(dev, PIPE_BULK, (endpoint) | USB_DIR_IN) + +#define usb_sndintpipe(dev,endpoint) \ + usb_find_host_endpoint(dev, PIPE_INTERRUPT, (endpoint) | USB_DIR_OUT) + +#define usb_rcvintpipe(dev,endpoint) \ + usb_find_host_endpoint(dev, PIPE_INTERRUPT, (endpoint) | USB_DIR_IN) + +/* + * The following structure is used to extend "struct urb" when we are + * dealing with an isochronous endpoint. It contains information about + * the data offset and data length of an isochronous packet. + * The "actual_length" field is updated before the "complete" + * callback in the "urb" structure is called. + */ +struct usb_iso_packet_descriptor { + uint32_t offset; /* depreciated buffer offset (the + * packets are usually back to back) */ + uint16_t length; /* expected length */ + uint16_t actual_length; + int16_t status; /* transfer status */ +}; + +/* + * The following structure holds various information about an USB + * transfer. This structure is used for all kinds of USB transfers. + * + * URB is short for USB Request Block. + */ +struct urb { + TAILQ_ENTRY(urb) bsd_urb_list; + struct cv cv_wait; + + struct usb_device *dev; /* (in) pointer to associated device */ + struct usb_host_endpoint *endpoint; /* (in) pipe pointer */ + uint8_t *setup_packet; /* (in) setup packet (control only) */ + uint8_t *bsd_data_ptr; + void *transfer_buffer; /* (in) associated data buffer */ + void *context; /* (in) context for completion */ + usb_complete_t *complete; /* (in) completion routine */ + + usb_size_t transfer_buffer_length;/* (in) data buffer length */ + usb_size_t bsd_length_rem; + usb_size_t actual_length; /* (return) actual transfer length */ + usb_timeout_t timeout; /* FreeBSD specific */ + + uint16_t transfer_flags; /* (in) */ +#define URB_SHORT_NOT_OK 0x0001 /* report short transfers like errors */ +#define URB_ISO_ASAP 0x0002 /* ignore "start_frame" field */ +#define URB_ZERO_PACKET 0x0004 /* the USB transfer ends with a short + * packet */ +#define URB_NO_TRANSFER_DMA_MAP 0x0008 /* "transfer_dma" is valid on submit */ +#define URB_WAIT_WAKEUP 0x0010 /* custom flags */ +#define URB_IS_SLEEPING 0x0020 /* custom flags */ + + usb_frcount_t start_frame; /* (modify) start frame (ISO) */ + usb_frcount_t number_of_packets; /* (in) number of ISO packets */ + uint16_t interval; /* (modify) transfer interval + * (INT/ISO) */ + uint16_t error_count; /* (return) number of ISO errors */ + int16_t status; /* (return) status */ + + uint8_t setup_dma; /* (in) not used on FreeBSD */ + uint8_t transfer_dma; /* (in) not used on FreeBSD */ + uint8_t bsd_isread; + uint8_t kill_count; /* FreeBSD specific */ + + struct usb_iso_packet_descriptor iso_frame_desc[]; /* (in) ISO ONLY */ +}; + +/* various prototypes */ + +int usb_submit_urb(struct urb *urb, uint16_t mem_flags); +int usb_unlink_urb(struct urb *urb); +int usb_clear_halt(struct usb_device *dev, struct usb_host_endpoint *uhe); +int usb_control_msg(struct usb_device *dev, struct usb_host_endpoint *ep, + uint8_t request, uint8_t requesttype, uint16_t value, + uint16_t index, void *data, uint16_t size, usb_timeout_t timeout); +int usb_set_interface(struct usb_device *dev, uint8_t ifnum, + uint8_t alternate); +int usb_setup_endpoint(struct usb_device *dev, + struct usb_host_endpoint *uhe, usb_frlength_t bufsize); + +struct usb_host_endpoint *usb_find_host_endpoint(struct usb_device *dev, + uint8_t type, uint8_t ep); +struct urb *usb_alloc_urb(uint16_t iso_packets, uint16_t mem_flags); +struct usb_host_interface *usb_altnum_to_altsetting( + const struct usb_interface *intf, uint8_t alt_index); +struct usb_interface *usb_ifnum_to_if(struct usb_device *dev, uint8_t iface_no); + +void *usb_buffer_alloc(struct usb_device *dev, usb_size_t size, + uint16_t mem_flags, uint8_t *dma_addr); +void *usbd_get_intfdata(struct usb_interface *intf); + +void usb_buffer_free(struct usb_device *dev, usb_size_t size, void *addr, uint8_t dma_addr); +void usb_free_urb(struct urb *urb); +void usb_init_urb(struct urb *urb); +void usb_kill_urb(struct urb *urb); +void usb_set_intfdata(struct usb_interface *intf, void *data); +void usb_linux_register(void *arg); +void usb_linux_deregister(void *arg); + +void usb_fill_bulk_urb(struct urb *, struct usb_device *, + struct usb_host_endpoint *, void *, int, usb_complete_t, void *); +int usb_bulk_msg(struct usb_device *, struct usb_host_endpoint *, + void *, int, uint16_t *, usb_timeout_t); + +#define interface_to_usbdev(intf) (intf)->linux_udev +#define interface_to_bsddev(intf) (intf)->linux_udev + +#endif /* _USB_COMPAT_LINUX_H */ diff --git a/sys/compat/linuxkpi/common/include/linux/vmalloc.h b/sys/compat/linuxkpi/common/include/linux/vmalloc.h new file mode 100644 index 000000000000..53178314c35a --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/vmalloc.h @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_VMALLOC_H_ +#define _LINUX_VMALLOC_H_ + +#include <linux/page.h> + +#define VM_MAP 0x0000 +#define PAGE_KERNEL 0x0000 + +void *vmap(struct page **pages, unsigned int count, unsigned long flags, + int prot); +void vunmap(void *addr); + +#endif /* _LINUX_VMALLOC_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/wait.h b/sys/compat/linuxkpi/common/include/linux/wait.h new file mode 100644 index 000000000000..348464fb27df --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/wait.h @@ -0,0 +1,313 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _LINUX_WAIT_H_ +#define _LINUX_WAIT_H_ + +#include <linux/compiler.h> +#include <linux/list.h> +#include <linux/spinlock.h> +#include <linux/sched.h> + +#include <asm/atomic.h> + +#include <sys/param.h> +#include <sys/systm.h> + +#define SKIP_SLEEP() (SCHEDULER_STOPPED() || kdb_active) + +#define might_sleep() \ + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "might_sleep()") + +#define might_sleep_if(cond) do { \ + if (cond) { might_sleep(); } \ +} while (0) + +struct wait_queue; +struct wait_queue_head; + +#define wait_queue_entry wait_queue + +typedef struct wait_queue wait_queue_t; +typedef struct wait_queue_entry wait_queue_entry_t; +typedef struct wait_queue_head wait_queue_head_t; + +typedef int wait_queue_func_t(wait_queue_t *, unsigned int, int, void *); + +/* + * Many API consumers directly reference these fields and those of + * wait_queue_head. + */ +struct wait_queue { + unsigned int flags; /* always 0 */ + void *private; + wait_queue_func_t *func; + union { + struct list_head task_list; /* < v4.13 */ + struct list_head entry; /* >= v4.13 */ + }; +}; + +struct wait_queue_head { + spinlock_t lock; + union { + struct list_head task_list; /* < v4.13 */ + struct list_head head; /* >= v4.13 */ + }; +}; + +/* + * This function is referenced by at least one DRM driver, so it may not be + * renamed and furthermore must be the default wait queue callback. + */ +extern wait_queue_func_t autoremove_wake_function; +extern wait_queue_func_t default_wake_function; + +#define DEFINE_WAIT_FUNC(name, function) \ + wait_queue_t name = { \ + .private = current, \ + .func = function, \ + .task_list = LINUX_LIST_HEAD_INIT(name.task_list) \ + } + +#define DEFINE_WAIT(name) \ + DEFINE_WAIT_FUNC(name, autoremove_wake_function) + +#define DECLARE_WAITQUEUE(name, task) \ + wait_queue_t name = { \ + .private = task, \ + .task_list = LINUX_LIST_HEAD_INIT(name.task_list) \ + } + +#define DECLARE_WAIT_QUEUE_HEAD(name) \ + wait_queue_head_t name = { \ + .task_list = LINUX_LIST_HEAD_INIT(name.task_list), \ + }; \ + MTX_SYSINIT(name, &(name).lock.m, spin_lock_name("wqhead"), MTX_DEF) + +#define init_waitqueue_head(wqh) do { \ + mtx_init(&(wqh)->lock.m, spin_lock_name("wqhead"), \ + NULL, MTX_DEF | MTX_NEW | MTX_NOWITNESS); \ + INIT_LIST_HEAD(&(wqh)->task_list); \ +} while (0) + +#define __init_waitqueue_head(wqh, name, lk) init_waitqueue_head(wqh) + +void linux_init_wait_entry(wait_queue_t *, int); +void linux_wake_up(wait_queue_head_t *, unsigned int, int, bool); + +#define init_wait_entry(wq, flags) \ + linux_init_wait_entry(wq, flags) +#define wake_up(wqh) \ + linux_wake_up(wqh, TASK_NORMAL, 1, false) +#define wake_up_all(wqh) \ + linux_wake_up(wqh, TASK_NORMAL, 0, false) +#define wake_up_locked(wqh) \ + linux_wake_up(wqh, TASK_NORMAL, 1, true) +#define wake_up_all_locked(wqh) \ + linux_wake_up(wqh, TASK_NORMAL, 0, true) +#define wake_up_interruptible(wqh) \ + linux_wake_up(wqh, TASK_INTERRUPTIBLE, 1, false) +#define wake_up_interruptible_all(wqh) \ + linux_wake_up(wqh, TASK_INTERRUPTIBLE, 0, false) + +int linux_wait_event_common(wait_queue_head_t *, wait_queue_t *, int, + unsigned int, spinlock_t *); + +/* + * Returns -ERESTARTSYS for a signal, 0 if cond is false after timeout, 1 if + * cond is true after timeout, remaining jiffies (> 0) if cond is true before + * timeout. + */ +#define __wait_event_common(wqh, cond, timeout, state, lock) ({ \ + DEFINE_WAIT(__wq); \ + const int __timeout = ((int)(timeout)) < 1 ? 1 : (timeout); \ + int __start = ticks; \ + int __ret = 0; \ + \ + for (;;) { \ + linux_prepare_to_wait(&(wqh), &__wq, state); \ + if (cond) \ + break; \ + __ret = linux_wait_event_common(&(wqh), &__wq, \ + __timeout, state, lock); \ + if (__ret != 0) \ + break; \ + } \ + linux_finish_wait(&(wqh), &__wq); \ + if (__timeout != MAX_SCHEDULE_TIMEOUT) { \ + if (__ret == -EWOULDBLOCK) \ + __ret = !!(cond); \ + else if (__ret != -ERESTARTSYS) { \ + __ret = __timeout + __start - ticks; \ + /* range check return value */ \ + if (__ret < 1) \ + __ret = 1; \ + else if (__ret > __timeout) \ + __ret = __timeout; \ + } \ + } \ + __ret; \ +}) + +#define wait_event(wqh, cond) do { \ + (void) __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ + TASK_UNINTERRUPTIBLE, NULL); \ +} while (0) + +#define wait_event_timeout(wqh, cond, timeout) ({ \ + __wait_event_common(wqh, cond, timeout, TASK_UNINTERRUPTIBLE, \ + NULL); \ +}) + +#define wait_event_killable(wqh, cond) ({ \ + __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ + TASK_INTERRUPTIBLE, NULL); \ +}) + +#define wait_event_interruptible(wqh, cond) ({ \ + __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ + TASK_INTERRUPTIBLE, NULL); \ +}) + +#define wait_event_interruptible_timeout(wqh, cond, timeout) ({ \ + __wait_event_common(wqh, cond, timeout, TASK_INTERRUPTIBLE, \ + NULL); \ +}) + +/* + * Wait queue is already locked. + */ +#define wait_event_interruptible_locked(wqh, cond) ({ \ + int __ret; \ + \ + spin_unlock(&(wqh).lock); \ + __ret = __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ + TASK_INTERRUPTIBLE, NULL); \ + spin_lock(&(wqh).lock); \ + __ret; \ +}) + +/* + * The passed spinlock is held when testing the condition. + */ +#define wait_event_interruptible_lock_irq(wqh, cond, lock) ({ \ + __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ + TASK_INTERRUPTIBLE, &(lock)); \ +}) + +/* + * The passed spinlock is held when testing the condition. + */ +#define wait_event_lock_irq(wqh, cond, lock) ({ \ + __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ + TASK_UNINTERRUPTIBLE, &(lock)); \ +}) + +static inline void +__add_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) +{ + list_add(&wq->task_list, &wqh->task_list); +} + +static inline void +add_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) +{ + + spin_lock(&wqh->lock); + __add_wait_queue(wqh, wq); + spin_unlock(&wqh->lock); +} + +static inline void +__add_wait_queue_tail(wait_queue_head_t *wqh, wait_queue_t *wq) +{ + list_add_tail(&wq->task_list, &wqh->task_list); +} + +static inline void +__add_wait_queue_entry_tail(wait_queue_head_t *wqh, wait_queue_entry_t *wq) +{ + list_add_tail(&wq->entry, &wqh->head); +} + +static inline void +__remove_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) +{ + list_del(&wq->task_list); +} + +static inline void +remove_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) +{ + + spin_lock(&wqh->lock); + __remove_wait_queue(wqh, wq); + spin_unlock(&wqh->lock); +} + +bool linux_waitqueue_active(wait_queue_head_t *); + +#define waitqueue_active(wqh) linux_waitqueue_active(wqh) + +void linux_prepare_to_wait(wait_queue_head_t *, wait_queue_t *, int); +void linux_finish_wait(wait_queue_head_t *, wait_queue_t *); + +#define prepare_to_wait(wqh, wq, state) linux_prepare_to_wait(wqh, wq, state) +#define finish_wait(wqh, wq) linux_finish_wait(wqh, wq) + +void linux_wake_up_bit(void *, int); +int linux_wait_on_bit_timeout(unsigned long *, int, unsigned int, int); +void linux_wake_up_atomic_t(atomic_t *); +int linux_wait_on_atomic_t(atomic_t *, unsigned int); + +#define wake_up_bit(word, bit) linux_wake_up_bit(word, bit) +#define wait_on_bit(word, bit, state) \ + linux_wait_on_bit_timeout(word, bit, state, MAX_SCHEDULE_TIMEOUT) +#define wait_on_bit_timeout(word, bit, state, timeout) \ + linux_wait_on_bit_timeout(word, bit, state, timeout) +#define wake_up_atomic_t(a) linux_wake_up_atomic_t(a) +/* + * All existing callers have a cb that just schedule()s. To avoid adding + * complexity, just emulate that internally. The prototype is different so that + * callers must be manually modified; a cb that does something other than call + * schedule() will require special treatment. + */ +#define wait_on_atomic_t(a, state) linux_wait_on_atomic_t(a, state) + +struct task_struct; +bool linux_wake_up_state(struct task_struct *, unsigned int); + +#define wake_up_process(task) linux_wake_up_state(task, TASK_NORMAL) +#define wake_up_state(task, state) linux_wake_up_state(task, state) + +#endif /* _LINUX_WAIT_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/wait_bit.h b/sys/compat/linuxkpi/common/include/linux/wait_bit.h new file mode 100644 index 000000000000..e3dddaade498 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/wait_bit.h @@ -0,0 +1,64 @@ +/*- + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Emmanuel Vadot under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __LINUX_WAITBIT_H__ +#define __LINUX_WAITBIT_H__ + +#include <linux/wait.h> +#include <linux/bitops.h> + +extern wait_queue_head_t linux_bit_waitq; +extern wait_queue_head_t linux_var_waitq; + +#define wait_var_event_killable(var, cond) \ + wait_event_killable(linux_var_waitq, cond) + +static inline void +clear_and_wake_up_bit(int bit, void *word) +{ + clear_bit_unlock(bit, word); + wake_up_bit(word, bit); +} + +static inline wait_queue_head_t * +bit_waitqueue(void *word, int bit) +{ + + return (&linux_bit_waitq); +} + +static inline void +wake_up_var(void *var) +{ + + wake_up(&linux_var_waitq); +} + +#endif /* __LINUX_WAITBIT_H__ */ diff --git a/sys/compat/linuxkpi/common/include/linux/workqueue.h b/sys/compat/linuxkpi/common/include/linux/workqueue.h new file mode 100644 index 000000000000..768ce33bb20d --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/workqueue.h @@ -0,0 +1,259 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_WORKQUEUE_H_ +#define _LINUX_WORKQUEUE_H_ + +#include <linux/types.h> +#include <linux/kernel.h> +#include <linux/timer.h> +#include <linux/slab.h> + +#include <asm/atomic.h> + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/taskqueue.h> +#include <sys/mutex.h> + +#define WORK_CPU_UNBOUND MAXCPU +#define WQ_UNBOUND (1 << 0) +#define WQ_HIGHPRI (1 << 1) + +struct work_struct; +typedef void (*work_func_t)(struct work_struct *); + +struct work_exec { + TAILQ_ENTRY(work_exec) entry; + struct work_struct *target; +}; + +struct workqueue_struct { + struct taskqueue *taskqueue; + struct mtx exec_mtx; + TAILQ_HEAD(, work_exec) exec_head; + atomic_t draining; +}; + +#define WQ_EXEC_LOCK(wq) mtx_lock(&(wq)->exec_mtx) +#define WQ_EXEC_UNLOCK(wq) mtx_unlock(&(wq)->exec_mtx) + +struct work_struct { + struct task work_task; + struct workqueue_struct *work_queue; + work_func_t func; + atomic_t state; +}; + +struct rcu_work { + struct work_struct work; + struct rcu_head rcu; + + struct workqueue_struct *wq; +}; + +#define DECLARE_WORK(name, fn) \ + struct work_struct name; \ + static void name##_init(void *arg) \ + { \ + INIT_WORK(&name, fn); \ + } \ + SYSINIT(name, SI_SUB_LOCK, SI_ORDER_SECOND, name##_init, NULL) + +struct delayed_work { + struct work_struct work; + struct { + struct callout callout; + struct mtx mtx; + int expires; + } timer; +}; + +#define DECLARE_DELAYED_WORK(name, fn) \ + struct delayed_work name; \ + static void __linux_delayed_ ## name ## _init(void *arg) \ + { \ + linux_init_delayed_work(&name, fn); \ + } \ + SYSINIT(name, SI_SUB_LOCK, SI_ORDER_SECOND, \ + __linux_delayed_ ## name##_init, NULL) + +static inline struct delayed_work * +to_delayed_work(struct work_struct *work) +{ + return (container_of(work, struct delayed_work, work)); +} + +#define INIT_WORK(work, fn) \ +do { \ + (work)->func = (fn); \ + (work)->work_queue = NULL; \ + atomic_set(&(work)->state, 0); \ + TASK_INIT(&(work)->work_task, 0, linux_work_fn, (work)); \ +} while (0) + +#define INIT_RCU_WORK(_work, _fn) \ + INIT_WORK(&(_work)->work, (_fn)) + +#define INIT_WORK_ONSTACK(work, fn) \ + INIT_WORK(work, fn) + +#define INIT_DELAYED_WORK(dwork, fn) \ + linux_init_delayed_work(dwork, fn) + +#define INIT_DELAYED_WORK_ONSTACK(dwork, fn) \ + linux_init_delayed_work(dwork, fn) + +#define INIT_DEFERRABLE_WORK(dwork, fn) \ + INIT_DELAYED_WORK(dwork, fn) + +#define flush_scheduled_work() \ + taskqueue_drain_all(system_wq->taskqueue) + +#define queue_work(wq, work) \ + linux_queue_work_on(WORK_CPU_UNBOUND, wq, work) + +#define schedule_work(work) \ + linux_queue_work_on(WORK_CPU_UNBOUND, system_wq, work) + +#define queue_delayed_work(wq, dwork, delay) \ + linux_queue_delayed_work_on(WORK_CPU_UNBOUND, wq, dwork, delay) + +#define schedule_delayed_work_on(cpu, dwork, delay) \ + linux_queue_delayed_work_on(cpu, system_wq, dwork, delay) + +#define queue_work_on(cpu, wq, work) \ + linux_queue_work_on(cpu, wq, work) + +#define schedule_delayed_work(dwork, delay) \ + linux_queue_delayed_work_on(WORK_CPU_UNBOUND, system_wq, dwork, delay) + +#define queue_delayed_work_on(cpu, wq, dwork, delay) \ + linux_queue_delayed_work_on(cpu, wq, dwork, delay) + +#define create_singlethread_workqueue(name) \ + linux_create_workqueue_common(name, 1) + +#define create_workqueue(name) \ + linux_create_workqueue_common(name, mp_ncpus) + +#define alloc_ordered_workqueue(name, flags) \ + linux_create_workqueue_common(name, 1) + +#define alloc_workqueue(name, flags, max_active) \ + linux_create_workqueue_common(name, max_active) + +#define flush_workqueue(wq) \ + taskqueue_drain_all((wq)->taskqueue) + +#define drain_workqueue(wq) do { \ + atomic_inc(&(wq)->draining); \ + taskqueue_drain_all((wq)->taskqueue); \ + atomic_dec(&(wq)->draining); \ +} while (0) + +#define mod_delayed_work(wq, dwork, delay) ({ \ + bool __retval; \ + __retval = linux_cancel_delayed_work(dwork); \ + linux_queue_delayed_work_on(WORK_CPU_UNBOUND, \ + wq, dwork, delay); \ + __retval; \ +}) + +#define delayed_work_pending(dwork) \ + linux_work_pending(&(dwork)->work) + +#define cancel_delayed_work(dwork) \ + linux_cancel_delayed_work(dwork) + +#define cancel_work_sync(work) \ + linux_cancel_work_sync(work) + +#define cancel_delayed_work_sync(dwork) \ + linux_cancel_delayed_work_sync(dwork) + +#define flush_work(work) \ + linux_flush_work(work) + +#define queue_rcu_work(wq, rwork) \ + linux_queue_rcu_work(wq, rwork) + +#define flush_rcu_work(rwork) \ + linux_flush_rcu_work(rwork) + +#define flush_delayed_work(dwork) \ + linux_flush_delayed_work(dwork) + +#define work_pending(work) \ + linux_work_pending(work) + +#define work_busy(work) \ + linux_work_busy(work) + +#define destroy_work_on_stack(work) \ + do { } while (0) + +#define destroy_delayed_work_on_stack(dwork) \ + do { } while (0) + +#define destroy_workqueue(wq) \ + linux_destroy_workqueue(wq) + +#define current_work() \ + linux_current_work() + +/* prototypes */ + +extern struct workqueue_struct *system_wq; +extern struct workqueue_struct *system_long_wq; +extern struct workqueue_struct *system_unbound_wq; +extern struct workqueue_struct *system_highpri_wq; +extern struct workqueue_struct *system_power_efficient_wq; + +extern void linux_init_delayed_work(struct delayed_work *, work_func_t); +extern void linux_work_fn(void *, int); +extern void linux_delayed_work_fn(void *, int); +extern struct workqueue_struct *linux_create_workqueue_common(const char *, int); +extern void linux_destroy_workqueue(struct workqueue_struct *); +extern bool linux_queue_work_on(int cpu, struct workqueue_struct *, struct work_struct *); +extern bool linux_queue_delayed_work_on(int cpu, struct workqueue_struct *, + struct delayed_work *, unsigned delay); +extern bool linux_cancel_delayed_work(struct delayed_work *); +extern bool linux_cancel_work_sync(struct work_struct *); +extern bool linux_cancel_delayed_work_sync(struct delayed_work *); +extern bool linux_flush_work(struct work_struct *); +extern bool linux_flush_delayed_work(struct delayed_work *); +extern bool linux_work_pending(struct work_struct *); +extern bool linux_work_busy(struct work_struct *); +extern struct work_struct *linux_current_work(void); +extern bool linux_queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork); +extern bool linux_flush_rcu_work(struct rcu_work *rwork); + +#endif /* _LINUX_WORKQUEUE_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/ww_mutex.h b/sys/compat/linuxkpi/common/include/linux/ww_mutex.h new file mode 100644 index 000000000000..c23d4dd6fa24 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/ww_mutex.h @@ -0,0 +1,143 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_WW_MUTEX_H_ +#define _LINUX_WW_MUTEX_H_ + +#include <sys/param.h> +#include <sys/proc.h> +#include <sys/condvar.h> +#include <sys/kernel.h> + +#include <linux/mutex.h> + +struct ww_class { + const char *mutex_name; +}; + +struct ww_acquire_ctx { +}; + +struct ww_mutex { + struct mutex base; + struct cv condvar; + struct ww_acquire_ctx *ctx; +}; + +#define DEFINE_WW_CLASS(name) \ + struct ww_class name = { \ + .mutex_name = mutex_name(#name "_mutex") \ + } + +#define DEFINE_WW_MUTEX(name, ww_class) \ + struct ww_mutex name; \ + static void name##_init(void *arg) \ + { \ + ww_mutex_init(&name, &ww_class); \ + } \ + SYSINIT(name, SI_SUB_LOCK, SI_ORDER_SECOND, name##_init, NULL) + +#define ww_mutex_is_locked(_m) \ + sx_xlocked(&(_m)->base.sx) + +#define ww_mutex_lock_slow(_m, _x) \ + ww_mutex_lock(_m, _x) + +#define ww_mutex_lock_slow_interruptible(_m, _x) \ + ww_mutex_lock_interruptible(_m, _x) + +static inline int __must_check +ww_mutex_trylock(struct ww_mutex *lock) +{ + return (mutex_trylock(&lock->base)); +} + +extern int linux_ww_mutex_lock_sub(struct ww_mutex *, int catch_signal); + +static inline int +ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) +{ + if (MUTEX_SKIP()) + return (0); + else if ((struct thread *)SX_OWNER(lock->base.sx.sx_lock) == curthread) + return (-EALREADY); + else + return (linux_ww_mutex_lock_sub(lock, 0)); +} + +static inline int +ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) +{ + if (MUTEX_SKIP()) + return (0); + else if ((struct thread *)SX_OWNER(lock->base.sx.sx_lock) == curthread) + return (-EALREADY); + else + return (linux_ww_mutex_lock_sub(lock, 1)); +} + +extern void linux_ww_mutex_unlock_sub(struct ww_mutex *); + +static inline void +ww_mutex_unlock(struct ww_mutex *lock) +{ + if (MUTEX_SKIP()) + return; + else + linux_ww_mutex_unlock_sub(lock); +} + +static inline void +ww_mutex_destroy(struct ww_mutex *lock) +{ + cv_destroy(&lock->condvar); + mutex_destroy(&lock->base); +} + +static inline void +ww_acquire_init(struct ww_acquire_ctx *ctx, struct ww_class *ww_class) +{ +} + +static inline void +ww_mutex_init(struct ww_mutex *lock, struct ww_class *ww_class) +{ + linux_mutex_init(&lock->base, ww_class->mutex_name, SX_NOWITNESS); + cv_init(&lock->condvar, "lkpi-ww"); +} + +static inline void +ww_acquire_fini(struct ww_acquire_ctx *ctx) +{ +} + +static inline void +ww_acquire_done(struct ww_acquire_ctx *ctx) +{ +} + +#endif /* _LINUX_WW_MUTEX_H_ */ diff --git a/sys/compat/linuxkpi/common/include/net/if_inet6.h b/sys/compat/linuxkpi/common/include/net/if_inet6.h new file mode 100644 index 000000000000..bb4df2615186 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/net/if_inet6.h @@ -0,0 +1,53 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _NET_IF_INET6_H_ +#define _NET_IF_INET6_H_ + +#include <sys/types.h> +#include <netinet/in.h> + +#include <asm/types.h> + +static inline void ipv6_eth_mc_map(const struct in6_addr *addr, char *buf) +{ +/* + * +-------+-------+-------+-------+-------+-------+ + * | 33 | 33 | DST13 | DST14 | DST15 | DST16 | + * +-------+-------+-------+-------+-------+-------+ + */ + + buf[0]= 0x33; + buf[1]= 0x33; + + memcpy(buf + 2, &addr->s6_addr32[3], sizeof(__u32)); +} + +#endif /* _NET_IF_INET6_H_ */ diff --git a/sys/compat/linuxkpi/common/include/net/ip.h b/sys/compat/linuxkpi/common/include/net/ip.h new file mode 100644 index 000000000000..1cfc568db323 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/net/ip.h @@ -0,0 +1,103 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_NET_IP_H_ +#define _LINUX_NET_IP_H_ + +#include "opt_inet.h" + +#include <sys/types.h> +#include <sys/socket.h> + +#include <net/if_types.h> +#include <net/if.h> +#include <net/if_var.h> + +#include <netinet/in.h> +#include <netinet/in_pcb.h> + +static inline void +inet_get_local_port_range(struct vnet *vnet, int *low, int *high) +{ +#ifdef INET + CURVNET_SET_QUIET(vnet); + *low = V_ipport_firstauto; + *high = V_ipport_lastauto; + CURVNET_RESTORE(); +#else + *low = IPPORT_EPHEMERALFIRST; /* 10000 */ + *high = IPPORT_EPHEMERALLAST; /* 65535 */ +#endif +} + +static inline void +ip_eth_mc_map(uint32_t addr, char *buf) +{ + + addr = ntohl(addr); + + buf[0] = 0x01; + buf[1] = 0x00; + buf[2] = 0x5e; + buf[3] = (addr >> 16) & 0x7f; + buf[4] = (addr >> 8) & 0xff; + buf[5] = (addr & 0xff); +} + +static inline void +ip_ib_mc_map(uint32_t addr, const unsigned char *bcast, char *buf) +{ + unsigned char scope; + + addr = ntohl(addr); + scope = bcast[5] & 0xF; + buf[0] = 0; + buf[1] = 0xff; + buf[2] = 0xff; + buf[3] = 0xff; + buf[4] = 0xff; + buf[5] = 0x10 | scope; + buf[6] = 0x40; + buf[7] = 0x1b; + buf[8] = bcast[8]; + buf[9] = bcast[9]; + buf[10] = 0; + buf[11] = 0; + buf[12] = 0; + buf[13] = 0; + buf[14] = 0; + buf[15] = 0; + buf[16] = (addr >> 24) & 0xff; + buf[17] = (addr >> 16) & 0xff; + buf[18] = (addr >> 8) & 0xff; + buf[19] = addr & 0xff; +} + +#endif /* _LINUX_NET_IP_H_ */ diff --git a/sys/compat/linuxkpi/common/include/net/ipv6.h b/sys/compat/linuxkpi/common/include/net/ipv6.h new file mode 100644 index 000000000000..feabd7360280 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/net/ipv6.h @@ -0,0 +1,114 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_NET_IPV6_H_ +#define _LINUX_NET_IPV6_H_ + +#include <sys/types.h> +#include <netinet/in.h> +#include <linux/types.h> + +#define IPV6_DEFAULT_HOPLIMIT 64 + +#define ipv6_addr_loopback(addr) IN6_IS_ADDR_LOOPBACK(addr) +#define ipv6_addr_any(addr) IN6_IS_ADDR_UNSPECIFIED(addr) + +#define ipv6_addr_copy(dst, src) \ + memcpy((dst), (src), sizeof(struct in6_addr)) + +static inline void +ipv6_ib_mc_map(const struct in6_addr *addr, const unsigned char *broadcast, + char *buf) +{ + unsigned char scope; + + scope = broadcast[5] & 0xF; + buf[0] = 0; + buf[1] = 0xff; + buf[2] = 0xff; + buf[3] = 0xff; + buf[4] = 0xff; + buf[5] = 0x10 | scope; + buf[6] = 0x60; + buf[7] = 0x1b; + buf[8] = broadcast[8]; + buf[9] = broadcast[9]; + memcpy(&buf[10], &addr->s6_addr[6], 10); +} + +static inline void __ipv6_addr_set_half(__be32 *addr, __be32 wh, __be32 wl) +{ +#if BITS_PER_LONG == 64 +#if defined(__BIG_ENDIAN) + if (__builtin_constant_p(wh) && __builtin_constant_p(wl)) { + *(__force u64 *)addr = ((__force u64)(wh) << 32 | (__force u64)(wl)); + return; + } +#elif defined(__LITTLE_ENDIAN) + if (__builtin_constant_p(wl) && __builtin_constant_p(wh)) { + *(__force u64 *)addr = ((__force u64)(wl) << 32 | (__force u64)(wh)); + return; + } +#endif +#endif + addr[0] = wh; + addr[1] = wl; +} + +static inline void ipv6_addr_set(struct in6_addr *addr, + __be32 w1, __be32 w2, + __be32 w3, __be32 w4) +{ + __ipv6_addr_set_half(&addr->s6_addr32[0], w1, w2); + __ipv6_addr_set_half(&addr->s6_addr32[2], w3, w4); +} + +static inline void ipv6_addr_set_v4mapped(const __be32 addr, + struct in6_addr *v4mapped) +{ + ipv6_addr_set(v4mapped, + 0, 0, + htonl(0x0000FFFF), + addr); +} + +static inline int ipv6_addr_v4mapped(const struct in6_addr *a) +{ + return ((a->s6_addr32[0] | a->s6_addr32[1] | + (a->s6_addr32[2] ^ htonl(0x0000ffff))) == 0); +} + +static inline int ipv6_addr_cmp(const struct in6_addr *a1, const struct in6_addr *a2) +{ + return memcmp(a1, a2, sizeof(struct in6_addr)); +} + + +#endif /* _LINUX_NET_IPV6_H_ */ diff --git a/sys/compat/linuxkpi/common/include/net/netevent.h b/sys/compat/linuxkpi/common/include/net/netevent.h new file mode 100644 index 000000000000..3c6fba2f28d6 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/net/netevent.h @@ -0,0 +1,75 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_NET_NETEVENT_H_ +#define _LINUX_NET_NETEVENT_H_ + +#include <sys/types.h> +#include <sys/eventhandler.h> + +#include <linux/notifier.h> + +enum netevent_notif_type { + NETEVENT_NEIGH_UPDATE = 0, +#if 0 /* Unsupported events. */ + NETEVENT_PMTU_UPDATE, + NETEVENT_REDIRECT, +#endif +}; + +struct llentry; + +static inline void +_handle_arp_update_event(void *arg, struct llentry *lle, int evt __unused) +{ + struct notifier_block *nb; + + nb = arg; + nb->notifier_call(nb, NETEVENT_NEIGH_UPDATE, lle); +} + +static inline int +register_netevent_notifier(struct notifier_block *nb) +{ + nb->tags[NETEVENT_NEIGH_UPDATE] = EVENTHANDLER_REGISTER( + lle_event, _handle_arp_update_event, nb, 0); + return (0); +} + +static inline int +unregister_netevent_notifier(struct notifier_block *nb) +{ + + EVENTHANDLER_DEREGISTER(lle_event, nb->tags[NETEVENT_NEIGH_UPDATE]); + + return (0); +} + +#endif /* _LINUX_NET_NETEVENT_H_ */ diff --git a/sys/compat/linuxkpi/common/include/net/tcp.h b/sys/compat/linuxkpi/common/include/net/tcp.h new file mode 100644 index 000000000000..3f156196aaaf --- /dev/null +++ b/sys/compat/linuxkpi/common/include/net/tcp.h @@ -0,0 +1,40 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ +#ifndef _LINUX_NET_TCP_H_ +#define _LINUX_NET_TCP_H_ + +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/kref.h> + +#include <net/ip.h> + +#endif /* _LINUX_NET_TCP_H_ */ diff --git a/sys/compat/linuxkpi/common/src/linux_compat.c b/sys/compat/linuxkpi/common/src/linux_compat.c new file mode 100644 index 000000000000..b3c2475b223f --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_compat.c @@ -0,0 +1,2554 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2018 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_stack.h" + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> +#include <sys/kernel.h> +#include <sys/sysctl.h> +#include <sys/proc.h> +#include <sys/sglist.h> +#include <sys/sleepqueue.h> +#include <sys/refcount.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/bus.h> +#include <sys/eventhandler.h> +#include <sys/fcntl.h> +#include <sys/file.h> +#include <sys/filio.h> +#include <sys/rwlock.h> +#include <sys/mman.h> +#include <sys/stack.h> +#include <sys/user.h> + +#include <vm/vm.h> +#include <vm/pmap.h> +#include <vm/vm_object.h> +#include <vm/vm_page.h> +#include <vm/vm_pager.h> + +#include <machine/stdarg.h> + +#if defined(__i386__) || defined(__amd64__) +#include <machine/md_var.h> +#endif + +#include <linux/kobject.h> +#include <linux/device.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/cdev.h> +#include <linux/file.h> +#include <linux/sysfs.h> +#include <linux/mm.h> +#include <linux/io.h> +#include <linux/vmalloc.h> +#include <linux/netdevice.h> +#include <linux/timer.h> +#include <linux/interrupt.h> +#include <linux/uaccess.h> +#include <linux/list.h> +#include <linux/kthread.h> +#include <linux/kernel.h> +#include <linux/compat.h> +#include <linux/poll.h> +#include <linux/smp.h> +#include <linux/wait_bit.h> + +#if defined(__i386__) || defined(__amd64__) +#include <asm/smp.h> +#endif + +SYSCTL_NODE(_compat, OID_AUTO, linuxkpi, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, + "LinuxKPI parameters"); + +int linuxkpi_debug; +SYSCTL_INT(_compat_linuxkpi, OID_AUTO, debug, CTLFLAG_RWTUN, + &linuxkpi_debug, 0, "Set to enable pr_debug() prints. Clear to disable."); + +MALLOC_DEFINE(M_KMALLOC, "linux", "Linux kmalloc compat"); + +#include <linux/rbtree.h> +/* Undo Linux compat changes. */ +#undef RB_ROOT +#undef file +#undef cdev +#define RB_ROOT(head) (head)->rbh_root + +static void linux_cdev_deref(struct linux_cdev *ldev); +static struct vm_area_struct *linux_cdev_handle_find(void *handle); + +struct kobject linux_class_root; +struct device linux_root_device; +struct class linux_class_misc; +struct list_head pci_drivers; +struct list_head pci_devices; +spinlock_t pci_lock; + +unsigned long linux_timer_hz_mask; + +wait_queue_head_t linux_bit_waitq; +wait_queue_head_t linux_var_waitq; + +int +panic_cmp(struct rb_node *one, struct rb_node *two) +{ + panic("no cmp"); +} + +RB_GENERATE(linux_root, rb_node, __entry, panic_cmp); + +int +kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args) +{ + va_list tmp_va; + int len; + char *old; + char *name; + char dummy; + + old = kobj->name; + + if (old && fmt == NULL) + return (0); + + /* compute length of string */ + va_copy(tmp_va, args); + len = vsnprintf(&dummy, 0, fmt, tmp_va); + va_end(tmp_va); + + /* account for zero termination */ + len++; + + /* check for error */ + if (len < 1) + return (-EINVAL); + + /* allocate memory for string */ + name = kzalloc(len, GFP_KERNEL); + if (name == NULL) + return (-ENOMEM); + vsnprintf(name, len, fmt, args); + kobj->name = name; + + /* free old string */ + kfree(old); + + /* filter new string */ + for (; *name != '\0'; name++) + if (*name == '/') + *name = '!'; + return (0); +} + +int +kobject_set_name(struct kobject *kobj, const char *fmt, ...) +{ + va_list args; + int error; + + va_start(args, fmt); + error = kobject_set_name_vargs(kobj, fmt, args); + va_end(args); + + return (error); +} + +static int +kobject_add_complete(struct kobject *kobj, struct kobject *parent) +{ + const struct kobj_type *t; + int error; + + kobj->parent = parent; + error = sysfs_create_dir(kobj); + if (error == 0 && kobj->ktype && kobj->ktype->default_attrs) { + struct attribute **attr; + t = kobj->ktype; + + for (attr = t->default_attrs; *attr != NULL; attr++) { + error = sysfs_create_file(kobj, *attr); + if (error) + break; + } + if (error) + sysfs_remove_dir(kobj); + + } + return (error); +} + +int +kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...) +{ + va_list args; + int error; + + va_start(args, fmt); + error = kobject_set_name_vargs(kobj, fmt, args); + va_end(args); + if (error) + return (error); + + return kobject_add_complete(kobj, parent); +} + +void +linux_kobject_release(struct kref *kref) +{ + struct kobject *kobj; + char *name; + + kobj = container_of(kref, struct kobject, kref); + sysfs_remove_dir(kobj); + name = kobj->name; + if (kobj->ktype && kobj->ktype->release) + kobj->ktype->release(kobj); + kfree(name); +} + +static void +linux_kobject_kfree(struct kobject *kobj) +{ + kfree(kobj); +} + +static void +linux_kobject_kfree_name(struct kobject *kobj) +{ + if (kobj) { + kfree(kobj->name); + } +} + +const struct kobj_type linux_kfree_type = { + .release = linux_kobject_kfree +}; + +static void +linux_device_release(struct device *dev) +{ + pr_debug("linux_device_release: %s\n", dev_name(dev)); + kfree(dev); +} + +static ssize_t +linux_class_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct class_attribute *dattr; + ssize_t error; + + dattr = container_of(attr, struct class_attribute, attr); + error = -EIO; + if (dattr->show) + error = dattr->show(container_of(kobj, struct class, kobj), + dattr, buf); + return (error); +} + +static ssize_t +linux_class_store(struct kobject *kobj, struct attribute *attr, const char *buf, + size_t count) +{ + struct class_attribute *dattr; + ssize_t error; + + dattr = container_of(attr, struct class_attribute, attr); + error = -EIO; + if (dattr->store) + error = dattr->store(container_of(kobj, struct class, kobj), + dattr, buf, count); + return (error); +} + +static void +linux_class_release(struct kobject *kobj) +{ + struct class *class; + + class = container_of(kobj, struct class, kobj); + if (class->class_release) + class->class_release(class); +} + +static const struct sysfs_ops linux_class_sysfs = { + .show = linux_class_show, + .store = linux_class_store, +}; + +const struct kobj_type linux_class_ktype = { + .release = linux_class_release, + .sysfs_ops = &linux_class_sysfs +}; + +static void +linux_dev_release(struct kobject *kobj) +{ + struct device *dev; + + dev = container_of(kobj, struct device, kobj); + /* This is the precedence defined by linux. */ + if (dev->release) + dev->release(dev); + else if (dev->class && dev->class->dev_release) + dev->class->dev_release(dev); +} + +static ssize_t +linux_dev_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct device_attribute *dattr; + ssize_t error; + + dattr = container_of(attr, struct device_attribute, attr); + error = -EIO; + if (dattr->show) + error = dattr->show(container_of(kobj, struct device, kobj), + dattr, buf); + return (error); +} + +static ssize_t +linux_dev_store(struct kobject *kobj, struct attribute *attr, const char *buf, + size_t count) +{ + struct device_attribute *dattr; + ssize_t error; + + dattr = container_of(attr, struct device_attribute, attr); + error = -EIO; + if (dattr->store) + error = dattr->store(container_of(kobj, struct device, kobj), + dattr, buf, count); + return (error); +} + +static const struct sysfs_ops linux_dev_sysfs = { + .show = linux_dev_show, + .store = linux_dev_store, +}; + +const struct kobj_type linux_dev_ktype = { + .release = linux_dev_release, + .sysfs_ops = &linux_dev_sysfs +}; + +struct device * +device_create(struct class *class, struct device *parent, dev_t devt, + void *drvdata, const char *fmt, ...) +{ + struct device *dev; + va_list args; + + dev = kzalloc(sizeof(*dev), M_WAITOK); + dev->parent = parent; + dev->class = class; + dev->devt = devt; + dev->driver_data = drvdata; + dev->release = linux_device_release; + va_start(args, fmt); + kobject_set_name_vargs(&dev->kobj, fmt, args); + va_end(args); + device_register(dev); + + return (dev); +} + +int +kobject_init_and_add(struct kobject *kobj, const struct kobj_type *ktype, + struct kobject *parent, const char *fmt, ...) +{ + va_list args; + int error; + + kobject_init(kobj, ktype); + kobj->ktype = ktype; + kobj->parent = parent; + kobj->name = NULL; + + va_start(args, fmt); + error = kobject_set_name_vargs(kobj, fmt, args); + va_end(args); + if (error) + return (error); + return kobject_add_complete(kobj, parent); +} + +static void +linux_kq_lock(void *arg) +{ + spinlock_t *s = arg; + + spin_lock(s); +} +static void +linux_kq_unlock(void *arg) +{ + spinlock_t *s = arg; + + spin_unlock(s); +} + +static void +linux_kq_lock_owned(void *arg) +{ +#ifdef INVARIANTS + spinlock_t *s = arg; + + mtx_assert(&s->m, MA_OWNED); +#endif +} + +static void +linux_kq_lock_unowned(void *arg) +{ +#ifdef INVARIANTS + spinlock_t *s = arg; + + mtx_assert(&s->m, MA_NOTOWNED); +#endif +} + +static void +linux_file_kqfilter_poll(struct linux_file *, int); + +struct linux_file * +linux_file_alloc(void) +{ + struct linux_file *filp; + + filp = kzalloc(sizeof(*filp), GFP_KERNEL); + + /* set initial refcount */ + filp->f_count = 1; + + /* setup fields needed by kqueue support */ + spin_lock_init(&filp->f_kqlock); + knlist_init(&filp->f_selinfo.si_note, &filp->f_kqlock, + linux_kq_lock, linux_kq_unlock, + linux_kq_lock_owned, linux_kq_lock_unowned); + + return (filp); +} + +void +linux_file_free(struct linux_file *filp) +{ + if (filp->_file == NULL) { + if (filp->f_shmem != NULL) + vm_object_deallocate(filp->f_shmem); + kfree(filp); + } else { + /* + * The close method of the character device or file + * will free the linux_file structure: + */ + _fdrop(filp->_file, curthread); + } +} + +static int +linux_cdev_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, + vm_page_t *mres) +{ + struct vm_area_struct *vmap; + + vmap = linux_cdev_handle_find(vm_obj->handle); + + MPASS(vmap != NULL); + MPASS(vmap->vm_private_data == vm_obj->handle); + + if (likely(vmap->vm_ops != NULL && offset < vmap->vm_len)) { + vm_paddr_t paddr = IDX_TO_OFF(vmap->vm_pfn) + offset; + vm_page_t page; + + if (((*mres)->flags & PG_FICTITIOUS) != 0) { + /* + * If the passed in result page is a fake + * page, update it with the new physical + * address. + */ + page = *mres; + vm_page_updatefake(page, paddr, vm_obj->memattr); + } else { + /* + * Replace the passed in "mres" page with our + * own fake page and free up the all of the + * original pages. + */ + VM_OBJECT_WUNLOCK(vm_obj); + page = vm_page_getfake(paddr, vm_obj->memattr); + VM_OBJECT_WLOCK(vm_obj); + + vm_page_replace(page, vm_obj, (*mres)->pindex, *mres); + *mres = page; + } + vm_page_valid(page); + return (VM_PAGER_OK); + } + return (VM_PAGER_FAIL); +} + +static int +linux_cdev_pager_populate(vm_object_t vm_obj, vm_pindex_t pidx, int fault_type, + vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last) +{ + struct vm_area_struct *vmap; + int err; + + /* get VM area structure */ + vmap = linux_cdev_handle_find(vm_obj->handle); + MPASS(vmap != NULL); + MPASS(vmap->vm_private_data == vm_obj->handle); + + VM_OBJECT_WUNLOCK(vm_obj); + + linux_set_current(curthread); + + down_write(&vmap->vm_mm->mmap_sem); + if (unlikely(vmap->vm_ops == NULL)) { + err = VM_FAULT_SIGBUS; + } else { + struct vm_fault vmf; + + /* fill out VM fault structure */ + vmf.virtual_address = (void *)(uintptr_t)IDX_TO_OFF(pidx); + vmf.flags = (fault_type & VM_PROT_WRITE) ? FAULT_FLAG_WRITE : 0; + vmf.pgoff = 0; + vmf.page = NULL; + vmf.vma = vmap; + + vmap->vm_pfn_count = 0; + vmap->vm_pfn_pcount = &vmap->vm_pfn_count; + vmap->vm_obj = vm_obj; + + err = vmap->vm_ops->fault(vmap, &vmf); + + while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE) { + kern_yield(PRI_USER); + err = vmap->vm_ops->fault(vmap, &vmf); + } + } + + /* translate return code */ + switch (err) { + case VM_FAULT_OOM: + err = VM_PAGER_AGAIN; + break; + case VM_FAULT_SIGBUS: + err = VM_PAGER_BAD; + break; + case VM_FAULT_NOPAGE: + /* + * By contract the fault handler will return having + * busied all the pages itself. If pidx is already + * found in the object, it will simply xbusy the first + * page and return with vm_pfn_count set to 1. + */ + *first = vmap->vm_pfn_first; + *last = *first + vmap->vm_pfn_count - 1; + err = VM_PAGER_OK; + break; + default: + err = VM_PAGER_ERROR; + break; + } + up_write(&vmap->vm_mm->mmap_sem); + VM_OBJECT_WLOCK(vm_obj); + return (err); +} + +static struct rwlock linux_vma_lock; +static TAILQ_HEAD(, vm_area_struct) linux_vma_head = + TAILQ_HEAD_INITIALIZER(linux_vma_head); + +static void +linux_cdev_handle_free(struct vm_area_struct *vmap) +{ + /* Drop reference on vm_file */ + if (vmap->vm_file != NULL) + fput(vmap->vm_file); + + /* Drop reference on mm_struct */ + mmput(vmap->vm_mm); + + kfree(vmap); +} + +static void +linux_cdev_handle_remove(struct vm_area_struct *vmap) +{ + rw_wlock(&linux_vma_lock); + TAILQ_REMOVE(&linux_vma_head, vmap, vm_entry); + rw_wunlock(&linux_vma_lock); +} + +static struct vm_area_struct * +linux_cdev_handle_find(void *handle) +{ + struct vm_area_struct *vmap; + + rw_rlock(&linux_vma_lock); + TAILQ_FOREACH(vmap, &linux_vma_head, vm_entry) { + if (vmap->vm_private_data == handle) + break; + } + rw_runlock(&linux_vma_lock); + return (vmap); +} + +static int +linux_cdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, + vm_ooffset_t foff, struct ucred *cred, u_short *color) +{ + + MPASS(linux_cdev_handle_find(handle) != NULL); + *color = 0; + return (0); +} + +static void +linux_cdev_pager_dtor(void *handle) +{ + const struct vm_operations_struct *vm_ops; + struct vm_area_struct *vmap; + + vmap = linux_cdev_handle_find(handle); + MPASS(vmap != NULL); + + /* + * Remove handle before calling close operation to prevent + * other threads from reusing the handle pointer. + */ + linux_cdev_handle_remove(vmap); + + down_write(&vmap->vm_mm->mmap_sem); + vm_ops = vmap->vm_ops; + if (likely(vm_ops != NULL)) + vm_ops->close(vmap); + up_write(&vmap->vm_mm->mmap_sem); + + linux_cdev_handle_free(vmap); +} + +static struct cdev_pager_ops linux_cdev_pager_ops[2] = { + { + /* OBJT_MGTDEVICE */ + .cdev_pg_populate = linux_cdev_pager_populate, + .cdev_pg_ctor = linux_cdev_pager_ctor, + .cdev_pg_dtor = linux_cdev_pager_dtor + }, + { + /* OBJT_DEVICE */ + .cdev_pg_fault = linux_cdev_pager_fault, + .cdev_pg_ctor = linux_cdev_pager_ctor, + .cdev_pg_dtor = linux_cdev_pager_dtor + }, +}; + +int +zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, + unsigned long size) +{ + vm_object_t obj; + vm_page_t m; + + obj = vma->vm_obj; + if (obj == NULL || (obj->flags & OBJ_UNMANAGED) != 0) + return (-ENOTSUP); + VM_OBJECT_RLOCK(obj); + for (m = vm_page_find_least(obj, OFF_TO_IDX(address)); + m != NULL && m->pindex < OFF_TO_IDX(address + size); + m = TAILQ_NEXT(m, listq)) + pmap_remove_all(m); + VM_OBJECT_RUNLOCK(obj); + return (0); +} + +static struct file_operations dummy_ldev_ops = { + /* XXXKIB */ +}; + +static struct linux_cdev dummy_ldev = { + .ops = &dummy_ldev_ops, +}; + +#define LDEV_SI_DTR 0x0001 +#define LDEV_SI_REF 0x0002 + +static void +linux_get_fop(struct linux_file *filp, const struct file_operations **fop, + struct linux_cdev **dev) +{ + struct linux_cdev *ldev; + u_int siref; + + ldev = filp->f_cdev; + *fop = filp->f_op; + if (ldev != NULL) { + for (siref = ldev->siref;;) { + if ((siref & LDEV_SI_DTR) != 0) { + ldev = &dummy_ldev; + siref = ldev->siref; + *fop = ldev->ops; + MPASS((ldev->siref & LDEV_SI_DTR) == 0); + } else if (atomic_fcmpset_int(&ldev->siref, &siref, + siref + LDEV_SI_REF)) { + break; + } + } + } + *dev = ldev; +} + +static void +linux_drop_fop(struct linux_cdev *ldev) +{ + + if (ldev == NULL) + return; + MPASS((ldev->siref & ~LDEV_SI_DTR) != 0); + atomic_subtract_int(&ldev->siref, LDEV_SI_REF); +} + +#define OPW(fp,td,code) ({ \ + struct file *__fpop; \ + __typeof(code) __retval; \ + \ + __fpop = (td)->td_fpop; \ + (td)->td_fpop = (fp); \ + __retval = (code); \ + (td)->td_fpop = __fpop; \ + __retval; \ +}) + +static int +linux_dev_fdopen(struct cdev *dev, int fflags, struct thread *td, + struct file *file) +{ + struct linux_cdev *ldev; + struct linux_file *filp; + const struct file_operations *fop; + int error; + + ldev = dev->si_drv1; + + filp = linux_file_alloc(); + filp->f_dentry = &filp->f_dentry_store; + filp->f_op = ldev->ops; + filp->f_mode = file->f_flag; + filp->f_flags = file->f_flag; + filp->f_vnode = file->f_vnode; + filp->_file = file; + refcount_acquire(&ldev->refs); + filp->f_cdev = ldev; + + linux_set_current(td); + linux_get_fop(filp, &fop, &ldev); + + if (fop->open != NULL) { + error = -fop->open(file->f_vnode, filp); + if (error != 0) { + linux_drop_fop(ldev); + linux_cdev_deref(filp->f_cdev); + kfree(filp); + return (error); + } + } + + /* hold on to the vnode - used for fstat() */ + vhold(filp->f_vnode); + + /* release the file from devfs */ + finit(file, filp->f_mode, DTYPE_DEV, filp, &linuxfileops); + linux_drop_fop(ldev); + return (ENXIO); +} + +#define LINUX_IOCTL_MIN_PTR 0x10000UL +#define LINUX_IOCTL_MAX_PTR (LINUX_IOCTL_MIN_PTR + IOCPARM_MAX) + +static inline int +linux_remap_address(void **uaddr, size_t len) +{ + uintptr_t uaddr_val = (uintptr_t)(*uaddr); + + if (unlikely(uaddr_val >= LINUX_IOCTL_MIN_PTR && + uaddr_val < LINUX_IOCTL_MAX_PTR)) { + struct task_struct *pts = current; + if (pts == NULL) { + *uaddr = NULL; + return (1); + } + + /* compute data offset */ + uaddr_val -= LINUX_IOCTL_MIN_PTR; + + /* check that length is within bounds */ + if ((len > IOCPARM_MAX) || + (uaddr_val + len) > pts->bsd_ioctl_len) { + *uaddr = NULL; + return (1); + } + + /* re-add kernel buffer address */ + uaddr_val += (uintptr_t)pts->bsd_ioctl_data; + + /* update address location */ + *uaddr = (void *)uaddr_val; + return (1); + } + return (0); +} + +int +linux_copyin(const void *uaddr, void *kaddr, size_t len) +{ + if (linux_remap_address(__DECONST(void **, &uaddr), len)) { + if (uaddr == NULL) + return (-EFAULT); + memcpy(kaddr, uaddr, len); + return (0); + } + return (-copyin(uaddr, kaddr, len)); +} + +int +linux_copyout(const void *kaddr, void *uaddr, size_t len) +{ + if (linux_remap_address(&uaddr, len)) { + if (uaddr == NULL) + return (-EFAULT); + memcpy(uaddr, kaddr, len); + return (0); + } + return (-copyout(kaddr, uaddr, len)); +} + +size_t +linux_clear_user(void *_uaddr, size_t _len) +{ + uint8_t *uaddr = _uaddr; + size_t len = _len; + + /* make sure uaddr is aligned before going into the fast loop */ + while (((uintptr_t)uaddr & 7) != 0 && len > 7) { + if (subyte(uaddr, 0)) + return (_len); + uaddr++; + len--; + } + + /* zero 8 bytes at a time */ + while (len > 7) { +#ifdef __LP64__ + if (suword64(uaddr, 0)) + return (_len); +#else + if (suword32(uaddr, 0)) + return (_len); + if (suword32(uaddr + 4, 0)) + return (_len); +#endif + uaddr += 8; + len -= 8; + } + + /* zero fill end, if any */ + while (len > 0) { + if (subyte(uaddr, 0)) + return (_len); + uaddr++; + len--; + } + return (0); +} + +int +linux_access_ok(const void *uaddr, size_t len) +{ + uintptr_t saddr; + uintptr_t eaddr; + + /* get start and end address */ + saddr = (uintptr_t)uaddr; + eaddr = (uintptr_t)uaddr + len; + + /* verify addresses are valid for userspace */ + return ((saddr == eaddr) || + (eaddr > saddr && eaddr <= VM_MAXUSER_ADDRESS)); +} + +/* + * This function should return either EINTR or ERESTART depending on + * the signal type sent to this thread: + */ +static int +linux_get_error(struct task_struct *task, int error) +{ + /* check for signal type interrupt code */ + if (error == EINTR || error == ERESTARTSYS || error == ERESTART) { + error = -linux_schedule_get_interrupt_value(task); + if (error == 0) + error = EINTR; + } + return (error); +} + +static int +linux_file_ioctl_sub(struct file *fp, struct linux_file *filp, + const struct file_operations *fop, u_long cmd, caddr_t data, + struct thread *td) +{ + struct task_struct *task = current; + unsigned size; + int error; + + size = IOCPARM_LEN(cmd); + /* refer to logic in sys_ioctl() */ + if (size > 0) { + /* + * Setup hint for linux_copyin() and linux_copyout(). + * + * Background: Linux code expects a user-space address + * while FreeBSD supplies a kernel-space address. + */ + task->bsd_ioctl_data = data; + task->bsd_ioctl_len = size; + data = (void *)LINUX_IOCTL_MIN_PTR; + } else { + /* fetch user-space pointer */ + data = *(void **)data; + } +#if defined(__amd64__) + if (td->td_proc->p_elf_machine == EM_386) { + /* try the compat IOCTL handler first */ + if (fop->compat_ioctl != NULL) { + error = -OPW(fp, td, fop->compat_ioctl(filp, + cmd, (u_long)data)); + } else { + error = ENOTTY; + } + + /* fallback to the regular IOCTL handler, if any */ + if (error == ENOTTY && fop->unlocked_ioctl != NULL) { + error = -OPW(fp, td, fop->unlocked_ioctl(filp, + cmd, (u_long)data)); + } + } else +#endif + { + if (fop->unlocked_ioctl != NULL) { + error = -OPW(fp, td, fop->unlocked_ioctl(filp, + cmd, (u_long)data)); + } else { + error = ENOTTY; + } + } + if (size > 0) { + task->bsd_ioctl_data = NULL; + task->bsd_ioctl_len = 0; + } + + if (error == EWOULDBLOCK) { + /* update kqfilter status, if any */ + linux_file_kqfilter_poll(filp, + LINUX_KQ_FLAG_HAS_READ | LINUX_KQ_FLAG_HAS_WRITE); + } else { + error = linux_get_error(task, error); + } + return (error); +} + +#define LINUX_POLL_TABLE_NORMAL ((poll_table *)1) + +/* + * This function atomically updates the poll wakeup state and returns + * the previous state at the time of update. + */ +static uint8_t +linux_poll_wakeup_state(atomic_t *v, const uint8_t *pstate) +{ + int c, old; + + c = v->counter; + + while ((old = atomic_cmpxchg(v, c, pstate[c])) != c) + c = old; + + return (c); +} + + +static int +linux_poll_wakeup_callback(wait_queue_t *wq, unsigned int wq_state, int flags, void *key) +{ + static const uint8_t state[LINUX_FWQ_STATE_MAX] = { + [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_INIT, /* NOP */ + [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_NOT_READY, /* NOP */ + [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_READY, + [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_READY, /* NOP */ + }; + struct linux_file *filp = container_of(wq, struct linux_file, f_wait_queue.wq); + + switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) { + case LINUX_FWQ_STATE_QUEUED: + linux_poll_wakeup(filp); + return (1); + default: + return (0); + } +} + +void +linux_poll_wait(struct linux_file *filp, wait_queue_head_t *wqh, poll_table *p) +{ + static const uint8_t state[LINUX_FWQ_STATE_MAX] = { + [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_NOT_READY, + [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_NOT_READY, /* NOP */ + [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_QUEUED, /* NOP */ + [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_QUEUED, + }; + + /* check if we are called inside the select system call */ + if (p == LINUX_POLL_TABLE_NORMAL) + selrecord(curthread, &filp->f_selinfo); + + switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) { + case LINUX_FWQ_STATE_INIT: + /* NOTE: file handles can only belong to one wait-queue */ + filp->f_wait_queue.wqh = wqh; + filp->f_wait_queue.wq.func = &linux_poll_wakeup_callback; + add_wait_queue(wqh, &filp->f_wait_queue.wq); + atomic_set(&filp->f_wait_queue.state, LINUX_FWQ_STATE_QUEUED); + break; + default: + break; + } +} + +static void +linux_poll_wait_dequeue(struct linux_file *filp) +{ + static const uint8_t state[LINUX_FWQ_STATE_MAX] = { + [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_INIT, /* NOP */ + [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_INIT, + [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_INIT, + [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_INIT, + }; + + seldrain(&filp->f_selinfo); + + switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) { + case LINUX_FWQ_STATE_NOT_READY: + case LINUX_FWQ_STATE_QUEUED: + case LINUX_FWQ_STATE_READY: + remove_wait_queue(filp->f_wait_queue.wqh, &filp->f_wait_queue.wq); + break; + default: + break; + } +} + +void +linux_poll_wakeup(struct linux_file *filp) +{ + /* this function should be NULL-safe */ + if (filp == NULL) + return; + + selwakeup(&filp->f_selinfo); + + spin_lock(&filp->f_kqlock); + filp->f_kqflags |= LINUX_KQ_FLAG_NEED_READ | + LINUX_KQ_FLAG_NEED_WRITE; + + /* make sure the "knote" gets woken up */ + KNOTE_LOCKED(&filp->f_selinfo.si_note, 1); + spin_unlock(&filp->f_kqlock); +} + +static void +linux_file_kqfilter_detach(struct knote *kn) +{ + struct linux_file *filp = kn->kn_hook; + + spin_lock(&filp->f_kqlock); + knlist_remove(&filp->f_selinfo.si_note, kn, 1); + spin_unlock(&filp->f_kqlock); +} + +static int +linux_file_kqfilter_read_event(struct knote *kn, long hint) +{ + struct linux_file *filp = kn->kn_hook; + + mtx_assert(&filp->f_kqlock.m, MA_OWNED); + + return ((filp->f_kqflags & LINUX_KQ_FLAG_NEED_READ) ? 1 : 0); +} + +static int +linux_file_kqfilter_write_event(struct knote *kn, long hint) +{ + struct linux_file *filp = kn->kn_hook; + + mtx_assert(&filp->f_kqlock.m, MA_OWNED); + + return ((filp->f_kqflags & LINUX_KQ_FLAG_NEED_WRITE) ? 1 : 0); +} + +static struct filterops linux_dev_kqfiltops_read = { + .f_isfd = 1, + .f_detach = linux_file_kqfilter_detach, + .f_event = linux_file_kqfilter_read_event, +}; + +static struct filterops linux_dev_kqfiltops_write = { + .f_isfd = 1, + .f_detach = linux_file_kqfilter_detach, + .f_event = linux_file_kqfilter_write_event, +}; + +static void +linux_file_kqfilter_poll(struct linux_file *filp, int kqflags) +{ + struct thread *td; + const struct file_operations *fop; + struct linux_cdev *ldev; + int temp; + + if ((filp->f_kqflags & kqflags) == 0) + return; + + td = curthread; + + linux_get_fop(filp, &fop, &ldev); + /* get the latest polling state */ + temp = OPW(filp->_file, td, fop->poll(filp, NULL)); + linux_drop_fop(ldev); + + spin_lock(&filp->f_kqlock); + /* clear kqflags */ + filp->f_kqflags &= ~(LINUX_KQ_FLAG_NEED_READ | + LINUX_KQ_FLAG_NEED_WRITE); + /* update kqflags */ + if ((temp & (POLLIN | POLLOUT)) != 0) { + if ((temp & POLLIN) != 0) + filp->f_kqflags |= LINUX_KQ_FLAG_NEED_READ; + if ((temp & POLLOUT) != 0) + filp->f_kqflags |= LINUX_KQ_FLAG_NEED_WRITE; + + /* make sure the "knote" gets woken up */ + KNOTE_LOCKED(&filp->f_selinfo.si_note, 0); + } + spin_unlock(&filp->f_kqlock); +} + +static int +linux_file_kqfilter(struct file *file, struct knote *kn) +{ + struct linux_file *filp; + struct thread *td; + int error; + + td = curthread; + filp = (struct linux_file *)file->f_data; + filp->f_flags = file->f_flag; + if (filp->f_op->poll == NULL) + return (EINVAL); + + spin_lock(&filp->f_kqlock); + switch (kn->kn_filter) { + case EVFILT_READ: + filp->f_kqflags |= LINUX_KQ_FLAG_HAS_READ; + kn->kn_fop = &linux_dev_kqfiltops_read; + kn->kn_hook = filp; + knlist_add(&filp->f_selinfo.si_note, kn, 1); + error = 0; + break; + case EVFILT_WRITE: + filp->f_kqflags |= LINUX_KQ_FLAG_HAS_WRITE; + kn->kn_fop = &linux_dev_kqfiltops_write; + kn->kn_hook = filp; + knlist_add(&filp->f_selinfo.si_note, kn, 1); + error = 0; + break; + default: + error = EINVAL; + break; + } + spin_unlock(&filp->f_kqlock); + + if (error == 0) { + linux_set_current(td); + + /* update kqfilter status, if any */ + linux_file_kqfilter_poll(filp, + LINUX_KQ_FLAG_HAS_READ | LINUX_KQ_FLAG_HAS_WRITE); + } + return (error); +} + +static int +linux_file_mmap_single(struct file *fp, const struct file_operations *fop, + vm_ooffset_t *offset, vm_size_t size, struct vm_object **object, + int nprot, struct thread *td) +{ + struct task_struct *task; + struct vm_area_struct *vmap; + struct mm_struct *mm; + struct linux_file *filp; + vm_memattr_t attr; + int error; + + filp = (struct linux_file *)fp->f_data; + filp->f_flags = fp->f_flag; + + if (fop->mmap == NULL) + return (EOPNOTSUPP); + + linux_set_current(td); + + /* + * The same VM object might be shared by multiple processes + * and the mm_struct is usually freed when a process exits. + * + * The atomic reference below makes sure the mm_struct is + * available as long as the vmap is in the linux_vma_head. + */ + task = current; + mm = task->mm; + if (atomic_inc_not_zero(&mm->mm_users) == 0) + return (EINVAL); + + vmap = kzalloc(sizeof(*vmap), GFP_KERNEL); + vmap->vm_start = 0; + vmap->vm_end = size; + vmap->vm_pgoff = *offset / PAGE_SIZE; + vmap->vm_pfn = 0; + vmap->vm_flags = vmap->vm_page_prot = (nprot & VM_PROT_ALL); + vmap->vm_ops = NULL; + vmap->vm_file = get_file(filp); + vmap->vm_mm = mm; + + if (unlikely(down_write_killable(&vmap->vm_mm->mmap_sem))) { + error = linux_get_error(task, EINTR); + } else { + error = -OPW(fp, td, fop->mmap(filp, vmap)); + error = linux_get_error(task, error); + up_write(&vmap->vm_mm->mmap_sem); + } + + if (error != 0) { + linux_cdev_handle_free(vmap); + return (error); + } + + attr = pgprot2cachemode(vmap->vm_page_prot); + + if (vmap->vm_ops != NULL) { + struct vm_area_struct *ptr; + void *vm_private_data; + bool vm_no_fault; + + if (vmap->vm_ops->open == NULL || + vmap->vm_ops->close == NULL || + vmap->vm_private_data == NULL) { + /* free allocated VM area struct */ + linux_cdev_handle_free(vmap); + return (EINVAL); + } + + vm_private_data = vmap->vm_private_data; + + rw_wlock(&linux_vma_lock); + TAILQ_FOREACH(ptr, &linux_vma_head, vm_entry) { + if (ptr->vm_private_data == vm_private_data) + break; + } + /* check if there is an existing VM area struct */ + if (ptr != NULL) { + /* check if the VM area structure is invalid */ + if (ptr->vm_ops == NULL || + ptr->vm_ops->open == NULL || + ptr->vm_ops->close == NULL) { + error = ESTALE; + vm_no_fault = 1; + } else { + error = EEXIST; + vm_no_fault = (ptr->vm_ops->fault == NULL); + } + } else { + /* insert VM area structure into list */ + TAILQ_INSERT_TAIL(&linux_vma_head, vmap, vm_entry); + error = 0; + vm_no_fault = (vmap->vm_ops->fault == NULL); + } + rw_wunlock(&linux_vma_lock); + + if (error != 0) { + /* free allocated VM area struct */ + linux_cdev_handle_free(vmap); + /* check for stale VM area struct */ + if (error != EEXIST) + return (error); + } + + /* check if there is no fault handler */ + if (vm_no_fault) { + *object = cdev_pager_allocate(vm_private_data, OBJT_DEVICE, + &linux_cdev_pager_ops[1], size, nprot, *offset, + td->td_ucred); + } else { + *object = cdev_pager_allocate(vm_private_data, OBJT_MGTDEVICE, + &linux_cdev_pager_ops[0], size, nprot, *offset, + td->td_ucred); + } + + /* check if allocating the VM object failed */ + if (*object == NULL) { + if (error == 0) { + /* remove VM area struct from list */ + linux_cdev_handle_remove(vmap); + /* free allocated VM area struct */ + linux_cdev_handle_free(vmap); + } + return (EINVAL); + } + } else { + struct sglist *sg; + + sg = sglist_alloc(1, M_WAITOK); + sglist_append_phys(sg, + (vm_paddr_t)vmap->vm_pfn << PAGE_SHIFT, vmap->vm_len); + + *object = vm_pager_allocate(OBJT_SG, sg, vmap->vm_len, + nprot, 0, td->td_ucred); + + linux_cdev_handle_free(vmap); + + if (*object == NULL) { + sglist_free(sg); + return (EINVAL); + } + } + + if (attr != VM_MEMATTR_DEFAULT) { + VM_OBJECT_WLOCK(*object); + vm_object_set_memattr(*object, attr); + VM_OBJECT_WUNLOCK(*object); + } + *offset = 0; + return (0); +} + +struct cdevsw linuxcdevsw = { + .d_version = D_VERSION, + .d_fdopen = linux_dev_fdopen, + .d_name = "lkpidev", +}; + +static int +linux_file_read(struct file *file, struct uio *uio, struct ucred *active_cred, + int flags, struct thread *td) +{ + struct linux_file *filp; + const struct file_operations *fop; + struct linux_cdev *ldev; + ssize_t bytes; + int error; + + error = 0; + filp = (struct linux_file *)file->f_data; + filp->f_flags = file->f_flag; + /* XXX no support for I/O vectors currently */ + if (uio->uio_iovcnt != 1) + return (EOPNOTSUPP); + if (uio->uio_resid > DEVFS_IOSIZE_MAX) + return (EINVAL); + linux_set_current(td); + linux_get_fop(filp, &fop, &ldev); + if (fop->read != NULL) { + bytes = OPW(file, td, fop->read(filp, + uio->uio_iov->iov_base, + uio->uio_iov->iov_len, &uio->uio_offset)); + if (bytes >= 0) { + uio->uio_iov->iov_base = + ((uint8_t *)uio->uio_iov->iov_base) + bytes; + uio->uio_iov->iov_len -= bytes; + uio->uio_resid -= bytes; + } else { + error = linux_get_error(current, -bytes); + } + } else + error = ENXIO; + + /* update kqfilter status, if any */ + linux_file_kqfilter_poll(filp, LINUX_KQ_FLAG_HAS_READ); + linux_drop_fop(ldev); + + return (error); +} + +static int +linux_file_write(struct file *file, struct uio *uio, struct ucred *active_cred, + int flags, struct thread *td) +{ + struct linux_file *filp; + const struct file_operations *fop; + struct linux_cdev *ldev; + ssize_t bytes; + int error; + + filp = (struct linux_file *)file->f_data; + filp->f_flags = file->f_flag; + /* XXX no support for I/O vectors currently */ + if (uio->uio_iovcnt != 1) + return (EOPNOTSUPP); + if (uio->uio_resid > DEVFS_IOSIZE_MAX) + return (EINVAL); + linux_set_current(td); + linux_get_fop(filp, &fop, &ldev); + if (fop->write != NULL) { + bytes = OPW(file, td, fop->write(filp, + uio->uio_iov->iov_base, + uio->uio_iov->iov_len, &uio->uio_offset)); + if (bytes >= 0) { + uio->uio_iov->iov_base = + ((uint8_t *)uio->uio_iov->iov_base) + bytes; + uio->uio_iov->iov_len -= bytes; + uio->uio_resid -= bytes; + error = 0; + } else { + error = linux_get_error(current, -bytes); + } + } else + error = ENXIO; + + /* update kqfilter status, if any */ + linux_file_kqfilter_poll(filp, LINUX_KQ_FLAG_HAS_WRITE); + + linux_drop_fop(ldev); + + return (error); +} + +static int +linux_file_poll(struct file *file, int events, struct ucred *active_cred, + struct thread *td) +{ + struct linux_file *filp; + const struct file_operations *fop; + struct linux_cdev *ldev; + int revents; + + filp = (struct linux_file *)file->f_data; + filp->f_flags = file->f_flag; + linux_set_current(td); + linux_get_fop(filp, &fop, &ldev); + if (fop->poll != NULL) { + revents = OPW(file, td, fop->poll(filp, + LINUX_POLL_TABLE_NORMAL)) & events; + } else { + revents = 0; + } + linux_drop_fop(ldev); + return (revents); +} + +static int +linux_file_close(struct file *file, struct thread *td) +{ + struct linux_file *filp; + int (*release)(struct inode *, struct linux_file *); + const struct file_operations *fop; + struct linux_cdev *ldev; + int error; + + filp = (struct linux_file *)file->f_data; + + KASSERT(file_count(filp) == 0, + ("File refcount(%d) is not zero", file_count(filp))); + + if (td == NULL) + td = curthread; + + error = 0; + filp->f_flags = file->f_flag; + linux_set_current(td); + linux_poll_wait_dequeue(filp); + linux_get_fop(filp, &fop, &ldev); + /* + * Always use the real release function, if any, to avoid + * leaking device resources: + */ + release = filp->f_op->release; + if (release != NULL) + error = -OPW(file, td, release(filp->f_vnode, filp)); + funsetown(&filp->f_sigio); + if (filp->f_vnode != NULL) + vdrop(filp->f_vnode); + linux_drop_fop(ldev); + if (filp->f_cdev != NULL) + linux_cdev_deref(filp->f_cdev); + kfree(filp); + + return (error); +} + +static int +linux_file_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *cred, + struct thread *td) +{ + struct linux_file *filp; + const struct file_operations *fop; + struct linux_cdev *ldev; + struct fiodgname_arg *fgn; + const char *p; + int error, i; + + error = 0; + filp = (struct linux_file *)fp->f_data; + filp->f_flags = fp->f_flag; + linux_get_fop(filp, &fop, &ldev); + + linux_set_current(td); + switch (cmd) { + case FIONBIO: + break; + case FIOASYNC: + if (fop->fasync == NULL) + break; + error = -OPW(fp, td, fop->fasync(0, filp, fp->f_flag & FASYNC)); + break; + case FIOSETOWN: + error = fsetown(*(int *)data, &filp->f_sigio); + if (error == 0) { + if (fop->fasync == NULL) + break; + error = -OPW(fp, td, fop->fasync(0, filp, + fp->f_flag & FASYNC)); + } + break; + case FIOGETOWN: + *(int *)data = fgetown(&filp->f_sigio); + break; + case FIODGNAME: +#ifdef COMPAT_FREEBSD32 + case FIODGNAME_32: +#endif + if (filp->f_cdev == NULL || filp->f_cdev->cdev == NULL) { + error = ENXIO; + break; + } + fgn = data; + p = devtoname(filp->f_cdev->cdev); + i = strlen(p) + 1; + if (i > fgn->len) { + error = EINVAL; + break; + } + error = copyout(p, fiodgname_buf_get_ptr(fgn, cmd), i); + break; + default: + error = linux_file_ioctl_sub(fp, filp, fop, cmd, data, td); + break; + } + linux_drop_fop(ldev); + return (error); +} + +static int +linux_file_mmap_sub(struct thread *td, vm_size_t objsize, vm_prot_t prot, + vm_prot_t *maxprotp, int *flagsp, struct file *fp, + vm_ooffset_t *foff, const struct file_operations *fop, vm_object_t *objp) +{ + /* + * Character devices do not provide private mappings + * of any kind: + */ + if ((*maxprotp & VM_PROT_WRITE) == 0 && + (prot & VM_PROT_WRITE) != 0) + return (EACCES); + if ((*flagsp & (MAP_PRIVATE | MAP_COPY)) != 0) + return (EINVAL); + + return (linux_file_mmap_single(fp, fop, foff, objsize, objp, + (int)prot, td)); +} + +static int +linux_file_mmap(struct file *fp, vm_map_t map, vm_offset_t *addr, vm_size_t size, + vm_prot_t prot, vm_prot_t cap_maxprot, int flags, vm_ooffset_t foff, + struct thread *td) +{ + struct linux_file *filp; + const struct file_operations *fop; + struct linux_cdev *ldev; + struct mount *mp; + struct vnode *vp; + vm_object_t object; + vm_prot_t maxprot; + int error; + + filp = (struct linux_file *)fp->f_data; + + vp = filp->f_vnode; + if (vp == NULL) + return (EOPNOTSUPP); + + /* + * Ensure that file and memory protections are + * compatible. + */ + mp = vp->v_mount; + if (mp != NULL && (mp->mnt_flag & MNT_NOEXEC) != 0) { + maxprot = VM_PROT_NONE; + if ((prot & VM_PROT_EXECUTE) != 0) + return (EACCES); + } else + maxprot = VM_PROT_EXECUTE; + if ((fp->f_flag & FREAD) != 0) + maxprot |= VM_PROT_READ; + else if ((prot & VM_PROT_READ) != 0) + return (EACCES); + + /* + * If we are sharing potential changes via MAP_SHARED and we + * are trying to get write permission although we opened it + * without asking for it, bail out. + * + * Note that most character devices always share mappings. + * + * Rely on linux_file_mmap_sub() to fail invalid MAP_PRIVATE + * requests rather than doing it here. + */ + if ((flags & MAP_SHARED) != 0) { + if ((fp->f_flag & FWRITE) != 0) + maxprot |= VM_PROT_WRITE; + else if ((prot & VM_PROT_WRITE) != 0) + return (EACCES); + } + maxprot &= cap_maxprot; + + linux_get_fop(filp, &fop, &ldev); + error = linux_file_mmap_sub(td, size, prot, &maxprot, &flags, fp, + &foff, fop, &object); + if (error != 0) + goto out; + + error = vm_mmap_object(map, addr, size, prot, maxprot, flags, object, + foff, FALSE, td); + if (error != 0) + vm_object_deallocate(object); +out: + linux_drop_fop(ldev); + return (error); +} + +static int +linux_file_stat(struct file *fp, struct stat *sb, struct ucred *active_cred, + struct thread *td) +{ + struct linux_file *filp; + struct vnode *vp; + int error; + + filp = (struct linux_file *)fp->f_data; + if (filp->f_vnode == NULL) + return (EOPNOTSUPP); + + vp = filp->f_vnode; + + vn_lock(vp, LK_SHARED | LK_RETRY); + error = VOP_STAT(vp, sb, td->td_ucred, NOCRED, td); + VOP_UNLOCK(vp); + + return (error); +} + +static int +linux_file_fill_kinfo(struct file *fp, struct kinfo_file *kif, + struct filedesc *fdp) +{ + struct linux_file *filp; + struct vnode *vp; + int error; + + filp = fp->f_data; + vp = filp->f_vnode; + if (vp == NULL) { + error = 0; + kif->kf_type = KF_TYPE_DEV; + } else { + vref(vp); + FILEDESC_SUNLOCK(fdp); + error = vn_fill_kinfo_vnode(vp, kif); + vrele(vp); + kif->kf_type = KF_TYPE_VNODE; + FILEDESC_SLOCK(fdp); + } + return (error); +} + +unsigned int +linux_iminor(struct inode *inode) +{ + struct linux_cdev *ldev; + + if (inode == NULL || inode->v_rdev == NULL || + inode->v_rdev->si_devsw != &linuxcdevsw) + return (-1U); + ldev = inode->v_rdev->si_drv1; + if (ldev == NULL) + return (-1U); + + return (minor(ldev->dev)); +} + +struct fileops linuxfileops = { + .fo_read = linux_file_read, + .fo_write = linux_file_write, + .fo_truncate = invfo_truncate, + .fo_kqfilter = linux_file_kqfilter, + .fo_stat = linux_file_stat, + .fo_fill_kinfo = linux_file_fill_kinfo, + .fo_poll = linux_file_poll, + .fo_close = linux_file_close, + .fo_ioctl = linux_file_ioctl, + .fo_mmap = linux_file_mmap, + .fo_chmod = invfo_chmod, + .fo_chown = invfo_chown, + .fo_sendfile = invfo_sendfile, + .fo_flags = DFLAG_PASSABLE, +}; + +/* + * Hash of vmmap addresses. This is infrequently accessed and does not + * need to be particularly large. This is done because we must store the + * caller's idea of the map size to properly unmap. + */ +struct vmmap { + LIST_ENTRY(vmmap) vm_next; + void *vm_addr; + unsigned long vm_size; +}; + +struct vmmaphd { + struct vmmap *lh_first; +}; +#define VMMAP_HASH_SIZE 64 +#define VMMAP_HASH_MASK (VMMAP_HASH_SIZE - 1) +#define VM_HASH(addr) ((uintptr_t)(addr) >> PAGE_SHIFT) & VMMAP_HASH_MASK +static struct vmmaphd vmmaphead[VMMAP_HASH_SIZE]; +static struct mtx vmmaplock; + +static void +vmmap_add(void *addr, unsigned long size) +{ + struct vmmap *vmmap; + + vmmap = kmalloc(sizeof(*vmmap), GFP_KERNEL); + mtx_lock(&vmmaplock); + vmmap->vm_size = size; + vmmap->vm_addr = addr; + LIST_INSERT_HEAD(&vmmaphead[VM_HASH(addr)], vmmap, vm_next); + mtx_unlock(&vmmaplock); +} + +static struct vmmap * +vmmap_remove(void *addr) +{ + struct vmmap *vmmap; + + mtx_lock(&vmmaplock); + LIST_FOREACH(vmmap, &vmmaphead[VM_HASH(addr)], vm_next) + if (vmmap->vm_addr == addr) + break; + if (vmmap) + LIST_REMOVE(vmmap, vm_next); + mtx_unlock(&vmmaplock); + + return (vmmap); +} + +#if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) || defined(__aarch64__) +void * +_ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr) +{ + void *addr; + + addr = pmap_mapdev_attr(phys_addr, size, attr); + if (addr == NULL) + return (NULL); + vmmap_add(addr, size); + + return (addr); +} +#endif + +void +iounmap(void *addr) +{ + struct vmmap *vmmap; + + vmmap = vmmap_remove(addr); + if (vmmap == NULL) + return; +#if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) || defined(__aarch64__) + pmap_unmapdev((vm_offset_t)addr, vmmap->vm_size); +#endif + kfree(vmmap); +} + + +void * +vmap(struct page **pages, unsigned int count, unsigned long flags, int prot) +{ + vm_offset_t off; + size_t size; + + size = count * PAGE_SIZE; + off = kva_alloc(size); + if (off == 0) + return (NULL); + vmmap_add((void *)off, size); + pmap_qenter(off, pages, count); + + return ((void *)off); +} + +void +vunmap(void *addr) +{ + struct vmmap *vmmap; + + vmmap = vmmap_remove(addr); + if (vmmap == NULL) + return; + pmap_qremove((vm_offset_t)addr, vmmap->vm_size / PAGE_SIZE); + kva_free((vm_offset_t)addr, vmmap->vm_size); + kfree(vmmap); +} + +char * +kvasprintf(gfp_t gfp, const char *fmt, va_list ap) +{ + unsigned int len; + char *p; + va_list aq; + + va_copy(aq, ap); + len = vsnprintf(NULL, 0, fmt, aq); + va_end(aq); + + p = kmalloc(len + 1, gfp); + if (p != NULL) + vsnprintf(p, len + 1, fmt, ap); + + return (p); +} + +char * +kasprintf(gfp_t gfp, const char *fmt, ...) +{ + va_list ap; + char *p; + + va_start(ap, fmt); + p = kvasprintf(gfp, fmt, ap); + va_end(ap); + + return (p); +} + +static void +linux_timer_callback_wrapper(void *context) +{ + struct timer_list *timer; + + linux_set_current(curthread); + + timer = context; + timer->function(timer->data); +} + +int +mod_timer(struct timer_list *timer, int expires) +{ + int ret; + + timer->expires = expires; + ret = callout_reset(&timer->callout, + linux_timer_jiffies_until(expires), + &linux_timer_callback_wrapper, timer); + + MPASS(ret == 0 || ret == 1); + + return (ret == 1); +} + +void +add_timer(struct timer_list *timer) +{ + + callout_reset(&timer->callout, + linux_timer_jiffies_until(timer->expires), + &linux_timer_callback_wrapper, timer); +} + +void +add_timer_on(struct timer_list *timer, int cpu) +{ + + callout_reset_on(&timer->callout, + linux_timer_jiffies_until(timer->expires), + &linux_timer_callback_wrapper, timer, cpu); +} + +int +del_timer(struct timer_list *timer) +{ + + if (callout_stop(&(timer)->callout) == -1) + return (0); + return (1); +} + +int +del_timer_sync(struct timer_list *timer) +{ + + if (callout_drain(&(timer)->callout) == -1) + return (0); + return (1); +} + +/* greatest common divisor, Euclid equation */ +static uint64_t +lkpi_gcd_64(uint64_t a, uint64_t b) +{ + uint64_t an; + uint64_t bn; + + while (b != 0) { + an = b; + bn = a % b; + a = an; + b = bn; + } + return (a); +} + +uint64_t lkpi_nsec2hz_rem; +uint64_t lkpi_nsec2hz_div = 1000000000ULL; +uint64_t lkpi_nsec2hz_max; + +uint64_t lkpi_usec2hz_rem; +uint64_t lkpi_usec2hz_div = 1000000ULL; +uint64_t lkpi_usec2hz_max; + +uint64_t lkpi_msec2hz_rem; +uint64_t lkpi_msec2hz_div = 1000ULL; +uint64_t lkpi_msec2hz_max; + +static void +linux_timer_init(void *arg) +{ + uint64_t gcd; + + /* + * Compute an internal HZ value which can divide 2**32 to + * avoid timer rounding problems when the tick value wraps + * around 2**32: + */ + linux_timer_hz_mask = 1; + while (linux_timer_hz_mask < (unsigned long)hz) + linux_timer_hz_mask *= 2; + linux_timer_hz_mask--; + + /* compute some internal constants */ + + lkpi_nsec2hz_rem = hz; + lkpi_usec2hz_rem = hz; + lkpi_msec2hz_rem = hz; + + gcd = lkpi_gcd_64(lkpi_nsec2hz_rem, lkpi_nsec2hz_div); + lkpi_nsec2hz_rem /= gcd; + lkpi_nsec2hz_div /= gcd; + lkpi_nsec2hz_max = -1ULL / lkpi_nsec2hz_rem; + + gcd = lkpi_gcd_64(lkpi_usec2hz_rem, lkpi_usec2hz_div); + lkpi_usec2hz_rem /= gcd; + lkpi_usec2hz_div /= gcd; + lkpi_usec2hz_max = -1ULL / lkpi_usec2hz_rem; + + gcd = lkpi_gcd_64(lkpi_msec2hz_rem, lkpi_msec2hz_div); + lkpi_msec2hz_rem /= gcd; + lkpi_msec2hz_div /= gcd; + lkpi_msec2hz_max = -1ULL / lkpi_msec2hz_rem; +} +SYSINIT(linux_timer, SI_SUB_DRIVERS, SI_ORDER_FIRST, linux_timer_init, NULL); + +void +linux_complete_common(struct completion *c, int all) +{ + int wakeup_swapper; + + sleepq_lock(c); + if (all) { + c->done = UINT_MAX; + wakeup_swapper = sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0); + } else { + if (c->done != UINT_MAX) + c->done++; + wakeup_swapper = sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); + } + sleepq_release(c); + if (wakeup_swapper) + kick_proc0(); +} + +/* + * Indefinite wait for done != 0 with or without signals. + */ +int +linux_wait_for_common(struct completion *c, int flags) +{ + struct task_struct *task; + int error; + + if (SCHEDULER_STOPPED()) + return (0); + + task = current; + + if (flags != 0) + flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; + else + flags = SLEEPQ_SLEEP; + error = 0; + for (;;) { + sleepq_lock(c); + if (c->done) + break; + sleepq_add(c, NULL, "completion", flags, 0); + if (flags & SLEEPQ_INTERRUPTIBLE) { + DROP_GIANT(); + error = -sleepq_wait_sig(c, 0); + PICKUP_GIANT(); + if (error != 0) { + linux_schedule_save_interrupt_value(task, error); + error = -ERESTARTSYS; + goto intr; + } + } else { + DROP_GIANT(); + sleepq_wait(c, 0); + PICKUP_GIANT(); + } + } + if (c->done != UINT_MAX) + c->done--; + sleepq_release(c); + +intr: + return (error); +} + +/* + * Time limited wait for done != 0 with or without signals. + */ +int +linux_wait_for_timeout_common(struct completion *c, int timeout, int flags) +{ + struct task_struct *task; + int end = jiffies + timeout; + int error; + + if (SCHEDULER_STOPPED()) + return (0); + + task = current; + + if (flags != 0) + flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; + else + flags = SLEEPQ_SLEEP; + + for (;;) { + sleepq_lock(c); + if (c->done) + break; + sleepq_add(c, NULL, "completion", flags, 0); + sleepq_set_timeout(c, linux_timer_jiffies_until(end)); + + DROP_GIANT(); + if (flags & SLEEPQ_INTERRUPTIBLE) + error = -sleepq_timedwait_sig(c, 0); + else + error = -sleepq_timedwait(c, 0); + PICKUP_GIANT(); + + if (error != 0) { + /* check for timeout */ + if (error == -EWOULDBLOCK) { + error = 0; /* timeout */ + } else { + /* signal happened */ + linux_schedule_save_interrupt_value(task, error); + error = -ERESTARTSYS; + } + goto done; + } + } + if (c->done != UINT_MAX) + c->done--; + sleepq_release(c); + + /* return how many jiffies are left */ + error = linux_timer_jiffies_until(end); +done: + return (error); +} + +int +linux_try_wait_for_completion(struct completion *c) +{ + int isdone; + + sleepq_lock(c); + isdone = (c->done != 0); + if (c->done != 0 && c->done != UINT_MAX) + c->done--; + sleepq_release(c); + return (isdone); +} + +int +linux_completion_done(struct completion *c) +{ + int isdone; + + sleepq_lock(c); + isdone = (c->done != 0); + sleepq_release(c); + return (isdone); +} + +static void +linux_cdev_deref(struct linux_cdev *ldev) +{ + + if (refcount_release(&ldev->refs)) + kfree(ldev); +} + +static void +linux_cdev_release(struct kobject *kobj) +{ + struct linux_cdev *cdev; + struct kobject *parent; + + cdev = container_of(kobj, struct linux_cdev, kobj); + parent = kobj->parent; + linux_destroy_dev(cdev); + linux_cdev_deref(cdev); + kobject_put(parent); +} + +static void +linux_cdev_static_release(struct kobject *kobj) +{ + struct linux_cdev *cdev; + struct kobject *parent; + + cdev = container_of(kobj, struct linux_cdev, kobj); + parent = kobj->parent; + linux_destroy_dev(cdev); + kobject_put(parent); +} + +void +linux_destroy_dev(struct linux_cdev *ldev) +{ + + if (ldev->cdev == NULL) + return; + + MPASS((ldev->siref & LDEV_SI_DTR) == 0); + atomic_set_int(&ldev->siref, LDEV_SI_DTR); + while ((atomic_load_int(&ldev->siref) & ~LDEV_SI_DTR) != 0) + pause("ldevdtr", hz / 4); + + destroy_dev(ldev->cdev); + ldev->cdev = NULL; +} + +const struct kobj_type linux_cdev_ktype = { + .release = linux_cdev_release, +}; + +const struct kobj_type linux_cdev_static_ktype = { + .release = linux_cdev_static_release, +}; + +static void +linux_handle_ifnet_link_event(void *arg, struct ifnet *ifp, int linkstate) +{ + struct notifier_block *nb; + + nb = arg; + if (linkstate == LINK_STATE_UP) + nb->notifier_call(nb, NETDEV_UP, ifp); + else + nb->notifier_call(nb, NETDEV_DOWN, ifp); +} + +static void +linux_handle_ifnet_arrival_event(void *arg, struct ifnet *ifp) +{ + struct notifier_block *nb; + + nb = arg; + nb->notifier_call(nb, NETDEV_REGISTER, ifp); +} + +static void +linux_handle_ifnet_departure_event(void *arg, struct ifnet *ifp) +{ + struct notifier_block *nb; + + nb = arg; + nb->notifier_call(nb, NETDEV_UNREGISTER, ifp); +} + +static void +linux_handle_iflladdr_event(void *arg, struct ifnet *ifp) +{ + struct notifier_block *nb; + + nb = arg; + nb->notifier_call(nb, NETDEV_CHANGEADDR, ifp); +} + +static void +linux_handle_ifaddr_event(void *arg, struct ifnet *ifp) +{ + struct notifier_block *nb; + + nb = arg; + nb->notifier_call(nb, NETDEV_CHANGEIFADDR, ifp); +} + +int +register_netdevice_notifier(struct notifier_block *nb) +{ + + nb->tags[NETDEV_UP] = EVENTHANDLER_REGISTER( + ifnet_link_event, linux_handle_ifnet_link_event, nb, 0); + nb->tags[NETDEV_REGISTER] = EVENTHANDLER_REGISTER( + ifnet_arrival_event, linux_handle_ifnet_arrival_event, nb, 0); + nb->tags[NETDEV_UNREGISTER] = EVENTHANDLER_REGISTER( + ifnet_departure_event, linux_handle_ifnet_departure_event, nb, 0); + nb->tags[NETDEV_CHANGEADDR] = EVENTHANDLER_REGISTER( + iflladdr_event, linux_handle_iflladdr_event, nb, 0); + + return (0); +} + +int +register_inetaddr_notifier(struct notifier_block *nb) +{ + + nb->tags[NETDEV_CHANGEIFADDR] = EVENTHANDLER_REGISTER( + ifaddr_event, linux_handle_ifaddr_event, nb, 0); + return (0); +} + +int +unregister_netdevice_notifier(struct notifier_block *nb) +{ + + EVENTHANDLER_DEREGISTER(ifnet_link_event, + nb->tags[NETDEV_UP]); + EVENTHANDLER_DEREGISTER(ifnet_arrival_event, + nb->tags[NETDEV_REGISTER]); + EVENTHANDLER_DEREGISTER(ifnet_departure_event, + nb->tags[NETDEV_UNREGISTER]); + EVENTHANDLER_DEREGISTER(iflladdr_event, + nb->tags[NETDEV_CHANGEADDR]); + + return (0); +} + +int +unregister_inetaddr_notifier(struct notifier_block *nb) +{ + + EVENTHANDLER_DEREGISTER(ifaddr_event, + nb->tags[NETDEV_CHANGEIFADDR]); + + return (0); +} + +struct list_sort_thunk { + int (*cmp)(void *, struct list_head *, struct list_head *); + void *priv; +}; + +static inline int +linux_le_cmp(void *priv, const void *d1, const void *d2) +{ + struct list_head *le1, *le2; + struct list_sort_thunk *thunk; + + thunk = priv; + le1 = *(__DECONST(struct list_head **, d1)); + le2 = *(__DECONST(struct list_head **, d2)); + return ((thunk->cmp)(thunk->priv, le1, le2)); +} + +void +list_sort(void *priv, struct list_head *head, int (*cmp)(void *priv, + struct list_head *a, struct list_head *b)) +{ + struct list_sort_thunk thunk; + struct list_head **ar, *le; + size_t count, i; + + count = 0; + list_for_each(le, head) + count++; + ar = malloc(sizeof(struct list_head *) * count, M_KMALLOC, M_WAITOK); + i = 0; + list_for_each(le, head) + ar[i++] = le; + thunk.cmp = cmp; + thunk.priv = priv; + qsort_r(ar, count, sizeof(struct list_head *), &thunk, linux_le_cmp); + INIT_LIST_HEAD(head); + for (i = 0; i < count; i++) + list_add_tail(ar[i], head); + free(ar, M_KMALLOC); +} + +void +linux_irq_handler(void *ent) +{ + struct irq_ent *irqe; + + linux_set_current(curthread); + + irqe = ent; + irqe->handler(irqe->irq, irqe->arg); +} + +#if defined(__i386__) || defined(__amd64__) +int +linux_wbinvd_on_all_cpus(void) +{ + + pmap_invalidate_cache(); + return (0); +} +#endif + +int +linux_on_each_cpu(void callback(void *), void *data) +{ + + smp_rendezvous(smp_no_rendezvous_barrier, callback, + smp_no_rendezvous_barrier, data); + return (0); +} + +int +linux_in_atomic(void) +{ + + return ((curthread->td_pflags & TDP_NOFAULTING) != 0); +} + +struct linux_cdev * +linux_find_cdev(const char *name, unsigned major, unsigned minor) +{ + dev_t dev = MKDEV(major, minor); + struct cdev *cdev; + + dev_lock(); + LIST_FOREACH(cdev, &linuxcdevsw.d_devs, si_list) { + struct linux_cdev *ldev = cdev->si_drv1; + if (ldev->dev == dev && + strcmp(kobject_name(&ldev->kobj), name) == 0) { + break; + } + } + dev_unlock(); + + return (cdev != NULL ? cdev->si_drv1 : NULL); +} + +int +__register_chrdev(unsigned int major, unsigned int baseminor, + unsigned int count, const char *name, + const struct file_operations *fops) +{ + struct linux_cdev *cdev; + int ret = 0; + int i; + + for (i = baseminor; i < baseminor + count; i++) { + cdev = cdev_alloc(); + cdev->ops = fops; + kobject_set_name(&cdev->kobj, name); + + ret = cdev_add(cdev, makedev(major, i), 1); + if (ret != 0) + break; + } + return (ret); +} + +int +__register_chrdev_p(unsigned int major, unsigned int baseminor, + unsigned int count, const char *name, + const struct file_operations *fops, uid_t uid, + gid_t gid, int mode) +{ + struct linux_cdev *cdev; + int ret = 0; + int i; + + for (i = baseminor; i < baseminor + count; i++) { + cdev = cdev_alloc(); + cdev->ops = fops; + kobject_set_name(&cdev->kobj, name); + + ret = cdev_add_ext(cdev, makedev(major, i), uid, gid, mode); + if (ret != 0) + break; + } + return (ret); +} + +void +__unregister_chrdev(unsigned int major, unsigned int baseminor, + unsigned int count, const char *name) +{ + struct linux_cdev *cdevp; + int i; + + for (i = baseminor; i < baseminor + count; i++) { + cdevp = linux_find_cdev(name, major, i); + if (cdevp != NULL) + cdev_del(cdevp); + } +} + +void +linux_dump_stack(void) +{ +#ifdef STACK + struct stack st; + + stack_zero(&st); + stack_save(&st); + stack_print(&st); +#endif +} + +#if defined(__i386__) || defined(__amd64__) +bool linux_cpu_has_clflush; +#endif + +static void +linux_compat_init(void *arg) +{ + struct sysctl_oid *rootoid; + int i; + +#if defined(__i386__) || defined(__amd64__) + linux_cpu_has_clflush = (cpu_feature & CPUID_CLFSH); +#endif + rw_init(&linux_vma_lock, "lkpi-vma-lock"); + + rootoid = SYSCTL_ADD_ROOT_NODE(NULL, + OID_AUTO, "sys", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "sys"); + kobject_init(&linux_class_root, &linux_class_ktype); + kobject_set_name(&linux_class_root, "class"); + linux_class_root.oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(rootoid), + OID_AUTO, "class", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "class"); + kobject_init(&linux_root_device.kobj, &linux_dev_ktype); + kobject_set_name(&linux_root_device.kobj, "device"); + linux_root_device.kobj.oidp = SYSCTL_ADD_NODE(NULL, + SYSCTL_CHILDREN(rootoid), OID_AUTO, "device", + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "device"); + linux_root_device.bsddev = root_bus; + linux_class_misc.name = "misc"; + class_register(&linux_class_misc); + INIT_LIST_HEAD(&pci_drivers); + INIT_LIST_HEAD(&pci_devices); + spin_lock_init(&pci_lock); + mtx_init(&vmmaplock, "IO Map lock", NULL, MTX_DEF); + for (i = 0; i < VMMAP_HASH_SIZE; i++) + LIST_INIT(&vmmaphead[i]); + init_waitqueue_head(&linux_bit_waitq); + init_waitqueue_head(&linux_var_waitq); +} +SYSINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_init, NULL); + +static void +linux_compat_uninit(void *arg) +{ + linux_kobject_kfree_name(&linux_class_root); + linux_kobject_kfree_name(&linux_root_device.kobj); + linux_kobject_kfree_name(&linux_class_misc.kobj); + + mtx_destroy(&vmmaplock); + spin_lock_destroy(&pci_lock); + rw_destroy(&linux_vma_lock); +} +SYSUNINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_uninit, NULL); + +/* + * NOTE: Linux frequently uses "unsigned long" for pointer to integer + * conversion and vice versa, where in FreeBSD "uintptr_t" would be + * used. Assert these types have the same size, else some parts of the + * LinuxKPI may not work like expected: + */ +CTASSERT(sizeof(unsigned long) == sizeof(uintptr_t)); diff --git a/sys/compat/linuxkpi/common/src/linux_current.c b/sys/compat/linuxkpi/common/src/linux_current.c new file mode 100644 index 000000000000..611d10df3383 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_current.c @@ -0,0 +1,263 @@ +/*- + * Copyright (c) 2017 Hans Petter Selasky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <linux/compat.h> +#include <linux/completion.h> +#include <linux/mm.h> +#include <linux/kthread.h> + +#include <sys/kernel.h> +#include <sys/eventhandler.h> +#include <sys/malloc.h> + +static eventhandler_tag linuxkpi_thread_dtor_tag; + +static MALLOC_DEFINE(M_LINUX_CURRENT, "linuxcurrent", "LinuxKPI task structure"); + +int +linux_alloc_current(struct thread *td, int flags) +{ + struct proc *proc; + struct thread *td_other; + struct task_struct *ts; + struct task_struct *ts_other; + struct mm_struct *mm; + struct mm_struct *mm_other; + + MPASS(td->td_lkpi_task == NULL); + + ts = malloc(sizeof(*ts), M_LINUX_CURRENT, flags | M_ZERO); + if (ts == NULL) + return (ENOMEM); + + mm = malloc(sizeof(*mm), M_LINUX_CURRENT, flags | M_ZERO); + if (mm == NULL) { + free(ts, M_LINUX_CURRENT); + return (ENOMEM); + } + + /* setup new task structure */ + atomic_set(&ts->kthread_flags, 0); + ts->task_thread = td; + ts->comm = td->td_name; + ts->pid = td->td_tid; + ts->group_leader = ts; + atomic_set(&ts->usage, 1); + atomic_set(&ts->state, TASK_RUNNING); + init_completion(&ts->parked); + init_completion(&ts->exited); + + proc = td->td_proc; + + /* check if another thread already has a mm_struct */ + PROC_LOCK(proc); + FOREACH_THREAD_IN_PROC(proc, td_other) { + ts_other = td_other->td_lkpi_task; + if (ts_other == NULL) + continue; + + mm_other = ts_other->mm; + if (mm_other == NULL) + continue; + + /* try to share other mm_struct */ + if (atomic_inc_not_zero(&mm_other->mm_users)) { + /* set mm_struct pointer */ + ts->mm = mm_other; + break; + } + } + + /* use allocated mm_struct as a fallback */ + if (ts->mm == NULL) { + /* setup new mm_struct */ + init_rwsem(&mm->mmap_sem); + atomic_set(&mm->mm_count, 1); + atomic_set(&mm->mm_users, 1); + /* set mm_struct pointer */ + ts->mm = mm; + /* clear pointer to not free memory */ + mm = NULL; + } + + /* store pointer to task struct */ + td->td_lkpi_task = ts; + PROC_UNLOCK(proc); + + /* free mm_struct pointer, if any */ + free(mm, M_LINUX_CURRENT); + + return (0); +} + +struct mm_struct * +linux_get_task_mm(struct task_struct *task) +{ + struct mm_struct *mm; + + mm = task->mm; + if (mm != NULL) { + atomic_inc(&mm->mm_users); + return (mm); + } + return (NULL); +} + +void +linux_mm_dtor(struct mm_struct *mm) +{ + free(mm, M_LINUX_CURRENT); +} + +void +linux_free_current(struct task_struct *ts) +{ + mmput(ts->mm); + free(ts, M_LINUX_CURRENT); +} + +static void +linuxkpi_thread_dtor(void *arg __unused, struct thread *td) +{ + struct task_struct *ts; + + ts = td->td_lkpi_task; + if (ts == NULL) + return; + + td->td_lkpi_task = NULL; + put_task_struct(ts); +} + +static struct task_struct * +linux_get_pid_task_int(pid_t pid, const bool do_get) +{ + struct thread *td; + struct proc *p; + struct task_struct *ts; + + if (pid > PID_MAX) { + /* try to find corresponding thread */ + td = tdfind(pid, -1); + if (td != NULL) { + ts = td->td_lkpi_task; + if (do_get && ts != NULL) + get_task_struct(ts); + PROC_UNLOCK(td->td_proc); + return (ts); + } + } else { + /* try to find corresponding procedure */ + p = pfind(pid); + if (p != NULL) { + FOREACH_THREAD_IN_PROC(p, td) { + ts = td->td_lkpi_task; + if (ts != NULL) { + if (do_get) + get_task_struct(ts); + PROC_UNLOCK(p); + return (ts); + } + } + PROC_UNLOCK(p); + } + } + return (NULL); +} + +struct task_struct * +linux_pid_task(pid_t pid) +{ + return (linux_get_pid_task_int(pid, false)); +} + +struct task_struct * +linux_get_pid_task(pid_t pid) +{ + return (linux_get_pid_task_int(pid, true)); +} + +bool +linux_task_exiting(struct task_struct *task) +{ + struct thread *td; + struct proc *p; + bool ret; + + ret = false; + + /* try to find corresponding thread */ + td = tdfind(task->pid, -1); + if (td != NULL) { + p = td->td_proc; + } else { + /* try to find corresponding procedure */ + p = pfind(task->pid); + } + + if (p != NULL) { + if ((p->p_flag & P_WEXIT) != 0) + ret = true; + PROC_UNLOCK(p); + } + return (ret); +} + +static void +linux_current_init(void *arg __unused) +{ + lkpi_alloc_current = linux_alloc_current; + linuxkpi_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, + linuxkpi_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); +} +SYSINIT(linux_current, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, linux_current_init, NULL); + +static void +linux_current_uninit(void *arg __unused) +{ + struct proc *p; + struct task_struct *ts; + struct thread *td; + + sx_slock(&allproc_lock); + FOREACH_PROC_IN_SYSTEM(p) { + PROC_LOCK(p); + FOREACH_THREAD_IN_PROC(p, td) { + if ((ts = td->td_lkpi_task) != NULL) { + td->td_lkpi_task = NULL; + put_task_struct(ts); + } + } + PROC_UNLOCK(p); + } + sx_sunlock(&allproc_lock); + EVENTHANDLER_DEREGISTER(thread_dtor, linuxkpi_thread_dtor_tag); + lkpi_alloc_current = linux_alloc_current_noop; +} +SYSUNINIT(linux_current, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, linux_current_uninit, NULL); diff --git a/sys/compat/linuxkpi/common/src/linux_hrtimer.c b/sys/compat/linuxkpi/common/src/linux_hrtimer.c new file mode 100644 index 000000000000..efd91bfdaed5 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_hrtimer.c @@ -0,0 +1,123 @@ +/*- + * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/time.h> + +#include <machine/cpu.h> + +#include <linux/hrtimer.h> + +static void +hrtimer_call_handler(void *arg) +{ + struct hrtimer *hrtimer; + enum hrtimer_restart ret; + + hrtimer = arg; + ret = hrtimer->function(hrtimer); + + if (ret == HRTIMER_RESTART) { + callout_schedule_sbt(&hrtimer->callout, + nstosbt(hrtimer->expires), nstosbt(hrtimer->precision), 0); + } else { + callout_deactivate(&hrtimer->callout); + } +} + +bool +linux_hrtimer_active(struct hrtimer *hrtimer) +{ + bool ret; + + mtx_lock(&hrtimer->mtx); + ret = callout_active(&hrtimer->callout); + mtx_unlock(&hrtimer->mtx); + + return (ret); +} + +/* + * Cancel active hrtimer. + * Return 1 if timer was active and cancellation succeeded, or 0 otherwise. + */ +int +linux_hrtimer_cancel(struct hrtimer *hrtimer) +{ + + return (callout_drain(&hrtimer->callout) > 0); +} + +void +linux_hrtimer_init(struct hrtimer *hrtimer) +{ + + memset(hrtimer, 0, sizeof(*hrtimer)); + mtx_init(&hrtimer->mtx, "hrtimer", NULL, + MTX_DEF | MTX_RECURSE | MTX_NOWITNESS); + callout_init_mtx(&hrtimer->callout, &hrtimer->mtx, 0); +} + +void +linux_hrtimer_set_expires(struct hrtimer *hrtimer, ktime_t time) +{ + hrtimer->expires = ktime_to_ns(time); +} + +void +linux_hrtimer_start(struct hrtimer *hrtimer, ktime_t time) +{ + + linux_hrtimer_start_range_ns(hrtimer, time, 0); +} + +void +linux_hrtimer_start_range_ns(struct hrtimer *hrtimer, ktime_t time, + int64_t nsec) +{ + + mtx_lock(&hrtimer->mtx); + hrtimer->precision = nsec; + callout_reset_sbt(&hrtimer->callout, nstosbt(ktime_to_ns(time)), + nstosbt(nsec), hrtimer_call_handler, hrtimer, 0); + mtx_unlock(&hrtimer->mtx); +} + +void +linux_hrtimer_forward_now(struct hrtimer *hrtimer, ktime_t interval) +{ + + mtx_lock(&hrtimer->mtx); + callout_reset_sbt(&hrtimer->callout, nstosbt(ktime_to_ns(interval)), + nstosbt(hrtimer->precision), hrtimer_call_handler, hrtimer, 0); + mtx_unlock(&hrtimer->mtx); +} + diff --git a/sys/compat/linuxkpi/common/src/linux_idr.c b/sys/compat/linuxkpi/common/src/linux_idr.c new file mode 100644 index 000000000000..773ebd3b8410 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_idr.c @@ -0,0 +1,810 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> +#include <sys/kernel.h> +#include <sys/sysctl.h> +#include <sys/lock.h> +#include <sys/mutex.h> + +#include <machine/stdarg.h> + +#include <linux/bitmap.h> +#include <linux/kobject.h> +#include <linux/slab.h> +#include <linux/idr.h> +#include <linux/err.h> + +#define MAX_IDR_LEVEL ((MAX_IDR_SHIFT + IDR_BITS - 1) / IDR_BITS) +#define MAX_IDR_FREE (MAX_IDR_LEVEL * 2) + +struct linux_idr_cache { + spinlock_t lock; + struct idr_layer *head; + unsigned count; +}; + +DPCPU_DEFINE_STATIC(struct linux_idr_cache, linux_idr_cache); + +/* + * IDR Implementation. + * + * This is quick and dirty and not as re-entrant as the linux version + * however it should be fairly fast. It is basically a radix tree with + * a builtin bitmap for allocation. + */ +static MALLOC_DEFINE(M_IDR, "idr", "Linux IDR compat"); + +static struct idr_layer * +idr_preload_dequeue_locked(struct linux_idr_cache *lic) +{ + struct idr_layer *retval; + + /* check if wrong thread is trying to dequeue */ + if (mtx_owned(&lic->lock.m) == 0) + return (NULL); + + retval = lic->head; + if (likely(retval != NULL)) { + lic->head = retval->ary[0]; + lic->count--; + retval->ary[0] = NULL; + } + return (retval); +} + +static void +idr_preload_init(void *arg) +{ + int cpu; + + CPU_FOREACH(cpu) { + struct linux_idr_cache *lic = + DPCPU_ID_PTR(cpu, linux_idr_cache); + + spin_lock_init(&lic->lock); + } +} +SYSINIT(idr_preload_init, SI_SUB_CPU, SI_ORDER_ANY, idr_preload_init, NULL); + +static void +idr_preload_uninit(void *arg) +{ + int cpu; + + CPU_FOREACH(cpu) { + struct idr_layer *cacheval; + struct linux_idr_cache *lic = + DPCPU_ID_PTR(cpu, linux_idr_cache); + + while (1) { + spin_lock(&lic->lock); + cacheval = idr_preload_dequeue_locked(lic); + spin_unlock(&lic->lock); + + if (cacheval == NULL) + break; + free(cacheval, M_IDR); + } + spin_lock_destroy(&lic->lock); + } +} +SYSUNINIT(idr_preload_uninit, SI_SUB_LOCK, SI_ORDER_FIRST, idr_preload_uninit, NULL); + +void +idr_preload(gfp_t gfp_mask) +{ + struct linux_idr_cache *lic; + struct idr_layer *cacheval; + + sched_pin(); + + lic = &DPCPU_GET(linux_idr_cache); + + /* fill up cache */ + spin_lock(&lic->lock); + while (lic->count < MAX_IDR_FREE) { + spin_unlock(&lic->lock); + cacheval = malloc(sizeof(*cacheval), M_IDR, M_ZERO | gfp_mask); + spin_lock(&lic->lock); + if (cacheval == NULL) + break; + cacheval->ary[0] = lic->head; + lic->head = cacheval; + lic->count++; + } +} + +void +idr_preload_end(void) +{ + struct linux_idr_cache *lic; + + lic = &DPCPU_GET(linux_idr_cache); + spin_unlock(&lic->lock); + sched_unpin(); +} + +static inline int +idr_max(struct idr *idr) +{ + return (1 << (idr->layers * IDR_BITS)) - 1; +} + +static inline int +idr_pos(int id, int layer) +{ + return (id >> (IDR_BITS * layer)) & IDR_MASK; +} + +void +idr_init(struct idr *idr) +{ + bzero(idr, sizeof(*idr)); + mtx_init(&idr->lock, "idr", NULL, MTX_DEF); +} + +/* Only frees cached pages. */ +void +idr_destroy(struct idr *idr) +{ + struct idr_layer *il, *iln; + + idr_remove_all(idr); + mtx_lock(&idr->lock); + for (il = idr->free; il != NULL; il = iln) { + iln = il->ary[0]; + free(il, M_IDR); + } + mtx_unlock(&idr->lock); + mtx_destroy(&idr->lock); +} + +static void +idr_remove_layer(struct idr_layer *il, int layer) +{ + int i; + + if (il == NULL) + return; + if (layer == 0) { + free(il, M_IDR); + return; + } + for (i = 0; i < IDR_SIZE; i++) + if (il->ary[i]) + idr_remove_layer(il->ary[i], layer - 1); +} + +void +idr_remove_all(struct idr *idr) +{ + + mtx_lock(&idr->lock); + idr_remove_layer(idr->top, idr->layers - 1); + idr->top = NULL; + idr->layers = 0; + mtx_unlock(&idr->lock); +} + +static void * +idr_remove_locked(struct idr *idr, int id) +{ + struct idr_layer *il; + void *res; + int layer; + int idx; + + id &= MAX_ID_MASK; + il = idr->top; + layer = idr->layers - 1; + if (il == NULL || id > idr_max(idr)) + return (NULL); + /* + * Walk down the tree to this item setting bitmaps along the way + * as we know at least one item will be free along this path. + */ + while (layer && il) { + idx = idr_pos(id, layer); + il->bitmap |= 1 << idx; + il = il->ary[idx]; + layer--; + } + idx = id & IDR_MASK; + /* + * At this point we've set free space bitmaps up the whole tree. + * We could make this non-fatal and unwind but linux dumps a stack + * and a warning so I don't think it's necessary. + */ + if (il == NULL || (il->bitmap & (1 << idx)) != 0) + panic("idr_remove: Item %d not allocated (%p, %p)\n", + id, idr, il); + res = il->ary[idx]; + il->ary[idx] = NULL; + il->bitmap |= 1 << idx; + + return (res); +} + +void * +idr_remove(struct idr *idr, int id) +{ + void *res; + + mtx_lock(&idr->lock); + res = idr_remove_locked(idr, id); + mtx_unlock(&idr->lock); + + return (res); +} + + +static inline struct idr_layer * +idr_find_layer_locked(struct idr *idr, int id) +{ + struct idr_layer *il; + int layer; + + id &= MAX_ID_MASK; + il = idr->top; + layer = idr->layers - 1; + if (il == NULL || id > idr_max(idr)) + return (NULL); + while (layer && il) { + il = il->ary[idr_pos(id, layer)]; + layer--; + } + return (il); +} + +void * +idr_replace(struct idr *idr, void *ptr, int id) +{ + struct idr_layer *il; + void *res; + int idx; + + mtx_lock(&idr->lock); + il = idr_find_layer_locked(idr, id); + idx = id & IDR_MASK; + + /* Replace still returns an error if the item was not allocated. */ + if (il == NULL || (il->bitmap & (1 << idx))) { + res = ERR_PTR(-ENOENT); + } else { + res = il->ary[idx]; + il->ary[idx] = ptr; + } + mtx_unlock(&idr->lock); + return (res); +} + +static inline void * +idr_find_locked(struct idr *idr, int id) +{ + struct idr_layer *il; + void *res; + + mtx_assert(&idr->lock, MA_OWNED); + il = idr_find_layer_locked(idr, id); + if (il != NULL) + res = il->ary[id & IDR_MASK]; + else + res = NULL; + return (res); +} + +void * +idr_find(struct idr *idr, int id) +{ + void *res; + + mtx_lock(&idr->lock); + res = idr_find_locked(idr, id); + mtx_unlock(&idr->lock); + return (res); +} + +void * +idr_get_next(struct idr *idr, int *nextidp) +{ + void *res = NULL; + int id = *nextidp; + + mtx_lock(&idr->lock); + for (; id <= idr_max(idr); id++) { + res = idr_find_locked(idr, id); + if (res == NULL) + continue; + *nextidp = id; + break; + } + mtx_unlock(&idr->lock); + return (res); +} + +int +idr_pre_get(struct idr *idr, gfp_t gfp_mask) +{ + struct idr_layer *il, *iln; + struct idr_layer *head; + int need; + + mtx_lock(&idr->lock); + for (;;) { + need = idr->layers + 1; + for (il = idr->free; il != NULL; il = il->ary[0]) + need--; + mtx_unlock(&idr->lock); + if (need <= 0) + break; + for (head = NULL; need; need--) { + iln = malloc(sizeof(*il), M_IDR, M_ZERO | gfp_mask); + if (iln == NULL) + break; + bitmap_fill(&iln->bitmap, IDR_SIZE); + if (head != NULL) { + il->ary[0] = iln; + il = iln; + } else + head = il = iln; + } + if (head == NULL) + return (0); + mtx_lock(&idr->lock); + il->ary[0] = idr->free; + idr->free = head; + } + return (1); +} + +static struct idr_layer * +idr_free_list_get(struct idr *idp) +{ + struct idr_layer *il; + + if ((il = idp->free) != NULL) { + idp->free = il->ary[0]; + il->ary[0] = NULL; + } + return (il); +} + +static inline struct idr_layer * +idr_get(struct idr *idp) +{ + struct idr_layer *il; + + if ((il = idr_free_list_get(idp)) != NULL) { + MPASS(il->bitmap != 0); + } else if ((il = malloc(sizeof(*il), M_IDR, M_ZERO | M_NOWAIT)) != NULL) { + bitmap_fill(&il->bitmap, IDR_SIZE); + } else if ((il = idr_preload_dequeue_locked(&DPCPU_GET(linux_idr_cache))) != NULL) { + bitmap_fill(&il->bitmap, IDR_SIZE); + } else { + return (NULL); + } + return (il); +} + +/* + * Could be implemented as get_new_above(idr, ptr, 0, idp) but written + * first for simplicity sake. + */ +static int +idr_get_new_locked(struct idr *idr, void *ptr, int *idp) +{ + struct idr_layer *stack[MAX_LEVEL]; + struct idr_layer *il; + int error; + int layer; + int idx; + int id; + + mtx_assert(&idr->lock, MA_OWNED); + + error = -EAGAIN; + /* + * Expand the tree until there is free space. + */ + if (idr->top == NULL || idr->top->bitmap == 0) { + if (idr->layers == MAX_LEVEL + 1) { + error = -ENOSPC; + goto out; + } + il = idr_get(idr); + if (il == NULL) + goto out; + il->ary[0] = idr->top; + if (idr->top) + il->bitmap &= ~1; + idr->top = il; + idr->layers++; + } + il = idr->top; + id = 0; + /* + * Walk the tree following free bitmaps, record our path. + */ + for (layer = idr->layers - 1;; layer--) { + stack[layer] = il; + idx = ffsl(il->bitmap); + if (idx == 0) + panic("idr_get_new: Invalid leaf state (%p, %p)\n", + idr, il); + idx--; + id |= idx << (layer * IDR_BITS); + if (layer == 0) + break; + if (il->ary[idx] == NULL) { + il->ary[idx] = idr_get(idr); + if (il->ary[idx] == NULL) + goto out; + } + il = il->ary[idx]; + } + /* + * Allocate the leaf to the consumer. + */ + il->bitmap &= ~(1 << idx); + il->ary[idx] = ptr; + *idp = id; + /* + * Clear bitmaps potentially up to the root. + */ + while (il->bitmap == 0 && ++layer < idr->layers) { + il = stack[layer]; + il->bitmap &= ~(1 << idr_pos(id, layer)); + } + error = 0; +out: +#ifdef INVARIANTS + if (error == 0 && idr_find_locked(idr, id) != ptr) { + panic("idr_get_new: Failed for idr %p, id %d, ptr %p\n", + idr, id, ptr); + } +#endif + return (error); +} + +int +idr_get_new(struct idr *idr, void *ptr, int *idp) +{ + int retval; + + mtx_lock(&idr->lock); + retval = idr_get_new_locked(idr, ptr, idp); + mtx_unlock(&idr->lock); + return (retval); +} + +static int +idr_get_new_above_locked(struct idr *idr, void *ptr, int starting_id, int *idp) +{ + struct idr_layer *stack[MAX_LEVEL]; + struct idr_layer *il; + int error; + int layer; + int idx, sidx; + int id; + + mtx_assert(&idr->lock, MA_OWNED); + + error = -EAGAIN; + /* + * Compute the layers required to support starting_id and the mask + * at the top layer. + */ +restart: + idx = starting_id; + layer = 0; + while (idx & ~IDR_MASK) { + layer++; + idx >>= IDR_BITS; + } + if (layer == MAX_LEVEL + 1) { + error = -ENOSPC; + goto out; + } + /* + * Expand the tree until there is free space at or beyond starting_id. + */ + while (idr->layers <= layer || + idr->top->bitmap < (1 << idr_pos(starting_id, idr->layers - 1))) { + if (idr->layers == MAX_LEVEL + 1) { + error = -ENOSPC; + goto out; + } + il = idr_get(idr); + if (il == NULL) + goto out; + il->ary[0] = idr->top; + if (idr->top && idr->top->bitmap == 0) + il->bitmap &= ~1; + idr->top = il; + idr->layers++; + } + il = idr->top; + id = 0; + /* + * Walk the tree following free bitmaps, record our path. + */ + for (layer = idr->layers - 1;; layer--) { + stack[layer] = il; + sidx = idr_pos(starting_id, layer); + /* Returns index numbered from 0 or size if none exists. */ + idx = find_next_bit(&il->bitmap, IDR_SIZE, sidx); + if (idx == IDR_SIZE && sidx == 0) + panic("idr_get_new: Invalid leaf state (%p, %p)\n", + idr, il); + /* + * We may have walked a path where there was a free bit but + * it was lower than what we wanted. Restart the search with + * a larger starting id. id contains the progress we made so + * far. Search the leaf one above this level. This may + * restart as many as MAX_LEVEL times but that is expected + * to be rare. + */ + if (idx == IDR_SIZE) { + starting_id = id + (1 << ((layer + 1) * IDR_BITS)); + goto restart; + } + if (idx > sidx) + starting_id = 0; /* Search the whole subtree. */ + id |= idx << (layer * IDR_BITS); + if (layer == 0) + break; + if (il->ary[idx] == NULL) { + il->ary[idx] = idr_get(idr); + if (il->ary[idx] == NULL) + goto out; + } + il = il->ary[idx]; + } + /* + * Allocate the leaf to the consumer. + */ + il->bitmap &= ~(1 << idx); + il->ary[idx] = ptr; + *idp = id; + /* + * Clear bitmaps potentially up to the root. + */ + while (il->bitmap == 0 && ++layer < idr->layers) { + il = stack[layer]; + il->bitmap &= ~(1 << idr_pos(id, layer)); + } + error = 0; +out: +#ifdef INVARIANTS + if (error == 0 && idr_find_locked(idr, id) != ptr) { + panic("idr_get_new_above: Failed for idr %p, id %d, ptr %p\n", + idr, id, ptr); + } +#endif + return (error); +} + +int +idr_get_new_above(struct idr *idr, void *ptr, int starting_id, int *idp) +{ + int retval; + + mtx_lock(&idr->lock); + retval = idr_get_new_above_locked(idr, ptr, starting_id, idp); + mtx_unlock(&idr->lock); + return (retval); +} + +int +ida_get_new_above(struct ida *ida, int starting_id, int *p_id) +{ + return (idr_get_new_above(&ida->idr, NULL, starting_id, p_id)); +} + +static int +idr_alloc_locked(struct idr *idr, void *ptr, int start, int end) +{ + int max = end > 0 ? end - 1 : INT_MAX; + int error; + int id; + + mtx_assert(&idr->lock, MA_OWNED); + + if (unlikely(start < 0)) + return (-EINVAL); + if (unlikely(max < start)) + return (-ENOSPC); + + if (start == 0) + error = idr_get_new_locked(idr, ptr, &id); + else + error = idr_get_new_above_locked(idr, ptr, start, &id); + + if (unlikely(error < 0)) + return (error); + if (unlikely(id > max)) { + idr_remove_locked(idr, id); + return (-ENOSPC); + } + return (id); +} + +int +idr_alloc(struct idr *idr, void *ptr, int start, int end, gfp_t gfp_mask) +{ + int retval; + + mtx_lock(&idr->lock); + retval = idr_alloc_locked(idr, ptr, start, end); + mtx_unlock(&idr->lock); + return (retval); +} + +int +idr_alloc_cyclic(struct idr *idr, void *ptr, int start, int end, gfp_t gfp_mask) +{ + int retval; + + mtx_lock(&idr->lock); + retval = idr_alloc_locked(idr, ptr, max(start, idr->next_cyclic_id), end); + if (unlikely(retval == -ENOSPC)) + retval = idr_alloc_locked(idr, ptr, start, end); + if (likely(retval >= 0)) + idr->next_cyclic_id = retval + 1; + mtx_unlock(&idr->lock); + return (retval); +} + +static int +idr_for_each_layer(struct idr_layer *il, int offset, int layer, + int (*f)(int id, void *p, void *data), void *data) +{ + int i, err; + + if (il == NULL) + return (0); + if (layer == 0) { + for (i = 0; i < IDR_SIZE; i++) { + if (il->ary[i] == NULL) + continue; + err = f(i + offset, il->ary[i], data); + if (err) + return (err); + } + return (0); + } + for (i = 0; i < IDR_SIZE; i++) { + if (il->ary[i] == NULL) + continue; + err = idr_for_each_layer(il->ary[i], + (i + offset) * IDR_SIZE, layer - 1, f, data); + if (err) + return (err); + } + return (0); +} + +/* NOTE: It is not allowed to modify the IDR tree while it is being iterated */ +int +idr_for_each(struct idr *idp, int (*f)(int id, void *p, void *data), void *data) +{ + return (idr_for_each_layer(idp->top, 0, idp->layers - 1, f, data)); +} + +static int +idr_has_entry(int id, void *p, void *data) +{ + + return (1); +} + +bool +idr_is_empty(struct idr *idp) +{ + + return (idr_for_each(idp, idr_has_entry, NULL) == 0); +} + +int +ida_pre_get(struct ida *ida, gfp_t flags) +{ + if (idr_pre_get(&ida->idr, flags) == 0) + return (0); + + if (ida->free_bitmap == NULL) { + ida->free_bitmap = + malloc(sizeof(struct ida_bitmap), M_IDR, flags); + } + return (ida->free_bitmap != NULL); +} + +int +ida_simple_get(struct ida *ida, unsigned int start, unsigned int end, + gfp_t flags) +{ + int ret, id; + unsigned int max; + + MPASS((int)start >= 0); + MPASS((int)end >= 0); + + if (end == 0) + max = 0x80000000; + else { + MPASS(end > start); + max = end - 1; + } +again: + if (!ida_pre_get(ida, flags)) + return (-ENOMEM); + + if ((ret = ida_get_new_above(ida, start, &id)) == 0) { + if (id > max) { + ida_remove(ida, id); + ret = -ENOSPC; + } else { + ret = id; + } + } + if (__predict_false(ret == -EAGAIN)) + goto again; + + return (ret); +} + +void +ida_simple_remove(struct ida *ida, unsigned int id) +{ + idr_remove(&ida->idr, id); +} + +void +ida_remove(struct ida *ida, int id) +{ + idr_remove(&ida->idr, id); +} + +void +ida_init(struct ida *ida) +{ + idr_init(&ida->idr); +} + +void +ida_destroy(struct ida *ida) +{ + idr_destroy(&ida->idr); + free(ida->free_bitmap, M_IDR); +} diff --git a/sys/compat/linuxkpi/common/src/linux_kmod.c b/sys/compat/linuxkpi/common/src/linux_kmod.c new file mode 100644 index 000000000000..efe16fbb1aa9 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_kmod.c @@ -0,0 +1,35 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/module.h> + +MODULE_VERSION(linuxkpi, 1); +MODULE_DEPEND(linuxkpi, pci, 1, 1, 1); + diff --git a/sys/compat/linuxkpi/common/src/linux_kthread.c b/sys/compat/linuxkpi/common/src/linux_kthread.c new file mode 100644 index 000000000000..26afe005ea59 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_kthread.c @@ -0,0 +1,167 @@ +/*- + * Copyright (c) 2017 Hans Petter Selasky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <linux/compat.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/wait.h> + +#include <sys/bus.h> +#include <sys/interrupt.h> +#include <sys/priority.h> + +enum { + KTHREAD_SHOULD_STOP_MASK = (1 << 0), + KTHREAD_SHOULD_PARK_MASK = (1 << 1), + KTHREAD_IS_PARKED_MASK = (1 << 2), +}; + +bool +linux_kthread_should_stop_task(struct task_struct *task) +{ + + return (atomic_read(&task->kthread_flags) & KTHREAD_SHOULD_STOP_MASK); +} + +bool +linux_kthread_should_stop(void) +{ + + return (atomic_read(¤t->kthread_flags) & KTHREAD_SHOULD_STOP_MASK); +} + +int +linux_kthread_stop(struct task_struct *task) +{ + int retval; + + /* + * Assume task is still alive else caller should not call + * kthread_stop(): + */ + atomic_or(KTHREAD_SHOULD_STOP_MASK, &task->kthread_flags); + kthread_unpark(task); + wake_up_process(task); + wait_for_completion(&task->exited); + + /* + * Get return code and free task structure: + */ + retval = task->task_ret; + put_task_struct(task); + + return (retval); +} + +int +linux_kthread_park(struct task_struct *task) +{ + + atomic_or(KTHREAD_SHOULD_PARK_MASK, &task->kthread_flags); + wake_up_process(task); + wait_for_completion(&task->parked); + return (0); +} + +void +linux_kthread_parkme(void) +{ + struct task_struct *task; + + task = current; + set_task_state(task, TASK_PARKED | TASK_UNINTERRUPTIBLE); + while (linux_kthread_should_park()) { + while ((atomic_fetch_or(KTHREAD_IS_PARKED_MASK, + &task->kthread_flags) & KTHREAD_IS_PARKED_MASK) == 0) + complete(&task->parked); + schedule(); + set_task_state(task, TASK_PARKED | TASK_UNINTERRUPTIBLE); + } + atomic_andnot(KTHREAD_IS_PARKED_MASK, &task->kthread_flags); + set_task_state(task, TASK_RUNNING); +} + +bool +linux_kthread_should_park(void) +{ + struct task_struct *task; + + task = current; + return (atomic_read(&task->kthread_flags) & KTHREAD_SHOULD_PARK_MASK); +} + +void +linux_kthread_unpark(struct task_struct *task) +{ + + atomic_andnot(KTHREAD_SHOULD_PARK_MASK, &task->kthread_flags); + if ((atomic_fetch_andnot(KTHREAD_IS_PARKED_MASK, &task->kthread_flags) & + KTHREAD_IS_PARKED_MASK) != 0) + wake_up_state(task, TASK_PARKED); +} + +struct task_struct * +linux_kthread_setup_and_run(struct thread *td, linux_task_fn_t *task_fn, void *arg) +{ + struct task_struct *task; + + linux_set_current(td); + + task = td->td_lkpi_task; + task->task_fn = task_fn; + task->task_data = arg; + + thread_lock(td); + /* make sure the scheduler priority is raised */ + sched_prio(td, PI_SWI(SWI_NET)); + /* put thread into run-queue */ + sched_add(td, SRQ_BORING); + + return (task); +} + +void +linux_kthread_fn(void *arg __unused) +{ + struct task_struct *task = current; + + if (linux_kthread_should_stop_task(task) == 0) + task->task_ret = task->task_fn(task->task_data); + + if (linux_kthread_should_stop_task(task) != 0) { + struct thread *td = curthread; + + /* let kthread_stop() free data */ + td->td_lkpi_task = NULL; + + /* wakeup kthread_stop() */ + complete(&task->exited); + } + kthread_exit(); +} diff --git a/sys/compat/linuxkpi/common/src/linux_lock.c b/sys/compat/linuxkpi/common/src/linux_lock.c new file mode 100644 index 000000000000..f037cd3a1eaa --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_lock.c @@ -0,0 +1,168 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include <sys/queue.h> + +#include <linux/sched.h> +#include <linux/ww_mutex.h> + +struct ww_mutex_thread { + TAILQ_ENTRY(ww_mutex_thread) entry; + struct thread *thread; + struct ww_mutex *lock; +}; + +static TAILQ_HEAD(, ww_mutex_thread) ww_mutex_head; +static struct mtx ww_mutex_global; + +static void +linux_ww_init(void *arg) +{ + TAILQ_INIT(&ww_mutex_head); + mtx_init(&ww_mutex_global, "lkpi-ww-mtx", NULL, MTX_DEF); +} + +SYSINIT(ww_init, SI_SUB_LOCK, SI_ORDER_SECOND, linux_ww_init, NULL); + +static void +linux_ww_uninit(void *arg) +{ + mtx_destroy(&ww_mutex_global); +} + +SYSUNINIT(ww_uninit, SI_SUB_LOCK, SI_ORDER_SECOND, linux_ww_uninit, NULL); + +static inline void +linux_ww_lock(void) +{ + mtx_lock(&ww_mutex_global); +} + +static inline void +linux_ww_unlock(void) +{ + mtx_unlock(&ww_mutex_global); +} + +/* lock a mutex with deadlock avoidance */ +int +linux_ww_mutex_lock_sub(struct ww_mutex *lock, int catch_signal) +{ + struct task_struct *task; + struct ww_mutex_thread entry; + struct ww_mutex_thread *other; + int retval = 0; + + task = current; + + linux_ww_lock(); + if (unlikely(sx_try_xlock(&lock->base.sx) == 0)) { + entry.thread = curthread; + entry.lock = lock; + TAILQ_INSERT_TAIL(&ww_mutex_head, &entry, entry); + + do { + struct thread *owner = (struct thread *) + SX_OWNER(lock->base.sx.sx_lock); + + /* scan for deadlock */ + TAILQ_FOREACH(other, &ww_mutex_head, entry) { + /* skip own thread */ + if (other == &entry) + continue; + /* + * If another thread is owning our + * lock and is at the same time trying + * to acquire a lock this thread owns, + * that means deadlock. + */ + if (other->thread == owner && + (struct thread *)SX_OWNER( + other->lock->base.sx.sx_lock) == curthread) { + retval = -EDEADLK; + goto done; + } + } + if (catch_signal) { + retval = -cv_wait_sig(&lock->condvar, &ww_mutex_global); + if (retval != 0) { + linux_schedule_save_interrupt_value(task, retval); + retval = -EINTR; + goto done; + } + } else { + cv_wait(&lock->condvar, &ww_mutex_global); + } + } while (sx_try_xlock(&lock->base.sx) == 0); +done: + TAILQ_REMOVE(&ww_mutex_head, &entry, entry); + + /* if the lock is free, wakeup next lock waiter, if any */ + if ((struct thread *)SX_OWNER(lock->base.sx.sx_lock) == NULL) + cv_signal(&lock->condvar); + } + linux_ww_unlock(); + return (retval); +} + +void +linux_ww_mutex_unlock_sub(struct ww_mutex *lock) +{ + /* protect ww_mutex ownership change */ + linux_ww_lock(); + sx_xunlock(&lock->base.sx); + /* wakeup a lock waiter, if any */ + cv_signal(&lock->condvar); + linux_ww_unlock(); +} + +int +linux_mutex_lock_interruptible(mutex_t *m) +{ + int error; + + error = -sx_xlock_sig(&m->sx); + if (error != 0) { + linux_schedule_save_interrupt_value(current, error); + error = -EINTR; + } + return (error); +} + +int +linux_down_write_killable(struct rw_semaphore *rw) +{ + int error; + + error = -sx_xlock_sig(&rw->sx); + if (error != 0) { + linux_schedule_save_interrupt_value(current, error); + error = -EINTR; + } + return (error); +} diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c new file mode 100644 index 000000000000..ee41366c53a6 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_page.c @@ -0,0 +1,277 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> +#include <sys/kernel.h> +#include <sys/sysctl.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/rwlock.h> +#include <sys/proc.h> +#include <sys/sched.h> + +#include <machine/bus.h> + +#include <vm/vm.h> +#include <vm/pmap.h> +#include <vm/vm_param.h> +#include <vm/vm_kern.h> +#include <vm/vm_object.h> +#include <vm/vm_map.h> +#include <vm/vm_page.h> +#include <vm/vm_pageout.h> +#include <vm/vm_pager.h> +#include <vm/vm_radix.h> +#include <vm/vm_reserv.h> +#include <vm/vm_extern.h> + +#include <vm/uma.h> +#include <vm/uma_int.h> + +#include <linux/gfp.h> +#include <linux/mm.h> +#include <linux/preempt.h> +#include <linux/fs.h> +#include <linux/shmem_fs.h> + +void +si_meminfo(struct sysinfo *si) +{ + si->totalram = physmem; + si->totalhigh = 0; + si->mem_unit = PAGE_SIZE; +} + +void * +linux_page_address(struct page *page) +{ + + if (page->object != kernel_object) { + return (PMAP_HAS_DMAP ? + ((void *)(uintptr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page))) : + NULL); + } + return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS + + IDX_TO_OFF(page->pindex))); +} + +vm_page_t +linux_alloc_pages(gfp_t flags, unsigned int order) +{ + vm_page_t page; + + if (PMAP_HAS_DMAP) { + unsigned long npages = 1UL << order; + int req = VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_NORMAL; + + if ((flags & M_ZERO) != 0) + req |= VM_ALLOC_ZERO; + if (order == 0 && (flags & GFP_DMA32) == 0) { + page = vm_page_alloc(NULL, 0, req); + if (page == NULL) + return (NULL); + } else { + vm_paddr_t pmax = (flags & GFP_DMA32) ? + BUS_SPACE_MAXADDR_32BIT : BUS_SPACE_MAXADDR; + retry: + page = vm_page_alloc_contig(NULL, 0, req, + npages, 0, pmax, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); + + if (page == NULL) { + if (flags & M_WAITOK) { + if (!vm_page_reclaim_contig(req, + npages, 0, pmax, PAGE_SIZE, 0)) { + vm_wait(NULL); + } + flags &= ~M_WAITOK; + goto retry; + } + return (NULL); + } + } + if (flags & M_ZERO) { + unsigned long x; + + for (x = 0; x != npages; x++) { + vm_page_t pgo = page + x; + + if ((pgo->flags & PG_ZERO) == 0) + pmap_zero_page(pgo); + } + } + } else { + vm_offset_t vaddr; + + vaddr = linux_alloc_kmem(flags, order); + if (vaddr == 0) + return (NULL); + + page = PHYS_TO_VM_PAGE(vtophys((void *)vaddr)); + + KASSERT(vaddr == (vm_offset_t)page_address(page), + ("Page address mismatch")); + } + + return (page); +} + +void +linux_free_pages(vm_page_t page, unsigned int order) +{ + if (PMAP_HAS_DMAP) { + unsigned long npages = 1UL << order; + unsigned long x; + + for (x = 0; x != npages; x++) { + vm_page_t pgo = page + x; + + if (vm_page_unwire_noq(pgo)) + vm_page_free(pgo); + } + } else { + vm_offset_t vaddr; + + vaddr = (vm_offset_t)page_address(page); + + linux_free_kmem(vaddr, order); + } +} + +vm_offset_t +linux_alloc_kmem(gfp_t flags, unsigned int order) +{ + size_t size = ((size_t)PAGE_SIZE) << order; + vm_offset_t addr; + + if ((flags & GFP_DMA32) == 0) { + addr = kmem_malloc(size, flags & GFP_NATIVE_MASK); + } else { + addr = kmem_alloc_contig(size, flags & GFP_NATIVE_MASK, 0, + BUS_SPACE_MAXADDR_32BIT, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); + } + return (addr); +} + +void +linux_free_kmem(vm_offset_t addr, unsigned int order) +{ + size_t size = ((size_t)PAGE_SIZE) << order; + + kmem_free(addr, size); +} + +static int +linux_get_user_pages_internal(vm_map_t map, unsigned long start, int nr_pages, + int write, struct page **pages) +{ + vm_prot_t prot; + size_t len; + int count; + + prot = write ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ; + len = ptoa((vm_offset_t)nr_pages); + count = vm_fault_quick_hold_pages(map, start, len, prot, pages, nr_pages); + return (count == -1 ? -EFAULT : nr_pages); +} + +int +__get_user_pages_fast(unsigned long start, int nr_pages, int write, + struct page **pages) +{ + vm_map_t map; + vm_page_t *mp; + vm_offset_t va; + vm_offset_t end; + vm_prot_t prot; + int count; + + if (nr_pages == 0 || in_interrupt()) + return (0); + + MPASS(pages != NULL); + map = &curthread->td_proc->p_vmspace->vm_map; + end = start + ptoa((vm_offset_t)nr_pages); + if (!vm_map_range_valid(map, start, end)) + return (-EINVAL); + prot = write ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ; + for (count = 0, mp = pages, va = start; va < end; + mp++, va += PAGE_SIZE, count++) { + *mp = pmap_extract_and_hold(map->pmap, va, prot); + if (*mp == NULL) + break; + + if ((prot & VM_PROT_WRITE) != 0 && + (*mp)->dirty != VM_PAGE_BITS_ALL) { + /* + * Explicitly dirty the physical page. Otherwise, the + * caller's changes may go unnoticed because they are + * performed through an unmanaged mapping or by a DMA + * operation. + * + * The object lock is not held here. + * See vm_page_clear_dirty_mask(). + */ + vm_page_dirty(*mp); + } + } + return (count); +} + +long +get_user_pages_remote(struct task_struct *task, struct mm_struct *mm, + unsigned long start, unsigned long nr_pages, int gup_flags, + struct page **pages, struct vm_area_struct **vmas) +{ + vm_map_t map; + + map = &task->task_thread->td_proc->p_vmspace->vm_map; + return (linux_get_user_pages_internal(map, start, nr_pages, + !!(gup_flags & FOLL_WRITE), pages)); +} + +long +get_user_pages(unsigned long start, unsigned long nr_pages, int gup_flags, + struct page **pages, struct vm_area_struct **vmas) +{ + vm_map_t map; + + map = &curthread->td_proc->p_vmspace->vm_map; + return (linux_get_user_pages_internal(map, start, nr_pages, + !!(gup_flags & FOLL_WRITE), pages)); +} + +int +is_vmalloc_addr(const void *addr) +{ + return (vtoslab((vm_offset_t)addr & ~UMA_SLAB_MASK) != NULL); +} diff --git a/sys/compat/linuxkpi/common/src/linux_pci.c b/sys/compat/linuxkpi/common/src/linux_pci.c new file mode 100644 index 000000000000..d3b64e07b980 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_pci.c @@ -0,0 +1,954 @@ +/*- + * Copyright (c) 2015-2016 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/malloc.h> +#include <sys/kernel.h> +#include <sys/sysctl.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/fcntl.h> +#include <sys/file.h> +#include <sys/filio.h> +#include <sys/pciio.h> +#include <sys/pctrie.h> +#include <sys/rwlock.h> + +#include <vm/vm.h> +#include <vm/pmap.h> + +#include <machine/stdarg.h> + +#include <dev/pci/pcivar.h> +#include <dev/pci/pci_private.h> +#include <dev/pci/pci_iov.h> + +#include <linux/kobject.h> +#include <linux/device.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/cdev.h> +#include <linux/file.h> +#include <linux/sysfs.h> +#include <linux/mm.h> +#include <linux/io.h> +#include <linux/vmalloc.h> +#include <linux/pci.h> +#include <linux/compat.h> + +static device_probe_t linux_pci_probe; +static device_attach_t linux_pci_attach; +static device_detach_t linux_pci_detach; +static device_suspend_t linux_pci_suspend; +static device_resume_t linux_pci_resume; +static device_shutdown_t linux_pci_shutdown; +static pci_iov_init_t linux_pci_iov_init; +static pci_iov_uninit_t linux_pci_iov_uninit; +static pci_iov_add_vf_t linux_pci_iov_add_vf; + +static device_method_t pci_methods[] = { + DEVMETHOD(device_probe, linux_pci_probe), + DEVMETHOD(device_attach, linux_pci_attach), + DEVMETHOD(device_detach, linux_pci_detach), + DEVMETHOD(device_suspend, linux_pci_suspend), + DEVMETHOD(device_resume, linux_pci_resume), + DEVMETHOD(device_shutdown, linux_pci_shutdown), + DEVMETHOD(pci_iov_init, linux_pci_iov_init), + DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit), + DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf), + DEVMETHOD_END +}; + +struct linux_dma_priv { + uint64_t dma_mask; + struct mtx lock; + bus_dma_tag_t dmat; + struct pctrie ptree; +}; +#define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock) +#define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock) + +static int +linux_pdev_dma_init(struct pci_dev *pdev) +{ + struct linux_dma_priv *priv; + int error; + + priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO); + pdev->dev.dma_priv = priv; + + mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF); + + pctrie_init(&priv->ptree); + + /* create a default DMA tag */ + error = linux_dma_tag_init(&pdev->dev, DMA_BIT_MASK(64)); + if (error) { + mtx_destroy(&priv->lock); + free(priv, M_DEVBUF); + pdev->dev.dma_priv = NULL; + } + return (error); +} + +static int +linux_pdev_dma_uninit(struct pci_dev *pdev) +{ + struct linux_dma_priv *priv; + + priv = pdev->dev.dma_priv; + if (priv->dmat) + bus_dma_tag_destroy(priv->dmat); + mtx_destroy(&priv->lock); + free(priv, M_DEVBUF); + pdev->dev.dma_priv = NULL; + return (0); +} + +int +linux_dma_tag_init(struct device *dev, u64 dma_mask) +{ + struct linux_dma_priv *priv; + int error; + + priv = dev->dma_priv; + + if (priv->dmat) { + if (priv->dma_mask == dma_mask) + return (0); + + bus_dma_tag_destroy(priv->dmat); + } + + priv->dma_mask = dma_mask; + + error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), + 1, 0, /* alignment, boundary */ + dma_mask, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filtfunc, filtfuncarg */ + BUS_SPACE_MAXSIZE, /* maxsize */ + 1, /* nsegments */ + BUS_SPACE_MAXSIZE, /* maxsegsz */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &priv->dmat); + return (-error); +} + +static struct pci_driver * +linux_pci_find(device_t dev, const struct pci_device_id **idp) +{ + const struct pci_device_id *id; + struct pci_driver *pdrv; + uint16_t vendor; + uint16_t device; + uint16_t subvendor; + uint16_t subdevice; + + vendor = pci_get_vendor(dev); + device = pci_get_device(dev); + subvendor = pci_get_subvendor(dev); + subdevice = pci_get_subdevice(dev); + + spin_lock(&pci_lock); + list_for_each_entry(pdrv, &pci_drivers, links) { + for (id = pdrv->id_table; id->vendor != 0; id++) { + if (vendor == id->vendor && + (PCI_ANY_ID == id->device || device == id->device) && + (PCI_ANY_ID == id->subvendor || subvendor == id->subvendor) && + (PCI_ANY_ID == id->subdevice || subdevice == id->subdevice)) { + *idp = id; + spin_unlock(&pci_lock); + return (pdrv); + } + } + } + spin_unlock(&pci_lock); + return (NULL); +} + +static int +linux_pci_probe(device_t dev) +{ + const struct pci_device_id *id; + struct pci_driver *pdrv; + + if ((pdrv = linux_pci_find(dev, &id)) == NULL) + return (ENXIO); + if (device_get_driver(dev) != &pdrv->bsddriver) + return (ENXIO); + device_set_desc(dev, pdrv->name); + return (0); +} + +static int +linux_pci_attach(device_t dev) +{ + const struct pci_device_id *id; + struct pci_driver *pdrv; + struct pci_dev *pdev; + + pdrv = linux_pci_find(dev, &id); + pdev = device_get_softc(dev); + + MPASS(pdrv != NULL); + MPASS(pdev != NULL); + + return (linux_pci_attach_device(dev, pdrv, id, pdev)); +} + +int +linux_pci_attach_device(device_t dev, struct pci_driver *pdrv, + const struct pci_device_id *id, struct pci_dev *pdev) +{ + struct resource_list_entry *rle; + struct pci_bus *pbus; + struct pci_devinfo *dinfo; + device_t parent; + int error; + + linux_set_current(curthread); + + if (pdrv != NULL && pdrv->isdrm) { + parent = device_get_parent(dev); + dinfo = device_get_ivars(parent); + device_set_ivars(dev, dinfo); + } else { + dinfo = device_get_ivars(dev); + } + + pdev->dev.parent = &linux_root_device; + pdev->dev.bsddev = dev; + INIT_LIST_HEAD(&pdev->dev.irqents); + pdev->devfn = PCI_DEVFN(pci_get_slot(dev), pci_get_function(dev)); + pdev->device = dinfo->cfg.device; + pdev->vendor = dinfo->cfg.vendor; + pdev->subsystem_vendor = dinfo->cfg.subvendor; + pdev->subsystem_device = dinfo->cfg.subdevice; + pdev->class = pci_get_class(dev); + pdev->revision = pci_get_revid(dev); + pdev->pdrv = pdrv; + kobject_init(&pdev->dev.kobj, &linux_dev_ktype); + kobject_set_name(&pdev->dev.kobj, device_get_nameunit(dev)); + kobject_add(&pdev->dev.kobj, &linux_root_device.kobj, + kobject_name(&pdev->dev.kobj)); + rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0); + if (rle != NULL) + pdev->dev.irq = rle->start; + else + pdev->dev.irq = LINUX_IRQ_INVALID; + pdev->irq = pdev->dev.irq; + error = linux_pdev_dma_init(pdev); + if (error) + goto out_dma_init; + + TAILQ_INIT(&pdev->mmio); + pbus = malloc(sizeof(*pbus), M_DEVBUF, M_WAITOK | M_ZERO); + pbus->self = pdev; + pbus->number = pci_get_bus(dev); + pbus->domain = pci_get_domain(dev); + pdev->bus = pbus; + + spin_lock(&pci_lock); + list_add(&pdev->links, &pci_devices); + spin_unlock(&pci_lock); + + if (pdrv != NULL) { + error = pdrv->probe(pdev, id); + if (error) + goto out_probe; + } + return (0); + +out_probe: + free(pdev->bus, M_DEVBUF); + linux_pdev_dma_uninit(pdev); +out_dma_init: + spin_lock(&pci_lock); + list_del(&pdev->links); + spin_unlock(&pci_lock); + put_device(&pdev->dev); + return (-error); +} + +static int +linux_pci_detach(device_t dev) +{ + struct pci_dev *pdev; + + pdev = device_get_softc(dev); + + MPASS(pdev != NULL); + + device_set_desc(dev, NULL); + + return (linux_pci_detach_device(pdev)); +} + +int +linux_pci_detach_device(struct pci_dev *pdev) +{ + + linux_set_current(curthread); + + if (pdev->pdrv != NULL) + pdev->pdrv->remove(pdev); + + free(pdev->bus, M_DEVBUF); + linux_pdev_dma_uninit(pdev); + + spin_lock(&pci_lock); + list_del(&pdev->links); + spin_unlock(&pci_lock); + put_device(&pdev->dev); + + return (0); +} + +static int +linux_pci_suspend(device_t dev) +{ + const struct dev_pm_ops *pmops; + struct pm_message pm = { }; + struct pci_dev *pdev; + int error; + + error = 0; + linux_set_current(curthread); + pdev = device_get_softc(dev); + pmops = pdev->pdrv->driver.pm; + + if (pdev->pdrv->suspend != NULL) + error = -pdev->pdrv->suspend(pdev, pm); + else if (pmops != NULL && pmops->suspend != NULL) { + error = -pmops->suspend(&pdev->dev); + if (error == 0 && pmops->suspend_late != NULL) + error = -pmops->suspend_late(&pdev->dev); + } + return (error); +} + +static int +linux_pci_resume(device_t dev) +{ + const struct dev_pm_ops *pmops; + struct pci_dev *pdev; + int error; + + error = 0; + linux_set_current(curthread); + pdev = device_get_softc(dev); + pmops = pdev->pdrv->driver.pm; + + if (pdev->pdrv->resume != NULL) + error = -pdev->pdrv->resume(pdev); + else if (pmops != NULL && pmops->resume != NULL) { + if (pmops->resume_early != NULL) + error = -pmops->resume_early(&pdev->dev); + if (error == 0 && pmops->resume != NULL) + error = -pmops->resume(&pdev->dev); + } + return (error); +} + +static int +linux_pci_shutdown(device_t dev) +{ + struct pci_dev *pdev; + + linux_set_current(curthread); + pdev = device_get_softc(dev); + if (pdev->pdrv->shutdown != NULL) + pdev->pdrv->shutdown(pdev); + return (0); +} + +static int +linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config) +{ + struct pci_dev *pdev; + int error; + + linux_set_current(curthread); + pdev = device_get_softc(dev); + if (pdev->pdrv->bsd_iov_init != NULL) + error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config); + else + error = EINVAL; + return (error); +} + +static void +linux_pci_iov_uninit(device_t dev) +{ + struct pci_dev *pdev; + + linux_set_current(curthread); + pdev = device_get_softc(dev); + if (pdev->pdrv->bsd_iov_uninit != NULL) + pdev->pdrv->bsd_iov_uninit(dev); +} + +static int +linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config) +{ + struct pci_dev *pdev; + int error; + + linux_set_current(curthread); + pdev = device_get_softc(dev); + if (pdev->pdrv->bsd_iov_add_vf != NULL) + error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config); + else + error = EINVAL; + return (error); +} + +static int +_linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc) +{ + int error; + + linux_set_current(curthread); + spin_lock(&pci_lock); + list_add(&pdrv->links, &pci_drivers); + spin_unlock(&pci_lock); + pdrv->bsddriver.name = pdrv->name; + pdrv->bsddriver.methods = pci_methods; + pdrv->bsddriver.size = sizeof(struct pci_dev); + + mtx_lock(&Giant); + error = devclass_add_driver(dc, &pdrv->bsddriver, + BUS_PASS_DEFAULT, &pdrv->bsdclass); + mtx_unlock(&Giant); + return (-error); +} + +int +linux_pci_register_driver(struct pci_driver *pdrv) +{ + devclass_t dc; + + dc = devclass_find("pci"); + if (dc == NULL) + return (-ENXIO); + pdrv->isdrm = false; + return (_linux_pci_register_driver(pdrv, dc)); +} + +unsigned long +pci_resource_start(struct pci_dev *pdev, int bar) +{ + struct resource_list_entry *rle; + rman_res_t newstart; + device_t dev; + + if ((rle = linux_pci_get_bar(pdev, bar)) == NULL) + return (0); + dev = pci_find_dbsf(pdev->bus->domain, pdev->bus->number, + PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); + MPASS(dev != NULL); + if (BUS_TRANSLATE_RESOURCE(dev, rle->type, rle->start, &newstart)) { + device_printf(pdev->dev.bsddev, "translate of %#jx failed\n", + (uintmax_t)rle->start); + return (0); + } + return (newstart); +} + +unsigned long +pci_resource_len(struct pci_dev *pdev, int bar) +{ + struct resource_list_entry *rle; + + if ((rle = linux_pci_get_bar(pdev, bar)) == NULL) + return (0); + return (rle->count); +} + +int +linux_pci_register_drm_driver(struct pci_driver *pdrv) +{ + devclass_t dc; + + dc = devclass_create("vgapci"); + if (dc == NULL) + return (-ENXIO); + pdrv->isdrm = true; + pdrv->name = "drmn"; + return (_linux_pci_register_driver(pdrv, dc)); +} + +void +linux_pci_unregister_driver(struct pci_driver *pdrv) +{ + devclass_t bus; + + bus = devclass_find("pci"); + + spin_lock(&pci_lock); + list_del(&pdrv->links); + spin_unlock(&pci_lock); + mtx_lock(&Giant); + if (bus != NULL) + devclass_delete_driver(bus, &pdrv->bsddriver); + mtx_unlock(&Giant); +} + +void +linux_pci_unregister_drm_driver(struct pci_driver *pdrv) +{ + devclass_t bus; + + bus = devclass_find("vgapci"); + + spin_lock(&pci_lock); + list_del(&pdrv->links); + spin_unlock(&pci_lock); + mtx_lock(&Giant); + if (bus != NULL) + devclass_delete_driver(bus, &pdrv->bsddriver); + mtx_unlock(&Giant); +} + +CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t)); + +struct linux_dma_obj { + void *vaddr; + uint64_t dma_addr; + bus_dmamap_t dmamap; +}; + +static uma_zone_t linux_dma_trie_zone; +static uma_zone_t linux_dma_obj_zone; + +static void +linux_dma_init(void *arg) +{ + + linux_dma_trie_zone = uma_zcreate("linux_dma_pctrie", + pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL, + UMA_ALIGN_PTR, 0); + linux_dma_obj_zone = uma_zcreate("linux_dma_object", + sizeof(struct linux_dma_obj), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); + +} +SYSINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_init, NULL); + +static void +linux_dma_uninit(void *arg) +{ + + uma_zdestroy(linux_dma_obj_zone); + uma_zdestroy(linux_dma_trie_zone); +} +SYSUNINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_uninit, NULL); + +static void * +linux_dma_trie_alloc(struct pctrie *ptree) +{ + + return (uma_zalloc(linux_dma_trie_zone, M_NOWAIT)); +} + +static void +linux_dma_trie_free(struct pctrie *ptree, void *node) +{ + + uma_zfree(linux_dma_trie_zone, node); +} + + +PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc, + linux_dma_trie_free); + +void * +linux_dma_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag) +{ + struct linux_dma_priv *priv; + vm_paddr_t high; + size_t align; + void *mem; + + if (dev == NULL || dev->dma_priv == NULL) { + *dma_handle = 0; + return (NULL); + } + priv = dev->dma_priv; + if (priv->dma_mask) + high = priv->dma_mask; + else if (flag & GFP_DMA32) + high = BUS_SPACE_MAXADDR_32BIT; + else + high = BUS_SPACE_MAXADDR; + align = PAGE_SIZE << get_order(size); + mem = (void *)kmem_alloc_contig(size, flag, 0, high, align, 0, + VM_MEMATTR_DEFAULT); + if (mem != NULL) { + *dma_handle = linux_dma_map_phys(dev, vtophys(mem), size); + if (*dma_handle == 0) { + kmem_free((vm_offset_t)mem, size); + mem = NULL; + } + } else { + *dma_handle = 0; + } + return (mem); +} + +#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) +dma_addr_t +linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) +{ + struct linux_dma_priv *priv; + struct linux_dma_obj *obj; + int error, nseg; + bus_dma_segment_t seg; + + priv = dev->dma_priv; + + /* + * If the resultant mapping will be entirely 1:1 with the + * physical address, short-circuit the remainder of the + * bus_dma API. This avoids tracking collisions in the pctrie + * with the additional benefit of reducing overhead. + */ + if (bus_dma_id_mapped(priv->dmat, phys, len)) + return (phys); + + obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT); + if (obj == NULL) { + return (0); + } + + DMA_PRIV_LOCK(priv); + if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0) { + DMA_PRIV_UNLOCK(priv); + uma_zfree(linux_dma_obj_zone, obj); + return (0); + } + + nseg = -1; + if (_bus_dmamap_load_phys(priv->dmat, obj->dmamap, phys, len, + BUS_DMA_NOWAIT, &seg, &nseg) != 0) { + bus_dmamap_destroy(priv->dmat, obj->dmamap); + DMA_PRIV_UNLOCK(priv); + uma_zfree(linux_dma_obj_zone, obj); + return (0); + } + + KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); + obj->dma_addr = seg.ds_addr; + + error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); + if (error != 0) { + bus_dmamap_unload(priv->dmat, obj->dmamap); + bus_dmamap_destroy(priv->dmat, obj->dmamap); + DMA_PRIV_UNLOCK(priv); + uma_zfree(linux_dma_obj_zone, obj); + return (0); + } + DMA_PRIV_UNLOCK(priv); + return (obj->dma_addr); +} +#else +dma_addr_t +linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) +{ + return (phys); +} +#endif + +#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) +void +linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) +{ + struct linux_dma_priv *priv; + struct linux_dma_obj *obj; + + priv = dev->dma_priv; + + if (pctrie_is_empty(&priv->ptree)) + return; + + DMA_PRIV_LOCK(priv); + obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); + if (obj == NULL) { + DMA_PRIV_UNLOCK(priv); + return; + } + LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); + bus_dmamap_unload(priv->dmat, obj->dmamap); + bus_dmamap_destroy(priv->dmat, obj->dmamap); + DMA_PRIV_UNLOCK(priv); + + uma_zfree(linux_dma_obj_zone, obj); +} +#else +void +linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) +{ +} +#endif + +int +linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + struct linux_dma_priv *priv; + struct scatterlist *sg; + int i, nseg; + bus_dma_segment_t seg; + + priv = dev->dma_priv; + + DMA_PRIV_LOCK(priv); + + /* create common DMA map in the first S/G entry */ + if (bus_dmamap_create(priv->dmat, 0, &sgl->dma_map) != 0) { + DMA_PRIV_UNLOCK(priv); + return (0); + } + + /* load all S/G list entries */ + for_each_sg(sgl, sg, nents, i) { + nseg = -1; + if (_bus_dmamap_load_phys(priv->dmat, sgl->dma_map, + sg_phys(sg), sg->length, BUS_DMA_NOWAIT, + &seg, &nseg) != 0) { + bus_dmamap_unload(priv->dmat, sgl->dma_map); + bus_dmamap_destroy(priv->dmat, sgl->dma_map); + DMA_PRIV_UNLOCK(priv); + return (0); + } + KASSERT(nseg == 0, + ("More than one segment (nseg=%d)", nseg + 1)); + + sg_dma_address(sg) = seg.ds_addr; + } + DMA_PRIV_UNLOCK(priv); + + return (nents); +} + +void +linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, struct dma_attrs *attrs) +{ + struct linux_dma_priv *priv; + + priv = dev->dma_priv; + + DMA_PRIV_LOCK(priv); + bus_dmamap_unload(priv->dmat, sgl->dma_map); + bus_dmamap_destroy(priv->dmat, sgl->dma_map); + DMA_PRIV_UNLOCK(priv); +} + +struct dma_pool { + struct device *pool_device; + uma_zone_t pool_zone; + struct mtx pool_lock; + bus_dma_tag_t pool_dmat; + size_t pool_entry_size; + struct pctrie pool_ptree; +}; + +#define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock) +#define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock) + +static inline int +dma_pool_obj_ctor(void *mem, int size, void *arg, int flags) +{ + struct linux_dma_obj *obj = mem; + struct dma_pool *pool = arg; + int error, nseg; + bus_dma_segment_t seg; + + nseg = -1; + DMA_POOL_LOCK(pool); + error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap, + vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT, + &seg, &nseg); + DMA_POOL_UNLOCK(pool); + if (error != 0) { + return (error); + } + KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); + obj->dma_addr = seg.ds_addr; + + return (0); +} + +static void +dma_pool_obj_dtor(void *mem, int size, void *arg) +{ + struct linux_dma_obj *obj = mem; + struct dma_pool *pool = arg; + + DMA_POOL_LOCK(pool); + bus_dmamap_unload(pool->pool_dmat, obj->dmamap); + DMA_POOL_UNLOCK(pool); +} + +static int +dma_pool_obj_import(void *arg, void **store, int count, int domain __unused, + int flags) +{ + struct dma_pool *pool = arg; + struct linux_dma_priv *priv; + struct linux_dma_obj *obj; + int error, i; + + priv = pool->pool_device->dma_priv; + for (i = 0; i < count; i++) { + obj = uma_zalloc(linux_dma_obj_zone, flags); + if (obj == NULL) + break; + + error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr, + BUS_DMA_NOWAIT, &obj->dmamap); + if (error!= 0) { + uma_zfree(linux_dma_obj_zone, obj); + break; + } + + store[i] = obj; + } + + return (i); +} + +static void +dma_pool_obj_release(void *arg, void **store, int count) +{ + struct dma_pool *pool = arg; + struct linux_dma_priv *priv; + struct linux_dma_obj *obj; + int i; + + priv = pool->pool_device->dma_priv; + for (i = 0; i < count; i++) { + obj = store[i]; + bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap); + uma_zfree(linux_dma_obj_zone, obj); + } +} + +struct dma_pool * +linux_dma_pool_create(char *name, struct device *dev, size_t size, + size_t align, size_t boundary) +{ + struct linux_dma_priv *priv; + struct dma_pool *pool; + + priv = dev->dma_priv; + + pool = kzalloc(sizeof(*pool), GFP_KERNEL); + pool->pool_device = dev; + pool->pool_entry_size = size; + + if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), + align, boundary, /* alignment, boundary */ + priv->dma_mask, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filtfunc, filtfuncarg */ + size, /* maxsize */ + 1, /* nsegments */ + size, /* maxsegsz */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &pool->pool_dmat)) { + kfree(pool); + return (NULL); + } + + pool->pool_zone = uma_zcache_create(name, -1, dma_pool_obj_ctor, + dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import, + dma_pool_obj_release, pool, 0); + + mtx_init(&pool->pool_lock, "lkpi-dma-pool", NULL, MTX_DEF); + pctrie_init(&pool->pool_ptree); + + return (pool); +} + +void +linux_dma_pool_destroy(struct dma_pool *pool) +{ + + uma_zdestroy(pool->pool_zone); + bus_dma_tag_destroy(pool->pool_dmat); + mtx_destroy(&pool->pool_lock); + kfree(pool); +} + +void * +linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, + dma_addr_t *handle) +{ + struct linux_dma_obj *obj; + + obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags); + if (obj == NULL) + return (NULL); + + DMA_POOL_LOCK(pool); + if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) { + DMA_POOL_UNLOCK(pool); + uma_zfree_arg(pool->pool_zone, obj, pool); + return (NULL); + } + DMA_POOL_UNLOCK(pool); + + *handle = obj->dma_addr; + return (obj->vaddr); +} + +void +linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr) +{ + struct linux_dma_obj *obj; + + DMA_POOL_LOCK(pool); + obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr); + if (obj == NULL) { + DMA_POOL_UNLOCK(pool); + return; + } + LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr); + DMA_POOL_UNLOCK(pool); + + uma_zfree_arg(pool->pool_zone, obj, pool); +} diff --git a/sys/compat/linuxkpi/common/src/linux_radix.c b/sys/compat/linuxkpi/common/src/linux_radix.c new file mode 100644 index 000000000000..17c7b4bb48e7 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_radix.c @@ -0,0 +1,387 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2010 iX Systems, Inc. + * Copyright (c) 2010 Panasas, Inc. + * Copyright (c) 2013-2020 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> +#include <sys/kernel.h> +#include <sys/sysctl.h> + +#include <linux/slab.h> +#include <linux/kernel.h> +#include <linux/radix-tree.h> +#include <linux/err.h> + +static MALLOC_DEFINE(M_RADIX, "radix", "Linux radix compat"); + +static inline unsigned long +radix_max(struct radix_tree_root *root) +{ + return ((1UL << (root->height * RADIX_TREE_MAP_SHIFT)) - 1UL); +} + +static inline int +radix_pos(long id, int height) +{ + return (id >> (RADIX_TREE_MAP_SHIFT * height)) & RADIX_TREE_MAP_MASK; +} + +static void +radix_tree_clean_root_node(struct radix_tree_root *root) +{ + /* Check if the root node should be freed */ + if (root->rnode->count == 0) { + free(root->rnode, M_RADIX); + root->rnode = NULL; + root->height = 0; + } +} + +void * +radix_tree_lookup(struct radix_tree_root *root, unsigned long index) +{ + struct radix_tree_node *node; + void *item; + int height; + + item = NULL; + node = root->rnode; + height = root->height - 1; + if (index > radix_max(root)) + goto out; + while (height && node) + node = node->slots[radix_pos(index, height--)]; + if (node) + item = node->slots[radix_pos(index, 0)]; + +out: + return (item); +} + +bool +radix_tree_iter_find(struct radix_tree_root *root, struct radix_tree_iter *iter, + void ***pppslot) +{ + struct radix_tree_node *node; + unsigned long index = iter->index; + int height; + +restart: + node = root->rnode; + if (node == NULL) + return (false); + height = root->height - 1; + if (height == -1 || index > radix_max(root)) + return (false); + do { + unsigned long mask = RADIX_TREE_MAP_MASK << (RADIX_TREE_MAP_SHIFT * height); + unsigned long step = 1UL << (RADIX_TREE_MAP_SHIFT * height); + int pos = radix_pos(index, height); + struct radix_tree_node *next; + + /* track last slot */ + *pppslot = node->slots + pos; + + next = node->slots[pos]; + if (next == NULL) { + index += step; + index &= -step; + if ((index & mask) == 0) + goto restart; + } else { + node = next; + height--; + } + } while (height != -1); + iter->index = index; + return (true); +} + +void * +radix_tree_delete(struct radix_tree_root *root, unsigned long index) +{ + struct radix_tree_node *stack[RADIX_TREE_MAX_HEIGHT]; + struct radix_tree_node *node; + void *item; + int height; + int idx; + + item = NULL; + node = root->rnode; + height = root->height - 1; + if (index > radix_max(root)) + goto out; + /* + * Find the node and record the path in stack. + */ + while (height && node) { + stack[height] = node; + node = node->slots[radix_pos(index, height--)]; + } + idx = radix_pos(index, 0); + if (node) + item = node->slots[idx]; + /* + * If we removed something reduce the height of the tree. + */ + if (item) + for (;;) { + node->slots[idx] = NULL; + node->count--; + if (node->count > 0) + break; + free(node, M_RADIX); + if (node == root->rnode) { + root->rnode = NULL; + root->height = 0; + break; + } + height++; + node = stack[height]; + idx = radix_pos(index, height); + } +out: + return (item); +} + +void +radix_tree_iter_delete(struct radix_tree_root *root, + struct radix_tree_iter *iter, void **slot) +{ + radix_tree_delete(root, iter->index); +} + +int +radix_tree_insert(struct radix_tree_root *root, unsigned long index, void *item) +{ + struct radix_tree_node *node; + struct radix_tree_node *temp[RADIX_TREE_MAX_HEIGHT - 1]; + int height; + int idx; + + /* bail out upon insertion of a NULL item */ + if (item == NULL) + return (-EINVAL); + + /* get root node, if any */ + node = root->rnode; + + /* allocate root node, if any */ + if (node == NULL) { + node = malloc(sizeof(*node), M_RADIX, root->gfp_mask | M_ZERO); + if (node == NULL) + return (-ENOMEM); + root->rnode = node; + root->height++; + } + + /* expand radix tree as needed */ + while (radix_max(root) < index) { + + /* check if the radix tree is getting too big */ + if (root->height == RADIX_TREE_MAX_HEIGHT) { + radix_tree_clean_root_node(root); + return (-E2BIG); + } + + /* + * If the root radix level is not empty, we need to + * allocate a new radix level: + */ + if (node->count != 0) { + node = malloc(sizeof(*node), M_RADIX, root->gfp_mask | M_ZERO); + if (node == NULL) { + /* + * Freeing the already allocated radix + * levels, if any, will be handled by + * the radix_tree_delete() function. + * This code path can only happen when + * the tree is not empty. + */ + return (-ENOMEM); + } + node->slots[0] = root->rnode; + node->count++; + root->rnode = node; + } + root->height++; + } + + /* get radix tree height index */ + height = root->height - 1; + + /* walk down the tree until the first missing node, if any */ + for ( ; height != 0; height--) { + idx = radix_pos(index, height); + if (node->slots[idx] == NULL) + break; + node = node->slots[idx]; + } + + /* allocate the missing radix levels, if any */ + for (idx = 0; idx != height; idx++) { + temp[idx] = malloc(sizeof(*node), M_RADIX, + root->gfp_mask | M_ZERO); + if (temp[idx] == NULL) { + while (idx--) + free(temp[idx], M_RADIX); + radix_tree_clean_root_node(root); + return (-ENOMEM); + } + } + + /* setup new radix levels, if any */ + for ( ; height != 0; height--) { + idx = radix_pos(index, height); + node->slots[idx] = temp[height - 1]; + node->count++; + node = node->slots[idx]; + } + + /* + * Insert and adjust count if the item does not already exist. + */ + idx = radix_pos(index, 0); + if (node->slots[idx]) + return (-EEXIST); + node->slots[idx] = item; + node->count++; + + return (0); +} + +int +radix_tree_store(struct radix_tree_root *root, unsigned long index, void **ppitem) +{ + struct radix_tree_node *node; + struct radix_tree_node *temp[RADIX_TREE_MAX_HEIGHT - 1]; + void *pitem; + int height; + int idx; + + /* + * Inserting a NULL item means delete it. The old pointer is + * stored at the location pointed to by "ppitem". + */ + if (*ppitem == NULL) { + *ppitem = radix_tree_delete(root, index); + return (0); + } + + /* get root node, if any */ + node = root->rnode; + + /* allocate root node, if any */ + if (node == NULL) { + node = malloc(sizeof(*node), M_RADIX, root->gfp_mask | M_ZERO); + if (node == NULL) + return (-ENOMEM); + root->rnode = node; + root->height++; + } + + /* expand radix tree as needed */ + while (radix_max(root) < index) { + + /* check if the radix tree is getting too big */ + if (root->height == RADIX_TREE_MAX_HEIGHT) { + radix_tree_clean_root_node(root); + return (-E2BIG); + } + + /* + * If the root radix level is not empty, we need to + * allocate a new radix level: + */ + if (node->count != 0) { + node = malloc(sizeof(*node), M_RADIX, root->gfp_mask | M_ZERO); + if (node == NULL) { + /* + * Freeing the already allocated radix + * levels, if any, will be handled by + * the radix_tree_delete() function. + * This code path can only happen when + * the tree is not empty. + */ + return (-ENOMEM); + } + node->slots[0] = root->rnode; + node->count++; + root->rnode = node; + } + root->height++; + } + + /* get radix tree height index */ + height = root->height - 1; + + /* walk down the tree until the first missing node, if any */ + for ( ; height != 0; height--) { + idx = radix_pos(index, height); + if (node->slots[idx] == NULL) + break; + node = node->slots[idx]; + } + + /* allocate the missing radix levels, if any */ + for (idx = 0; idx != height; idx++) { + temp[idx] = malloc(sizeof(*node), M_RADIX, + root->gfp_mask | M_ZERO); + if (temp[idx] == NULL) { + while (idx--) + free(temp[idx], M_RADIX); + radix_tree_clean_root_node(root); + return (-ENOMEM); + } + } + + /* setup new radix levels, if any */ + for ( ; height != 0; height--) { + idx = radix_pos(index, height); + node->slots[idx] = temp[height - 1]; + node->count++; + node = node->slots[idx]; + } + + /* + * Insert and adjust count if the item does not already exist. + */ + idx = radix_pos(index, 0); + /* swap */ + pitem = node->slots[idx]; + node->slots[idx] = *ppitem; + *ppitem = pitem; + + if (pitem == NULL) + node->count++; + return (0); +} diff --git a/sys/compat/linuxkpi/common/src/linux_rcu.c b/sys/compat/linuxkpi/common/src/linux_rcu.c new file mode 100644 index 000000000000..5e698739ab09 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_rcu.c @@ -0,0 +1,422 @@ +/*- + * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) + * Copyright (c) 2017-2020 Hans Petter Selasky (hselasky@freebsd.org) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/types.h> +#include <sys/systm.h> +#include <sys/malloc.h> +#include <sys/kernel.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/proc.h> +#include <sys/sched.h> +#include <sys/smp.h> +#include <sys/queue.h> +#include <sys/taskqueue.h> +#include <sys/kdb.h> + +#include <ck_epoch.h> + +#include <linux/rcupdate.h> +#include <linux/srcu.h> +#include <linux/slab.h> +#include <linux/kernel.h> +#include <linux/compat.h> + +/* + * By defining CONFIG_NO_RCU_SKIP LinuxKPI RCU locks and asserts will + * not be skipped during panic(). + */ +#ifdef CONFIG_NO_RCU_SKIP +#define RCU_SKIP(void) 0 +#else +#define RCU_SKIP(void) unlikely(SCHEDULER_STOPPED() || kdb_active) +#endif + +struct callback_head { + STAILQ_ENTRY(callback_head) entry; + rcu_callback_t func; +}; + +struct linux_epoch_head { + STAILQ_HEAD(, callback_head) cb_head; + struct mtx lock; + struct task task; +} __aligned(CACHE_LINE_SIZE); + +struct linux_epoch_record { + ck_epoch_record_t epoch_record; + TAILQ_HEAD(, task_struct) ts_head; + int cpuid; + int type; +} __aligned(CACHE_LINE_SIZE); + +/* + * Verify that "struct rcu_head" is big enough to hold "struct + * callback_head". This has been done to avoid having to add special + * compile flags for including ck_epoch.h to all clients of the + * LinuxKPI. + */ +CTASSERT(sizeof(struct rcu_head) == sizeof(struct callback_head)); + +/* + * Verify that "epoch_record" is at beginning of "struct + * linux_epoch_record": + */ +CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0); + +CTASSERT(TS_RCU_TYPE_MAX == RCU_TYPE_MAX); + +static ck_epoch_t linux_epoch[RCU_TYPE_MAX]; +static struct linux_epoch_head linux_epoch_head[RCU_TYPE_MAX]; +DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record[RCU_TYPE_MAX]); + +static void linux_rcu_cleaner_func(void *, int); + +static void +linux_rcu_runtime_init(void *arg __unused) +{ + struct linux_epoch_head *head; + int i; + int j; + + for (j = 0; j != RCU_TYPE_MAX; j++) { + ck_epoch_init(&linux_epoch[j]); + + head = &linux_epoch_head[j]; + + mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF); + TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head); + STAILQ_INIT(&head->cb_head); + + CPU_FOREACH(i) { + struct linux_epoch_record *record; + + record = &DPCPU_ID_GET(i, linux_epoch_record[j]); + + record->cpuid = i; + record->type = j; + ck_epoch_register(&linux_epoch[j], + &record->epoch_record, NULL); + TAILQ_INIT(&record->ts_head); + } + } +} +SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL); + +static void +linux_rcu_runtime_uninit(void *arg __unused) +{ + struct linux_epoch_head *head; + int j; + + for (j = 0; j != RCU_TYPE_MAX; j++) { + head = &linux_epoch_head[j]; + + mtx_destroy(&head->lock); + } +} +SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL); + +static void +linux_rcu_cleaner_func(void *context, int pending __unused) +{ + struct linux_epoch_head *head; + struct callback_head *rcu; + STAILQ_HEAD(, callback_head) tmp_head; + uintptr_t offset; + + linux_set_current(curthread); + + head = context; + + /* move current callbacks into own queue */ + mtx_lock(&head->lock); + STAILQ_INIT(&tmp_head); + STAILQ_CONCAT(&tmp_head, &head->cb_head); + mtx_unlock(&head->lock); + + /* synchronize */ + linux_synchronize_rcu(head - linux_epoch_head); + + /* dispatch all callbacks, if any */ + while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) { + + STAILQ_REMOVE_HEAD(&tmp_head, entry); + + offset = (uintptr_t)rcu->func; + + if (offset < LINUX_KFREE_RCU_OFFSET_MAX) + kfree((char *)rcu - offset); + else + rcu->func((struct rcu_head *)rcu); + } +} + +void +linux_rcu_read_lock(unsigned type) +{ + struct linux_epoch_record *record; + struct task_struct *ts; + + MPASS(type < RCU_TYPE_MAX); + + if (RCU_SKIP()) + return; + + /* + * Pin thread to current CPU so that the unlock code gets the + * same per-CPU epoch record: + */ + sched_pin(); + + record = &DPCPU_GET(linux_epoch_record[type]); + ts = current; + + /* + * Use a critical section to prevent recursion inside + * ck_epoch_begin(). Else this function supports recursion. + */ + critical_enter(); + ck_epoch_begin(&record->epoch_record, NULL); + ts->rcu_recurse[type]++; + if (ts->rcu_recurse[type] == 1) + TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]); + critical_exit(); +} + +void +linux_rcu_read_unlock(unsigned type) +{ + struct linux_epoch_record *record; + struct task_struct *ts; + + MPASS(type < RCU_TYPE_MAX); + + if (RCU_SKIP()) + return; + + record = &DPCPU_GET(linux_epoch_record[type]); + ts = current; + + /* + * Use a critical section to prevent recursion inside + * ck_epoch_end(). Else this function supports recursion. + */ + critical_enter(); + ck_epoch_end(&record->epoch_record, NULL); + ts->rcu_recurse[type]--; + if (ts->rcu_recurse[type] == 0) + TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]); + critical_exit(); + + sched_unpin(); +} + +static void +linux_synchronize_rcu_cb(ck_epoch_t *epoch __unused, ck_epoch_record_t *epoch_record, void *arg __unused) +{ + struct linux_epoch_record *record = + container_of(epoch_record, struct linux_epoch_record, epoch_record); + struct thread *td = curthread; + struct task_struct *ts; + + /* check if blocked on the current CPU */ + if (record->cpuid == PCPU_GET(cpuid)) { + bool is_sleeping = 0; + u_char prio = 0; + + /* + * Find the lowest priority or sleeping thread which + * is blocking synchronization on this CPU core. All + * the threads in the queue are CPU-pinned and cannot + * go anywhere while the current thread is locked. + */ + TAILQ_FOREACH(ts, &record->ts_head, rcu_entry[record->type]) { + if (ts->task_thread->td_priority > prio) + prio = ts->task_thread->td_priority; + is_sleeping |= (ts->task_thread->td_inhibitors != 0); + } + + if (is_sleeping) { + thread_unlock(td); + pause("W", 1); + thread_lock(td); + } else { + /* set new thread priority */ + sched_prio(td, prio); + /* task switch */ + mi_switch(SW_VOL | SWT_RELINQUISH); + /* + * It is important the thread lock is dropped + * while yielding to allow other threads to + * acquire the lock pointed to by + * TDQ_LOCKPTR(td). Currently mi_switch() will + * unlock the thread lock before + * returning. Else a deadlock like situation + * might happen. + */ + thread_lock(td); + } + } else { + /* + * To avoid spinning move execution to the other CPU + * which is blocking synchronization. Set highest + * thread priority so that code gets run. The thread + * priority will be restored later. + */ + sched_prio(td, 0); + sched_bind(td, record->cpuid); + } +} + +void +linux_synchronize_rcu(unsigned type) +{ + struct thread *td; + int was_bound; + int old_cpu; + int old_pinned; + u_char old_prio; + + MPASS(type < RCU_TYPE_MAX); + + if (RCU_SKIP()) + return; + + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "linux_synchronize_rcu() can sleep"); + + td = curthread; + DROP_GIANT(); + + /* + * Synchronizing RCU might change the CPU core this function + * is running on. Save current values: + */ + thread_lock(td); + + old_cpu = PCPU_GET(cpuid); + old_pinned = td->td_pinned; + old_prio = td->td_priority; + was_bound = sched_is_bound(td); + sched_unbind(td); + td->td_pinned = 0; + sched_bind(td, old_cpu); + + ck_epoch_synchronize_wait(&linux_epoch[type], + &linux_synchronize_rcu_cb, NULL); + + /* restore CPU binding, if any */ + if (was_bound != 0) { + sched_bind(td, old_cpu); + } else { + /* get thread back to initial CPU, if any */ + if (old_pinned != 0) + sched_bind(td, old_cpu); + sched_unbind(td); + } + /* restore pinned after bind */ + td->td_pinned = old_pinned; + + /* restore thread priority */ + sched_prio(td, old_prio); + thread_unlock(td); + + PICKUP_GIANT(); +} + +void +linux_rcu_barrier(unsigned type) +{ + struct linux_epoch_head *head; + + MPASS(type < RCU_TYPE_MAX); + + linux_synchronize_rcu(type); + + head = &linux_epoch_head[type]; + + /* wait for callbacks to complete */ + taskqueue_drain(taskqueue_fast, &head->task); +} + +void +linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func) +{ + struct callback_head *rcu; + struct linux_epoch_head *head; + + MPASS(type < RCU_TYPE_MAX); + + rcu = (struct callback_head *)context; + head = &linux_epoch_head[type]; + + mtx_lock(&head->lock); + rcu->func = func; + STAILQ_INSERT_TAIL(&head->cb_head, rcu, entry); + taskqueue_enqueue(taskqueue_fast, &head->task); + mtx_unlock(&head->lock); +} + +int +init_srcu_struct(struct srcu_struct *srcu) +{ + return (0); +} + +void +cleanup_srcu_struct(struct srcu_struct *srcu) +{ +} + +int +srcu_read_lock(struct srcu_struct *srcu) +{ + linux_rcu_read_lock(RCU_TYPE_SLEEPABLE); + return (0); +} + +void +srcu_read_unlock(struct srcu_struct *srcu, int key __unused) +{ + linux_rcu_read_unlock(RCU_TYPE_SLEEPABLE); +} + +void +synchronize_srcu(struct srcu_struct *srcu) +{ + linux_synchronize_rcu(RCU_TYPE_SLEEPABLE); +} + +void +srcu_barrier(struct srcu_struct *srcu) +{ + linux_rcu_barrier(RCU_TYPE_SLEEPABLE); +} diff --git a/sys/compat/linuxkpi/common/src/linux_schedule.c b/sys/compat/linuxkpi/common/src/linux_schedule.c new file mode 100644 index 000000000000..656d8697d169 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_schedule.c @@ -0,0 +1,432 @@ +/*- + * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conds + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conds, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conds and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/proc.h> +#include <sys/signalvar.h> +#include <sys/sleepqueue.h> + +#include <linux/delay.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/sched.h> +#include <linux/spinlock.h> +#include <linux/wait.h> + +static int +linux_add_to_sleepqueue(void *wchan, struct task_struct *task, + const char *wmesg, int timeout, int state) +{ + int flags, ret; + + MPASS((state & ~(TASK_PARKED | TASK_NORMAL)) == 0); + + flags = SLEEPQ_SLEEP | ((state & TASK_INTERRUPTIBLE) != 0 ? + SLEEPQ_INTERRUPTIBLE : 0); + + sleepq_add(wchan, NULL, wmesg, flags, 0); + if (timeout != 0) + sleepq_set_timeout(wchan, timeout); + + DROP_GIANT(); + if ((state & TASK_INTERRUPTIBLE) != 0) { + if (timeout == 0) + ret = -sleepq_wait_sig(wchan, 0); + else + ret = -sleepq_timedwait_sig(wchan, 0); + } else { + if (timeout == 0) { + sleepq_wait(wchan, 0); + ret = 0; + } else + ret = -sleepq_timedwait(wchan, 0); + } + PICKUP_GIANT(); + + /* filter return value */ + if (ret != 0 && ret != -EWOULDBLOCK) { + linux_schedule_save_interrupt_value(task, ret); + ret = -ERESTARTSYS; + } + return (ret); +} + +unsigned int +linux_msleep_interruptible(unsigned int ms) +{ + int ret; + + /* guard against invalid values */ + if (ms == 0) + ms = 1; + ret = -pause_sbt("lnxsleep", mstosbt(ms), 0, C_HARDCLOCK | C_CATCH); + + switch (ret) { + case -EWOULDBLOCK: + return (0); + default: + linux_schedule_save_interrupt_value(current, ret); + return (ms); + } +} + +static int +wake_up_task(struct task_struct *task, unsigned int state) +{ + int ret, wakeup_swapper; + + ret = wakeup_swapper = 0; + sleepq_lock(task); + if ((atomic_read(&task->state) & state) != 0) { + set_task_state(task, TASK_WAKING); + wakeup_swapper = sleepq_signal(task, SLEEPQ_SLEEP, 0, 0); + ret = 1; + } + sleepq_release(task); + if (wakeup_swapper) + kick_proc0(); + return (ret); +} + +bool +linux_signal_pending(struct task_struct *task) +{ + struct thread *td; + sigset_t pending; + + td = task->task_thread; + PROC_LOCK(td->td_proc); + pending = td->td_siglist; + SIGSETOR(pending, td->td_proc->p_siglist); + SIGSETNAND(pending, td->td_sigmask); + PROC_UNLOCK(td->td_proc); + return (!SIGISEMPTY(pending)); +} + +bool +linux_fatal_signal_pending(struct task_struct *task) +{ + struct thread *td; + bool ret; + + td = task->task_thread; + PROC_LOCK(td->td_proc); + ret = SIGISMEMBER(td->td_siglist, SIGKILL) || + SIGISMEMBER(td->td_proc->p_siglist, SIGKILL); + PROC_UNLOCK(td->td_proc); + return (ret); +} + +bool +linux_signal_pending_state(long state, struct task_struct *task) +{ + + MPASS((state & ~TASK_NORMAL) == 0); + + if ((state & TASK_INTERRUPTIBLE) == 0) + return (false); + return (linux_signal_pending(task)); +} + +void +linux_send_sig(int signo, struct task_struct *task) +{ + struct thread *td; + + td = task->task_thread; + PROC_LOCK(td->td_proc); + tdsignal(td, signo); + PROC_UNLOCK(td->td_proc); +} + +int +autoremove_wake_function(wait_queue_t *wq, unsigned int state, int flags, + void *key __unused) +{ + struct task_struct *task; + int ret; + + task = wq->private; + if ((ret = wake_up_task(task, state)) != 0) + list_del_init(&wq->task_list); + return (ret); +} + +int +default_wake_function(wait_queue_t *wq, unsigned int state, int flags, + void *key __unused) +{ + return (wake_up_task(wq->private, state)); +} + +void +linux_init_wait_entry(wait_queue_t *wq, int flags) +{ + + memset(wq, 0, sizeof(*wq)); + wq->flags = flags; + wq->private = current; + wq->func = autoremove_wake_function; + INIT_LIST_HEAD(&wq->task_list); +} + +void +linux_wake_up(wait_queue_head_t *wqh, unsigned int state, int nr, bool locked) +{ + wait_queue_t *pos, *next; + + if (!locked) + spin_lock(&wqh->lock); + list_for_each_entry_safe(pos, next, &wqh->task_list, task_list) { + if (pos->func == NULL) { + if (wake_up_task(pos->private, state) != 0 && --nr == 0) + break; + } else { + if (pos->func(pos, state, 0, NULL) != 0 && --nr == 0) + break; + } + } + if (!locked) + spin_unlock(&wqh->lock); +} + +void +linux_prepare_to_wait(wait_queue_head_t *wqh, wait_queue_t *wq, int state) +{ + + spin_lock(&wqh->lock); + if (list_empty(&wq->task_list)) + __add_wait_queue(wqh, wq); + set_task_state(current, state); + spin_unlock(&wqh->lock); +} + +void +linux_finish_wait(wait_queue_head_t *wqh, wait_queue_t *wq) +{ + + spin_lock(&wqh->lock); + set_task_state(current, TASK_RUNNING); + if (!list_empty(&wq->task_list)) { + __remove_wait_queue(wqh, wq); + INIT_LIST_HEAD(&wq->task_list); + } + spin_unlock(&wqh->lock); +} + +bool +linux_waitqueue_active(wait_queue_head_t *wqh) +{ + bool ret; + + spin_lock(&wqh->lock); + ret = !list_empty(&wqh->task_list); + spin_unlock(&wqh->lock); + return (ret); +} + +int +linux_wait_event_common(wait_queue_head_t *wqh, wait_queue_t *wq, int timeout, + unsigned int state, spinlock_t *lock) +{ + struct task_struct *task; + int ret; + + if (lock != NULL) + spin_unlock_irq(lock); + + /* range check timeout */ + if (timeout < 1) + timeout = 1; + else if (timeout == MAX_SCHEDULE_TIMEOUT) + timeout = 0; + + task = current; + + /* + * Our wait queue entry is on the stack - make sure it doesn't + * get swapped out while we sleep. + */ + PHOLD(task->task_thread->td_proc); + sleepq_lock(task); + if (atomic_read(&task->state) != TASK_WAKING) { + ret = linux_add_to_sleepqueue(task, task, "wevent", timeout, + state); + } else { + sleepq_release(task); + ret = 0; + } + PRELE(task->task_thread->td_proc); + + if (lock != NULL) + spin_lock_irq(lock); + return (ret); +} + +int +linux_schedule_timeout(int timeout) +{ + struct task_struct *task; + int ret; + int state; + int remainder; + + task = current; + + /* range check timeout */ + if (timeout < 1) + timeout = 1; + else if (timeout == MAX_SCHEDULE_TIMEOUT) + timeout = 0; + + remainder = ticks + timeout; + + sleepq_lock(task); + state = atomic_read(&task->state); + if (state != TASK_WAKING) { + ret = linux_add_to_sleepqueue(task, task, "sched", timeout, + state); + } else { + sleepq_release(task); + ret = 0; + } + set_task_state(task, TASK_RUNNING); + + if (timeout == 0) + return (MAX_SCHEDULE_TIMEOUT); + + /* range check return value */ + remainder -= ticks; + + /* range check return value */ + if (ret == -ERESTARTSYS && remainder < 1) + remainder = 1; + else if (remainder < 0) + remainder = 0; + else if (remainder > timeout) + remainder = timeout; + return (remainder); +} + +static void +wake_up_sleepers(void *wchan) +{ + int wakeup_swapper; + + sleepq_lock(wchan); + wakeup_swapper = sleepq_signal(wchan, SLEEPQ_SLEEP, 0, 0); + sleepq_release(wchan); + if (wakeup_swapper) + kick_proc0(); +} + +#define bit_to_wchan(word, bit) ((void *)(((uintptr_t)(word) << 6) | (bit))) + +void +linux_wake_up_bit(void *word, int bit) +{ + + wake_up_sleepers(bit_to_wchan(word, bit)); +} + +int +linux_wait_on_bit_timeout(unsigned long *word, int bit, unsigned int state, + int timeout) +{ + struct task_struct *task; + void *wchan; + int ret; + + /* range check timeout */ + if (timeout < 1) + timeout = 1; + else if (timeout == MAX_SCHEDULE_TIMEOUT) + timeout = 0; + + task = current; + wchan = bit_to_wchan(word, bit); + for (;;) { + sleepq_lock(wchan); + if ((*word & (1 << bit)) == 0) { + sleepq_release(wchan); + ret = 0; + break; + } + set_task_state(task, state); + ret = linux_add_to_sleepqueue(wchan, task, "wbit", timeout, + state); + if (ret != 0) + break; + } + set_task_state(task, TASK_RUNNING); + + return (ret); +} + +void +linux_wake_up_atomic_t(atomic_t *a) +{ + + wake_up_sleepers(a); +} + +int +linux_wait_on_atomic_t(atomic_t *a, unsigned int state) +{ + struct task_struct *task; + void *wchan; + int ret; + + task = current; + wchan = a; + for (;;) { + sleepq_lock(wchan); + if (atomic_read(a) == 0) { + sleepq_release(wchan); + ret = 0; + break; + } + set_task_state(task, state); + ret = linux_add_to_sleepqueue(wchan, task, "watomic", 0, state); + if (ret != 0) + break; + } + set_task_state(task, TASK_RUNNING); + + return (ret); +} + +bool +linux_wake_up_state(struct task_struct *task, unsigned int state) +{ + + return (wake_up_task(task, state) != 0); +} diff --git a/sys/compat/linuxkpi/common/src/linux_seq_file.c b/sys/compat/linuxkpi/common/src/linux_seq_file.c new file mode 100644 index 000000000000..b9eb15cdb47e --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_seq_file.c @@ -0,0 +1,157 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2016-2018, Matthew Macy <mmacy@freebsd.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/types.h> +#include <sys/systm.h> +#include <sys/param.h> +#include <sys/sbuf.h> +#include <sys/syslog.h> +#include <sys/vnode.h> + +#include <linux/seq_file.h> +#include <linux/file.h> + +#undef file +MALLOC_DEFINE(M_LSEQ, "seq_file", "seq_file"); + +ssize_t +seq_read(struct linux_file *f, char *ubuf, size_t size, off_t *ppos) +{ + struct seq_file *m = f->private_data; + void *p; + int rc; + off_t pos = 0; + + p = m->op->start(m, &pos); + rc = m->op->show(m, p); + if (rc) + return (rc); + return (size); +} + +int +seq_write(struct seq_file *seq, const void *data, size_t len) +{ + + return (sbuf_bcpy(seq->buf, data, len)); +} + +/* + * This only needs to be a valid address for lkpi + * drivers it should never actually be called + */ +off_t +seq_lseek(struct linux_file *file, off_t offset, int whence) +{ + + panic("%s not supported\n", __FUNCTION__); + return (0); +} + +static void * +single_start(struct seq_file *p, off_t *pos) +{ + + return ((void *)(uintptr_t)(*pos == 0)); +} + +static void * +single_next(struct seq_file *p, void *v, off_t *pos) +{ + + ++*pos; + return (NULL); +} + +static void +single_stop(struct seq_file *p, void *v) +{ +} + +int +seq_open(struct linux_file *f, const struct seq_operations *op) +{ + struct seq_file *p; + + if (f->private_data != NULL) + log(LOG_WARNING, "%s private_data not NULL", __func__); + + if ((p = malloc(sizeof(*p), M_LSEQ, M_NOWAIT|M_ZERO)) == NULL) + return (-ENOMEM); + + f->private_data = p; + p->op = op; + p->file = f; + return (0); +} + +int +single_open(struct linux_file *f, int (*show)(struct seq_file *, void *), void *d) +{ + struct seq_operations *op; + int rc = -ENOMEM; + + op = malloc(sizeof(*op), M_LSEQ, M_NOWAIT); + if (op) { + op->start = single_start; + op->next = single_next; + op->stop = single_stop; + op->show = show; + rc = seq_open(f, op); + if (rc) + free(op, M_LSEQ); + else + ((struct seq_file *)f->private_data)->private = d; + + } + return (rc); +} + +int +seq_release(struct inode *inode __unused, struct linux_file *file) +{ + struct seq_file *m; + + m = file->private_data; + free(m, M_LSEQ); + return (0); +} + +int +single_release(struct vnode *v, struct linux_file *f) +{ + const struct seq_operations *op = ((struct seq_file *)f->private_data)->op; + int rc; + + rc = seq_release(v, f); + free(__DECONST(void *, op), M_LSEQ); + return (rc); +} diff --git a/sys/compat/linuxkpi/common/src/linux_shmemfs.c b/sys/compat/linuxkpi/common/src/linux_shmemfs.c new file mode 100644 index 000000000000..ead9cc9d9f40 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_shmemfs.c @@ -0,0 +1,128 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/rwlock.h> + +#include <vm/vm.h> +#include <vm/pmap.h> +#include <vm/vm_object.h> +#include <vm/vm_map.h> +#include <vm/vm_page.h> +#include <vm/vm_pager.h> + +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/shmem_fs.h> + +struct page * +linux_shmem_read_mapping_page_gfp(vm_object_t obj, int pindex, gfp_t gfp) +{ + vm_page_t page; + int rv; + + if ((gfp & GFP_NOWAIT) != 0) + panic("GFP_NOWAIT is unimplemented"); + + VM_OBJECT_WLOCK(obj); + rv = vm_page_grab_valid(&page, obj, pindex, VM_ALLOC_NORMAL | + VM_ALLOC_NOBUSY | VM_ALLOC_WIRED); + VM_OBJECT_WUNLOCK(obj); + if (rv != VM_PAGER_OK) + return (ERR_PTR(-EINVAL)); + return (page); +} + +struct linux_file * +linux_shmem_file_setup(const char *name, loff_t size, unsigned long flags) +{ + struct fileobj { + struct linux_file file __aligned(sizeof(void *)); + struct vnode vnode __aligned(sizeof(void *)); + }; + struct fileobj *fileobj; + struct linux_file *filp; + struct vnode *vp; + int error; + + fileobj = kzalloc(sizeof(*fileobj), GFP_KERNEL); + if (fileobj == NULL) { + error = -ENOMEM; + goto err_0; + } + filp = &fileobj->file; + vp = &fileobj->vnode; + + filp->f_count = 1; + filp->f_vnode = vp; + filp->f_shmem = vm_pager_allocate(OBJT_DEFAULT, NULL, size, + VM_PROT_READ | VM_PROT_WRITE, 0, curthread->td_ucred); + if (filp->f_shmem == NULL) { + error = -ENOMEM; + goto err_1; + } + return (filp); +err_1: + kfree(filp); +err_0: + return (ERR_PTR(error)); +} + +static vm_ooffset_t +linux_invalidate_mapping_pages_sub(vm_object_t obj, vm_pindex_t start, + vm_pindex_t end, int flags) +{ + int start_count, end_count; + + VM_OBJECT_WLOCK(obj); + start_count = obj->resident_page_count; + vm_object_page_remove(obj, start, end, flags); + end_count = obj->resident_page_count; + VM_OBJECT_WUNLOCK(obj); + return (start_count - end_count); +} + +unsigned long +linux_invalidate_mapping_pages(vm_object_t obj, pgoff_t start, pgoff_t end) +{ + + return (linux_invalidate_mapping_pages_sub(obj, start, end, OBJPR_CLEANONLY)); +} + +void +linux_shmem_truncate_range(vm_object_t obj, loff_t lstart, loff_t lend) +{ + vm_pindex_t start = OFF_TO_IDX(lstart + PAGE_SIZE - 1); + vm_pindex_t end = OFF_TO_IDX(lend + 1); + + (void) linux_invalidate_mapping_pages_sub(obj, start, end, 0); +} diff --git a/sys/compat/linuxkpi/common/src/linux_slab.c b/sys/compat/linuxkpi/common/src/linux_slab.c new file mode 100644 index 000000000000..c8deab01731a --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_slab.c @@ -0,0 +1,128 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <linux/slab.h> +#include <linux/rcupdate.h> +#include <linux/kernel.h> + +struct linux_kmem_rcu { + struct rcu_head rcu_head; + struct linux_kmem_cache *cache; +}; + +#define LINUX_KMEM_TO_RCU(c, m) \ + ((struct linux_kmem_rcu *)((char *)(m) + \ + (c)->cache_size - sizeof(struct linux_kmem_rcu))) + +#define LINUX_RCU_TO_KMEM(r) \ + ((void *)((char *)(r) + sizeof(struct linux_kmem_rcu) - \ + (r)->cache->cache_size)) + +static int +linux_kmem_ctor(void *mem, int size, void *arg, int flags) +{ + struct linux_kmem_cache *c = arg; + + if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU)) { + struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, mem); + + /* duplicate cache pointer */ + rcu->cache = c; + } + + /* check for constructor */ + if (likely(c->cache_ctor != NULL)) + c->cache_ctor(mem); + + return (0); +} + +static void +linux_kmem_cache_free_rcu_callback(struct rcu_head *head) +{ + struct linux_kmem_rcu *rcu = + container_of(head, struct linux_kmem_rcu, rcu_head); + + uma_zfree(rcu->cache->cache_zone, LINUX_RCU_TO_KMEM(rcu)); +} + +struct linux_kmem_cache * +linux_kmem_cache_create(const char *name, size_t size, size_t align, + unsigned flags, linux_kmem_ctor_t *ctor) +{ + struct linux_kmem_cache *c; + + c = malloc(sizeof(*c), M_KMALLOC, M_WAITOK); + + if (flags & SLAB_HWCACHE_ALIGN) + align = UMA_ALIGN_CACHE; + else if (align != 0) + align--; + + if (flags & SLAB_TYPESAFE_BY_RCU) { + /* make room for RCU structure */ + size = ALIGN(size, sizeof(void *)); + size += sizeof(struct linux_kmem_rcu); + + /* create cache_zone */ + c->cache_zone = uma_zcreate(name, size, + linux_kmem_ctor, NULL, NULL, NULL, + align, UMA_ZONE_ZINIT); + } else { + /* create cache_zone */ + c->cache_zone = uma_zcreate(name, size, + ctor ? linux_kmem_ctor : NULL, NULL, + NULL, NULL, align, 0); + } + + c->cache_flags = flags; + c->cache_ctor = ctor; + c->cache_size = size; + return (c); +} + +void +linux_kmem_cache_free_rcu(struct linux_kmem_cache *c, void *m) +{ + struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, m); + + call_rcu(&rcu->rcu_head, linux_kmem_cache_free_rcu_callback); +} + +void +linux_kmem_cache_destroy(struct linux_kmem_cache *c) +{ + if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU)) { + /* make sure all free callbacks have been called */ + rcu_barrier(); + } + + uma_zdestroy(c->cache_zone); + free(c, M_KMALLOC); +} diff --git a/sys/compat/linuxkpi/common/src/linux_tasklet.c b/sys/compat/linuxkpi/common/src/linux_tasklet.c new file mode 100644 index 000000000000..5cac8ffd994c --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_tasklet.c @@ -0,0 +1,253 @@ +/*- + * Copyright (c) 2017 Hans Petter Selasky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/types.h> +#include <sys/malloc.h> +#include <sys/gtaskqueue.h> +#include <sys/proc.h> +#include <sys/sched.h> + +#include <linux/compiler.h> +#include <linux/interrupt.h> +#include <linux/compat.h> + +#define TASKLET_ST_IDLE 0 +#define TASKLET_ST_BUSY 1 +#define TASKLET_ST_EXEC 2 +#define TASKLET_ST_LOOP 3 + +#define TASKLET_ST_CMPSET(ts, old, new) \ + atomic_cmpset_int((volatile u_int *)&(ts)->tasklet_state, old, new) + +#define TASKLET_ST_SET(ts, new) \ + WRITE_ONCE(*(volatile u_int *)&(ts)->tasklet_state, new) + +#define TASKLET_ST_GET(ts) \ + READ_ONCE(*(volatile u_int *)&(ts)->tasklet_state) + +struct tasklet_worker { + struct mtx mtx; + TAILQ_HEAD(tasklet_list, tasklet_struct) head; + struct grouptask gtask; +} __aligned(CACHE_LINE_SIZE); + +#define TASKLET_WORKER_LOCK(tw) mtx_lock(&(tw)->mtx) +#define TASKLET_WORKER_UNLOCK(tw) mtx_unlock(&(tw)->mtx) + +DPCPU_DEFINE_STATIC(struct tasklet_worker, tasklet_worker); + +static void +tasklet_handler(void *arg) +{ + struct tasklet_worker *tw = (struct tasklet_worker *)arg; + struct tasklet_struct *ts; + struct tasklet_struct *last; + + linux_set_current(curthread); + + TASKLET_WORKER_LOCK(tw); + last = TAILQ_LAST(&tw->head, tasklet_list); + while (1) { + ts = TAILQ_FIRST(&tw->head); + if (ts == NULL) + break; + TAILQ_REMOVE(&tw->head, ts, entry); + + if (!atomic_read(&ts->count)) { + TASKLET_WORKER_UNLOCK(tw); + do { + /* reset executing state */ + TASKLET_ST_SET(ts, TASKLET_ST_EXEC); + + ts->func(ts->data); + + } while (TASKLET_ST_CMPSET(ts, TASKLET_ST_EXEC, + TASKLET_ST_IDLE) == 0); + TASKLET_WORKER_LOCK(tw); + } else { + TAILQ_INSERT_TAIL(&tw->head, ts, entry); + } + if (ts == last) + break; + } + TASKLET_WORKER_UNLOCK(tw); +} + +static void +tasklet_subsystem_init(void *arg __unused) +{ + struct tasklet_worker *tw; + char buf[32]; + int i; + + CPU_FOREACH(i) { + if (CPU_ABSENT(i)) + continue; + + tw = DPCPU_ID_PTR(i, tasklet_worker); + + mtx_init(&tw->mtx, "linux_tasklet", NULL, MTX_DEF); + TAILQ_INIT(&tw->head); + GROUPTASK_INIT(&tw->gtask, 0, tasklet_handler, tw); + snprintf(buf, sizeof(buf), "softirq%d", i); + taskqgroup_attach_cpu(qgroup_softirq, &tw->gtask, + "tasklet", i, NULL, NULL, buf); + } +} +SYSINIT(linux_tasklet, SI_SUB_TASKQ, SI_ORDER_THIRD, tasklet_subsystem_init, NULL); + +static void +tasklet_subsystem_uninit(void *arg __unused) +{ + struct tasklet_worker *tw; + int i; + + CPU_FOREACH(i) { + if (CPU_ABSENT(i)) + continue; + + tw = DPCPU_ID_PTR(i, tasklet_worker); + + taskqgroup_detach(qgroup_softirq, &tw->gtask); + mtx_destroy(&tw->mtx); + } +} +SYSUNINIT(linux_tasklet, SI_SUB_TASKQ, SI_ORDER_THIRD, tasklet_subsystem_uninit, NULL); + +void +tasklet_init(struct tasklet_struct *ts, + tasklet_func_t *func, unsigned long data) +{ + ts->entry.tqe_prev = NULL; + ts->entry.tqe_next = NULL; + ts->func = func; + ts->data = data; + atomic_set_int(&ts->tasklet_state, TASKLET_ST_IDLE); + atomic_set(&ts->count, 0); +} + +void +local_bh_enable(void) +{ + sched_unpin(); +} + +void +local_bh_disable(void) +{ + sched_pin(); +} + +void +tasklet_schedule(struct tasklet_struct *ts) +{ + + /* tasklet is paused */ + if (atomic_read(&ts->count)) + return; + + if (TASKLET_ST_CMPSET(ts, TASKLET_ST_EXEC, TASKLET_ST_LOOP)) { + /* tasklet_handler() will loop */ + } else if (TASKLET_ST_CMPSET(ts, TASKLET_ST_IDLE, TASKLET_ST_BUSY)) { + struct tasklet_worker *tw; + + tw = &DPCPU_GET(tasklet_worker); + + /* tasklet_handler() was not queued */ + TASKLET_WORKER_LOCK(tw); + /* enqueue tasklet */ + TAILQ_INSERT_TAIL(&tw->head, ts, entry); + /* schedule worker */ + GROUPTASK_ENQUEUE(&tw->gtask); + TASKLET_WORKER_UNLOCK(tw); + } else { + /* + * tasklet_handler() is already executing + * + * If the state is neither EXEC nor IDLE, it is either + * LOOP or BUSY. If the state changed between the two + * CMPSET's above the only possible transitions by + * elimination are LOOP->EXEC and BUSY->EXEC. If a + * EXEC->LOOP transition was missed that is not a + * problem because the callback function is then + * already about to be called again. + */ + } +} + +void +tasklet_kill(struct tasklet_struct *ts) +{ + + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "tasklet_kill() can sleep"); + + /* wait until tasklet is no longer busy */ + while (TASKLET_ST_GET(ts) != TASKLET_ST_IDLE) + pause("W", 1); +} + +void +tasklet_enable(struct tasklet_struct *ts) +{ + + atomic_dec(&ts->count); +} + +void +tasklet_disable(struct tasklet_struct *ts) +{ + + atomic_inc(&ts->count); + tasklet_unlock_wait(ts); +} + +int +tasklet_trylock(struct tasklet_struct *ts) +{ + + return (TASKLET_ST_CMPSET(ts, TASKLET_ST_IDLE, TASKLET_ST_BUSY)); +} + +void +tasklet_unlock(struct tasklet_struct *ts) +{ + + TASKLET_ST_SET(ts, TASKLET_ST_IDLE); +} + +void +tasklet_unlock_wait(struct tasklet_struct *ts) +{ + + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "tasklet_kill() can sleep"); + + /* wait until tasklet is no longer busy */ + while (TASKLET_ST_GET(ts) != TASKLET_ST_IDLE) + pause("W", 1); +} diff --git a/sys/compat/linuxkpi/common/src/linux_usb.c b/sys/compat/linuxkpi/common/src/linux_usb.c new file mode 100644 index 000000000000..abef31e20a07 --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_usb.c @@ -0,0 +1,1740 @@ +/* $FreeBSD$ */ +/*- + * Copyright (c) 2007 Luigi Rizzo - Universita` di Pisa. All rights reserved. + * Copyright (c) 2007 Hans Petter Selasky. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifdef USB_GLOBAL_INCLUDE_FILE +#include USB_GLOBAL_INCLUDE_FILE +#else +#include <sys/stdint.h> +#include <sys/stddef.h> +#include <sys/param.h> +#include <sys/queue.h> +#include <sys/types.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/bus.h> +#include <sys/module.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/condvar.h> +#include <sys/sysctl.h> +#include <sys/sx.h> +#include <sys/unistd.h> +#include <sys/callout.h> +#include <sys/malloc.h> +#include <sys/priv.h> + +#include <dev/usb/usb.h> +#include <dev/usb/usbdi.h> +#include <dev/usb/usbdi_util.h> + +#define USB_DEBUG_VAR usb_debug + +#include <dev/usb/usb_core.h> +#include <linux/usb.h> +#include <dev/usb/usb_process.h> +#include <dev/usb/usb_device.h> +#include <dev/usb/usb_util.h> +#include <dev/usb/usb_busdma.h> +#include <dev/usb/usb_transfer.h> +#include <dev/usb/usb_hub.h> +#include <dev/usb/usb_request.h> +#include <dev/usb/usb_debug.h> +#include <dev/usb/usb_dynamic.h> +#endif /* USB_GLOBAL_INCLUDE_FILE */ + +struct usb_linux_softc { + LIST_ENTRY(usb_linux_softc) sc_attached_list; + + device_t sc_fbsd_dev; + struct usb_device *sc_fbsd_udev; + struct usb_interface *sc_ui; + struct usb_driver *sc_udrv; +}; + +/* prototypes */ +static device_probe_t usb_linux_probe; +static device_attach_t usb_linux_attach; +static device_detach_t usb_linux_detach; +static device_suspend_t usb_linux_suspend; +static device_resume_t usb_linux_resume; + +static usb_callback_t usb_linux_isoc_callback; +static usb_callback_t usb_linux_non_isoc_callback; + +static usb_complete_t usb_linux_wait_complete; + +static uint16_t usb_max_isoc_frames(struct usb_device *); +static int usb_start_wait_urb(struct urb *, usb_timeout_t, uint16_t *); +static const struct usb_device_id *usb_linux_lookup_id( + const struct usb_device_id *, struct usb_attach_arg *); +static struct usb_driver *usb_linux_get_usb_driver(struct usb_linux_softc *); +static int usb_linux_create_usb_device(struct usb_device *, device_t); +static void usb_linux_cleanup_interface(struct usb_device *, + struct usb_interface *); +static void usb_linux_complete(struct usb_xfer *); +static int usb_unlink_urb_sub(struct urb *, uint8_t); + +/*------------------------------------------------------------------------* + * FreeBSD USB interface + *------------------------------------------------------------------------*/ + +static LIST_HEAD(, usb_linux_softc) usb_linux_attached_list; +static LIST_HEAD(, usb_driver) usb_linux_driver_list; + +static device_method_t usb_linux_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, usb_linux_probe), + DEVMETHOD(device_attach, usb_linux_attach), + DEVMETHOD(device_detach, usb_linux_detach), + DEVMETHOD(device_suspend, usb_linux_suspend), + DEVMETHOD(device_resume, usb_linux_resume), + + DEVMETHOD_END +}; + +static driver_t usb_linux_driver = { + .name = "usb_linux", + .methods = usb_linux_methods, + .size = sizeof(struct usb_linux_softc), +}; + +static devclass_t usb_linux_devclass; + +DRIVER_MODULE(usb_linux, uhub, usb_linux_driver, usb_linux_devclass, NULL, 0); +MODULE_VERSION(usb_linux, 1); + +/*------------------------------------------------------------------------* + * usb_linux_lookup_id + * + * This functions takes an array of "struct usb_device_id" and tries + * to match the entries with the information in "struct usb_attach_arg". + * If it finds a match the matching entry will be returned. + * Else "NULL" will be returned. + *------------------------------------------------------------------------*/ +static const struct usb_device_id * +usb_linux_lookup_id(const struct usb_device_id *id, struct usb_attach_arg *uaa) +{ + if (id == NULL) { + goto done; + } + /* + * Keep on matching array entries until we find one with + * "match_flags" equal to zero, which indicates the end of the + * array: + */ + for (; id->match_flags; id++) { + + if ((id->match_flags & USB_DEVICE_ID_MATCH_VENDOR) && + (id->idVendor != uaa->info.idVendor)) { + continue; + } + if ((id->match_flags & USB_DEVICE_ID_MATCH_PRODUCT) && + (id->idProduct != uaa->info.idProduct)) { + continue; + } + if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_LO) && + (id->bcdDevice_lo > uaa->info.bcdDevice)) { + continue; + } + if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_HI) && + (id->bcdDevice_hi < uaa->info.bcdDevice)) { + continue; + } + if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_CLASS) && + (id->bDeviceClass != uaa->info.bDeviceClass)) { + continue; + } + if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_SUBCLASS) && + (id->bDeviceSubClass != uaa->info.bDeviceSubClass)) { + continue; + } + if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_PROTOCOL) && + (id->bDeviceProtocol != uaa->info.bDeviceProtocol)) { + continue; + } + if ((uaa->info.bDeviceClass == 0xFF) && + !(id->match_flags & USB_DEVICE_ID_MATCH_VENDOR) && + (id->match_flags & (USB_DEVICE_ID_MATCH_INT_CLASS | + USB_DEVICE_ID_MATCH_INT_SUBCLASS | + USB_DEVICE_ID_MATCH_INT_PROTOCOL))) { + continue; + } + if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_CLASS) && + (id->bInterfaceClass != uaa->info.bInterfaceClass)) { + continue; + } + if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_SUBCLASS) && + (id->bInterfaceSubClass != uaa->info.bInterfaceSubClass)) { + continue; + } + if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_PROTOCOL) && + (id->bInterfaceProtocol != uaa->info.bInterfaceProtocol)) { + continue; + } + /* we found a match! */ + return (id); + } + +done: + return (NULL); +} + +/*------------------------------------------------------------------------* + * usb_linux_probe + * + * This function is the FreeBSD probe callback. It is called from the + * FreeBSD USB stack through the "device_probe_and_attach()" function. + *------------------------------------------------------------------------*/ +static int +usb_linux_probe(device_t dev) +{ + struct usb_attach_arg *uaa = device_get_ivars(dev); + struct usb_driver *udrv; + int err = ENXIO; + + if (uaa->usb_mode != USB_MODE_HOST) { + return (ENXIO); + } + mtx_lock(&Giant); + LIST_FOREACH(udrv, &usb_linux_driver_list, linux_driver_list) { + if (usb_linux_lookup_id(udrv->id_table, uaa)) { + err = 0; + break; + } + } + mtx_unlock(&Giant); + + return (err); +} + +/*------------------------------------------------------------------------* + * usb_linux_get_usb_driver + * + * This function returns the pointer to the "struct usb_driver" where + * the Linux USB device driver "struct usb_device_id" match was found. + * We apply a lock before reading out the pointer to avoid races. + *------------------------------------------------------------------------*/ +static struct usb_driver * +usb_linux_get_usb_driver(struct usb_linux_softc *sc) +{ + struct usb_driver *udrv; + + mtx_lock(&Giant); + udrv = sc->sc_udrv; + mtx_unlock(&Giant); + return (udrv); +} + +/*------------------------------------------------------------------------* + * usb_linux_attach + * + * This function is the FreeBSD attach callback. It is called from the + * FreeBSD USB stack through the "device_probe_and_attach()" function. + * This function is called when "usb_linux_probe()" returns zero. + *------------------------------------------------------------------------*/ +static int +usb_linux_attach(device_t dev) +{ + struct usb_attach_arg *uaa = device_get_ivars(dev); + struct usb_linux_softc *sc = device_get_softc(dev); + struct usb_driver *udrv; + const struct usb_device_id *id = NULL; + + mtx_lock(&Giant); + LIST_FOREACH(udrv, &usb_linux_driver_list, linux_driver_list) { + id = usb_linux_lookup_id(udrv->id_table, uaa); + if (id) + break; + } + mtx_unlock(&Giant); + + if (id == NULL) { + return (ENXIO); + } + if (usb_linux_create_usb_device(uaa->device, dev) != 0) + return (ENOMEM); + device_set_usb_desc(dev); + + sc->sc_fbsd_udev = uaa->device; + sc->sc_fbsd_dev = dev; + sc->sc_udrv = udrv; + sc->sc_ui = usb_ifnum_to_if(uaa->device, uaa->info.bIfaceNum); + if (sc->sc_ui == NULL) { + return (EINVAL); + } + if (udrv->probe) { + if ((udrv->probe) (sc->sc_ui, id)) { + return (ENXIO); + } + } + mtx_lock(&Giant); + LIST_INSERT_HEAD(&usb_linux_attached_list, sc, sc_attached_list); + mtx_unlock(&Giant); + + /* success */ + return (0); +} + +/*------------------------------------------------------------------------* + * usb_linux_detach + * + * This function is the FreeBSD detach callback. It is called from the + * FreeBSD USB stack through the "device_detach()" function. + *------------------------------------------------------------------------*/ +static int +usb_linux_detach(device_t dev) +{ + struct usb_linux_softc *sc = device_get_softc(dev); + struct usb_driver *udrv = NULL; + + mtx_lock(&Giant); + if (sc->sc_attached_list.le_prev) { + LIST_REMOVE(sc, sc_attached_list); + sc->sc_attached_list.le_prev = NULL; + udrv = sc->sc_udrv; + sc->sc_udrv = NULL; + } + mtx_unlock(&Giant); + + if (udrv && udrv->disconnect) { + (udrv->disconnect) (sc->sc_ui); + } + /* + * Make sure that we free all FreeBSD USB transfers belonging to + * this Linux "usb_interface", hence they will most likely not be + * needed any more. + */ + usb_linux_cleanup_interface(sc->sc_fbsd_udev, sc->sc_ui); + return (0); +} + +/*------------------------------------------------------------------------* + * usb_linux_suspend + * + * This function is the FreeBSD suspend callback. Usually it does nothing. + *------------------------------------------------------------------------*/ +static int +usb_linux_suspend(device_t dev) +{ + struct usb_linux_softc *sc = device_get_softc(dev); + struct usb_driver *udrv = usb_linux_get_usb_driver(sc); + int err; + + if (udrv && udrv->suspend) { + err = (udrv->suspend) (sc->sc_ui, 0); + } + return (0); +} + +/*------------------------------------------------------------------------* + * usb_linux_resume + * + * This function is the FreeBSD resume callback. Usually it does nothing. + *------------------------------------------------------------------------*/ +static int +usb_linux_resume(device_t dev) +{ + struct usb_linux_softc *sc = device_get_softc(dev); + struct usb_driver *udrv = usb_linux_get_usb_driver(sc); + int err; + + if (udrv && udrv->resume) { + err = (udrv->resume) (sc->sc_ui); + } + return (0); +} + +/*------------------------------------------------------------------------* + * Linux emulation layer + *------------------------------------------------------------------------*/ + +/*------------------------------------------------------------------------* + * usb_max_isoc_frames + * + * The following function returns the maximum number of isochronous + * frames that we support per URB. It is not part of the Linux USB API. + *------------------------------------------------------------------------*/ +static uint16_t +usb_max_isoc_frames(struct usb_device *dev) +{ + ; /* indent fix */ + switch (usbd_get_speed(dev)) { + case USB_SPEED_LOW: + case USB_SPEED_FULL: + return (USB_MAX_FULL_SPEED_ISOC_FRAMES); + default: + return (USB_MAX_HIGH_SPEED_ISOC_FRAMES); + } +} + +/*------------------------------------------------------------------------* + * usb_submit_urb + * + * This function is used to queue an URB after that it has been + * initialized. If it returns non-zero, it means that the URB was not + * queued. + *------------------------------------------------------------------------*/ +int +usb_submit_urb(struct urb *urb, uint16_t mem_flags) +{ + struct usb_host_endpoint *uhe; + uint8_t do_unlock; + int err; + + if (urb == NULL) + return (-EINVAL); + + do_unlock = mtx_owned(&Giant) ? 0 : 1; + if (do_unlock) + mtx_lock(&Giant); + + if (urb->endpoint == NULL) { + err = -EINVAL; + goto done; + } + + /* + * Check to see if the urb is in the process of being killed + * and stop a urb that is in the process of being killed from + * being re-submitted (e.g. from its completion callback + * function). + */ + if (urb->kill_count != 0) { + err = -EPERM; + goto done; + } + + uhe = urb->endpoint; + + /* + * Check that we have got a FreeBSD USB transfer that will dequeue + * the URB structure and do the real transfer. If there are no USB + * transfers, then we return an error. + */ + if (uhe->bsd_xfer[0] || + uhe->bsd_xfer[1]) { + /* we are ready! */ + + TAILQ_INSERT_TAIL(&uhe->bsd_urb_list, urb, bsd_urb_list); + + urb->status = -EINPROGRESS; + + usbd_transfer_start(uhe->bsd_xfer[0]); + usbd_transfer_start(uhe->bsd_xfer[1]); + err = 0; + } else { + /* no pipes have been setup yet! */ + urb->status = -EINVAL; + err = -EINVAL; + } +done: + if (do_unlock) + mtx_unlock(&Giant); + return (err); +} + +/*------------------------------------------------------------------------* + * usb_unlink_urb + * + * This function is used to stop an URB after that it is been + * submitted, but before the "complete" callback has been called. On + *------------------------------------------------------------------------*/ +int +usb_unlink_urb(struct urb *urb) +{ + return (usb_unlink_urb_sub(urb, 0)); +} + +static void +usb_unlink_bsd(struct usb_xfer *xfer, + struct urb *urb, uint8_t drain) +{ + if (xfer == NULL) + return; + if (!usbd_transfer_pending(xfer)) + return; + if (xfer->priv_fifo == (void *)urb) { + if (drain) { + mtx_unlock(&Giant); + usbd_transfer_drain(xfer); + mtx_lock(&Giant); + } else { + usbd_transfer_stop(xfer); + } + usbd_transfer_start(xfer); + } +} + +static int +usb_unlink_urb_sub(struct urb *urb, uint8_t drain) +{ + struct usb_host_endpoint *uhe; + uint16_t x; + uint8_t do_unlock; + int err; + + if (urb == NULL) + return (-EINVAL); + + do_unlock = mtx_owned(&Giant) ? 0 : 1; + if (do_unlock) + mtx_lock(&Giant); + if (drain) + urb->kill_count++; + + if (urb->endpoint == NULL) { + err = -EINVAL; + goto done; + } + uhe = urb->endpoint; + + if (urb->bsd_urb_list.tqe_prev) { + + /* not started yet, just remove it from the queue */ + TAILQ_REMOVE(&uhe->bsd_urb_list, urb, bsd_urb_list); + urb->bsd_urb_list.tqe_prev = NULL; + urb->status = -ECONNRESET; + urb->actual_length = 0; + + for (x = 0; x < urb->number_of_packets; x++) { + urb->iso_frame_desc[x].actual_length = 0; + } + + if (urb->complete) { + (urb->complete) (urb); + } + } else { + + /* + * If the URB is not on the URB list, then check if one of + * the FreeBSD USB transfer are processing the current URB. + * If so, re-start that transfer, which will lead to the + * termination of that URB: + */ + usb_unlink_bsd(uhe->bsd_xfer[0], urb, drain); + usb_unlink_bsd(uhe->bsd_xfer[1], urb, drain); + } + err = 0; +done: + if (drain) + urb->kill_count--; + if (do_unlock) + mtx_unlock(&Giant); + return (err); +} + +/*------------------------------------------------------------------------* + * usb_clear_halt + * + * This function must always be used to clear the stall. Stall is when + * an USB endpoint returns a stall message to the USB host controller. + * Until the stall is cleared, no data can be transferred. + *------------------------------------------------------------------------*/ +int +usb_clear_halt(struct usb_device *dev, struct usb_host_endpoint *uhe) +{ + struct usb_config cfg[1]; + struct usb_endpoint *ep; + uint8_t type; + uint8_t addr; + + if (uhe == NULL) + return (-EINVAL); + + type = uhe->desc.bmAttributes & UE_XFERTYPE; + addr = uhe->desc.bEndpointAddress; + + memset(cfg, 0, sizeof(cfg)); + + cfg[0].type = type; + cfg[0].endpoint = addr & UE_ADDR; + cfg[0].direction = addr & (UE_DIR_OUT | UE_DIR_IN); + + ep = usbd_get_endpoint(dev, uhe->bsd_iface_index, cfg); + if (ep == NULL) + return (-EINVAL); + + usbd_clear_data_toggle(dev, ep); + + return (usb_control_msg(dev, &dev->ep0, + UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT, + UF_ENDPOINT_HALT, addr, NULL, 0, 1000)); +} + +/*------------------------------------------------------------------------* + * usb_start_wait_urb + * + * This is an internal function that is used to perform synchronous + * Linux USB transfers. + *------------------------------------------------------------------------*/ +static int +usb_start_wait_urb(struct urb *urb, usb_timeout_t timeout, uint16_t *p_actlen) +{ + int err; + uint8_t do_unlock; + + /* you must have a timeout! */ + if (timeout == 0) { + timeout = 1; + } + urb->complete = &usb_linux_wait_complete; + urb->timeout = timeout; + urb->transfer_flags |= URB_WAIT_WAKEUP; + urb->transfer_flags &= ~URB_IS_SLEEPING; + + do_unlock = mtx_owned(&Giant) ? 0 : 1; + if (do_unlock) + mtx_lock(&Giant); + err = usb_submit_urb(urb, 0); + if (err) + goto done; + + /* + * the URB might have completed before we get here, so check that by + * using some flags! + */ + while (urb->transfer_flags & URB_WAIT_WAKEUP) { + urb->transfer_flags |= URB_IS_SLEEPING; + cv_wait(&urb->cv_wait, &Giant); + urb->transfer_flags &= ~URB_IS_SLEEPING; + } + + err = urb->status; + +done: + if (do_unlock) + mtx_unlock(&Giant); + if (p_actlen != NULL) { + if (err) + *p_actlen = 0; + else + *p_actlen = urb->actual_length; + } + return (err); +} + +/*------------------------------------------------------------------------* + * usb_control_msg + * + * The following function performs a control transfer sequence one any + * control, bulk or interrupt endpoint, specified by "uhe". A control + * transfer means that you transfer an 8-byte header first followed by + * a data-phase as indicated by the 8-byte header. The "timeout" is + * given in milliseconds. + * + * Return values: + * 0: Success + * < 0: Failure + * > 0: Actual length + *------------------------------------------------------------------------*/ +int +usb_control_msg(struct usb_device *dev, struct usb_host_endpoint *uhe, + uint8_t request, uint8_t requesttype, + uint16_t value, uint16_t index, void *data, + uint16_t size, usb_timeout_t timeout) +{ + struct usb_device_request req; + struct urb *urb; + int err; + uint16_t actlen; + uint8_t type; + uint8_t addr; + + req.bmRequestType = requesttype; + req.bRequest = request; + USETW(req.wValue, value); + USETW(req.wIndex, index); + USETW(req.wLength, size); + + if (uhe == NULL) { + return (-EINVAL); + } + type = (uhe->desc.bmAttributes & UE_XFERTYPE); + addr = (uhe->desc.bEndpointAddress & UE_ADDR); + + if (type != UE_CONTROL) { + return (-EINVAL); + } + if (addr == 0) { + /* + * The FreeBSD USB stack supports standard control + * transfers on control endpoint zero: + */ + err = usbd_do_request_flags(dev, + NULL, &req, data, USB_SHORT_XFER_OK, + &actlen, timeout); + if (err) { + err = -EPIPE; + } else { + err = actlen; + } + return (err); + } + if (dev->flags.usb_mode != USB_MODE_HOST) { + /* not supported */ + return (-EINVAL); + } + err = usb_setup_endpoint(dev, uhe, 1 /* dummy */ ); + + /* + * NOTE: we need to allocate real memory here so that we don't + * transfer data to/from the stack! + * + * 0xFFFF is a FreeBSD specific magic value. + */ + urb = usb_alloc_urb(0xFFFF, size); + + urb->dev = dev; + urb->endpoint = uhe; + + memcpy(urb->setup_packet, &req, sizeof(req)); + + if (size && (!(req.bmRequestType & UT_READ))) { + /* move the data to a real buffer */ + memcpy(USB_ADD_BYTES(urb->setup_packet, sizeof(req)), + data, size); + } + err = usb_start_wait_urb(urb, timeout, &actlen); + + if (req.bmRequestType & UT_READ) { + if (actlen) { + bcopy(USB_ADD_BYTES(urb->setup_packet, + sizeof(req)), data, actlen); + } + } + usb_free_urb(urb); + + if (err == 0) { + err = actlen; + } + return (err); +} + +/*------------------------------------------------------------------------* + * usb_set_interface + * + * The following function will select which alternate setting of an + * USB interface you plan to use. By default alternate setting with + * index zero is selected. Note that "iface_no" is not the interface + * index, but rather the value of "bInterfaceNumber". + *------------------------------------------------------------------------*/ +int +usb_set_interface(struct usb_device *dev, uint8_t iface_no, uint8_t alt_index) +{ + struct usb_interface *p_ui = usb_ifnum_to_if(dev, iface_no); + int err; + + if (p_ui == NULL) + return (-EINVAL); + if (alt_index >= p_ui->num_altsetting) + return (-EINVAL); + usb_linux_cleanup_interface(dev, p_ui); + err = -usbd_set_alt_interface_index(dev, + p_ui->bsd_iface_index, alt_index); + if (err == 0) { + p_ui->cur_altsetting = p_ui->altsetting + alt_index; + } + return (err); +} + +/*------------------------------------------------------------------------* + * usb_setup_endpoint + * + * The following function is an extension to the Linux USB API that + * allows you to set a maximum buffer size for a given USB endpoint. + * The maximum buffer size is per URB. If you don't call this function + * to set a maximum buffer size, the endpoint will not be functional. + * Note that for isochronous endpoints the maximum buffer size must be + * a non-zero dummy, hence this function will base the maximum buffer + * size on "wMaxPacketSize". + *------------------------------------------------------------------------*/ +int +usb_setup_endpoint(struct usb_device *dev, + struct usb_host_endpoint *uhe, usb_size_t bufsize) +{ + struct usb_config cfg[2]; + uint8_t type = uhe->desc.bmAttributes & UE_XFERTYPE; + uint8_t addr = uhe->desc.bEndpointAddress; + + if (uhe->fbsd_buf_size == bufsize) { + /* optimize */ + return (0); + } + usbd_transfer_unsetup(uhe->bsd_xfer, 2); + + uhe->fbsd_buf_size = bufsize; + + if (bufsize == 0) { + return (0); + } + memset(cfg, 0, sizeof(cfg)); + + if (type == UE_ISOCHRONOUS) { + + /* + * Isochronous transfers are special in that they don't fit + * into the BULK/INTR/CONTROL transfer model. + */ + + cfg[0].type = type; + cfg[0].endpoint = addr & UE_ADDR; + cfg[0].direction = addr & (UE_DIR_OUT | UE_DIR_IN); + cfg[0].callback = &usb_linux_isoc_callback; + cfg[0].bufsize = 0; /* use wMaxPacketSize */ + cfg[0].frames = usb_max_isoc_frames(dev); + cfg[0].flags.proxy_buffer = 1; +#if 0 + /* + * The Linux USB API allows non back-to-back + * isochronous frames which we do not support. If the + * isochronous frames are not back-to-back we need to + * do a copy, and then we need a buffer for + * that. Enable this at your own risk. + */ + cfg[0].flags.ext_buffer = 1; +#endif + cfg[0].flags.short_xfer_ok = 1; + + bcopy(cfg, cfg + 1, sizeof(*cfg)); + + /* Allocate and setup two generic FreeBSD USB transfers */ + + if (usbd_transfer_setup(dev, &uhe->bsd_iface_index, + uhe->bsd_xfer, cfg, 2, uhe, &Giant)) { + return (-EINVAL); + } + } else { + if (bufsize > (1 << 22)) { + /* limit buffer size */ + bufsize = (1 << 22); + } + /* Allocate and setup one generic FreeBSD USB transfer */ + + cfg[0].type = type; + cfg[0].endpoint = addr & UE_ADDR; + cfg[0].direction = addr & (UE_DIR_OUT | UE_DIR_IN); + cfg[0].callback = &usb_linux_non_isoc_callback; + cfg[0].bufsize = bufsize; + cfg[0].flags.ext_buffer = 1; /* enable zero-copy */ + cfg[0].flags.proxy_buffer = 1; + cfg[0].flags.short_xfer_ok = 1; + + if (usbd_transfer_setup(dev, &uhe->bsd_iface_index, + uhe->bsd_xfer, cfg, 1, uhe, &Giant)) { + return (-EINVAL); + } + } + return (0); +} + +/*------------------------------------------------------------------------* + * usb_linux_create_usb_device + * + * The following function is used to build up a per USB device + * structure tree, that mimics the Linux one. The root structure + * is returned by this function. + *------------------------------------------------------------------------*/ +static int +usb_linux_create_usb_device(struct usb_device *udev, device_t dev) +{ + struct usb_config_descriptor *cd = usbd_get_config_descriptor(udev); + struct usb_descriptor *desc; + struct usb_interface_descriptor *id; + struct usb_endpoint_descriptor *ed; + struct usb_interface *p_ui = NULL; + struct usb_host_interface *p_uhi = NULL; + struct usb_host_endpoint *p_uhe = NULL; + usb_size_t size; + uint16_t niface_total; + uint16_t nedesc; + uint16_t iface_no_curr; + uint16_t iface_index; + uint8_t pass; + uint8_t iface_no; + + /* + * We do two passes. One pass for computing necessary memory size + * and one pass to initialize all the allocated memory structures. + */ + for (pass = 0; pass < 2; pass++) { + + iface_no_curr = 0xFFFF; + niface_total = 0; + iface_index = 0; + nedesc = 0; + desc = NULL; + + /* + * Iterate over all the USB descriptors. Use the USB config + * descriptor pointer provided by the FreeBSD USB stack. + */ + while ((desc = usb_desc_foreach(cd, desc))) { + + /* + * Build up a tree according to the descriptors we + * find: + */ + switch (desc->bDescriptorType) { + case UDESC_DEVICE: + break; + + case UDESC_ENDPOINT: + ed = (void *)desc; + if ((ed->bLength < sizeof(*ed)) || + (iface_index == 0)) + break; + if (p_uhe) { + bcopy(ed, &p_uhe->desc, sizeof(p_uhe->desc)); + p_uhe->bsd_iface_index = iface_index - 1; + TAILQ_INIT(&p_uhe->bsd_urb_list); + p_uhe++; + } + if (p_uhi) { + (p_uhi - 1)->desc.bNumEndpoints++; + } + nedesc++; + break; + + case UDESC_INTERFACE: + id = (void *)desc; + if (id->bLength < sizeof(*id)) + break; + if (p_uhi) { + bcopy(id, &p_uhi->desc, sizeof(p_uhi->desc)); + p_uhi->desc.bNumEndpoints = 0; + p_uhi->endpoint = p_uhe; + p_uhi->string = ""; + p_uhi->bsd_iface_index = iface_index; + p_uhi++; + } + iface_no = id->bInterfaceNumber; + niface_total++; + if (iface_no_curr != iface_no) { + if (p_ui) { + p_ui->altsetting = p_uhi - 1; + p_ui->cur_altsetting = p_uhi - 1; + p_ui->num_altsetting = 1; + p_ui->bsd_iface_index = iface_index; + p_ui->linux_udev = udev; + p_ui++; + } + iface_no_curr = iface_no; + iface_index++; + } else { + if (p_ui) { + (p_ui - 1)->num_altsetting++; + } + } + break; + + default: + break; + } + } + + if (pass == 0) { + + size = (sizeof(*p_uhe) * nedesc) + + (sizeof(*p_ui) * iface_index) + + (sizeof(*p_uhi) * niface_total); + + p_uhe = malloc(size, M_USBDEV, M_WAITOK | M_ZERO); + p_ui = (void *)(p_uhe + nedesc); + p_uhi = (void *)(p_ui + iface_index); + + udev->linux_iface_start = p_ui; + udev->linux_iface_end = p_ui + iface_index; + udev->linux_endpoint_start = p_uhe; + udev->linux_endpoint_end = p_uhe + nedesc; + udev->devnum = device_get_unit(dev); + bcopy(&udev->ddesc, &udev->descriptor, + sizeof(udev->descriptor)); + bcopy(udev->ctrl_ep.edesc, &udev->ep0.desc, + sizeof(udev->ep0.desc)); + } + } + return (0); +} + +/*------------------------------------------------------------------------* + * usb_alloc_urb + * + * This function should always be used when you allocate an URB for + * use with the USB Linux stack. In case of an isochronous transfer + * you must specifiy the maximum number of "iso_packets" which you + * plan to transfer per URB. This function is always blocking, and + * "mem_flags" are not regarded like on Linux. + *------------------------------------------------------------------------*/ +struct urb * +usb_alloc_urb(uint16_t iso_packets, uint16_t mem_flags) +{ + struct urb *urb; + usb_size_t size; + + if (iso_packets == 0xFFFF) { + /* + * FreeBSD specific magic value to ask for control transfer + * memory allocation: + */ + size = sizeof(*urb) + sizeof(struct usb_device_request) + mem_flags; + } else { + size = sizeof(*urb) + (iso_packets * sizeof(urb->iso_frame_desc[0])); + } + + urb = malloc(size, M_USBDEV, M_WAITOK | M_ZERO); + + cv_init(&urb->cv_wait, "URBWAIT"); + if (iso_packets == 0xFFFF) { + urb->setup_packet = (void *)(urb + 1); + urb->transfer_buffer = (void *)(urb->setup_packet + + sizeof(struct usb_device_request)); + } else { + urb->number_of_packets = iso_packets; + } + return (urb); +} + +/*------------------------------------------------------------------------* + * usb_find_host_endpoint + * + * The following function will return the Linux USB host endpoint + * structure that matches the given endpoint type and endpoint + * value. If no match is found, NULL is returned. This function is not + * part of the Linux USB API and is only used internally. + *------------------------------------------------------------------------*/ +struct usb_host_endpoint * +usb_find_host_endpoint(struct usb_device *dev, uint8_t type, uint8_t ep) +{ + struct usb_host_endpoint *uhe; + struct usb_host_endpoint *uhe_end; + struct usb_host_interface *uhi; + struct usb_interface *ui; + uint8_t ea; + uint8_t at; + uint8_t mask; + + if (dev == NULL) { + return (NULL); + } + if (type == UE_CONTROL) { + mask = UE_ADDR; + } else { + mask = (UE_DIR_IN | UE_DIR_OUT | UE_ADDR); + } + + ep &= mask; + + /* + * Iterate over all the interfaces searching the selected alternate + * setting only, and all belonging endpoints. + */ + for (ui = dev->linux_iface_start; + ui != dev->linux_iface_end; + ui++) { + uhi = ui->cur_altsetting; + if (uhi) { + uhe_end = uhi->endpoint + uhi->desc.bNumEndpoints; + for (uhe = uhi->endpoint; + uhe != uhe_end; + uhe++) { + ea = uhe->desc.bEndpointAddress; + at = uhe->desc.bmAttributes; + + if (((ea & mask) == ep) && + ((at & UE_XFERTYPE) == type)) { + return (uhe); + } + } + } + } + + if ((type == UE_CONTROL) && ((ep & UE_ADDR) == 0)) { + return (&dev->ep0); + } + return (NULL); +} + +/*------------------------------------------------------------------------* + * usb_altnum_to_altsetting + * + * The following function returns a pointer to an alternate setting by + * index given a "usb_interface" pointer. If the alternate setting by + * index does not exist, NULL is returned. And alternate setting is a + * variant of an interface, but usually with slightly different + * characteristics. + *------------------------------------------------------------------------*/ +struct usb_host_interface * +usb_altnum_to_altsetting(const struct usb_interface *intf, uint8_t alt_index) +{ + if (alt_index >= intf->num_altsetting) { + return (NULL); + } + return (intf->altsetting + alt_index); +} + +/*------------------------------------------------------------------------* + * usb_ifnum_to_if + * + * The following function searches up an USB interface by + * "bInterfaceNumber". If no match is found, NULL is returned. + *------------------------------------------------------------------------*/ +struct usb_interface * +usb_ifnum_to_if(struct usb_device *dev, uint8_t iface_no) +{ + struct usb_interface *p_ui; + + for (p_ui = dev->linux_iface_start; + p_ui != dev->linux_iface_end; + p_ui++) { + if ((p_ui->num_altsetting > 0) && + (p_ui->altsetting->desc.bInterfaceNumber == iface_no)) { + return (p_ui); + } + } + return (NULL); +} + +/*------------------------------------------------------------------------* + * usb_buffer_alloc + *------------------------------------------------------------------------*/ +void * +usb_buffer_alloc(struct usb_device *dev, usb_size_t size, uint16_t mem_flags, uint8_t *dma_addr) +{ + return (malloc(size, M_USBDEV, M_WAITOK | M_ZERO)); +} + +/*------------------------------------------------------------------------* + * usbd_get_intfdata + *------------------------------------------------------------------------*/ +void * +usbd_get_intfdata(struct usb_interface *intf) +{ + return (intf->bsd_priv_sc); +} + +/*------------------------------------------------------------------------* + * usb_linux_register + * + * The following function is used by the "USB_DRIVER_EXPORT()" macro, + * and is used to register a Linux USB driver, so that its + * "usb_device_id" structures gets searched a probe time. This + * function is not part of the Linux USB API, and is for internal use + * only. + *------------------------------------------------------------------------*/ +void +usb_linux_register(void *arg) +{ + struct usb_driver *drv = arg; + + mtx_lock(&Giant); + LIST_INSERT_HEAD(&usb_linux_driver_list, drv, linux_driver_list); + mtx_unlock(&Giant); + + usb_needs_explore_all(); +} + +/*------------------------------------------------------------------------* + * usb_linux_deregister + * + * The following function is used by the "USB_DRIVER_EXPORT()" macro, + * and is used to deregister a Linux USB driver. This function will + * ensure that all driver instances belonging to the Linux USB device + * driver in question, gets detached before the driver is + * unloaded. This function is not part of the Linux USB API, and is + * for internal use only. + *------------------------------------------------------------------------*/ +void +usb_linux_deregister(void *arg) +{ + struct usb_driver *drv = arg; + struct usb_linux_softc *sc; + +repeat: + mtx_lock(&Giant); + LIST_FOREACH(sc, &usb_linux_attached_list, sc_attached_list) { + if (sc->sc_udrv == drv) { + mtx_unlock(&Giant); + device_detach(sc->sc_fbsd_dev); + goto repeat; + } + } + LIST_REMOVE(drv, linux_driver_list); + mtx_unlock(&Giant); +} + +/*------------------------------------------------------------------------* + * usb_linux_free_device + * + * The following function is only used by the FreeBSD USB stack, to + * cleanup and free memory after that a Linux USB device was attached. + *------------------------------------------------------------------------*/ +void +usb_linux_free_device(struct usb_device *dev) +{ + struct usb_host_endpoint *uhe; + struct usb_host_endpoint *uhe_end; + int err; + + uhe = dev->linux_endpoint_start; + uhe_end = dev->linux_endpoint_end; + while (uhe != uhe_end) { + err = usb_setup_endpoint(dev, uhe, 0); + uhe++; + } + err = usb_setup_endpoint(dev, &dev->ep0, 0); + free(dev->linux_endpoint_start, M_USBDEV); +} + +/*------------------------------------------------------------------------* + * usb_buffer_free + *------------------------------------------------------------------------*/ +void +usb_buffer_free(struct usb_device *dev, usb_size_t size, + void *addr, uint8_t dma_addr) +{ + free(addr, M_USBDEV); +} + +/*------------------------------------------------------------------------* + * usb_free_urb + *------------------------------------------------------------------------*/ +void +usb_free_urb(struct urb *urb) +{ + if (urb == NULL) { + return; + } + /* make sure that the current URB is not active */ + usb_kill_urb(urb); + + /* destroy condition variable */ + cv_destroy(&urb->cv_wait); + + /* just free it */ + free(urb, M_USBDEV); +} + +/*------------------------------------------------------------------------* + * usb_init_urb + * + * The following function can be used to initialize a custom URB. It + * is not recommended to use this function. Use "usb_alloc_urb()" + * instead. + *------------------------------------------------------------------------*/ +void +usb_init_urb(struct urb *urb) +{ + if (urb == NULL) { + return; + } + memset(urb, 0, sizeof(*urb)); +} + +/*------------------------------------------------------------------------* + * usb_kill_urb + *------------------------------------------------------------------------*/ +void +usb_kill_urb(struct urb *urb) +{ + usb_unlink_urb_sub(urb, 1); +} + +/*------------------------------------------------------------------------* + * usb_set_intfdata + * + * The following function sets the per Linux USB interface private + * data pointer. It is used by most Linux USB device drivers. + *------------------------------------------------------------------------*/ +void +usb_set_intfdata(struct usb_interface *intf, void *data) +{ + intf->bsd_priv_sc = data; +} + +/*------------------------------------------------------------------------* + * usb_linux_cleanup_interface + * + * The following function will release all FreeBSD USB transfers + * associated with a Linux USB interface. It is for internal use only. + *------------------------------------------------------------------------*/ +static void +usb_linux_cleanup_interface(struct usb_device *dev, struct usb_interface *iface) +{ + struct usb_host_interface *uhi; + struct usb_host_interface *uhi_end; + struct usb_host_endpoint *uhe; + struct usb_host_endpoint *uhe_end; + int err; + + uhi = iface->altsetting; + uhi_end = iface->altsetting + iface->num_altsetting; + while (uhi != uhi_end) { + uhe = uhi->endpoint; + uhe_end = uhi->endpoint + uhi->desc.bNumEndpoints; + while (uhe != uhe_end) { + err = usb_setup_endpoint(dev, uhe, 0); + uhe++; + } + uhi++; + } +} + +/*------------------------------------------------------------------------* + * usb_linux_wait_complete + * + * The following function is used by "usb_start_wait_urb()" to wake it + * up, when an USB transfer has finished. + *------------------------------------------------------------------------*/ +static void +usb_linux_wait_complete(struct urb *urb) +{ + if (urb->transfer_flags & URB_IS_SLEEPING) { + cv_signal(&urb->cv_wait); + } + urb->transfer_flags &= ~URB_WAIT_WAKEUP; +} + +/*------------------------------------------------------------------------* + * usb_linux_complete + *------------------------------------------------------------------------*/ +static void +usb_linux_complete(struct usb_xfer *xfer) +{ + struct urb *urb; + + urb = usbd_xfer_get_priv(xfer); + usbd_xfer_set_priv(xfer, NULL); + if (urb->complete) { + (urb->complete) (urb); + } +} + +/*------------------------------------------------------------------------* + * usb_linux_isoc_callback + * + * The following is the FreeBSD isochronous USB callback. Isochronous + * frames are USB packets transferred 1000 or 8000 times per second, + * depending on whether a full- or high- speed USB transfer is + * used. + *------------------------------------------------------------------------*/ +static void +usb_linux_isoc_callback(struct usb_xfer *xfer, usb_error_t error) +{ + usb_frlength_t max_frame = xfer->max_frame_size; + usb_frlength_t offset; + usb_frcount_t x; + struct urb *urb = usbd_xfer_get_priv(xfer); + struct usb_host_endpoint *uhe = usbd_xfer_softc(xfer); + struct usb_iso_packet_descriptor *uipd; + + DPRINTF("\n"); + + switch (USB_GET_STATE(xfer)) { + case USB_ST_TRANSFERRED: + + if (urb->bsd_isread) { + + /* copy in data with regard to the URB */ + + offset = 0; + + for (x = 0; x < urb->number_of_packets; x++) { + uipd = urb->iso_frame_desc + x; + if (uipd->length > xfer->frlengths[x]) { + if (urb->transfer_flags & URB_SHORT_NOT_OK) { + /* XXX should be EREMOTEIO */ + uipd->status = -EPIPE; + } else { + uipd->status = 0; + } + } else { + uipd->status = 0; + } + uipd->actual_length = xfer->frlengths[x]; + if (!xfer->flags.ext_buffer) { + usbd_copy_out(xfer->frbuffers, offset, + USB_ADD_BYTES(urb->transfer_buffer, + uipd->offset), uipd->actual_length); + } + offset += max_frame; + } + } else { + for (x = 0; x < urb->number_of_packets; x++) { + uipd = urb->iso_frame_desc + x; + uipd->actual_length = xfer->frlengths[x]; + uipd->status = 0; + } + } + + urb->actual_length = xfer->actlen; + + /* check for short transfer */ + if (xfer->actlen < xfer->sumlen) { + /* short transfer */ + if (urb->transfer_flags & URB_SHORT_NOT_OK) { + /* XXX should be EREMOTEIO */ + urb->status = -EPIPE; + } else { + urb->status = 0; + } + } else { + /* success */ + urb->status = 0; + } + + /* call callback */ + usb_linux_complete(xfer); + + case USB_ST_SETUP: +tr_setup: + + if (xfer->priv_fifo == NULL) { + + /* get next transfer */ + urb = TAILQ_FIRST(&uhe->bsd_urb_list); + if (urb == NULL) { + /* nothing to do */ + return; + } + TAILQ_REMOVE(&uhe->bsd_urb_list, urb, bsd_urb_list); + urb->bsd_urb_list.tqe_prev = NULL; + + x = xfer->max_frame_count; + if (urb->number_of_packets > x) { + /* XXX simply truncate the transfer */ + urb->number_of_packets = x; + } + } else { + DPRINTF("Already got a transfer\n"); + + /* already got a transfer (should not happen) */ + urb = usbd_xfer_get_priv(xfer); + } + + urb->bsd_isread = (uhe->desc.bEndpointAddress & UE_DIR_IN) ? 1 : 0; + + if (xfer->flags.ext_buffer) { + /* set virtual address to load */ + usbd_xfer_set_frame_data(xfer, 0, urb->transfer_buffer, 0); + } + if (!(urb->bsd_isread)) { + + /* copy out data with regard to the URB */ + + offset = 0; + + for (x = 0; x < urb->number_of_packets; x++) { + uipd = urb->iso_frame_desc + x; + usbd_xfer_set_frame_len(xfer, x, uipd->length); + if (!xfer->flags.ext_buffer) { + usbd_copy_in(xfer->frbuffers, offset, + USB_ADD_BYTES(urb->transfer_buffer, + uipd->offset), uipd->length); + } + offset += uipd->length; + } + } else { + + /* + * compute the transfer length into the "offset" + * variable + */ + + offset = urb->number_of_packets * max_frame; + + /* setup "frlengths" array */ + + for (x = 0; x < urb->number_of_packets; x++) { + uipd = urb->iso_frame_desc + x; + usbd_xfer_set_frame_len(xfer, x, max_frame); + } + } + usbd_xfer_set_priv(xfer, urb); + xfer->flags.force_short_xfer = 0; + xfer->timeout = urb->timeout; + xfer->nframes = urb->number_of_packets; + usbd_transfer_submit(xfer); + return; + + default: /* Error */ + if (xfer->error == USB_ERR_CANCELLED) { + urb->status = -ECONNRESET; + } else { + urb->status = -EPIPE; /* stalled */ + } + + /* Set zero for "actual_length" */ + urb->actual_length = 0; + + /* Set zero for "actual_length" */ + for (x = 0; x < urb->number_of_packets; x++) { + urb->iso_frame_desc[x].actual_length = 0; + urb->iso_frame_desc[x].status = urb->status; + } + + /* call callback */ + usb_linux_complete(xfer); + + if (xfer->error == USB_ERR_CANCELLED) { + /* we need to return in this case */ + return; + } + goto tr_setup; + + } +} + +/*------------------------------------------------------------------------* + * usb_linux_non_isoc_callback + * + * The following is the FreeBSD BULK/INTERRUPT and CONTROL USB + * callback. It dequeues Linux USB stack compatible URB's, transforms + * the URB fields into a FreeBSD USB transfer, and defragments the USB + * transfer as required. When the transfer is complete the "complete" + * callback is called. + *------------------------------------------------------------------------*/ +static void +usb_linux_non_isoc_callback(struct usb_xfer *xfer, usb_error_t error) +{ + enum { + REQ_SIZE = sizeof(struct usb_device_request) + }; + struct urb *urb = usbd_xfer_get_priv(xfer); + struct usb_host_endpoint *uhe = usbd_xfer_softc(xfer); + uint8_t *ptr; + usb_frlength_t max_bulk = usbd_xfer_max_len(xfer); + uint8_t data_frame = xfer->flags_int.control_xfr ? 1 : 0; + + DPRINTF("\n"); + + switch (USB_GET_STATE(xfer)) { + case USB_ST_TRANSFERRED: + + if (xfer->flags_int.control_xfr) { + + /* don't transfer the setup packet again: */ + + usbd_xfer_set_frame_len(xfer, 0, 0); + } + if (urb->bsd_isread && (!xfer->flags.ext_buffer)) { + /* copy in data with regard to the URB */ + usbd_copy_out(xfer->frbuffers + data_frame, 0, + urb->bsd_data_ptr, xfer->frlengths[data_frame]); + } + urb->bsd_length_rem -= xfer->frlengths[data_frame]; + urb->bsd_data_ptr += xfer->frlengths[data_frame]; + urb->actual_length += xfer->frlengths[data_frame]; + + /* check for short transfer */ + if (xfer->actlen < xfer->sumlen) { + urb->bsd_length_rem = 0; + + /* short transfer */ + if (urb->transfer_flags & URB_SHORT_NOT_OK) { + urb->status = -EPIPE; + } else { + urb->status = 0; + } + } else { + /* check remainder */ + if (urb->bsd_length_rem > 0) { + goto setup_bulk; + } + /* success */ + urb->status = 0; + } + + /* call callback */ + usb_linux_complete(xfer); + + case USB_ST_SETUP: +tr_setup: + /* get next transfer */ + urb = TAILQ_FIRST(&uhe->bsd_urb_list); + if (urb == NULL) { + /* nothing to do */ + return; + } + TAILQ_REMOVE(&uhe->bsd_urb_list, urb, bsd_urb_list); + urb->bsd_urb_list.tqe_prev = NULL; + + usbd_xfer_set_priv(xfer, urb); + xfer->flags.force_short_xfer = 0; + xfer->timeout = urb->timeout; + + if (xfer->flags_int.control_xfr) { + + /* + * USB control transfers need special handling. + * First copy in the header, then copy in data! + */ + if (!xfer->flags.ext_buffer) { + usbd_copy_in(xfer->frbuffers, 0, + urb->setup_packet, REQ_SIZE); + usbd_xfer_set_frame_len(xfer, 0, REQ_SIZE); + } else { + /* set virtual address to load */ + usbd_xfer_set_frame_data(xfer, 0, + urb->setup_packet, REQ_SIZE); + } + + ptr = urb->setup_packet; + + /* setup data transfer direction and length */ + urb->bsd_isread = (ptr[0] & UT_READ) ? 1 : 0; + urb->bsd_length_rem = ptr[6] | (ptr[7] << 8); + + } else { + + /* setup data transfer direction */ + + urb->bsd_length_rem = urb->transfer_buffer_length; + urb->bsd_isread = (uhe->desc.bEndpointAddress & + UE_DIR_IN) ? 1 : 0; + } + + urb->bsd_data_ptr = urb->transfer_buffer; + urb->actual_length = 0; + +setup_bulk: + if (max_bulk > urb->bsd_length_rem) { + max_bulk = urb->bsd_length_rem; + } + /* check if we need to force a short transfer */ + + if ((max_bulk == urb->bsd_length_rem) && + (urb->transfer_flags & URB_ZERO_PACKET) && + (!xfer->flags_int.control_xfr)) { + xfer->flags.force_short_xfer = 1; + } + /* check if we need to copy in data */ + + if (xfer->flags.ext_buffer) { + /* set virtual address to load */ + usbd_xfer_set_frame_data(xfer, data_frame, + urb->bsd_data_ptr, max_bulk); + } else if (!urb->bsd_isread) { + /* copy out data with regard to the URB */ + usbd_copy_in(xfer->frbuffers + data_frame, 0, + urb->bsd_data_ptr, max_bulk); + usbd_xfer_set_frame_len(xfer, data_frame, max_bulk); + } + if (xfer->flags_int.control_xfr) { + if (max_bulk > 0) { + xfer->nframes = 2; + } else { + xfer->nframes = 1; + } + } else { + xfer->nframes = 1; + } + usbd_transfer_submit(xfer); + return; + + default: + if (xfer->error == USB_ERR_CANCELLED) { + urb->status = -ECONNRESET; + } else { + urb->status = -EPIPE; + } + + /* Set zero for "actual_length" */ + urb->actual_length = 0; + + /* call callback */ + usb_linux_complete(xfer); + + if (xfer->error == USB_ERR_CANCELLED) { + /* we need to return in this case */ + return; + } + goto tr_setup; + } +} + +/*------------------------------------------------------------------------* + * usb_fill_bulk_urb + *------------------------------------------------------------------------*/ +void +usb_fill_bulk_urb(struct urb *urb, struct usb_device *udev, + struct usb_host_endpoint *uhe, void *buf, + int length, usb_complete_t callback, void *arg) +{ + urb->dev = udev; + urb->endpoint = uhe; + urb->transfer_buffer = buf; + urb->transfer_buffer_length = length; + urb->complete = callback; + urb->context = arg; +} + +/*------------------------------------------------------------------------* + * usb_bulk_msg + * + * NOTE: This function can also be used for interrupt endpoints! + * + * Return values: + * 0: Success + * Else: Failure + *------------------------------------------------------------------------*/ +int +usb_bulk_msg(struct usb_device *udev, struct usb_host_endpoint *uhe, + void *data, int len, uint16_t *pactlen, usb_timeout_t timeout) +{ + struct urb *urb; + int err; + + if (uhe == NULL) + return (-EINVAL); + if (len < 0) + return (-EINVAL); + + err = usb_setup_endpoint(udev, uhe, 4096 /* bytes */); + if (err) + return (err); + + urb = usb_alloc_urb(0, 0); + + usb_fill_bulk_urb(urb, udev, uhe, data, len, + usb_linux_wait_complete, NULL); + + err = usb_start_wait_urb(urb, timeout, pactlen); + + usb_free_urb(urb); + + return (err); +} +MODULE_DEPEND(linuxkpi, usb, 1, 1, 1); + +static void +usb_linux_init(void *arg) +{ + /* register our function */ + usb_linux_free_device_p = &usb_linux_free_device; +} +SYSINIT(usb_linux_init, SI_SUB_LOCK, SI_ORDER_FIRST, usb_linux_init, NULL); +SYSUNINIT(usb_linux_unload, SI_SUB_LOCK, SI_ORDER_ANY, usb_linux_unload, NULL); diff --git a/sys/compat/linuxkpi/common/src/linux_work.c b/sys/compat/linuxkpi/common/src/linux_work.c new file mode 100644 index 000000000000..043c5b7d1aff --- /dev/null +++ b/sys/compat/linuxkpi/common/src/linux_work.c @@ -0,0 +1,685 @@ +/*- + * Copyright (c) 2017-2019 Hans Petter Selasky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <linux/workqueue.h> +#include <linux/wait.h> +#include <linux/compat.h> +#include <linux/spinlock.h> +#include <linux/rcupdate.h> + +#include <sys/kernel.h> + +/* + * Define all work struct states + */ +enum { + WORK_ST_IDLE, /* idle - not started */ + WORK_ST_TIMER, /* timer is being started */ + WORK_ST_TASK, /* taskqueue is being queued */ + WORK_ST_EXEC, /* callback is being called */ + WORK_ST_CANCEL, /* cancel is being requested */ + WORK_ST_MAX, +}; + +/* + * Define global workqueues + */ +static struct workqueue_struct *linux_system_short_wq; +static struct workqueue_struct *linux_system_long_wq; + +struct workqueue_struct *system_wq; +struct workqueue_struct *system_long_wq; +struct workqueue_struct *system_unbound_wq; +struct workqueue_struct *system_highpri_wq; +struct workqueue_struct *system_power_efficient_wq; + +static int linux_default_wq_cpus = 4; + +static void linux_delayed_work_timer_fn(void *); + +/* + * This function atomically updates the work state and returns the + * previous state at the time of update. + */ +static uint8_t +linux_update_state(atomic_t *v, const uint8_t *pstate) +{ + int c, old; + + c = v->counter; + + while ((old = atomic_cmpxchg(v, c, pstate[c])) != c) + c = old; + + return (c); +} + +/* + * A LinuxKPI task is allowed to free itself inside the callback function + * and cannot safely be referred after the callback function has + * completed. This function gives the linux_work_fn() function a hint, + * that the task is not going away and can have its state checked + * again. Without this extra hint LinuxKPI tasks cannot be serialized + * accross multiple worker threads. + */ +static bool +linux_work_exec_unblock(struct work_struct *work) +{ + struct workqueue_struct *wq; + struct work_exec *exec; + bool retval = false; + + wq = work->work_queue; + if (unlikely(wq == NULL)) + goto done; + + WQ_EXEC_LOCK(wq); + TAILQ_FOREACH(exec, &wq->exec_head, entry) { + if (exec->target == work) { + exec->target = NULL; + retval = true; + break; + } + } + WQ_EXEC_UNLOCK(wq); +done: + return (retval); +} + +static void +linux_delayed_work_enqueue(struct delayed_work *dwork) +{ + struct taskqueue *tq; + + tq = dwork->work.work_queue->taskqueue; + taskqueue_enqueue(tq, &dwork->work.work_task); +} + +/* + * This function queues the given work structure on the given + * workqueue. It returns non-zero if the work was successfully + * [re-]queued. Else the work is already pending for completion. + */ +bool +linux_queue_work_on(int cpu __unused, struct workqueue_struct *wq, + struct work_struct *work) +{ + static const uint8_t states[WORK_ST_MAX] __aligned(8) = { + [WORK_ST_IDLE] = WORK_ST_TASK, /* start queuing task */ + [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */ + [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */ + [WORK_ST_EXEC] = WORK_ST_TASK, /* queue task another time */ + [WORK_ST_CANCEL] = WORK_ST_TASK, /* start queuing task again */ + }; + + if (atomic_read(&wq->draining) != 0) + return (!work_pending(work)); + + switch (linux_update_state(&work->state, states)) { + case WORK_ST_EXEC: + case WORK_ST_CANCEL: + if (linux_work_exec_unblock(work) != 0) + return (true); + /* FALLTHROUGH */ + case WORK_ST_IDLE: + work->work_queue = wq; + taskqueue_enqueue(wq->taskqueue, &work->work_task); + return (true); + default: + return (false); /* already on a queue */ + } +} + +/* + * Callback func for linux_queue_rcu_work + */ +static void +rcu_work_func(struct rcu_head *rcu) +{ + struct rcu_work *rwork; + + rwork = container_of(rcu, struct rcu_work, rcu); + linux_queue_work_on(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); +} + +/* + * This function queue a work after a grace period + * If the work was already pending it returns false, + * if not it calls call_rcu and returns true. + */ +bool +linux_queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) +{ + + if (!linux_work_pending(&rwork->work)) { + rwork->wq = wq; + linux_call_rcu(RCU_TYPE_REGULAR, &rwork->rcu, rcu_work_func); + return (true); + } + return (false); +} + +/* + * This function waits for the last execution of a work and then + * flush the work. + * It returns true if the work was pending and we waited, it returns + * false otherwise. + */ +bool +linux_flush_rcu_work(struct rcu_work *rwork) +{ + + if (linux_work_pending(&rwork->work)) { + linux_rcu_barrier(RCU_TYPE_REGULAR); + linux_flush_work(&rwork->work); + return (true); + } + return (linux_flush_work(&rwork->work)); +} + +/* + * This function queues the given work structure on the given + * workqueue after a given delay in ticks. It returns non-zero if the + * work was successfully [re-]queued. Else the work is already pending + * for completion. + */ +bool +linux_queue_delayed_work_on(int cpu, struct workqueue_struct *wq, + struct delayed_work *dwork, unsigned delay) +{ + static const uint8_t states[WORK_ST_MAX] __aligned(8) = { + [WORK_ST_IDLE] = WORK_ST_TIMER, /* start timeout */ + [WORK_ST_TIMER] = WORK_ST_TIMER, /* NOP */ + [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */ + [WORK_ST_EXEC] = WORK_ST_TIMER, /* start timeout */ + [WORK_ST_CANCEL] = WORK_ST_TIMER, /* start timeout */ + }; + + if (atomic_read(&wq->draining) != 0) + return (!work_pending(&dwork->work)); + + switch (linux_update_state(&dwork->work.state, states)) { + case WORK_ST_EXEC: + case WORK_ST_CANCEL: + if (delay == 0 && linux_work_exec_unblock(&dwork->work) != 0) { + dwork->timer.expires = jiffies; + return (true); + } + /* FALLTHROUGH */ + case WORK_ST_IDLE: + dwork->work.work_queue = wq; + dwork->timer.expires = jiffies + delay; + + if (delay == 0) { + linux_delayed_work_enqueue(dwork); + } else if (unlikely(cpu != WORK_CPU_UNBOUND)) { + mtx_lock(&dwork->timer.mtx); + callout_reset_on(&dwork->timer.callout, delay, + &linux_delayed_work_timer_fn, dwork, cpu); + mtx_unlock(&dwork->timer.mtx); + } else { + mtx_lock(&dwork->timer.mtx); + callout_reset(&dwork->timer.callout, delay, + &linux_delayed_work_timer_fn, dwork); + mtx_unlock(&dwork->timer.mtx); + } + return (true); + default: + return (false); /* already on a queue */ + } +} + +void +linux_work_fn(void *context, int pending) +{ + static const uint8_t states[WORK_ST_MAX] __aligned(8) = { + [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ + [WORK_ST_TIMER] = WORK_ST_EXEC, /* delayed work w/o timeout */ + [WORK_ST_TASK] = WORK_ST_EXEC, /* call callback */ + [WORK_ST_EXEC] = WORK_ST_IDLE, /* complete callback */ + [WORK_ST_CANCEL] = WORK_ST_EXEC, /* failed to cancel */ + }; + struct work_struct *work; + struct workqueue_struct *wq; + struct work_exec exec; + struct task_struct *task; + + task = current; + + /* setup local variables */ + work = context; + wq = work->work_queue; + + /* store target pointer */ + exec.target = work; + + /* insert executor into list */ + WQ_EXEC_LOCK(wq); + TAILQ_INSERT_TAIL(&wq->exec_head, &exec, entry); + while (1) { + switch (linux_update_state(&work->state, states)) { + case WORK_ST_TIMER: + case WORK_ST_TASK: + case WORK_ST_CANCEL: + WQ_EXEC_UNLOCK(wq); + + /* set current work structure */ + task->work = work; + + /* call work function */ + work->func(work); + + /* set current work structure */ + task->work = NULL; + + WQ_EXEC_LOCK(wq); + /* check if unblocked */ + if (exec.target != work) { + /* reapply block */ + exec.target = work; + break; + } + /* FALLTHROUGH */ + default: + goto done; + } + } +done: + /* remove executor from list */ + TAILQ_REMOVE(&wq->exec_head, &exec, entry); + WQ_EXEC_UNLOCK(wq); +} + +void +linux_delayed_work_fn(void *context, int pending) +{ + struct delayed_work *dwork = context; + + /* + * Make sure the timer belonging to the delayed work gets + * drained before invoking the work function. Else the timer + * mutex may still be in use which can lead to use-after-free + * situations, because the work function might free the work + * structure before returning. + */ + callout_drain(&dwork->timer.callout); + + linux_work_fn(&dwork->work, pending); +} + +static void +linux_delayed_work_timer_fn(void *arg) +{ + static const uint8_t states[WORK_ST_MAX] __aligned(8) = { + [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ + [WORK_ST_TIMER] = WORK_ST_TASK, /* start queueing task */ + [WORK_ST_TASK] = WORK_ST_TASK, /* NOP */ + [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */ + [WORK_ST_CANCEL] = WORK_ST_TASK, /* failed to cancel */ + }; + struct delayed_work *dwork = arg; + + switch (linux_update_state(&dwork->work.state, states)) { + case WORK_ST_TIMER: + case WORK_ST_CANCEL: + linux_delayed_work_enqueue(dwork); + break; + default: + break; + } +} + +/* + * This function cancels the given work structure in a synchronous + * fashion. It returns non-zero if the work was successfully + * cancelled. Else the work was already cancelled. + */ +bool +linux_cancel_work_sync(struct work_struct *work) +{ + static const uint8_t states[WORK_ST_MAX] __aligned(8) = { + [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ + [WORK_ST_TIMER] = WORK_ST_TIMER, /* can't happen */ + [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */ + [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */ + [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */ + }; + struct taskqueue *tq; + bool retval = false; + + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "linux_cancel_work_sync() might sleep"); +retry: + switch (linux_update_state(&work->state, states)) { + case WORK_ST_IDLE: + case WORK_ST_TIMER: + return (retval); + case WORK_ST_EXEC: + tq = work->work_queue->taskqueue; + if (taskqueue_cancel(tq, &work->work_task, NULL) != 0) + taskqueue_drain(tq, &work->work_task); + goto retry; /* work may have restarted itself */ + default: + tq = work->work_queue->taskqueue; + if (taskqueue_cancel(tq, &work->work_task, NULL) != 0) + taskqueue_drain(tq, &work->work_task); + retval = true; + goto retry; + } +} + +/* + * This function atomically stops the timer and callback. The timer + * callback will not be called after this function returns. This + * functions returns true when the timeout was cancelled. Else the + * timeout was not started or has already been called. + */ +static inline bool +linux_cancel_timer(struct delayed_work *dwork, bool drain) +{ + bool cancelled; + + mtx_lock(&dwork->timer.mtx); + cancelled = (callout_stop(&dwork->timer.callout) == 1); + mtx_unlock(&dwork->timer.mtx); + + /* check if we should drain */ + if (drain) + callout_drain(&dwork->timer.callout); + return (cancelled); +} + +/* + * This function cancels the given delayed work structure in a + * non-blocking fashion. It returns non-zero if the work was + * successfully cancelled. Else the work may still be busy or already + * cancelled. + */ +bool +linux_cancel_delayed_work(struct delayed_work *dwork) +{ + static const uint8_t states[WORK_ST_MAX] __aligned(8) = { + [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ + [WORK_ST_TIMER] = WORK_ST_CANCEL, /* try to cancel */ + [WORK_ST_TASK] = WORK_ST_CANCEL, /* try to cancel */ + [WORK_ST_EXEC] = WORK_ST_EXEC, /* NOP */ + [WORK_ST_CANCEL] = WORK_ST_CANCEL, /* NOP */ + }; + struct taskqueue *tq; + + switch (linux_update_state(&dwork->work.state, states)) { + case WORK_ST_TIMER: + case WORK_ST_CANCEL: + if (linux_cancel_timer(dwork, 0)) { + atomic_cmpxchg(&dwork->work.state, + WORK_ST_CANCEL, WORK_ST_IDLE); + return (true); + } + /* FALLTHROUGH */ + case WORK_ST_TASK: + tq = dwork->work.work_queue->taskqueue; + if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) == 0) { + atomic_cmpxchg(&dwork->work.state, + WORK_ST_CANCEL, WORK_ST_IDLE); + return (true); + } + /* FALLTHROUGH */ + default: + return (false); + } +} + +/* + * This function cancels the given work structure in a synchronous + * fashion. It returns non-zero if the work was successfully + * cancelled. Else the work was already cancelled. + */ +bool +linux_cancel_delayed_work_sync(struct delayed_work *dwork) +{ + static const uint8_t states[WORK_ST_MAX] __aligned(8) = { + [WORK_ST_IDLE] = WORK_ST_IDLE, /* NOP */ + [WORK_ST_TIMER] = WORK_ST_IDLE, /* cancel and drain */ + [WORK_ST_TASK] = WORK_ST_IDLE, /* cancel and drain */ + [WORK_ST_EXEC] = WORK_ST_IDLE, /* too late, drain */ + [WORK_ST_CANCEL] = WORK_ST_IDLE, /* cancel and drain */ + }; + struct taskqueue *tq; + bool retval = false; + + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "linux_cancel_delayed_work_sync() might sleep"); +retry: + switch (linux_update_state(&dwork->work.state, states)) { + case WORK_ST_IDLE: + return (retval); + case WORK_ST_EXEC: + tq = dwork->work.work_queue->taskqueue; + if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) != 0) + taskqueue_drain(tq, &dwork->work.work_task); + goto retry; /* work may have restarted itself */ + case WORK_ST_TIMER: + case WORK_ST_CANCEL: + if (linux_cancel_timer(dwork, 1)) { + /* + * Make sure taskqueue is also drained before + * returning: + */ + tq = dwork->work.work_queue->taskqueue; + taskqueue_drain(tq, &dwork->work.work_task); + retval = true; + goto retry; + } + /* FALLTHROUGH */ + default: + tq = dwork->work.work_queue->taskqueue; + if (taskqueue_cancel(tq, &dwork->work.work_task, NULL) != 0) + taskqueue_drain(tq, &dwork->work.work_task); + retval = true; + goto retry; + } +} + +/* + * This function waits until the given work structure is completed. + * It returns non-zero if the work was successfully + * waited for. Else the work was not waited for. + */ +bool +linux_flush_work(struct work_struct *work) +{ + struct taskqueue *tq; + bool retval; + + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "linux_flush_work() might sleep"); + + switch (atomic_read(&work->state)) { + case WORK_ST_IDLE: + return (false); + default: + tq = work->work_queue->taskqueue; + retval = taskqueue_poll_is_busy(tq, &work->work_task); + taskqueue_drain(tq, &work->work_task); + return (retval); + } +} + +/* + * This function waits until the given delayed work structure is + * completed. It returns non-zero if the work was successfully waited + * for. Else the work was not waited for. + */ +bool +linux_flush_delayed_work(struct delayed_work *dwork) +{ + struct taskqueue *tq; + bool retval; + + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "linux_flush_delayed_work() might sleep"); + + switch (atomic_read(&dwork->work.state)) { + case WORK_ST_IDLE: + return (false); + case WORK_ST_TIMER: + if (linux_cancel_timer(dwork, 1)) + linux_delayed_work_enqueue(dwork); + /* FALLTHROUGH */ + default: + tq = dwork->work.work_queue->taskqueue; + retval = taskqueue_poll_is_busy(tq, &dwork->work.work_task); + taskqueue_drain(tq, &dwork->work.work_task); + return (retval); + } +} + +/* + * This function returns true if the given work is pending, and not + * yet executing: + */ +bool +linux_work_pending(struct work_struct *work) +{ + switch (atomic_read(&work->state)) { + case WORK_ST_TIMER: + case WORK_ST_TASK: + case WORK_ST_CANCEL: + return (true); + default: + return (false); + } +} + +/* + * This function returns true if the given work is busy. + */ +bool +linux_work_busy(struct work_struct *work) +{ + struct taskqueue *tq; + + switch (atomic_read(&work->state)) { + case WORK_ST_IDLE: + return (false); + case WORK_ST_EXEC: + tq = work->work_queue->taskqueue; + return (taskqueue_poll_is_busy(tq, &work->work_task)); + default: + return (true); + } +} + +struct workqueue_struct * +linux_create_workqueue_common(const char *name, int cpus) +{ + struct workqueue_struct *wq; + + /* + * If zero CPUs are specified use the default number of CPUs: + */ + if (cpus == 0) + cpus = linux_default_wq_cpus; + + wq = kmalloc(sizeof(*wq), M_WAITOK | M_ZERO); + wq->taskqueue = taskqueue_create(name, M_WAITOK, + taskqueue_thread_enqueue, &wq->taskqueue); + atomic_set(&wq->draining, 0); + taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name); + TAILQ_INIT(&wq->exec_head); + mtx_init(&wq->exec_mtx, "linux_wq_exec", NULL, MTX_DEF); + + return (wq); +} + +void +linux_destroy_workqueue(struct workqueue_struct *wq) +{ + atomic_inc(&wq->draining); + drain_workqueue(wq); + taskqueue_free(wq->taskqueue); + mtx_destroy(&wq->exec_mtx); + kfree(wq); +} + +void +linux_init_delayed_work(struct delayed_work *dwork, work_func_t func) +{ + memset(dwork, 0, sizeof(*dwork)); + dwork->work.func = func; + TASK_INIT(&dwork->work.work_task, 0, linux_delayed_work_fn, dwork); + mtx_init(&dwork->timer.mtx, spin_lock_name("lkpi-dwork"), NULL, + MTX_DEF | MTX_NOWITNESS); + callout_init_mtx(&dwork->timer.callout, &dwork->timer.mtx, 0); +} + +struct work_struct * +linux_current_work(void) +{ + return (current->work); +} + +static void +linux_work_init(void *arg) +{ + int max_wq_cpus = mp_ncpus + 1; + + /* avoid deadlock when there are too few threads */ + if (max_wq_cpus < 4) + max_wq_cpus = 4; + + /* set default number of CPUs */ + linux_default_wq_cpus = max_wq_cpus; + + linux_system_short_wq = alloc_workqueue("linuxkpi_short_wq", 0, max_wq_cpus); + linux_system_long_wq = alloc_workqueue("linuxkpi_long_wq", 0, max_wq_cpus); + + /* populate the workqueue pointers */ + system_long_wq = linux_system_long_wq; + system_wq = linux_system_short_wq; + system_power_efficient_wq = linux_system_short_wq; + system_unbound_wq = linux_system_short_wq; + system_highpri_wq = linux_system_short_wq; +} +SYSINIT(linux_work_init, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_init, NULL); + +static void +linux_work_uninit(void *arg) +{ + destroy_workqueue(linux_system_short_wq); + destroy_workqueue(linux_system_long_wq); + + /* clear workqueue pointers */ + system_long_wq = NULL; + system_wq = NULL; + system_power_efficient_wq = NULL; + system_unbound_wq = NULL; + system_highpri_wq = NULL; +} +SYSUNINIT(linux_work_uninit, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_uninit, NULL); |