aboutsummaryrefslogtreecommitdiff
path: root/sys
diff options
context:
space:
mode:
authorMateusz Guzik <mjg@FreeBSD.org>2020-02-12 11:12:13 +0000
committerMateusz Guzik <mjg@FreeBSD.org>2020-02-12 11:12:13 +0000
commitfb886947d97375ea9906fa1396f83573b6d0674b (patch)
treebc47d67f771fbc25572b79c4d056fe3e80f9d2cc /sys
parent3acb6572fc0ef19119240a319873fa75fd597f28 (diff)
downloadsrc-fb886947d97375ea9906fa1396f83573b6d0674b.tar.gz
src-fb886947d97375ea9906fa1396f83573b6d0674b.zip
amd64: store per-cpu allocations subtracted by __pcpu
This eliminates a runtime subtraction from counter_u64_add. before: mov 0x4f00ed(%rip),%rax # 0xffffffff80c01788 <numfullpathfail4> sub 0x808ff6(%rip),%rax # 0xffffffff80f1a698 <__pcpu> addq $0x1,%gs:(%rax) after: mov 0x4f02fd(%rip),%rax # 0xffffffff80c01788 <numfullpathfail4> addq $0x1,%gs:(%rax) Reviewed by: jeff Differential Revision: https://reviews.freebsd.org/D23570
Notes
Notes: svn path=/head/; revision=357805
Diffstat (limited to 'sys')
-rw-r--r--sys/amd64/include/counter.h6
-rw-r--r--sys/amd64/include/pcpu.h4
2 files changed, 8 insertions, 2 deletions
diff --git a/sys/amd64/include/counter.h b/sys/amd64/include/counter.h
index e1b159482a6e..a3c7cecbc919 100644
--- a/sys/amd64/include/counter.h
+++ b/sys/amd64/include/counter.h
@@ -33,7 +33,7 @@
#include <sys/pcpu.h>
-#define EARLY_COUNTER &temp_bsp_pcpu.pc_early_dummy_counter
+#define EARLY_COUNTER (void *)__offsetof(struct pcpu, pc_early_dummy_counter)
#define counter_enter() do {} while (0)
#define counter_exit() do {} while (0)
@@ -43,6 +43,7 @@ static inline uint64_t
counter_u64_read_one(counter_u64_t c, int cpu)
{
+ MPASS(c != EARLY_COUNTER);
return (*zpcpu_get_cpu(c, cpu));
}
@@ -65,6 +66,7 @@ counter_u64_zero_one_cpu(void *arg)
counter_u64_t c;
c = arg;
+ MPASS(c != EARLY_COUNTER);
*(zpcpu_get(c)) = 0;
}
@@ -86,7 +88,7 @@ counter_u64_add(counter_u64_t c, int64_t inc)
KASSERT(IS_BSP() || c != EARLY_COUNTER, ("EARLY_COUNTER used on AP"));
__asm __volatile("addq\t%1,%%gs:(%0)"
:
- : "r" ((char *)c - (char *)&__pcpu[0]), "ri" (inc)
+ : "r" (c), "ri" (inc)
: "memory", "cc");
}
diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
index 24388e204df0..718c36d792fb 100644
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@@ -240,6 +240,10 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
#define IS_BSP() (PCPU_GET(cpuid) == 0)
+#define zpcpu_offset_cpu(cpu) ((uintptr_t)&__pcpu[0] + UMA_PCPU_ALLOC_SIZE * cpu)
+#define zpcpu_base_to_offset(base) (void *)((uintptr_t)(base) - (uintptr_t)&__pcpu[0])
+#define zpcpu_offset_to_base(base) (void *)((uintptr_t)(base) + (uintptr_t)&__pcpu[0])
+
#else /* !__GNUCLIKE_ASM || !__GNUCLIKE___TYPEOF */
#error "this file needs to be ported to your compiler"