Implement a facility for dynamic per-cpu variables.

 - Modules and kernel code alike may use DPCPU_DEFINE(),
   DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined
   PCPU_*.  Requires only one extra instruction more than PCPU_* and is
   virtually the same as __thread for builtin and much faster for shared
   objects.  DPCPU variables can be initialized when defined.
 - Modules are supported by relocating the module's per-cpu linker set
   over space reserved in the kernel.  Modules may fail to load if there
   is insufficient space available.
 - Track space available for modules with a one-off extent allocator.
   Free may block for memory to allocate space for an extent.

Reviewed by:    jhb, rwatson, kan, sam, grehan, marius, marcel, stas

1 files changed, 52 insertions, 0 deletions

diff --git a/sys/kern/link_elf.c b/sys/kern/link_elf.c
index 57f0206fb609..cd0f3e909d5b 100644
--- a/sys/kern/link_elf.c
+++ b/sys/kern/link_elf.c
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/mount.h>
+#include <sys/pcpu.h>
 #include <sys/proc.h>
 #include <sys/namei.h>
 #include <sys/fcntl.h>
@@ -107,6 +108,9 @@ typedef struct elf_file {
     caddr_t		ctfoff;		/* CTF offset table */
     caddr_t		typoff;		/* Type offset table */
     long		typlen;		/* Number of type entries. */
+    Elf_Addr		pcpu_start;	/* Pre-relocation pcpu set start. */
+    Elf_Addr		pcpu_stop;	/* Pre-relocation pcpu set stop. */
+    Elf_Addr		pcpu_base;	/* Relocated pcpu set address. */
 #ifdef GDB
     struct link_map	gdb;		/* hooks for gdb */
 #endif
@@ -475,6 +479,34 @@ parse_dynamic(elf_file_t ef)
 }
 
 static int
+parse_dpcpu(elf_file_t ef)
+{ 
+    int count;
+    int error;
+
+    ef->pcpu_start = 0;
+    ef->pcpu_stop = 0;
+    error = link_elf_lookup_set(&ef->lf, "pcpu", (void ***)&ef->pcpu_start,
+                               (void ***)&ef->pcpu_stop, &count);
+    /* Error just means there is no pcpu set to relocate. */
+    if (error)
+        return (0);
+    count *= sizeof(void *);
+    /*
+     * Allocate space in the primary pcpu area.  Copy in our initialization
+     * from the data section and then initialize all per-cpu storage from
+     * that.
+     */
+    ef->pcpu_base = (Elf_Addr)(uintptr_t)dpcpu_alloc(count);
+    if (ef->pcpu_base == (Elf_Addr)NULL)
+        return (ENOSPC);
+    memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, count);
+    dpcpu_copy((void *)ef->pcpu_base, count);
+
+    return (0);
+}
+
+static int
 link_elf_link_preload(linker_class_t cls,
 		      const char* filename, linker_file_t *result)
 {
@@ -519,6 +551,8 @@ link_elf_link_preload(linker_class_t cls,
     lf->size = *(size_t *)sizeptr;
 
     error = parse_dynamic(ef);
+    if (error == 0)
+        error = parse_dpcpu(ef);
     if (error) {
 	linker_file_unload(lf, LINKER_UNLOAD_FORCE);
 	return error;
@@ -801,6 +835,9 @@ link_elf_load_file(linker_class_t cls, const char* filename,
     error = parse_dynamic(ef);
     if (error)
 	goto out;
+    error = parse_dpcpu(ef);
+    if (error)
+        goto out;
     link_elf_reloc_local(lf);
 
     VOP_UNLOCK(nd.ni_vp, 0);
@@ -897,11 +934,26 @@ out:
     return error;
 }
 
+Elf_Addr
+elf_relocaddr(linker_file_t lf, Elf_Addr x)
+{
+    elf_file_t ef;
+
+    ef = (elf_file_t)lf;
+    if (x >= ef->pcpu_start && x < ef->pcpu_stop)
+	return ((x - ef->pcpu_start) + ef->pcpu_base);
+    return (x);
+}
+
+
 static void
 link_elf_unload_file(linker_file_t file)
 {
     elf_file_t ef = (elf_file_t) file;
 
+    if (ef->pcpu_base) {
+        dpcpu_free((void *)ef->pcpu_base, ef->pcpu_stop - ef->pcpu_start);
+    }
 #ifdef GDB
     if (ef->gdb.l_ld) {
 	GDB_STATE(RT_DELETE);