mmap

include/memkind/internal/memkind_memtier.h

@@ -96,6 +96,11 @@ void memtier_delete_memtier_memory(struct memtier_memory *memory);
 ///
 void *memtier_malloc(struct memtier_memory *memory, size_t size);
 
+// comment TODO
+void *memtier_mmap(struct memtier_memory *memory,
+    void *addr, size_t length, int prot, int flags, int fd, off_t offset);
+void memtier_munmap(void* ptr);
+
 ///
 /// \brief Allocates size bytes of uninitialized storage of the specified kind
 /// \note STANDARD API

include/memkind/internal/memkind_private.h

@@ -109,6 +109,7 @@ typedef enum memkind_node_variant_t
 void memkind_init(memkind_t kind, bool check_numa);
 
 void *kind_mmap(struct memkind *kind, void *addr, size_t size);
+void *sys_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t off);
 
 char *memkind_get_env(const char *name);
 

jemalloc/src/pages.c

@@ -76,7 +76,7 @@ os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
 	{
 		int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
 
-		ret = mmap(addr, size, prot, mmap_flags, -1, 0);
+		ret = sys_mmap(addr, size, prot, mmap_flags, -1, 0);
 	}
 	assert(ret != NULL);
 
@@ -201,7 +201,7 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
 			flags |= MAP_ALIGNED(alignment_bits - 1);
 		}
 
-		void *ret = mmap(addr, size, prot, flags, -1, 0);
+		void *ret = sys_mmap(addr, size, prot, flags, -1, 0);
 		if (ret == MAP_FAILED) {
 			ret = NULL;
 		}
@@ -260,7 +260,7 @@ pages_commit_impl(void *addr, size_t size, bool commit) {
 #else
 	{
 		int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
-		void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED,
+		void *result = sys_mmap(addr, size, prot, mmap_flags | MAP_FIXED,
 		    -1, 0);
 		if (result == MAP_FAILED) {
 			return true;

src/bigary.c

@@ -7,6 +7,7 @@
 #include <unistd.h>
 
 #include "memkind/internal/bigary.h"
+#include "memkind/internal/memkind_private.h"
 // default max: 16 GB
 #define BIGARY_DEFAULT_MAX (16 * 1024 * 1024 * 1024ULL)
 #define BIGARY_PAGESIZE 2097152
@@ -42,9 +43,9 @@ void bigary_init(bigary *restrict ba, int fd, int flags, size_t max)
     ba->declared = max;
     ba->fd = fd;
     ba->flags = flags;
-    if ((ba->area = mmap(0, max, PROT_NONE, flags, fd, 0)) == MAP_FAILED)
+    if ((ba->area = sys_mmap(0, max, PROT_NONE, flags, fd, 0)) == MAP_FAILED)
         die("mmapping bigary(%zd) failed: %m\n", max);
-    if (mmap(ba->area, BIGARY_PAGESIZE, PROT_READ|PROT_WRITE,
+    if (sys_mmap(ba->area, BIGARY_PAGESIZE, PROT_READ|PROT_WRITE,
         MAP_FIXED|flags, fd, 0) == MAP_FAILED)
     {
         die("bigary alloc of %zd failed: %m\n", BIGARY_PAGESIZE);
@@ -75,7 +76,7 @@ void bigary_alloc(bigary *restrict ba, size_t top)
     // printf("extending to %zd\n", top);
     if (top > ba->declared)
         die("bigary's max is %zd, %zd requested.\n", ba->declared, top);
-    if (mmap(ba->area + ba->top, top - ba->top, PROT_READ|PROT_WRITE,
+    if (sys_mmap(ba->area + ba->top, top - ba->top, PROT_READ|PROT_WRITE,
         MAP_FIXED|ba->flags, ba->fd, ba->top) == MAP_FAILED)
     {
         die("in-bigary alloc of %zd to %zd failed: %m\n", top - ba->top, top);

src/memkind.c

@@ -38,6 +38,8 @@
 #include <sys/mman.h>
 #include <sys/param.h>
 #include <unistd.h>
+#include <pthread.h>
+#include <threads.h>
 
 // clang-format off
 #ifdef MEMKIND_ENABLE_HEAP_MANAGER

src/memkind_default.c

@@ -125,7 +125,7 @@ MEMKIND_EXPORT void *memkind_default_mmap(struct memkind *kind, void *addr,
         err = memkind_default_get_mmap_flags(kind, &flags);
     }
     if (MEMKIND_LIKELY(!err)) {
-        result = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0);
+        result = sys_mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0);
         if (result == MAP_FAILED) {
             log_err("syscall mmap() returned: %p", result);
             return result;

src/memkind_memtier.c

@@ -17,6 +17,7 @@
 #include <pthread.h>
 #include <threads.h>
 #include <execinfo.h>
+#include <numaif.h>
 
 #ifdef HAVE_STDATOMIC_H
 #include <stdatomic.h>
@@ -284,6 +285,7 @@ memtier_policy_static_ratio_get_kind(struct memtier_memory *memory,
             dest_tier = i;
         }
     }
+    //log_info("kind: %d", dest_tier);
     return cfg[dest_tier].kind;
 }
 
@@ -1189,14 +1191,85 @@ MEMKIND_EXPORT void *memtier_malloc(struct memtier_memory *memory, size_t size)
     void *ptr;
     uint64_t data;
 
-    ptr = memtier_kind_malloc(memory->get_kind(memory, size, &data), size);
+    memkind_t kind = memory->get_kind(memory, size, &data);
+    ptr = memtier_kind_malloc(kind, size);
     memory->post_alloc(data, ptr, size);
     memory->update_cfg(memory);
     print_memory_statistics(memory);
 
     return ptr;
 }
 
+void *sys_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t off)
+{
+    long ret = syscall(SYS_mmap, addr, length, prot, flags, fd, off);
+    if (ret == -EPERM && !off && (flags&MAP_ANON) && !(flags&MAP_FIXED))
+        ret = -ENOMEM;
+    if (ret > -4096 && ret < 0) {
+        errno = -ret;
+        return MAP_FAILED;
+    }
+
+    return (void*)ret;
+}
+
+int munmap(void *addr, size_t length)
+{
+    long ret = syscall(SYS_munmap, addr, length);
+    if (!ret)
+        return 0;
+    errno = -ret;
+    return -1;
+}
+
+#define SLICE (2*1048576)
+
+static void split_into_slices(struct memtier_memory *memory, void *addr, size_t length)
+{
+    size_t todo = length;
+    void *sladdr = addr;
+
+    while (todo > 0) {
+        size_t len = (todo > SLICE) ? SLICE : todo;
+
+        uint64_t data = 0;
+        memkind_t kind = memory->get_kind(memory, len, &data);
+        memory->post_alloc(data, sladdr, len);
+        if (kind == MEMKIND_DEFAULT)
+            mbind(sladdr, len, MPOL_DEFAULT, 0, 0 ,0);
+        else {
+            unsigned long nodemask;
+            if (kind->ops->get_mbind_nodemask
+                && !kind->ops->get_mbind_nodemask(kind, &nodemask, sizeof(nodemask)*8))
+            {
+                mbind(sladdr, len, MPOL_PREFERRED, &nodemask, sizeof(nodemask)*8, 0);
+            } else
+                fprintf(stderr, "Kind without nodemask\n");
+        }
+
+        todo -= len;
+        sladdr += len;
+    }
+}
+
+MEMKIND_EXPORT void* memtier_mmap(struct memtier_memory *memory, void *addr, size_t length, int prot, int flags, int fd, off_t offset)
+{
+    if (memory == NULL)
+        return sys_mmap(addr, length, prot, flags, fd, offset);
+
+    void* ret = memtier_malloc(memory, length);
+
+    if (ret)
+        split_into_slices(memory, ret, length);
+
+    return ret;
+}
+
+MEMKIND_EXPORT void memtier_munmap(void* ptr)
+{
+    memtier_free(ptr);
+}
+
 MEMKIND_EXPORT void *memtier_kind_malloc(memkind_t kind, size_t size)
 {
 //     static atomic_uint_fast16_t counter=0;
@@ -1277,9 +1350,8 @@ MEMKIND_EXPORT void *memtier_realloc(struct memtier_memory *memory, void *ptr,
         return ptr;
     }
 
-    if (size == 0) {
+    if (size == 0)
         return NULL;
-    }
 
     return memtier_malloc(memory, size);
 }

src/memkind_pmem.c

@@ -332,7 +332,7 @@ MEMKIND_EXPORT void *memkind_pmem_mmap(struct memkind *kind, void *addr,
         }
     }
 
-    if ((result = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, priv->fd,
+    if ((result = sys_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, priv->fd,
                        priv->offset)) != MAP_FAILED) {
         priv->offset += size;
         priv->current_size += size;
@@ -367,7 +367,7 @@ int memkind_pmem_validate_dir(const char *dir)
         goto end;
     }
 
-    void *addr = mmap(NULL, size, PROT_READ | PROT_WRITE,
+    void *addr = sys_mmap(NULL, size, PROT_READ | PROT_WRITE,
                       MAP_SHARED_VALIDATE | MAP_SYNC, fd, 0);
     if (addr == MAP_FAILED) {
         ret = MEMKIND_ERROR_MMAP;

src/pebs.c

@@ -459,7 +459,7 @@ void pebs_init(pid_t pid)
     if (pebs_fd != -1) {
         int mmap_pages = 1 + MMAP_DATA_SIZE;
         int map_size = mmap_pages * getpagesize();
-        pebs_mmap = mmap(NULL, map_size,
+        pebs_mmap = sys_mmap(NULL, map_size,
                         PROT_READ | PROT_WRITE, MAP_SHARED, pebs_fd, 0);
 
 #if PRINT_PEBS_BASIC_INFO

tiering/memtier.c

@@ -11,6 +11,19 @@
 #include <pthread.h>
 #include <string.h>
 #include <dlfcn.h>
+#include <sys/mman.h>
+
+#include <pthread.h>
+#include <threads.h>
+#include <execinfo.h>
+
+#ifdef HAVE_STDATOMIC_H
+#include <stdatomic.h>
+#define MEMKIND_ATOMIC _Atomic
+#else
+#define MEMKIND_ATOMIC
+#endif
+
 
 #define MEMTIER_EXPORT __attribute__((visibility("default")))
 #define MEMTIER_INIT   __attribute__((constructor))
@@ -85,6 +98,9 @@ static int destructed;
 
 static struct memtier_memory *current_memory;
 
+void *sys_mmap(void *, size_t, int, int, int, off_t);
+int sys_munmap(void *, size_t);
+
 MEMTIER_EXPORT void *malloc(size_t size)
 {
     if (MEMTIER_LIKELY(current_memory)) {
@@ -143,6 +159,40 @@ MEMTIER_EXPORT size_t malloc_usable_size(void *ptr)
     return memtier_usable_size(ptr);
 }
 
+MEMTIER_EXPORT void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset)
+{
+    // TODO tweaked for MLC - find other valid flag combinations
+    if ((current_memory == 0) || (
+        (addr == NULL) &&
+        (prot == (PROT_READ | PROT_WRITE)) && 
+        (flags == (MAP_ANONYMOUS | MAP_PRIVATE)))) 
+    {
+        //log_err("mmap: start:%p, length:%lu, prot:%d, flags:%d, fd:%d, offset:%ld", 
+        //    addr, length, prot, flags, fd, offset);
+
+        return memtier_mmap(current_memory, addr, length, prot, flags, fd, offset);
+    }
+
+    return sys_mmap(addr, length, prot, flags, fd, offset);
+}
+
+/*
+MEMTIER_EXPORT int munmap(void *addr, size_t length)
+{
+    int i;
+    for(i = 0; i < num_mmaps; i++)
+    {
+        if (mmap_map[i] == addr) {
+            //log_err("munmap: start:%p, length:%lu", addr, length);
+            memtier_munmap(addr);
+            return 0;
+        }
+    }
+
+    return sys_munmap(addr, length);
+}
+*/
+
 static pthread_once_t init_once = PTHREAD_ONCE_INIT;
 
 static MEMTIER_INIT void memtier_init(void)