diff --git a/include/memkind/internal/memkind_memtier.h b/include/memkind/internal/memkind_memtier.h index 0697d97d..22b3cb92 100644 --- a/include/memkind/internal/memkind_memtier.h +++ b/include/memkind/internal/memkind_memtier.h @@ -96,6 +96,11 @@ void memtier_delete_memtier_memory(struct memtier_memory *memory); /// void *memtier_malloc(struct memtier_memory *memory, size_t size); +// comment TODO +void *memtier_mmap(struct memtier_memory *memory, + void *addr, size_t length, int prot, int flags, int fd, off_t offset); +void memtier_munmap(void* ptr); + /// /// \brief Allocates size bytes of uninitialized storage of the specified kind /// \note STANDARD API diff --git a/include/memkind/internal/memkind_private.h b/include/memkind/internal/memkind_private.h index dd1cf94f..186db5f3 100644 --- a/include/memkind/internal/memkind_private.h +++ b/include/memkind/internal/memkind_private.h @@ -109,6 +109,7 @@ typedef enum memkind_node_variant_t void memkind_init(memkind_t kind, bool check_numa); void *kind_mmap(struct memkind *kind, void *addr, size_t size); +void *sys_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t off); char *memkind_get_env(const char *name); diff --git a/jemalloc/src/pages.c b/jemalloc/src/pages.c index b6015780..0c54c243 100644 --- a/jemalloc/src/pages.c +++ b/jemalloc/src/pages.c @@ -76,7 +76,7 @@ os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) { { int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; - ret = mmap(addr, size, prot, mmap_flags, -1, 0); + ret = sys_mmap(addr, size, prot, mmap_flags, -1, 0); } assert(ret != NULL); @@ -201,7 +201,7 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) { flags |= MAP_ALIGNED(alignment_bits - 1); } - void *ret = mmap(addr, size, prot, flags, -1, 0); + void *ret = sys_mmap(addr, size, prot, flags, -1, 0); if (ret == MAP_FAILED) { ret = NULL; } @@ -260,7 +260,7 @@ pages_commit_impl(void *addr, size_t size, bool commit) { #else { int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; - void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED, + void *result = sys_mmap(addr, size, prot, mmap_flags | MAP_FIXED, -1, 0); if (result == MAP_FAILED) { return true; diff --git a/src/bigary.c b/src/bigary.c index bc08eb40..cfc2a6ed 100644 --- a/src/bigary.c +++ b/src/bigary.c @@ -7,6 +7,7 @@ #include #include "memkind/internal/bigary.h" +#include "memkind/internal/memkind_private.h" // default max: 16 GB #define BIGARY_DEFAULT_MAX (16 * 1024 * 1024 * 1024ULL) #define BIGARY_PAGESIZE 2097152 @@ -42,9 +43,9 @@ void bigary_init(bigary *restrict ba, int fd, int flags, size_t max) ba->declared = max; ba->fd = fd; ba->flags = flags; - if ((ba->area = mmap(0, max, PROT_NONE, flags, fd, 0)) == MAP_FAILED) + if ((ba->area = sys_mmap(0, max, PROT_NONE, flags, fd, 0)) == MAP_FAILED) die("mmapping bigary(%zd) failed: %m\n", max); - if (mmap(ba->area, BIGARY_PAGESIZE, PROT_READ|PROT_WRITE, + if (sys_mmap(ba->area, BIGARY_PAGESIZE, PROT_READ|PROT_WRITE, MAP_FIXED|flags, fd, 0) == MAP_FAILED) { die("bigary alloc of %zd failed: %m\n", BIGARY_PAGESIZE); @@ -75,7 +76,7 @@ void bigary_alloc(bigary *restrict ba, size_t top) // printf("extending to %zd\n", top); if (top > ba->declared) die("bigary's max is %zd, %zd requested.\n", ba->declared, top); - if (mmap(ba->area + ba->top, top - ba->top, PROT_READ|PROT_WRITE, + if (sys_mmap(ba->area + ba->top, top - ba->top, PROT_READ|PROT_WRITE, MAP_FIXED|ba->flags, ba->fd, ba->top) == MAP_FAILED) { die("in-bigary alloc of %zd to %zd failed: %m\n", top - ba->top, top); diff --git a/src/memkind.c b/src/memkind.c index 90bd5adc..4be68ea3 100644 --- a/src/memkind.c +++ b/src/memkind.c @@ -38,6 +38,8 @@ #include #include #include +#include +#include // clang-format off #ifdef MEMKIND_ENABLE_HEAP_MANAGER diff --git a/src/memkind_default.c b/src/memkind_default.c index 6f76f44b..728d82f1 100644 --- a/src/memkind_default.c +++ b/src/memkind_default.c @@ -125,7 +125,7 @@ MEMKIND_EXPORT void *memkind_default_mmap(struct memkind *kind, void *addr, err = memkind_default_get_mmap_flags(kind, &flags); } if (MEMKIND_LIKELY(!err)) { - result = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0); + result = sys_mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0); if (result == MAP_FAILED) { log_err("syscall mmap() returned: %p", result); return result; diff --git a/src/memkind_memtier.c b/src/memkind_memtier.c index badf5371..8267fbee 100644 --- a/src/memkind_memtier.c +++ b/src/memkind_memtier.c @@ -17,6 +17,7 @@ #include #include #include +#include #ifdef HAVE_STDATOMIC_H #include @@ -284,6 +285,7 @@ memtier_policy_static_ratio_get_kind(struct memtier_memory *memory, dest_tier = i; } } + //log_info("kind: %d", dest_tier); return cfg[dest_tier].kind; } @@ -1189,7 +1191,8 @@ MEMKIND_EXPORT void *memtier_malloc(struct memtier_memory *memory, size_t size) void *ptr; uint64_t data; - ptr = memtier_kind_malloc(memory->get_kind(memory, size, &data), size); + memkind_t kind = memory->get_kind(memory, size, &data); + ptr = memtier_kind_malloc(kind, size); memory->post_alloc(data, ptr, size); memory->update_cfg(memory); print_memory_statistics(memory); @@ -1197,6 +1200,76 @@ MEMKIND_EXPORT void *memtier_malloc(struct memtier_memory *memory, size_t size) return ptr; } +void *sys_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t off) +{ + long ret = syscall(SYS_mmap, addr, length, prot, flags, fd, off); + if (ret == -EPERM && !off && (flags&MAP_ANON) && !(flags&MAP_FIXED)) + ret = -ENOMEM; + if (ret > -4096 && ret < 0) { + errno = -ret; + return MAP_FAILED; + } + + return (void*)ret; +} + +int munmap(void *addr, size_t length) +{ + long ret = syscall(SYS_munmap, addr, length); + if (!ret) + return 0; + errno = -ret; + return -1; +} + +#define SLICE (2*1048576) + +static void split_into_slices(struct memtier_memory *memory, void *addr, size_t length) +{ + size_t todo = length; + void *sladdr = addr; + + while (todo > 0) { + size_t len = (todo > SLICE) ? SLICE : todo; + + uint64_t data = 0; + memkind_t kind = memory->get_kind(memory, len, &data); + memory->post_alloc(data, sladdr, len); + if (kind == MEMKIND_DEFAULT) + mbind(sladdr, len, MPOL_DEFAULT, 0, 0 ,0); + else { + unsigned long nodemask; + if (kind->ops->get_mbind_nodemask + && !kind->ops->get_mbind_nodemask(kind, &nodemask, sizeof(nodemask)*8)) + { + mbind(sladdr, len, MPOL_PREFERRED, &nodemask, sizeof(nodemask)*8, 0); + } else + fprintf(stderr, "Kind without nodemask\n"); + } + + todo -= len; + sladdr += len; + } +} + +MEMKIND_EXPORT void* memtier_mmap(struct memtier_memory *memory, void *addr, size_t length, int prot, int flags, int fd, off_t offset) +{ + if (memory == NULL) + return sys_mmap(addr, length, prot, flags, fd, offset); + + void* ret = memtier_malloc(memory, length); + + if (ret) + split_into_slices(memory, ret, length); + + return ret; +} + +MEMKIND_EXPORT void memtier_munmap(void* ptr) +{ + memtier_free(ptr); +} + MEMKIND_EXPORT void *memtier_kind_malloc(memkind_t kind, size_t size) { // static atomic_uint_fast16_t counter=0; @@ -1277,9 +1350,8 @@ MEMKIND_EXPORT void *memtier_realloc(struct memtier_memory *memory, void *ptr, return ptr; } - if (size == 0) { + if (size == 0) return NULL; - } return memtier_malloc(memory, size); } diff --git a/src/memkind_pmem.c b/src/memkind_pmem.c index 0704d30f..feb11231 100644 --- a/src/memkind_pmem.c +++ b/src/memkind_pmem.c @@ -332,7 +332,7 @@ MEMKIND_EXPORT void *memkind_pmem_mmap(struct memkind *kind, void *addr, } } - if ((result = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, priv->fd, + if ((result = sys_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, priv->fd, priv->offset)) != MAP_FAILED) { priv->offset += size; priv->current_size += size; @@ -367,7 +367,7 @@ int memkind_pmem_validate_dir(const char *dir) goto end; } - void *addr = mmap(NULL, size, PROT_READ | PROT_WRITE, + void *addr = sys_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED_VALIDATE | MAP_SYNC, fd, 0); if (addr == MAP_FAILED) { ret = MEMKIND_ERROR_MMAP; diff --git a/src/pebs.c b/src/pebs.c index d6f05a09..1cd86a4d 100644 --- a/src/pebs.c +++ b/src/pebs.c @@ -459,7 +459,7 @@ void pebs_init(pid_t pid) if (pebs_fd != -1) { int mmap_pages = 1 + MMAP_DATA_SIZE; int map_size = mmap_pages * getpagesize(); - pebs_mmap = mmap(NULL, map_size, + pebs_mmap = sys_mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, pebs_fd, 0); #if PRINT_PEBS_BASIC_INFO diff --git a/tiering/memtier.c b/tiering/memtier.c index 36fcf757..5455c866 100644 --- a/tiering/memtier.c +++ b/tiering/memtier.c @@ -11,6 +11,19 @@ #include #include #include +#include + +#include +#include +#include + +#ifdef HAVE_STDATOMIC_H +#include +#define MEMKIND_ATOMIC _Atomic +#else +#define MEMKIND_ATOMIC +#endif + #define MEMTIER_EXPORT __attribute__((visibility("default"))) #define MEMTIER_INIT __attribute__((constructor)) @@ -85,6 +98,9 @@ static int destructed; static struct memtier_memory *current_memory; +void *sys_mmap(void *, size_t, int, int, int, off_t); +int sys_munmap(void *, size_t); + MEMTIER_EXPORT void *malloc(size_t size) { if (MEMTIER_LIKELY(current_memory)) { @@ -143,6 +159,40 @@ MEMTIER_EXPORT size_t malloc_usable_size(void *ptr) return memtier_usable_size(ptr); } +MEMTIER_EXPORT void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) +{ + // TODO tweaked for MLC - find other valid flag combinations + if ((current_memory == 0) || ( + (addr == NULL) && + (prot == (PROT_READ | PROT_WRITE)) && + (flags == (MAP_ANONYMOUS | MAP_PRIVATE)))) + { + //log_err("mmap: start:%p, length:%lu, prot:%d, flags:%d, fd:%d, offset:%ld", + // addr, length, prot, flags, fd, offset); + + return memtier_mmap(current_memory, addr, length, prot, flags, fd, offset); + } + + return sys_mmap(addr, length, prot, flags, fd, offset); +} + +/* +MEMTIER_EXPORT int munmap(void *addr, size_t length) +{ + int i; + for(i = 0; i < num_mmaps; i++) + { + if (mmap_map[i] == addr) { + //log_err("munmap: start:%p, length:%lu", addr, length); + memtier_munmap(addr); + return 0; + } + } + + return sys_munmap(addr, length); +} +*/ + static pthread_once_t init_once = PTHREAD_ONCE_INIT; static MEMTIER_INIT void memtier_init(void)