From 8cc1d429c56b73545f783d67fdea6d2a3b8fc8f0 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 10 Mar 2025 14:22:47 +0100 Subject: [PATCH 1/7] Use atomics in find_successor() like in find_predecessor() Use atomics in find_successor() like in find_predecessor(). Ref: #1175 Signed-off-by: Lukasz Dorau --- src/critnib/critnib.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index feb492e20..5625781d3 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -645,7 +645,9 @@ static struct critnib_leaf *find_successor(struct critnib_node *__restrict n) { while (1) { unsigned nib; for (nib = 0; nib <= NIB; nib++) { - if (n->child[nib]) { + struct critnib_node *m; + utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&m); + if (m) { break; } } @@ -654,7 +656,7 @@ static struct critnib_leaf *find_successor(struct critnib_node *__restrict n) { return NULL; } - n = n->child[nib]; + utils_atomic_load_acquire_ptr((void **)&n->child[nib], (void **)&n); if (!n) { return NULL; From 965fc672d75f1d4e1be8534bd35063b940487902 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 4 Mar 2025 12:21:32 +0100 Subject: [PATCH 2/7] Add utils_atomic_store_release_u64() to utils Signed-off-by: Lukasz Dorau --- src/utils/utils_concurrency.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/utils/utils_concurrency.h b/src/utils/utils_concurrency.h index e8a601ecd..31e5793b9 100644 --- a/src/utils/utils_concurrency.h +++ b/src/utils/utils_concurrency.h @@ -152,6 +152,17 @@ static inline bool utils_compare_exchange_u64(uint64_t *ptr, uint64_t *expected, return false; } +static inline void utils_atomic_store_release_u64(void *ptr, uint64_t val) { + ASSERT_IS_ALIGNED((uintptr_t)ptr, 8); + LONG64 out; + LONG64 desired = (LONG64)val; + LONG64 expected = 0; + while (expected != (out = InterlockedCompareExchange64( + (LONG64 volatile *)ptr, desired, expected))) { + expected = out; + } +} + #else // !defined(_WIN32) static inline void utils_atomic_load_acquire_u64(uint64_t *ptr, uint64_t *out) { @@ -168,6 +179,11 @@ static inline void utils_atomic_load_acquire_ptr(void **ptr, void **out) { utils_annotate_acquire(ptr); } +static inline void utils_atomic_store_release_u64(void *ptr, uint64_t val) { + ASSERT_IS_ALIGNED((uintptr_t)ptr, 8); + __atomic_store_n((uintptr_t *)ptr, (uintptr_t)val, memory_order_release); +} + static inline void utils_atomic_store_release_ptr(void **ptr, void *val) { ASSERT_IS_ALIGNED((uintptr_t)ptr, 8); utils_annotate_release(ptr); From d437d6c24581fe2fbd9f72891fa102281ff0533d Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 6 Mar 2025 09:43:51 +0100 Subject: [PATCH 3/7] Use eight level of critnibs in the tracker Multilevel maps are needed to support the case when one memory pool acts as a memory provider for another memory pool (nested memory pooling). Signed-off-by: Lukasz Dorau --- include/umf/base.h | 3 +- src/provider/provider_tracking.c | 430 +++++++++++++++++++++++++------ 2 files changed, 351 insertions(+), 82 deletions(-) diff --git a/include/umf/base.h b/include/umf/base.h index 8dad184f2..12e99aa2b 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -47,7 +47,8 @@ typedef enum umf_result_t { 6, ///< Failure in user provider code (i.e in user provided callback) UMF_RESULT_ERROR_DEPENDENCY_UNAVAILABLE = 7, ///< External required dependency is unavailable or missing - UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown or internal error + UMF_RESULT_ERROR_OUT_OF_RESOURCES = 8, ///< Out of internal resources + UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown error } umf_result_t; #ifdef __cplusplus diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index 4696bc562..bc560304c 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -7,73 +7,220 @@ * */ -#include "provider_tracking.h" +#include +#include +#include +#include +#include + +#include +#include +#include + #include "base_alloc_global.h" #include "critnib.h" #include "ipc_cache.h" #include "ipc_internal.h" +#include "provider_tracking.h" #include "utils_common.h" #include "utils_concurrency.h" #include "utils_log.h" -#include -#include -#include - -#include -#include -#include -#include -#include +// TODO: we need to support an arbitrary amount of layers in the future +#define MAX_LEVELS_OF_ALLOC_SEGMENT_MAP 8 uint64_t IPC_HANDLE_ID = 0; struct umf_memory_tracker_t { umf_ba_pool_t *alloc_info_allocator; - critnib *alloc_segments_map; + // Multilevel maps are needed to support the case + // when one memory pool acts as a memory provider + // for another memory pool (nested memory pooling). + critnib *alloc_segments_map[MAX_LEVELS_OF_ALLOC_SEGMENT_MAP]; utils_mutex_t splitMergeMutex; }; typedef struct tracker_alloc_info_t { umf_memory_pool_handle_t pool; size_t size; + // number of overlapping memory regions + // in the next level of map + // falling within the current range + size_t n_children; } tracker_alloc_info_t; -static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, - umf_memory_pool_handle_t pool, - const void *ptr, size_t size) { +// Get the most nested (on the highest level) allocation segment in the map with the `ptr` key. +// If `no_children` is set to 1, the function will return the entry +// only if it has no children on the higher level. +// The function returns the entry if found, otherwise NULL. +static tracker_alloc_info_t *get_most_nested_alloc_segment( + umf_memory_tracker_handle_t hTracker, const void *ptr, int *_level, + uintptr_t *_parent_key, tracker_alloc_info_t **_parent_value, + int no_children) { assert(ptr); + tracker_alloc_info_t *parent_value = NULL; + tracker_alloc_info_t *rvalue = NULL; + uintptr_t parent_key = 0; + uintptr_t rkey = 0; + uint64_t rsize = 0; + int level = 0; + int found = 0; + + do { + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + found = + critnib_find(hTracker->alloc_segments_map[level], (uintptr_t)ptr, + FIND_LE, (void *)&rkey, (void **)&rvalue); + if (!found || !rvalue) { + break; + } + + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + + if (found && (uintptr_t)ptr < rkey + rsize) { + if (rvalue->n_children) { + if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { + break; + } + level++; + parent_key = rkey; + parent_value = rvalue; + } + } + } while (found && ((uintptr_t)ptr < rkey + rsize) && rvalue->n_children); + + if (!rvalue || rkey != (uintptr_t)ptr) { + return NULL; + } + + if (no_children && (rvalue->n_children > 0)) { + return NULL; + } + + if (_level) { + *_level = level; + } + if (_parent_key) { + *_parent_key = parent_key; + } + if (_parent_value) { + *_parent_value = parent_value; + } + + assert(!no_children || rvalue->n_children == 0); + + return rvalue; +} + +static umf_result_t +umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, + umf_memory_pool_handle_t pool, const void *ptr, + size_t size, uintptr_t parent_key, + tracker_alloc_info_t *parent_value) { + assert(ptr); + + umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; + tracker_alloc_info_t *value = umf_ba_alloc(hTracker->alloc_info_allocator); if (value == NULL) { - LOG_ERR("failed to allocate tracker value, ptr=%p, size=%zu", ptr, + LOG_ERR("failed to allocate a tracker value, ptr=%p, size=%zu", ptr, size); return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } value->pool = pool; - value->size = size; - - int ret = - critnib_insert(hTracker->alloc_segments_map, (uintptr_t)ptr, value, 0); + utils_atomic_store_release_u64(&value->size, size); + value->n_children = 0; + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + int ret = critnib_insert(hTracker->alloc_segments_map[level], + (uintptr_t)ptr, value, 0); if (ret == 0) { - LOG_DEBUG( - "memory region is added, tracker=%p, ptr=%p, pool=%p, size=%zu", - (void *)hTracker, ptr, (void *)pool, size); + LOG_DEBUG("memory region is added, tracker=%p, level=%i, pool=%p, " + "ptr=%p, size=%zu", + (void *)hTracker, level, (void *)pool, ptr, size); + + if (parent_value) { + parent_value->n_children++; + LOG_DEBUG( + "child #%zu added to memory region: tracker=%p, level=%i, " + "pool=%p, ptr=%p, size=%zu", + parent_value->n_children, (void *)hTracker, level - 1, + (void *)parent_value->pool, (void *)parent_key, + parent_value->size); + } return UMF_RESULT_SUCCESS; } + if (ret == ENOMEM) { + umf_result = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } - LOG_ERR("failed to insert tracker value, ret=%d, ptr=%p, pool=%p, size=%zu", - ret, ptr, (void *)pool, size); + LOG_ERR( + "failed to insert the tracker value: pool=%p, ptr=%p, size=%zu, ret=%d", + (void *)pool, ptr, size, ret); umf_ba_free(hTracker->alloc_info_allocator, value); - if (ret == ENOMEM) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + return umf_result; +} + +static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, + umf_memory_pool_handle_t pool, + const void *ptr, size_t size) { + assert(ptr); + + umf_result_t umf_result = UMF_RESULT_ERROR_UNKNOWN; + tracker_alloc_info_t *parent_value = NULL; + tracker_alloc_info_t *rvalue = NULL; + uintptr_t parent_key = 0; + uintptr_t rkey = 0; + uint64_t rsize = 0; + int level = 0; + int found = 0; + + // Find the most nested (in the highest level) entry + // in the critnib maps that contains the given 'ptr' pointer. + do { + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + found = + critnib_find(hTracker->alloc_segments_map[level], (uintptr_t)ptr, + FIND_LE, (void *)&rkey, (void **)&rvalue); + if (!found || !rvalue) { + break; + } + + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + + if ((uintptr_t)ptr < rkey + rsize) { + if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { + // TODO: we need to support an arbitrary amount of layers in the future + LOG_ERR("tracker level is too high, ptr=%p, size=%zu", ptr, + size); + return UMF_RESULT_ERROR_OUT_OF_RESOURCES; + } + if (((uintptr_t)ptr + size) > (rkey + rsize)) { + LOG_ERR( + "cannot insert to the tracker value (pool=%p, ptr=%p, " + "size=%zu) " + "that exceeds the parent value (pool=%p, ptr=%p, size=%zu)", + (void *)pool, ptr, size, (void *)rvalue->pool, (void *)rkey, + (size_t)rsize); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + parent_key = rkey; + parent_value = rvalue; + level++; + } + } while (found && ((uintptr_t)ptr < rkey + rsize) && rvalue->n_children); + + umf_result = umfMemoryTrackerAddAtLevel(hTracker, level, pool, ptr, size, + parent_key, parent_value); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; } - return UMF_RESULT_ERROR_UNKNOWN; + return UMF_RESULT_SUCCESS; } static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, @@ -85,16 +232,35 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, // Every umfMemoryTrackerAdd(..., ptr, ...) should have a corresponding // umfMemoryTrackerRemove call with the same ptr value. - void *value = critnib_remove(hTracker->alloc_segments_map, (uintptr_t)ptr); + tracker_alloc_info_t *parent_value = NULL; + uintptr_t parent_key = 0; + int level = 0; + + // Find the most nested (on the highest level) entry in the map + // with the `ptr` key and with no children - only such entry can be removed. + tracker_alloc_info_t *value = get_most_nested_alloc_segment( + hTracker, ptr, &level, &parent_key, &parent_value, 1 /* no_children */); if (!value) { LOG_ERR("pointer %p not found in the alloc_segments_map", ptr); return UMF_RESULT_ERROR_UNKNOWN; } - tracker_alloc_info_t *v = value; + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + value = critnib_remove(hTracker->alloc_segments_map[level], (uintptr_t)ptr); + assert(value); - LOG_DEBUG("memory region removed: tracker=%p, ptr=%p, size=%zu", - (void *)hTracker, ptr, v->size); + LOG_DEBUG("memory region removed: tracker=%p, level=%i, pool=%p, ptr=%p, " + "size=%zu", + (void *)hTracker, level, value->pool, ptr, value->size); + + if (parent_value) { + LOG_DEBUG( + "child #%zu removed from memory region: tracker=%p, level=%i, " + "pool=%p, ptr=%p, size=%zu", + parent_value->n_children, (void *)hTracker, level - 1, + (void *)parent_value->pool, (void *)parent_key, parent_value->size); + parent_value->n_children--; + } umf_ba_free(hTracker->alloc_info_allocator, value); @@ -124,24 +290,43 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, return UMF_RESULT_ERROR_NOT_SUPPORTED; } - if (TRACKER->alloc_segments_map == NULL) { + if (TRACKER->alloc_segments_map[0] == NULL) { LOG_ERR("tracker's alloc_segments_map does not exist"); return UMF_RESULT_ERROR_NOT_SUPPORTED; } - uintptr_t rkey; - tracker_alloc_info_t *rvalue; - int found = critnib_find(TRACKER->alloc_segments_map, (uintptr_t)ptr, + tracker_alloc_info_t *top_most_value = NULL; + tracker_alloc_info_t *rvalue = NULL; + uintptr_t top_most_key = 0; + uintptr_t rkey = 0; + int level = 0; + int found = 0; + + do { + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + found = critnib_find(TRACKER->alloc_segments_map[level], (uintptr_t)ptr, FIND_LE, (void *)&rkey, (void **)&rvalue); - if (!found || (uintptr_t)ptr >= rkey + rvalue->size) { + if (found && (uintptr_t)ptr < rkey + rvalue->size) { + top_most_key = rkey; + top_most_value = rvalue; + if (rvalue->n_children == 0 || + level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { + break; + } + level++; + } + } while (found && (uintptr_t)ptr < rkey + rvalue->size && + rvalue->n_children); + + if (!top_most_value) { LOG_DEBUG("pointer %p not found in the tracker, TRACKER=%p", ptr, (void *)TRACKER); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - pAllocInfo->base = (void *)rkey; - pAllocInfo->baseSize = rvalue->size; - pAllocInfo->pool = rvalue->pool; + pAllocInfo->base = (void *)top_most_key; + pAllocInfo->baseSize = top_most_value->size; + pAllocInfo->pool = top_most_value->pool; return UMF_RESULT_SUCCESS; } @@ -166,26 +351,38 @@ typedef struct umf_tracking_memory_provider_t { typedef struct umf_tracking_memory_provider_t umf_tracking_memory_provider_t; static umf_result_t trackingAlloc(void *hProvider, size_t size, - size_t alignment, void **ptr) { + size_t alignment, void **_ptr) { umf_tracking_memory_provider_t *p = (umf_tracking_memory_provider_t *)hProvider; umf_result_t ret = UMF_RESULT_SUCCESS; + void *ptr; assert(p->hUpstream); - ret = umfMemoryProviderAlloc(p->hUpstream, size, alignment, ptr); - if (ret != UMF_RESULT_SUCCESS || !*ptr) { + *_ptr = NULL; + + ret = umfMemoryProviderAlloc(p->hUpstream, size, alignment, &ptr); + if (ret != UMF_RESULT_SUCCESS || !ptr) { return ret; } - umf_result_t ret2 = umfMemoryTrackerAdd(p->hTracker, p->pool, *ptr, size); - if (ret2 != UMF_RESULT_SUCCESS) { + ret = umfMemoryTrackerAdd(p->hTracker, p->pool, ptr, size); + if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("failed to add allocated region to the tracker, ptr = %p, size " "= %zu, ret = %d", - *ptr, size, ret2); + ptr, size, ret); + umf_result_t ret2 = umfMemoryProviderFree(p->hUpstream, ptr, size); + if (ret2 != UMF_RESULT_SUCCESS) { + LOG_ERR("upstream provider failed to free the memory: ptr = %p, " + "size = %zu, ret = %d", + ptr, size, ret2); + } + return ret; } - return ret; + *_ptr = ptr; + + return UMF_RESULT_SUCCESS; } static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, @@ -194,6 +391,8 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, umf_result_t ret = UMF_RESULT_ERROR_UNKNOWN; umf_tracking_memory_provider_t *provider = (umf_tracking_memory_provider_t *)hProvider; + tracker_alloc_info_t *parent_value = NULL; + uintptr_t parent_key = 0; tracker_alloc_info_t *splitValue = umf_ba_alloc(provider->hTracker->alloc_info_allocator); @@ -203,21 +402,27 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, splitValue->pool = provider->pool; splitValue->size = firstSize; + splitValue->n_children = 0; int r = utils_mutex_lock(&provider->hTracker->splitMergeMutex); if (r) { goto err_lock; } - tracker_alloc_info_t *value = (tracker_alloc_info_t *)critnib_get( - provider->hTracker->alloc_segments_map, (uintptr_t)ptr); + int level = 0; + + // Find the most nested (on the highest level) entry in the map + // with the `ptr` key and with no children - only such entry can be split. + tracker_alloc_info_t *value = get_most_nested_alloc_segment( + provider->hTracker, ptr, &level, &parent_key, &parent_value, + 1 /* no_children */); if (!value) { LOG_ERR("region for split is not found in the tracker"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; } if (value->size != totalSize) { - LOG_ERR("tracked size %zu does not match requested size to split: %zu", + LOG_ERR("tracked size=%zu does not match requested size to split: %zu", value->size, totalSize); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; @@ -230,40 +435,58 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, goto err; } + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + int cret = + critnib_insert(provider->hTracker->alloc_segments_map[level], + (uintptr_t)ptr, (void *)splitValue, 1 /* update */); + // this cannot fail since we know the element exists (nothing to allocate) + assert(cret == 0); + (void)cret; + void *highPtr = (void *)(((uintptr_t)ptr) + firstSize); size_t secondSize = totalSize - firstSize; // We'll have a duplicate entry for the range [highPtr, highValue->size] but this is fine, // the value is the same anyway and we forbid removing that range concurrently - ret = umfMemoryTrackerAdd(provider->hTracker, provider->pool, highPtr, - secondSize); + ret = umfMemoryTrackerAddAtLevel(provider->hTracker, level, provider->pool, + highPtr, secondSize, parent_key, + parent_value); if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("failed to add split region to the tracker, ptr = %p, size " - "= %zu, ret = %d", + LOG_ERR("failed to add the split region to the tracker, ptr=%p, " + "size=%zu, ret=%d", highPtr, secondSize, ret); + // revert the split + assert(level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP); + cret = critnib_insert(provider->hTracker->alloc_segments_map[level], + (uintptr_t)ptr, (void *)value, 1 /* update */); + // this cannot fail since we know the element exists (nothing to allocate) + assert(cret == 0); + (void)cret; // TODO: what now? should we rollback the split? This can only happen due to ENOMEM // so it's unlikely but probably the best solution would be to try to preallocate everything // (value and critnib nodes) before calling umfMemoryProviderAllocationSplit. goto err; } - int cret = - critnib_insert(provider->hTracker->alloc_segments_map, (uintptr_t)ptr, - (void *)splitValue, 1 /* update */); - // this cannot fail since we know the element exists (nothing to allocate) - assert(cret == 0); - (void)cret; - // free the original value umf_ba_free(provider->hTracker->alloc_info_allocator, value); utils_mutex_unlock(&provider->hTracker->splitMergeMutex); + LOG_DEBUG( + "split memory region (level=%i): ptr=%p, totalSize=%zu, firstSize=%zu", + level, ptr, totalSize, firstSize); + return UMF_RESULT_SUCCESS; err: utils_mutex_unlock(&provider->hTracker->splitMergeMutex); err_lock: umf_ba_free(provider->hTracker->alloc_info_allocator, splitValue); + + LOG_ERR( + "failed to split memory region: ptr=%p, totalSize=%zu, firstSize=%zu", + ptr, totalSize, firstSize); + return ret; } @@ -282,26 +505,38 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, mergedValue->pool = provider->pool; mergedValue->size = totalSize; + mergedValue->n_children = 0; + + // any different negative values + int lowLevel = -2; + int highLevel = -1; int r = utils_mutex_lock(&provider->hTracker->splitMergeMutex); if (r) { goto err_lock; } - tracker_alloc_info_t *lowValue = (tracker_alloc_info_t *)critnib_get( - provider->hTracker->alloc_segments_map, (uintptr_t)lowPtr); + tracker_alloc_info_t *lowValue = get_most_nested_alloc_segment( + provider->hTracker, lowPtr, &lowLevel, NULL, NULL, + 0 /* no_children */); // can have children if (!lowValue) { LOG_FATAL("no left value"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_assert; } - tracker_alloc_info_t *highValue = (tracker_alloc_info_t *)critnib_get( - provider->hTracker->alloc_segments_map, (uintptr_t)highPtr); + tracker_alloc_info_t *highValue = get_most_nested_alloc_segment( + provider->hTracker, highPtr, &highLevel, NULL, NULL, + 0 /* no_children */); // can have children if (!highValue) { LOG_FATAL("no right value"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_assert; } + if (lowLevel != highLevel) { + LOG_FATAL("tracker level mismatch"); + ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; + goto err_assert; + } if (lowValue->pool != highValue->pool) { LOG_FATAL("pool mismatch"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; @@ -313,6 +548,8 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, goto err_assert; } + mergedValue->n_children = lowValue->n_children + highValue->n_children; + ret = umfMemoryProviderAllocationMerge(provider->hUpstream, lowPtr, highPtr, totalSize); if (ret != UMF_RESULT_SUCCESS) { @@ -320,10 +557,13 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, goto not_merged; } + size_t lno = lowValue->n_children; + size_t hno = highValue->n_children; + // We'll have a duplicate entry for the range [highPtr, highValue->size] but this is fine, // the value is the same anyway and we forbid removing that range concurrently int cret = - critnib_insert(provider->hTracker->alloc_segments_map, + critnib_insert(provider->hTracker->alloc_segments_map[lowLevel], (uintptr_t)lowPtr, (void *)mergedValue, 1 /* update */); // this cannot fail since we know the element exists (nothing to allocate) assert(cret == 0); @@ -333,16 +573,23 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, umf_ba_free(provider->hTracker->alloc_info_allocator, lowValue); void *erasedhighValue = critnib_remove( - provider->hTracker->alloc_segments_map, (uintptr_t)highPtr); + provider->hTracker->alloc_segments_map[highLevel], (uintptr_t)highPtr); assert(erasedhighValue == highValue); umf_ba_free(provider->hTracker->alloc_info_allocator, erasedhighValue); utils_mutex_unlock(&provider->hTracker->splitMergeMutex); + LOG_DEBUG("merged memory regions (level=%i): lowPtr=%p (child=%zu), " + "highPtr=%p (child=%zu), totalSize=%zu", + lowLevel, lowPtr, lno, highPtr, hno, totalSize); + return UMF_RESULT_SUCCESS; err_assert: + LOG_FATAL("failed to merge memory regions: lowPtr=%p (level=%i), " + "highPtr=%p (level=%i), totalSize=%zu", + lowPtr, lowLevel, highPtr, highLevel, totalSize); assert(0); not_merged: @@ -350,6 +597,11 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, err_lock: umf_ba_free(provider->hTracker->alloc_info_allocator, mergedValue); + + LOG_ERR("failed to merge memory regions: lowPtr=%p (level=%i), highPtr=%p " + "(level=%i), totalSize=%zu", + lowPtr, lowLevel, highPtr, highLevel, totalSize); + return ret; } @@ -428,19 +680,21 @@ static umf_result_t trackingInitialize(void *params, void **ret) { #ifndef NDEBUG static void check_if_tracker_is_empty(umf_memory_tracker_handle_t hTracker, umf_memory_pool_handle_t pool) { - uintptr_t rkey; - void *rvalue; size_t n_items = 0; - uintptr_t last_key = 0; - while (1 == critnib_find((critnib *)hTracker->alloc_segments_map, last_key, - FIND_G, &rkey, &rvalue)) { - tracker_alloc_info_t *value = (tracker_alloc_info_t *)rvalue; - if (value->pool == pool || pool == NULL) { - n_items++; - } + for (int i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + uintptr_t last_key = 0; + uintptr_t rkey; + tracker_alloc_info_t *rvalue; + + while (1 == critnib_find(hTracker->alloc_segments_map[i], last_key, + FIND_G, &rkey, (void **)&rvalue)) { + if (rvalue->pool == pool || pool == NULL) { + n_items++; + } - last_key = rkey; + last_key = rkey; + } } if (n_items) { @@ -813,6 +1067,8 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { return NULL; } + memset(handle, 0, sizeof(struct umf_memory_tracker_t)); + umf_ba_pool_t *alloc_info_allocator = umf_ba_create(sizeof(struct tracker_alloc_info_t)); if (!alloc_info_allocator) { @@ -826,9 +1082,12 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { goto err_destroy_alloc_info_allocator; } - handle->alloc_segments_map = critnib_new(); - if (!handle->alloc_segments_map) { - goto err_destroy_mutex; + int i; + for (i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + handle->alloc_segments_map[i] = critnib_new(); + if (!handle->alloc_segments_map[i]) { + goto err_destroy_mutex; + } } LOG_DEBUG("tracker created, handle=%p, alloc_segments_map=%p", @@ -837,6 +1096,11 @@ umf_memory_tracker_handle_t umfMemoryTrackerCreate(void) { return handle; err_destroy_mutex: + for (int j = i; j >= 0; j--) { + if (handle->alloc_segments_map[j]) { + critnib_delete(handle->alloc_segments_map[j]); + } + } utils_mutex_destroy_not_free(&handle->splitMergeMutex); err_destroy_alloc_info_allocator: umf_ba_destroy(alloc_info_allocator); @@ -864,8 +1128,12 @@ void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle) { // We have to zero all inner pointers, // because the tracker handle can be copied // and used in many places. - critnib_delete(handle->alloc_segments_map); - handle->alloc_segments_map = NULL; + for (int i = 0; i < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; i++) { + if (handle->alloc_segments_map[i]) { + critnib_delete(handle->alloc_segments_map[i]); + handle->alloc_segments_map[i] = NULL; + } + } utils_mutex_destroy_not_free(&handle->splitMergeMutex); umf_ba_destroy(handle->alloc_info_allocator); handle->alloc_info_allocator = NULL; From 775ac129021e92bd04a2e385ba4cb2ee2cb553bb Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 25 Feb 2025 13:54:44 +0100 Subject: [PATCH 4/7] Add tests for pool from pointer to Fixed provider tests Signed-off-by: Lukasz Dorau --- test/provider_fixed_memory.cpp | 112 ++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/test/provider_fixed_memory.cpp b/test/provider_fixed_memory.cpp index 7f976a1f5..1760ca4f7 100644 --- a/test/provider_fixed_memory.cpp +++ b/test/provider_fixed_memory.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2024 Intel Corporation +// Copyright (C) 2024-2025 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -11,10 +11,12 @@ #endif #include +#include #include using umf_test::test; +#define FIXED_BUFFER_SIZE (10 * utils_get_page_size()) #define INVALID_PTR ((void *)0x01) typedef enum purge_t { @@ -59,7 +61,7 @@ struct FixedProviderTest test::SetUp(); // Allocate a memory buffer to use with the fixed memory provider - memory_size = utils_get_page_size() * 10; // Allocate 10 pages + memory_size = FIXED_BUFFER_SIZE; // Allocate 10 pages memory_buffer = malloc(memory_size); ASSERT_NE(memory_buffer, nullptr); @@ -391,3 +393,109 @@ TEST_P(FixedProviderTest, split) { umf_result = umfMemoryProviderFree(provider.get(), ptr2, size); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); } + +TEST_P(FixedProviderTest, pool_from_ptr_whole_size_success) { + umf_result_t umf_result; + size_t size_of_first_alloc; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t proxyFixedPool = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider.get(), nullptr, 0, + &proxyFixedPool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_of_first_alloc = FIXED_BUFFER_SIZE - (2 * page_size); + ptr_for_pool = umfPoolMalloc(proxyFixedPool, size_of_first_alloc); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc; // whole size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(proxyFixedPool, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(proxyFixedPool); +} + +TEST_P(FixedProviderTest, pool_from_ptr_half_size_success) { + umf_result_t umf_result; + size_t size_of_first_alloc; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t proxyFixedPool = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider.get(), nullptr, 0, + &proxyFixedPool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_of_first_alloc = FIXED_BUFFER_SIZE - (2 * page_size); + ptr_for_pool = umfPoolMalloc(proxyFixedPool, size_of_first_alloc); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc / 2; // half size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(proxyFixedPool, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(proxyFixedPool); +} From cd61c1e6187966fede302fc98d33c2e6ca6e7449 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 25 Feb 2025 17:38:11 +0100 Subject: [PATCH 5/7] Add tests for pool from pointer to poolFixtures.hpp Signed-off-by: Lukasz Dorau --- test/poolFixtures.hpp | 149 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 144 insertions(+), 5 deletions(-) diff --git a/test/poolFixtures.hpp b/test/poolFixtures.hpp index 6f18664f9..6b01769f1 100644 --- a/test/poolFixtures.hpp +++ b/test/poolFixtures.hpp @@ -5,11 +5,6 @@ #ifndef UMF_TEST_POOL_FIXTURES_HPP #define UMF_TEST_POOL_FIXTURES_HPP 1 -#include "pool.hpp" -#include "provider.hpp" -#include "umf/providers/provider_devdax_memory.h" -#include "utils/utils_sanitizers.h" - #include #include #include @@ -17,7 +12,14 @@ #include #include +#include +#include +#include + #include "../malloc_compliance_tests.hpp" +#include "pool.hpp" +#include "provider.hpp" +#include "utils/utils_sanitizers.h" typedef void *(*pfnPoolParamsCreate)(); typedef umf_result_t (*pfnPoolParamsDestroy)(void *); @@ -493,4 +495,141 @@ TEST_P(umfPoolTest, mallocUsableSize) { } } +TEST_P(umfPoolTest, umfPoolAlignedMalloc) { +#ifdef _WIN32 + // TODO: implement support for windows + GTEST_SKIP() << "umfPoolAlignedMalloc() is not supported on Windows"; +#else /* !_WIN32 */ + umf_result_t umf_result; + void *ptr = nullptr; + const size_t size = 2 * 1024 * 1024; // 2MB + + umf_memory_pool_handle_t pool_get = pool.get(); + + if (!umf_test::isAlignedAllocSupported(pool_get)) { + GTEST_SKIP(); + } + + ptr = umfPoolAlignedMalloc(pool_get, size, utils_get_page_size()); + ASSERT_NE(ptr, nullptr); + + umf_result = umfPoolFree(pool_get, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +#endif /* !_WIN32 */ +} + +TEST_P(umfPoolTest, pool_from_ptr_whole_size_success) { +#ifdef _WIN32 + // TODO: implement support for windows + GTEST_SKIP() << "umfPoolAlignedMalloc() is not supported on Windows"; +#else /* !_WIN32 */ + umf_result_t umf_result; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t pool_get = pool.get(); + const size_t size_of_first_alloc = 2 * 1024 * 1024; // 2MB + + if (!umf_test::isAlignedAllocSupported(pool_get)) { + GTEST_SKIP(); + } + + ptr_for_pool = umfPoolAlignedMalloc(pool_get, size_of_first_alloc, + utils_get_page_size()); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc; // whole size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(pool_get, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +#endif /* !_WIN32 */ +} + +TEST_P(umfPoolTest, pool_from_ptr_half_size_success) { +#ifdef _WIN32 + // TODO: implement support for windows + GTEST_SKIP() << "umfPoolAlignedMalloc() is not supported on Windows"; +#else /* !_WIN32 */ + umf_result_t umf_result; + size_t size_of_pool_from_ptr; + void *ptr_for_pool = nullptr; + void *ptr = nullptr; + + umf_memory_pool_handle_t pool_get = pool.get(); + const size_t size_of_first_alloc = 2 * 1024 * 1024; // 2MB + + if (!umf_test::isAlignedAllocSupported(pool_get)) { + GTEST_SKIP(); + } + + ptr_for_pool = umfPoolAlignedMalloc(pool_get, size_of_first_alloc, + utils_get_page_size()); + ASSERT_NE(ptr_for_pool, nullptr); + + // Create provider parameters + size_of_pool_from_ptr = size_of_first_alloc / 2; // half size + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr_for_pool, + size_of_pool_from_ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t providerFromPtr = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &providerFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(providerFromPtr, nullptr); + + umf_memory_pool_handle_t poolFromPtr = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), providerFromPtr, nullptr, 0, + &poolFromPtr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + ptr = umfPoolMalloc(poolFromPtr, size_of_pool_from_ptr); + ASSERT_NE(ptr, nullptr); + + memset(ptr, 0xFF, size_of_pool_from_ptr); + + umf_result = umfPoolFree(poolFromPtr, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(poolFromPtr); + umfMemoryProviderDestroy(providerFromPtr); + umfFixedMemoryProviderParamsDestroy(params); + + umf_result = umfPoolFree(pool_get, ptr_for_pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +#endif /* !_WIN32 */ +} + #endif /* UMF_TEST_POOL_FIXTURES_HPP */ From bb4a5e48f9e903fddc1fa92d88db9a61c7ce2331 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 28 Feb 2025 16:43:39 +0100 Subject: [PATCH 6/7] Add tests for tracking provider Signed-off-by: Lukasz Dorau --- test/CMakeLists.txt | 4 + test/provider_tracking.cpp | 374 +++++++++++++++++++++++++++++++++++++ 2 files changed, 378 insertions(+) create mode 100644 test/provider_tracking.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 37f4c809e..e47ce5a39 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -343,6 +343,10 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented NAME provider_fixed_memory SRCS provider_fixed_memory.cpp LIBS ${UMF_UTILS_FOR_TEST}) + add_umf_test( + NAME provider_tracking + SRCS provider_tracking.cpp + LIBS ${UMF_UTILS_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) diff --git a/test/provider_tracking.cpp b/test/provider_tracking.cpp new file mode 100644 index 000000000..864c15564 --- /dev/null +++ b/test/provider_tracking.cpp @@ -0,0 +1,374 @@ +// Copyright (C) 2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "base.hpp" + +#include "cpp_helpers.hpp" +#include "test_helpers.h" +#ifndef _WIN32 +#include "test_helpers_linux.h" +#endif + +#include +#include +#include + +using umf_test::test; + +#define FIXED_BUFFER_SIZE (512 * utils_get_page_size()) +#define INVALID_PTR ((void *)0x01) + +using providerCreateExtParams = std::tuple; + +static void providerCreateExt(providerCreateExtParams params, + umf::provider_unique_handle_t *handle) { + umf_memory_provider_handle_t hProvider = nullptr; + auto [provider_ops, provider_params] = params; + + auto ret = + umfMemoryProviderCreate(provider_ops, provider_params, &hProvider); + ASSERT_EQ(ret, UMF_RESULT_SUCCESS); + ASSERT_NE(hProvider, nullptr); + + *handle = + umf::provider_unique_handle_t(hProvider, &umfMemoryProviderDestroy); +} + +struct TrackingProviderTest + : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + + // Allocate a memory buffer to use with the fixed memory provider + memory_size = FIXED_BUFFER_SIZE; + memory_buffer = malloc(memory_size); + ASSERT_NE(memory_buffer, nullptr); + + // Create provider parameters + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result_t res = umfFixedMemoryProviderParamsCreate( + ¶ms, memory_buffer, memory_size); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + providerCreateExt(std::make_tuple(umfFixedMemoryProviderOps(), params), + &provider); + + umfFixedMemoryProviderParamsDestroy(params); + umf_result_t umf_result = + umfMemoryProviderGetMinPageSize(provider.get(), NULL, &page_size); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + page_plus_64 = page_size + 64; + + umf_memory_pool_handle_t hPool = nullptr; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider.get(), nullptr, + 0, &hPool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + pool = umf::pool_unique_handle_t(hPool, &umfPoolDestroy); + } + + void TearDown() override { + if (memory_buffer) { + free(memory_buffer); + memory_buffer = nullptr; + } + test::TearDown(); + } + + umf::provider_unique_handle_t provider; + umf::pool_unique_handle_t pool; + size_t page_size; + size_t page_plus_64; + void *memory_buffer = nullptr; + size_t memory_size = 0; +}; + +static void +createPoolFromAllocation(void *ptr0, size_t size1, + umf_memory_provider_handle_t *_providerFromPtr, + umf_memory_pool_handle_t *_poolFromPtr) { + umf_result_t umf_result; + + // Create provider parameters + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result = umfFixedMemoryProviderParamsCreate(¶ms, ptr0, size1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_result = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + &provider1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider1, nullptr); + + umf_memory_pool_handle_t pool1 = nullptr; + umf_result = + umfPoolCreate(umfProxyPoolOps(), provider1, nullptr, 0, &pool1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfFixedMemoryProviderParamsDestroy(params); + + *_providerFromPtr = provider1; + *_poolFromPtr = pool1; +} + +// TESTS + +INSTANTIATE_TEST_SUITE_P(trackingProviderTest, TrackingProviderTest, + ::testing::Values(providerCreateExtParams{ + umfFixedMemoryProviderOps(), nullptr})); + +TEST_P(TrackingProviderTest, create_destroy) { + // Creation and destruction are handled in SetUp and TearDown +} + +TEST_P(TrackingProviderTest, whole_size_success) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + size1 = size0; // whole size + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size1, &provider1, &pool1); + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(TrackingProviderTest, half_size_success) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + size1 = size0 / 2; // half size + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size1, &provider1, &pool1); + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(TrackingProviderTest, failure_exceeding_size) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + size1 = FIXED_BUFFER_SIZE - page_size; // exceeding size + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size1, &provider1, &pool1); + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_EQ(ptr1, nullptr); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + umf_result = umfPoolFree(pool0, ptr0); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +#define MAX_ARRAY 9 +#define TEST_LEVEL_SUCCESS 7 +#define TEST_LEVEL_FAILURE 8 + +TEST_P(TrackingProviderTest, success_max_levels) { + umf_result_t umf_result; + size_t size; + void *ptr[MAX_ARRAY] = {0}; + umf_memory_provider_handle_t providers[MAX_ARRAY] = {0}; + umf_memory_pool_handle_t pools[MAX_ARRAY] = {0}; + + size = FIXED_BUFFER_SIZE - (2 * page_size); + pools[0] = pool.get(); + + for (int i = 0; i < TEST_LEVEL_SUCCESS; i++) { + fprintf(stderr, "Alloc #%d\n", i); + ptr[i] = umfPoolAlignedMalloc(pools[i], size, utils_get_page_size()); + ASSERT_NE(ptr[i], nullptr); + + createPoolFromAllocation(ptr[i], size, &providers[i + 1], + &pools[i + 1]); + } + + int s = TEST_LEVEL_SUCCESS; + fprintf(stderr, "Alloc #%d\n", s); + ptr[s] = umfPoolAlignedMalloc(pools[s], size, utils_get_page_size()); + ASSERT_NE(ptr[s], nullptr); + + fprintf(stderr, "Free #%d\n", s); + umf_result = umfPoolFree(pools[s], ptr[s]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + for (int i = TEST_LEVEL_SUCCESS - 1; i >= 0; i--) { + umfPoolDestroy(pools[i + 1]); + umfMemoryProviderDestroy(providers[i + 1]); + + fprintf(stderr, "Free #%d\n", i); + umf_result = umfPoolFree(pools[i], ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } +} + +TEST_P(TrackingProviderTest, failure_exceeding_levels) { + umf_result_t umf_result; + size_t size; + void *ptr[MAX_ARRAY] = {0}; + umf_memory_provider_handle_t providers[MAX_ARRAY] = {0}; + umf_memory_pool_handle_t pools[MAX_ARRAY] = {0}; + + size = FIXED_BUFFER_SIZE - (2 * page_size); + pools[0] = pool.get(); + + for (int i = 0; i < TEST_LEVEL_FAILURE; i++) { + fprintf(stderr, "Alloc #%d\n", i); + ptr[i] = umfPoolAlignedMalloc(pools[i], size, utils_get_page_size()); + ASSERT_NE(ptr[i], nullptr); + + createPoolFromAllocation(ptr[i], size, &providers[i + 1], + &pools[i + 1]); + } + + // tracker level is too high + int f = TEST_LEVEL_FAILURE; + fprintf(stderr, "Alloc #%d\n", f); + ptr[f] = umfPoolAlignedMalloc(pools[f], size, utils_get_page_size()); + ASSERT_EQ(ptr[f], nullptr); + + for (int i = TEST_LEVEL_FAILURE - 1; i >= 0; i--) { + umfPoolDestroy(pools[i + 1]); + umfMemoryProviderDestroy(providers[i + 1]); + + fprintf(stderr, "Free #%d\n", i); + umf_result = umfPoolFree(pools[i], ptr[i]); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } +} + +TEST_P(TrackingProviderTest, reverted_free_half_size) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size0, &provider1, &pool1); + + size1 = size0 / 2; // half size + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + // Freeing the "busy" pointer from the first pool is an Undefined Behavior + // It fails now if the sizes are different. + // see: https://github.com/oneapi-src/unified-memory-framework/pull/1161 + umf_result = umfPoolFree(pool0, ptr0); + + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + // It could have been freed above, + // so we cannot verify the result here. + umf_result = umfPoolFree(pool0, ptr0); +} + +TEST_P(TrackingProviderTest, reverted_free_the_same_size) { + umf_result_t umf_result; + size_t size0; + size_t size1; + void *ptr0 = nullptr; + void *ptr1 = nullptr; + + umf_memory_pool_handle_t pool0 = pool.get(); + + size0 = FIXED_BUFFER_SIZE - (2 * page_size); + ptr0 = umfPoolAlignedMalloc(pool0, size0, utils_get_page_size()); + ASSERT_NE(ptr0, nullptr); + + umf_memory_provider_handle_t provider1 = nullptr; + umf_memory_pool_handle_t pool1 = nullptr; + createPoolFromAllocation(ptr0, size0, &provider1, &pool1); + + size1 = size0; // the same size + + ptr1 = umfPoolMalloc(pool1, size1); + ASSERT_NE(ptr1, nullptr); + + // Freeing the "busy" pointer from the first pool is an Undefined Behavior + // It succeeds now if the sizes are equal. + // see: https://github.com/oneapi-src/unified-memory-framework/pull/1161 + umf_result = umfPoolFree(pool0, ptr0); + + // try to free the pointer from the second pool (the same size) + umf_result = umfPoolFree(pool1, ptr1); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umfPoolDestroy(pool1); + umfMemoryProviderDestroy(provider1); + + // It could have been freed above, + // so we cannot verify the result here. + umf_result = umfPoolFree(pool0, ptr0); +} From c1b9f1bdc019d769a716f19b356b10c15ba22cdf Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 3 Mar 2025 12:34:48 +0100 Subject: [PATCH 7/7] Add provider_tracking_fixture_tests Signed-off-by: Lukasz Dorau --- test/CMakeLists.txt | 5 ++ test/provider_tracking_fixture_tests.cpp | 91 ++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 test/provider_tracking_fixture_tests.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e47ce5a39..5f244b60e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -347,6 +347,11 @@ if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented NAME provider_tracking SRCS provider_tracking.cpp LIBS ${UMF_UTILS_FOR_TEST}) + add_umf_test( + NAME provider_tracking_fixture_tests + SRCS provider_tracking_fixture_tests.cpp malloc_compliance_tests.cpp + ${BA_SOURCES_FOR_TEST} + LIBS ${UMF_UTILS_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) diff --git a/test/provider_tracking_fixture_tests.cpp b/test/provider_tracking_fixture_tests.cpp new file mode 100644 index 000000000..05b87f87f --- /dev/null +++ b/test/provider_tracking_fixture_tests.cpp @@ -0,0 +1,91 @@ +// Copyright (C) 2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include + +#include "base.hpp" +#include "provider.hpp" + +#include "cpp_helpers.hpp" +#include "test_helpers.h" +#ifndef _WIN32 +#include "test_helpers_linux.h" +#endif + +#include "poolFixtures.hpp" + +#define FILE_PATH ((char *)"tmp_file") + +struct provider_from_pool : public umf_test::provider_base_t { + umf_memory_pool_handle_t pool; + umf_result_t initialize(umf_memory_pool_handle_t _pool) noexcept { + if (!_pool) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + pool = _pool; + return UMF_RESULT_SUCCESS; + } + umf_result_t alloc(size_t size, size_t align, void **ptr) noexcept { + *ptr = umfPoolAlignedMalloc(pool, size, align); + return (*ptr) ? UMF_RESULT_SUCCESS + : UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + umf_result_t free(void *ptr, size_t) noexcept { + return umfPoolFree(pool, ptr); + } + const char *get_name() noexcept { return "provider_from_pool"; } + + virtual ~provider_from_pool() { + if (pool) { + umfPoolDestroy(pool); + pool = nullptr; + } + } +}; + +umf_memory_provider_ops_t PROVIDER_FROM_POOL_OPS = + umf::providerMakeCOps(); + +static void *providerFromPoolParamsCreate(void) { + umf_file_memory_provider_params_handle_t paramsFile = NULL; + umf_result_t umf_result = + umfFileMemoryProviderParamsCreate(¶msFile, FILE_PATH); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_NE(paramsFile, nullptr); + + umf_memory_provider_handle_t providerFile = nullptr; + umf_result = umfMemoryProviderCreate(umfFileMemoryProviderOps(), paramsFile, + &providerFile); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_NE(providerFile, nullptr); + + umf_memory_pool_handle_t poolProxyFile = nullptr; + umf_result = + umfPoolCreate(umfProxyPoolOps(), providerFile, nullptr, + UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &poolProxyFile); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + EXPECT_NE(poolProxyFile, nullptr); + + umf_result = umfFileMemoryProviderParamsDestroy(paramsFile); + EXPECT_EQ(umf_result, UMF_RESULT_SUCCESS); + paramsFile = nullptr; + + return poolProxyFile; +} + +// TESTS + +INSTANTIATE_TEST_SUITE_P(TrackingProviderPoolTest, umfPoolTest, + ::testing::Values(poolCreateExtParams{ + umfProxyPoolOps(), nullptr, nullptr, + &PROVIDER_FROM_POOL_OPS, + providerFromPoolParamsCreate, nullptr})); + +INSTANTIATE_TEST_SUITE_P(TrackingProviderMultiPoolTest, umfMultiPoolTest, + ::testing::Values(poolCreateExtParams{ + umfProxyPoolOps(), nullptr, nullptr, + &PROVIDER_FROM_POOL_OPS, + providerFromPoolParamsCreate, nullptr}));