Enable IPC cache for Open/Close functions

Author: vinser52

src/CMakeLists.txt

@@ -104,6 +104,7 @@ set(UMF_SOURCES
     ${BA_SOURCES}
     libumf.c
     ipc.c
+    ipc_cache.c
     memory_pool.c
     memory_provider.c
     memory_provider_get_last_failed.c
@@ -270,6 +271,7 @@ target_include_directories(
            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/provider>
            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/memspaces>
            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/memtargets>
+           $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/uthash>
            $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
 
 install(TARGETS umf EXPORT ${PROJECT_NAME}-targets)

src/ipc.c

@@ -90,6 +90,8 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle,
         return ret;
     }
 
+    // ipcData->handle_id is filled by tracking provider
+    ipcData->base = allocInfo.base;
     ipcData->pid = utils_getpid();
     ipcData->baseSize = allocInfo.baseSize;
     ipcData->offset = (uintptr_t)ptr - (uintptr_t)allocInfo.base;

src/ipc_cache.c

@@ -0,0 +1,138 @@
+/*
+ *
+ * Copyright (C) 2024 Intel Corporation
+ *
+ * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ */
+#include "ipc_cache.h"
+
+#include <stdbool.h>
+
+#include "base_alloc_global.h"
+#include "utils_common.h"
+#include "utils_concurrency.h"
+#include "utils_log.h"
+#include "utlist.h"
+
+ipc_handle_mmaped_cache_handle_t umfIpcHandleMmapedCacheCreate(void) {
+    ipc_handle_mmaped_cache_t *cache = umf_ba_global_alloc(sizeof(*cache));
+    if (!cache) {
+        return NULL;
+    }
+
+    if (NULL == utils_mutex_init(&(cache->cache_lock))) {
+        LOG_ERR("Failed to initialize mutex for the IPC cache");
+        umf_ba_global_free(cache);
+        return NULL;
+    }
+
+    cache->cache_allocator = umf_ba_create(sizeof(ipc_handle_cache_entry_t));
+    if (!cache->cache_allocator) {
+        LOG_ERR("Failed to create IPC cache allocator");
+        umf_ba_global_free(cache);
+        return NULL;
+    }
+
+    cache->max_size = 0;
+    cache->cur_size = 0;
+    cache->hash_table = NULL;
+    cache->frequency_list = NULL;
+
+    return cache;
+}
+
+void umfIpcHandleMmapedCacheDestroy(ipc_handle_mmaped_cache_handle_t cache) {
+    if (!cache) {
+        return;
+    }
+
+    ipc_handle_cache_entry_t *entry, *tmp;
+    HASH_ITER(hh, cache->hash_table, entry, tmp) {
+        DL_DELETE(cache->frequency_list, entry);
+        HASH_DEL(cache->hash_table, entry);
+        cache->cur_size -= 1;
+        umf_ba_free(cache->cache_allocator, entry);
+    }
+    HASH_CLEAR(hh, cache->hash_table);
+
+    umf_ba_destroy(cache->cache_allocator);
+    umf_ba_global_free(cache);
+}
+
+umf_result_t
+umfIpcHandleMmapedCacheGet(ipc_handle_mmaped_cache_handle_t cache,
+                           const ipc_mmaped_handle_cache_key_t *key,
+                           uint64_t handle_id,
+                           ipc_handle_mmaped_cache_eviction_cb_t eviction_cb,
+                           ipc_mmaped_handle_cache_value_t **retEntry) {
+    ipc_handle_cache_entry_t *entry = NULL;
+    umf_result_t ret = UMF_RESULT_SUCCESS;
+    bool evicted = false;
+    ipc_mmaped_handle_cache_value_t evicted_value;
+
+    if (!cache) {
+        return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    utils_mutex_lock(&(cache->cache_lock));
+
+    HASH_FIND(hh, cache->hash_table, key, sizeof(*key), entry);
+    if (entry && entry->handle_id == handle_id) { // cache hit
+        // update frequency list
+        DL_DELETE(cache->frequency_list, entry);
+        DL_PREPEND(cache->frequency_list, entry);
+    } else { //cache miss
+        // Look for eviction candidate
+        if (entry == NULL && cache->max_size != 0 &&
+            cache->cur_size >= cache->max_size) {
+            entry = cache->frequency_list->prev;
+        }
+
+        if (entry) { // we have eviction candidate
+            DL_DELETE(cache->frequency_list, entry);
+            HASH_DEL(cache->hash_table, entry);
+            cache->cur_size -= 1;
+            evicted_value.mmaped_base_ptr = entry->value.mmaped_base_ptr;
+            evicted_value.mmaped_size = entry->value.mmaped_size;
+            evicted = true;
+        } else {
+            entry = umf_ba_alloc(cache->cache_allocator);
+            if (!entry) {
+                ret = UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY;
+                goto exit;
+            }
+            if (NULL == utils_mutex_init(&(entry->value.mmap_lock))) {
+                LOG_ERR("Failed to initialize mutex for the IPC cache entry");
+                umf_ba_global_free(entry);
+                ret = UMF_RESULT_ERROR_UNKNOWN;
+                goto exit;
+            }
+        }
+
+        entry->key = *key;
+        entry->ref_count = 0;
+        entry->handle_id = handle_id;
+        entry->value.mmaped_size = 0;
+        entry->value.mmaped_base_ptr = NULL;
+
+        HASH_ADD(hh, cache->hash_table, key, sizeof(entry->key), entry);
+        DL_PREPEND(cache->frequency_list, entry);
+        cache->cur_size += 1;
+    }
+
+exit:
+    if (ret == UMF_RESULT_SUCCESS) {
+        utils_atomic_increment(&entry->ref_count);
+        *retEntry = &entry->value;
+    }
+
+    utils_mutex_unlock(&(cache->cache_lock));
+
+    if (evicted) {
+        eviction_cb(key, &evicted_value);
+    }
+
+    return ret;
+}

src/ipc_cache.h

@@ -0,0 +1,67 @@
+/*
+ *
+ * Copyright (C) 2024 Intel Corporation
+ *
+ * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ */
+
+#ifndef UMF_IPC_CACHE_H
+#define UMF_IPC_CACHE_H 1
+
+#include "umf/memory_provider.h"
+
+#include "base_alloc.h"
+#include "uthash.h"
+#include "utils_concurrency.h"
+
+typedef struct ipc_mmaped_handle_cache_key_t {
+    void *remote_base_ptr;
+    umf_memory_provider_handle_t local_provider;
+    int remote_pid;
+} ipc_mmaped_handle_cache_key_t;
+
+typedef struct ipc_mmaped_handle_cache_value_t {
+    void *mmaped_base_ptr;
+    size_t mmaped_size;
+    utils_mutex_t mmap_lock;
+} ipc_mmaped_handle_cache_value_t;
+
+typedef struct ipc_handle_cache_entry_t {
+    UT_hash_handle hh;
+    struct ipc_handle_cache_entry_t *next, *prev;
+    ipc_mmaped_handle_cache_key_t key;
+    uint64_t ref_count;
+    uint64_t handle_id;
+    ipc_mmaped_handle_cache_value_t value;
+} ipc_handle_cache_entry_t;
+
+typedef struct ipc_handle_mmaped_cache_t {
+    utils_mutex_t cache_lock;
+    umf_ba_pool_t *cache_allocator;
+    size_t max_size;
+    size_t cur_size;
+    ipc_handle_cache_entry_t *hash_table;
+    ipc_handle_cache_entry_t *frequency_list;
+} ipc_handle_mmaped_cache_t;
+
+typedef ipc_handle_mmaped_cache_t *ipc_handle_mmaped_cache_handle_t;
+
+ipc_handle_mmaped_cache_handle_t umfIpcHandleMmapedCacheCreate(void);
+
+void umfIpcHandleMmapedCacheDestroy(ipc_handle_mmaped_cache_handle_t cache);
+
+// define pointer to the eviction callback function
+typedef void (*ipc_handle_mmaped_cache_eviction_cb_t)(
+    const ipc_mmaped_handle_cache_key_t *key,
+    const ipc_mmaped_handle_cache_value_t *value);
+
+umf_result_t
+umfIpcHandleMmapedCacheGet(ipc_handle_mmaped_cache_handle_t cache,
+                           const ipc_mmaped_handle_cache_key_t *key,
+                           uint64_t handle_id,
+                           ipc_handle_mmaped_cache_eviction_cb_t eviction_cb,
+                           ipc_mmaped_handle_cache_value_t **retEntry);
+
+#endif /* UMF_IPC_CACHE_H */

src/ipc_internal.h

@@ -21,8 +21,10 @@ extern "C" {
 // providerIpcData is a Flexible Array Member because its size varies
 // depending on the provider.
 typedef struct umf_ipc_data_t {
-    int pid;         // process ID of the process that allocated the memory
-    size_t baseSize; // size of base (coarse-grain) allocation
+    uint64_t handle_id; // unique handle ID
+    void *base;         // base address of the memory
+    int pid;            // process ID of the process that allocated the memory
+    size_t baseSize;    // size of base (coarse-grain) allocation
     uint64_t offset;
     char providerIpcData[];
 } umf_ipc_data_t;

src/libumf.c

@@ -10,6 +10,7 @@
 #include <stddef.h>
 
 #include "base_alloc_global.h"
+#include "ipc_cache.h"
 #include "memspace_internal.h"
 #include "provider_tracking.h"
 #include "utils_log.h"
@@ -18,13 +19,15 @@
 #endif
 
 umf_memory_tracker_handle_t TRACKER = NULL;
+ipc_handle_mmaped_cache_handle_t IPC_MMAPED_CACHE = NULL;
 
 static unsigned long long umfRefCount = 0;
 
 int umfInit(void) {
     if (utils_fetch_and_add64(&umfRefCount, 1) == 0) {
         utils_log_init();
         TRACKER = umfMemoryTrackerCreate();
+        IPC_MMAPED_CACHE = umfIpcHandleMmapedCacheCreate();
     }
 
     return (TRACKER) ? 0 : -1;
@@ -39,6 +42,9 @@ void umfTearDown(void) {
         umfMemspaceLowestLatencyDestroy();
         umfDestroyTopology();
 #endif
+        ipc_handle_mmaped_cache_handle_t ipc_mmaped_cache = IPC_MMAPED_CACHE;
+        IPC_MMAPED_CACHE = NULL;
+        umfIpcHandleMmapedCacheDestroy(ipc_mmaped_cache);
         // make sure TRACKER is not used after being destroyed
         umf_memory_tracker_handle_t t = TRACKER;
         TRACKER = NULL;