{"action":"created","comment":{"url":"https://api.github.com/repos/openucx/ucx/pulls/comments/48591527","id":48591527,"diff_hunk":"@@ -0,0 +1,438 @@\n+/**\n+* Copyright (C) Mellanox Technologies Ltd. 2001-2015.  ALL RIGHTS RESERVED.\n+*\n+* See file LICENSE for terms.\n+*/\n+\n+#include \"malloc_hook.h\"\n+\n+#include <malloc.h>\n+#undef M_TRIM_THRESHOLD\n+#undef M_MMAP_THRESHOLD\n+\n+#include <ucm/api/ucm.h>\n+#include <ucm/event/event.h>\n+#include <ucm/mmap/mmap.h>\n+#include <ucm/ptmalloc3/malloc-2.8.3.h>\n+#include <ucm/util/log.h>\n+#include \"../util/reloc.h\"\n+#include <ucm/util/ucm_config.h>\n+#include <ucs/datastruct/queue.h>\n+#include <ucs/type/component.h>\n+#include <ucs/type/spinlock.h>\n+#include <ucs/sys/compiler.h>\n+#include <ucs/sys/sys.h>\n+\n+#include <string.h>\n+\n+\n+\n+/* Flags for install_state */\n+#define UCM_MALLOC_INSTALLED_HOOKS      UCS_BIT(0)  /* Installed malloc hooks */\n+#define UCM_MALLOC_INSTALLED_SBRK_EVH   UCS_BIT(1)  /* Installed sbrk event handler */\n+#define UCM_MALLOC_INSTALLED_OPT_SYMS   UCS_BIT(2)  /* Installed optional symbols */\n+#define UCM_MALLOC_INSTALLED_MALL_SYMS  UCS_BIT(3)  /* Installed malloc symbols */\n+\n+\n+typedef struct ucm_malloc_hook_state {\n+    /*\n+     * State of hook installment\n+     */\n+    pthread_mutex_t       install_mutex; /* Protect hooks installation */\n+    int                   install_state; /* State of hook installation */\n+    int                   installed_events; /* Which events are working */\n+    int                   hook_called; /* Our malloc hook was called */\n+    size_t                (*usable_size)(void*); /* function pointer to get usable size */\n+#if !NVALGRIND\n+    void                  (*free)(void*); /*function pointer to release memory */\n+#endif\n+\n+    /*\n+     * Track record of which pointers are ours\n+     */\n+    ucs_spinlock_t        lock;       /* Protect heap counters.\n+                                         Note: Cannot modify events when this lock\n+                                         is held - may deadlock */\n+    /* Our heap address range. Used to identify whether a released pointer is ours,\n+     * or was allocated by the previous heap manager. */\n+    void                  *heap_start;\n+    void                  *heap_end;\n+\n+    /* Save the pointers that we have allocated with mmap, so when they are\n+     * released we would know they are ours, despite the fact they are not in the\n+     * heap address range. */\n+    void                 **ptrs;\n+    unsigned             num_ptrs;\n+    unsigned             max_ptrs;\n+} ucm_malloc_hook_state_t;\n+\n+\n+static ucm_malloc_hook_state_t ucm_malloc_hook_state = {\n+    .install_mutex    = PTHREAD_MUTEX_INITIALIZER,\n+    .install_state    = 0,\n+    .installed_events = 0,\n+    .hook_called      = 0,\n+    .usable_size      = malloc_usable_size,\n+#if !NVALGRIND\n+    .free             = free,\n+#endif\n+    .heap_start       = (void*)-1,\n+    .heap_end         = (void*)-1,\n+    .ptrs             = NULL,\n+    .num_ptrs         = 0,\n+    .max_ptrs         = 0\n+};\n+\n+\n+static void ucm_malloc_mmaped_ptr_add(void *ptr)\n+{\n+    ucs_spin_lock(&ucm_malloc_hook_state.lock);\n+\n+    if (ucm_malloc_hook_state.num_ptrs == ucm_malloc_hook_state.max_ptrs) {\n+        /* Enlarge the array if needed */\n+        if (ucm_malloc_hook_state.max_ptrs == 0) {\n+            ucm_malloc_hook_state.max_ptrs = 256;\n+        } else {\n+            ucm_malloc_hook_state.max_ptrs *= 2;\n+        }\n+\n+        ucm_malloc_hook_state.ptrs =\n+                        dlrealloc(ucm_malloc_hook_state.ptrs,\n+                                  ucm_malloc_hook_state.max_ptrs * sizeof(void*));\n+    }\n+\n+    ucm_malloc_hook_state.ptrs[ucm_malloc_hook_state.num_ptrs] = ptr;\n+    ++ucm_malloc_hook_state.num_ptrs;\n+\n+    ucs_spin_unlock(&ucm_malloc_hook_state.lock);\n+}\n+\n+static int ucm_malloc_mmaped_ptr_remove_if_exists(void *ptr)\n+{\n+    unsigned i;\n+\n+    ucs_spin_lock(&ucm_malloc_hook_state.lock);\n+    for (i = 0; i < ucm_malloc_hook_state.num_ptrs; ++i) {\n+        if (ucm_malloc_hook_state.ptrs[i] == ptr) {\n+            --ucm_malloc_hook_state.num_ptrs;\n+            ucm_malloc_hook_state.ptrs[i] =\n+                            ucm_malloc_hook_state.ptrs[ucm_malloc_hook_state.num_ptrs];\n+            ucs_spin_unlock(&ucm_malloc_hook_state.lock);\n+            return 1;\n+        }\n+    }\n+    ucs_spin_unlock(&ucm_malloc_hook_state.lock);\n+    return 0;\n+}\n+\n+static int ucm_malloc_is_address_in_heap(void *ptr)\n+{\n+    int in_heap;\n+\n+    ucs_spin_lock(&ucm_malloc_hook_state.lock);\n+    in_heap = (ptr >= ucm_malloc_hook_state.heap_start) &&\n+              (ptr < ucm_malloc_hook_state.heap_end);\n+    ucs_spin_unlock(&ucm_malloc_hook_state.lock);\n+    return in_heap;\n+}\n+\n+static int ucm_malloc_address_remove_if_mapped(void *ptr, const char *caller)\n+{\n+    int is_mapped;\n+\n+    if (ucm_malloc_is_address_in_heap(ptr)) {\n+        is_mapped = 1;\n+    } else {\n+        is_mapped = ucm_malloc_mmaped_ptr_remove_if_exists(ptr);\n+    }\n+\n+    ucm_trace(\"%s: %p is%s ours (heap [%p..%p])\",  caller, ptr,\n+              is_mapped ? \"\" : \" not\",\n+              ucm_malloc_hook_state.heap_start, ucm_malloc_hook_state.heap_end);\n+    return is_mapped;\n+}\n+\n+static void ucm_malloc_allocated(void *ptr, const char *caller)\n+{\n+    if (ucm_malloc_is_address_in_heap(ptr)) {\n+        ucm_trace(\"%s: %p in heap [%p..%p]\", caller, ptr,\n+                  ucm_malloc_hook_state.heap_start, ucm_malloc_hook_state.heap_end);\n+    } else {\n+        ucm_trace(\"%s: %p is mmapped\", caller, ptr);\n+        ucm_malloc_mmaped_ptr_add(ptr);\n+    }\n+}\n+\n+static void ucm_release_foreign_block(void *ptr)\n+{\n+#if !NVALGRIND\n+    if (RUNNING_ON_VALGRIND) {\n+        /* We want to keep valgrind happy and release foreign memory as well.\n+         * Otherwise, it's safer to do nothing.\n+         */\n+        ucm_malloc_hook_state.free(ptr);\n+    }\n+#endif\n+}\n+\n+void *ucm_malloc(size_t size, const void *caller)\n+{\n+    void *ptr;\n+\n+    ucm_malloc_hook_state.hook_called = 1;\n+    if (ucm_global_config.alloc_alignment > 1) {\n+        ptr = dlmemalign(ucm_global_config.alloc_alignment, size);\n+    } else {\n+        ptr = dlmalloc(size);\n+    }\n+    ucm_malloc_allocated(ptr, \"malloc\");\n+    return ptr;\n+}\n+\n+void *ucm_realloc(void *oldptr, size_t size, const void *caller)\n+{\n+    void *newptr;\n+\n+    ucm_malloc_hook_state.hook_called = 1;\n+\n+    if ((oldptr == NULL) || ucm_malloc_address_remove_if_mapped(oldptr, \"realloc\")) {\n+        newptr = dlrealloc(oldptr, size);\n+        goto out;\n+    }\n+\n+    /* If pointer was created by original malloc(), allocate the new pointer\n+     * with the new heap, and copy out the data. Then, release the old pointer.\n+     */\n+    newptr = dlmalloc(size);","path":"src/ucm/malloc/malloc_hook.c","position":206,"original_position":206,"commit_id":"d005ba295652e08deb352d8d579c542875803dd1","original_commit_id":"d005ba295652e08deb352d8d579c542875803dd1","user":{"login":"miked-mellanox","id":2255576,"avatar_url":"https://avatars.githubusercontent.com/u/2255576?v=3","gravatar_id":"","url":"https://api.github.com/users/miked-mellanox","html_url":"https://github.com/miked-mellanox","followers_url":"https://api.github.com/users/miked-mellanox/followers","following_url":"https://api.github.com/users/miked-mellanox/following{/other_user}","gists_url":"https://api.github.com/users/miked-mellanox/gists{/gist_id}","starred_url":"https://api.github.com/users/miked-mellanox/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/miked-mellanox/subscriptions","organizations_url":"https://api.github.com/users/miked-mellanox/orgs","repos_url":"https://api.github.com/users/miked-mellanox/repos","events_url":"https://api.github.com/users/miked-mellanox/events{/privacy}","received_events_url":"https://api.github.com/users/miked-mellanox/received_events","type":"User","site_admin":false},"body":"is it safe to realloc w/ new allocator while memory was originally allocated by \"old\" allocator?\r\nThe ptr vm will be in another range?\r\nwhat if ULP counts on VM continuity after realloc?","created_at":"2015-12-30T08:25:22Z","updated_at":"2015-12-30T08:25:22Z","html_url":"https://github.com/openucx/ucx/pull/529#discussion_r48591527","pull_request_url":"https://api.github.com/repos/openucx/ucx/pulls/529","_links":{"self":{"href":"https://api.github.com/repos/openucx/ucx/pulls/comments/48591527"},"html":{"href":"https://github.com/openucx/ucx/pull/529#discussion_r48591527"},"pull_request":{"href":"https://api.github.com/repos/openucx/ucx/pulls/529"}}},"pull_request":{"url":"https://api.github.com/repos/openucx/ucx/pulls/529","id":54716934,"html_url":"https://github.com/openucx/ucx/pull/529","diff_url":"https://github.com/openucx/ucx/pull/529.diff","patch_url":"https://github.com/openucx/ucx/pull/529.patch","issue_url":"https://api.github.com/repos/openucx/ucx/issues/529","number":529,"state":"open","locked":false,"title":"UCM/TEST: Add UCM library which provides memory (un)mapping hooks.","user":{"login":"yosefe","id":2255631,"avatar_url":"https://avatars.githubusercontent.com/u/2255631?v=3","gravatar_id":"","url":"https://api.github.com/users/yosefe","html_url":"https://github.com/yosefe","followers_url":"https://api.github.com/users/yosefe/followers","following_url":"https://api.github.com/users/yosefe/following{/other_user}","gists_url":"https://api.github.com/users/yosefe/gists{/gist_id}","starred_url":"https://api.github.com/users/yosefe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yosefe/subscriptions","organizations_url":"https://api.github.com/users/yosefe/orgs","repos_url":"https://api.github.com/users/yosefe/repos","events_url":"https://api.github.com/users/yosefe/events{/privacy}","received_events_url":"https://api.github.com/users/yosefe/received_events","type":"User","site_admin":false},"body":"See description here: https://github.com/openucx/ucx/wiki/Memory-hooks","created_at":"2015-12-29T10:36:22Z","updated_at":"2015-12-30T08:25:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"82d9a69dc206a967e6f273ddea75e5251de119dd","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/openucx/ucx/pulls/529/commits","review_comments_url":"https://api.github.com/repos/openucx/ucx/pulls/529/comments","review_comment_url":"https://api.github.com/repos/openucx/ucx/pulls/comments{/number}","comments_url":"https://api.github.com/repos/openucx/ucx/issues/529/comments","statuses_url":"https://api.github.com/repos/openucx/ucx/statuses/d005ba295652e08deb352d8d579c542875803dd1","head":{"label":"yosefe:topic/memory-hooks","ref":"topic/memory-hooks","sha":"d005ba295652e08deb352d8d579c542875803dd1","user":{"login":"yosefe","id":2255631,"avatar_url":"https://avatars.githubusercontent.com/u/2255631?v=3","gravatar_id":"","url":"https://api.github.com/users/yosefe","html_url":"https://github.com/yosefe","followers_url":"https://api.github.com/users/yosefe/followers","following_url":"https://api.github.com/users/yosefe/following{/other_user}","gists_url":"https://api.github.com/users/yosefe/gists{/gist_id}","starred_url":"https://api.github.com/users/yosefe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yosefe/subscriptions","organizations_url":"https://api.github.com/users/yosefe/orgs","repos_url":"https://api.github.com/users/yosefe/repos","events_url":"https://api.github.com/users/yosefe/events{/privacy}","received_events_url":"https://api.github.com/users/yosefe/received_events","type":"User","site_admin":false},"repo":{"id":25532870,"name":"ucx","full_name":"yosefe/ucx","owner":{"login":"yosefe","id":2255631,"avatar_url":"https://avatars.githubusercontent.com/u/2255631?v=3","gravatar_id":"","url":"https://api.github.com/users/yosefe","html_url":"https://github.com/yosefe","followers_url":"https://api.github.com/users/yosefe/followers","following_url":"https://api.github.com/users/yosefe/following{/other_user}","gists_url":"https://api.github.com/users/yosefe/gists{/gist_id}","starred_url":"https://api.github.com/users/yosefe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yosefe/subscriptions","organizations_url":"https://api.github.com/users/yosefe/orgs","repos_url":"https://api.github.com/users/yosefe/repos","events_url":"https://api.github.com/users/yosefe/events{/privacy}","received_events_url":"https://api.github.com/users/yosefe/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/yosefe/ucx","description":"The UCX","fork":true,"url":"https://api.github.com/repos/yosefe/ucx","forks_url":"https://api.github.com/repos/yosefe/ucx/forks","keys_url":"https://api.github.com/repos/yosefe/ucx/keys{/key_id}","collaborators_url":"https://api.github.com/repos/yosefe/ucx/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/yosefe/ucx/teams","hooks_url":"https://api.github.com/repos/yosefe/ucx/hooks","issue_events_url":"https://api.github.com/repos/yosefe/ucx/issues/events{/number}","events_url":"https://api.github.com/repos/yosefe/ucx/events","assignees_url":"https://api.github.com/repos/yosefe/ucx/assignees{/user}","branches_url":"https://api.github.com/repos/yosefe/ucx/branches{/branch}","tags_url":"https://api.github.com/repos/yosefe/ucx/tags","blobs_url":"https://api.github.com/repos/yosefe/ucx/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/yosefe/ucx/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/yosefe/ucx/git/refs{/sha}","trees_url":"https://api.github.com/repos/yosefe/ucx/git/trees{/sha}","statuses_url":"https://api.github.com/repos/yosefe/ucx/statuses/{sha}","languages_url":"https://api.github.com/repos/yosefe/ucx/languages","stargazers_url":"https://api.github.com/repos/yosefe/ucx/stargazers","contributors_url":"https://api.github.com/repos/yosefe/ucx/contributors","subscribers_url":"https://api.github.com/repos/yosefe/ucx/subscribers","subscription_url":"https://api.github.com/repos/yosefe/ucx/subscription","commits_url":"https://api.github.com/repos/yosefe/ucx/commits{/sha}","git_commits_url":"https://api.github.com/repos/yosefe/ucx/git/commits{/sha}","comments_url":"https://api.github.com/repos/yosefe/ucx/comments{/number}","issue_comment_url":"https://api.github.com/repos/yosefe/ucx/issues/comments{/number}","contents_url":"https://api.github.com/repos/yosefe/ucx/contents/{+path}","compare_url":"https://api.github.com/repos/yosefe/ucx/compare/{base}...{head}","merges_url":"https://api.github.com/repos/yosefe/ucx/merges","archive_url":"https://api.github.com/repos/yosefe/ucx/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/yosefe/ucx/downloads","issues_url":"https://api.github.com/repos/yosefe/ucx/issues{/number}","pulls_url":"https://api.github.com/repos/yosefe/ucx/pulls{/number}","milestones_url":"https://api.github.com/repos/yosefe/ucx/milestones{/number}","notifications_url":"https://api.github.com/repos/yosefe/ucx/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/yosefe/ucx/labels{/name}","releases_url":"https://api.github.com/repos/yosefe/ucx/releases{/id}","created_at":"2014-10-21T16:22:47Z","updated_at":"2015-07-13T17:18:28Z","pushed_at":"2015-12-29T21:09:19Z","git_url":"git://github.com/yosefe/ucx.git","ssh_url":"git@github.com:yosefe/ucx.git","clone_url":"https://github.com/yosefe/ucx.git","svn_url":"https://github.com/yosefe/ucx","homepage":"","size":3270,"stargazers_count":0,"watchers_count":0,"language":"C","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"openucx:master","ref":"master","sha":"24a2b7fad79e125defabe09477b1ea31514fb3bc","user":{"login":"openucx","id":13118737,"avatar_url":"https://avatars.githubusercontent.com/u/13118737?v=3","gravatar_id":"","url":"https://api.github.com/users/openucx","html_url":"https://github.com/openucx","followers_url":"https://api.github.com/users/openucx/followers","following_url":"https://api.github.com/users/openucx/following{/other_user}","gists_url":"https://api.github.com/users/openucx/gists{/gist_id}","starred_url":"https://api.github.com/users/openucx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/openucx/subscriptions","organizations_url":"https://api.github.com/users/openucx/orgs","repos_url":"https://api.github.com/users/openucx/repos","events_url":"https://api.github.com/users/openucx/events{/privacy}","received_events_url":"https://api.github.com/users/openucx/received_events","type":"Organization","site_admin":false},"repo":{"id":25379390,"name":"ucx","full_name":"openucx/ucx","owner":{"login":"openucx","id":13118737,"avatar_url":"https://avatars.githubusercontent.com/u/13118737?v=3","gravatar_id":"","url":"https://api.github.com/users/openucx","html_url":"https://github.com/openucx","followers_url":"https://api.github.com/users/openucx/followers","following_url":"https://api.github.com/users/openucx/following{/other_user}","gists_url":"https://api.github.com/users/openucx/gists{/gist_id}","starred_url":"https://api.github.com/users/openucx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/openucx/subscriptions","organizations_url":"https://api.github.com/users/openucx/orgs","repos_url":"https://api.github.com/users/openucx/repos","events_url":"https://api.github.com/users/openucx/events{/privacy}","received_events_url":"https://api.github.com/users/openucx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/openucx/ucx","description":"Unified Communication X  (https://elist.ornl.gov/mailman/listinfo/ucx-group): ","fork":false,"url":"https://api.github.com/repos/openucx/ucx","forks_url":"https://api.github.com/repos/openucx/ucx/forks","keys_url":"https://api.github.com/repos/openucx/ucx/keys{/key_id}","collaborators_url":"https://api.github.com/repos/openucx/ucx/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/openucx/ucx/teams","hooks_url":"https://api.github.com/repos/openucx/ucx/hooks","issue_events_url":"https://api.github.com/repos/openucx/ucx/issues/events{/number}","events_url":"https://api.github.com/repos/openucx/ucx/events","assignees_url":"https://api.github.com/repos/openucx/ucx/assignees{/user}","branches_url":"https://api.github.com/repos/openucx/ucx/branches{/branch}","tags_url":"https://api.github.com/repos/openucx/ucx/tags","blobs_url":"https://api.github.com/repos/openucx/ucx/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/openucx/ucx/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/openucx/ucx/git/refs{/sha}","trees_url":"https://api.github.com/repos/openucx/ucx/git/trees{/sha}","statuses_url":"https://api.github.com/repos/openucx/ucx/statuses/{sha}","languages_url":"https://api.github.com/repos/openucx/ucx/languages","stargazers_url":"https://api.github.com/repos/openucx/ucx/stargazers","contributors_url":"https://api.github.com/repos/openucx/ucx/contributors","subscribers_url":"https://api.github.com/repos/openucx/ucx/subscribers","subscription_url":"https://api.github.com/repos/openucx/ucx/subscription","commits_url":"https://api.github.com/repos/openucx/ucx/commits{/sha}","git_commits_url":"https://api.github.com/repos/openucx/ucx/git/commits{/sha}","comments_url":"https://api.github.com/repos/openucx/ucx/comments{/number}","issue_comment_url":"https://api.github.com/repos/openucx/ucx/issues/comments{/number}","contents_url":"https://api.github.com/repos/openucx/ucx/contents/{+path}","compare_url":"https://api.github.com/repos/openucx/ucx/compare/{base}...{head}","merges_url":"https://api.github.com/repos/openucx/ucx/merges","archive_url":"https://api.github.com/repos/openucx/ucx/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/openucx/ucx/downloads","issues_url":"https://api.github.com/repos/openucx/ucx/issues{/number}","pulls_url":"https://api.github.com/repos/openucx/ucx/pulls{/number}","milestones_url":"https://api.github.com/repos/openucx/ucx/milestones{/number}","notifications_url":"https://api.github.com/repos/openucx/ucx/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/openucx/ucx/labels{/name}","releases_url":"https://api.github.com/repos/openucx/ucx/releases{/id}","created_at":"2014-10-17T22:17:24Z","updated_at":"2015-12-23T19:06:35Z","pushed_at":"2015-12-30T03:25:56Z","git_url":"git://github.com/openucx/ucx.git","ssh_url":"git@github.com:openucx/ucx.git","clone_url":"https://github.com/openucx/ucx.git","svn_url":"https://github.com/openucx/ucx","homepage":" http://www.openucx.org","size":4941,"stargazers_count":26,"watchers_count":26,"language":"C","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":19,"mirror_url":null,"open_issues_count":56,"forks":19,"open_issues":56,"watchers":26,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/openucx/ucx/pulls/529"},"html":{"href":"https://github.com/openucx/ucx/pull/529"},"issue":{"href":"https://api.github.com/repos/openucx/ucx/issues/529"},"comments":{"href":"https://api.github.com/repos/openucx/ucx/issues/529/comments"},"review_comments":{"href":"https://api.github.com/repos/openucx/ucx/pulls/529/comments"},"review_comment":{"href":"https://api.github.com/repos/openucx/ucx/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/openucx/ucx/pulls/529/commits"},"statuses":{"href":"https://api.github.com/repos/openucx/ucx/statuses/d005ba295652e08deb352d8d579c542875803dd1"}}}}