{"action":"created","comment":{"url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/comments/42733384","id":42733384,"diff_hunk":"@@ -1099,6 +1107,138 @@ bool HeapletBase::hasDestructor(const void *ptr)\n         return false;\n }\n \n+//================================================================================\n+\n+#ifdef _USE_CPPUNIT\n+static size_t parallelSyncReleaseGranularity = DEFAULT_PARALLEL_SYNC_RELEASE_GRANULARITY;\n+static size_t parallelSyncReleaseThreshold = DEFAULT_PARALLEL_SYNC_RELEASE_THRESHOLD;\n+\n+//Only used by the tuning unit tests, should possibly use a different flag to fix them as constant\n+static void setParallelSyncReleaseGranularity(size_t granularity, unsigned scaling)\n+{\n+    parallelSyncReleaseGranularity = granularity;\n+    parallelSyncReleaseThreshold = granularity * scaling;\n+}\n+#else\n+const static size_t parallelSyncReleaseGranularity = DEFAULT_PARALLEL_SYNC_RELEASE_GRANULARITY;\n+const static size_t parallelSyncReleaseThreshold = DEFAULT_PARALLEL_SYNC_RELEASE_THRESHOLD;\n+#endif\n+\n+\n+inline void inlineReleaseRoxieRowArray(size_t count, const void * * rows)\n+{\n+    for (size_t i = 0; i < count; i++)\n+        ReleaseRoxieRow(rows[i]);\n+}\n+\n+#ifdef PARALLEL_SYNC_RELEASE\n+class sync_releaser_task : public tbb::task\n+{\n+public:\n+    sync_releaser_task(size_t _num, const void * * _rows) : num(_num), rows(_rows) {}\n+\n+    virtual task * execute()\n+    {\n+        inlineReleaseRoxieRowArray(num, rows);\n+        return NULL;\n+    }\n+\n+public:\n+    size_t num;\n+    const void * * rows;\n+};\n+\n+void ParallelReleaseRoxieRowArray(size_t count, const void * * rows)\n+{\n+    //Two different implementations of the same code.  The task list is simpler and the recommended approach, but\n+    //appears to be slower - possibly because work doesn't start until all tasks created.  Needs more testing to verify.\n+#if 1\n+    tbb::task * completed_task = new (tbb::task::allocate_root()) tbb::empty_task();\n+    unsigned numTasks = (unsigned)((count + parallelSyncReleaseGranularity -1) / parallelSyncReleaseGranularity);\n+    completed_task->set_ref_count(1+numTasks);\n+\n+    //Arrange the blocks that are freed from the array so that if the rows are allocated in order, adjacent blocks\n+    //are not released at the same time, therefore reducing contention.\n+    const unsigned stride = 8;\n+    for (unsigned j= 0; j < stride; j++)\n+    {\n+        size_t i = j * parallelSyncReleaseGranularity;\n+        for (; i < count; i += parallelSyncReleaseGranularity * stride)\n+        {\n+            size_t remain = count - i;\n+            size_t blockRows = (remain > parallelSyncReleaseGranularity) ? parallelSyncReleaseGranularity : remain;\n+            tbb::task * next = new (completed_task->allocate_child()) sync_releaser_task(blockRows, rows + i);\n+            next->spawn(*next); // static member in tbb 3.0\n+        }\n+    }\n+\n+    completed_task->wait_for_all();\n+    completed_task->destroy(*completed_task);       // static member in tbb 3.0\n+#else\n+    //Arrange the blocks that are freed from the array so that if they are allocated in order adjacent blocks\n+    //are not released at the same time.\n+    tbb::task_list tasks;\n+    const unsigned stride = 8;\n+    for (unsigned j= 0; j < stride; j++)\n+    {\n+        size_t i = j * parallelSyncReleaseGranularity;\n+        for (; i < count; i += parallelSyncReleaseGranularity * stride)\n+        {\n+            size_t remain = count - i;\n+            size_t blockRows = (remain > parallelSyncReleaseGranularity) ? parallelSyncReleaseGranularity : remain;\n+            tasks.push_back(* new (tbb::task::allocate_root()) sync_releaser_task(blockRows, rows + i));\n+        }\n+    }\n+\n+    tbb::task::spawn_root_and_wait(tasks);\n+#endif\n+\n+}\n+#endif\n+\n+\n+void ReleaseRoxieRowArray(size_t count, const void * * rows)\n+{\n+#ifdef PARALLEL_SYNC_RELEASE","path":"roxie/roxiemem/roxiemem.cpp","position":136,"original_position":136,"commit_id":"2edefb3177ef66f3e5689ef54ee2c5665e0e0499","original_commit_id":"2edefb3177ef66f3e5689ef54ee2c5665e0e0499","user":{"login":"richardkchapman","id":900820,"avatar_url":"https://avatars.githubusercontent.com/u/900820?v=3","gravatar_id":"","url":"https://api.github.com/users/richardkchapman","html_url":"https://github.com/richardkchapman","followers_url":"https://api.github.com/users/richardkchapman/followers","following_url":"https://api.github.com/users/richardkchapman/following{/other_user}","gists_url":"https://api.github.com/users/richardkchapman/gists{/gist_id}","starred_url":"https://api.github.com/users/richardkchapman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/richardkchapman/subscriptions","organizations_url":"https://api.github.com/users/richardkchapman/orgs","repos_url":"https://api.github.com/users/richardkchapman/repos","events_url":"https://api.github.com/users/richardkchapman/events{/privacy}","received_events_url":"https://api.github.com/users/richardkchapman/received_events","type":"User","site_admin":false},"body":"This should be set somewhere (in the CMAKE file?)","created_at":"2015-10-22T10:41:53Z","updated_at":"2015-10-22T10:41:53Z","html_url":"https://github.com/hpcc-systems/HPCC-Platform/pull/7594#discussion_r42733384","pull_request_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7594","_links":{"self":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/comments/42733384"},"html":{"href":"https://github.com/hpcc-systems/HPCC-Platform/pull/7594#discussion_r42733384"},"pull_request":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7594"}}},"pull_request":{"url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7594","id":40548575,"html_url":"https://github.com/hpcc-systems/HPCC-Platform/pull/7594","diff_url":"https://github.com/hpcc-systems/HPCC-Platform/pull/7594.diff","patch_url":"https://github.com/hpcc-systems/HPCC-Platform/pull/7594.patch","issue_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/7594","number":7594,"state":"open","locked":false,"title":"HPCC-13690 Parallel row release (synchronous and async)","user":{"login":"ghalliday","id":902490,"avatar_url":"https://avatars.githubusercontent.com/u/902490?v=3","gravatar_id":"","url":"https://api.github.com/users/ghalliday","html_url":"https://github.com/ghalliday","followers_url":"https://api.github.com/users/ghalliday/followers","following_url":"https://api.github.com/users/ghalliday/following{/other_user}","gists_url":"https://api.github.com/users/ghalliday/gists{/gist_id}","starred_url":"https://api.github.com/users/ghalliday/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ghalliday/subscriptions","organizations_url":"https://api.github.com/users/ghalliday/orgs","repos_url":"https://api.github.com/users/ghalliday/repos","events_url":"https://api.github.com/users/ghalliday/events{/privacy}","received_events_url":"https://api.github.com/users/ghalliday/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-07-22T10:02:31Z","updated_at":"2015-10-22T10:41:53Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6b16c723ee760625bc1c0524656d64dd0255bc63","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7594/commits","review_comments_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7594/comments","review_comment_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/7594/comments","statuses_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/statuses/2edefb3177ef66f3e5689ef54ee2c5665e0e0499","head":{"label":"ghalliday:parallelrelease","ref":"parallelrelease","sha":"2edefb3177ef66f3e5689ef54ee2c5665e0e0499","user":{"login":"ghalliday","id":902490,"avatar_url":"https://avatars.githubusercontent.com/u/902490?v=3","gravatar_id":"","url":"https://api.github.com/users/ghalliday","html_url":"https://github.com/ghalliday","followers_url":"https://api.github.com/users/ghalliday/followers","following_url":"https://api.github.com/users/ghalliday/following{/other_user}","gists_url":"https://api.github.com/users/ghalliday/gists{/gist_id}","starred_url":"https://api.github.com/users/ghalliday/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ghalliday/subscriptions","organizations_url":"https://api.github.com/users/ghalliday/orgs","repos_url":"https://api.github.com/users/ghalliday/repos","events_url":"https://api.github.com/users/ghalliday/events{/privacy}","received_events_url":"https://api.github.com/users/ghalliday/received_events","type":"User","site_admin":false},"repo":{"id":2034867,"name":"HPCC-Platform","full_name":"ghalliday/HPCC-Platform","owner":{"login":"ghalliday","id":902490,"avatar_url":"https://avatars.githubusercontent.com/u/902490?v=3","gravatar_id":"","url":"https://api.github.com/users/ghalliday","html_url":"https://github.com/ghalliday","followers_url":"https://api.github.com/users/ghalliday/followers","following_url":"https://api.github.com/users/ghalliday/following{/other_user}","gists_url":"https://api.github.com/users/ghalliday/gists{/gist_id}","starred_url":"https://api.github.com/users/ghalliday/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ghalliday/subscriptions","organizations_url":"https://api.github.com/users/ghalliday/orgs","repos_url":"https://api.github.com/users/ghalliday/repos","events_url":"https://api.github.com/users/ghalliday/events{/privacy}","received_events_url":"https://api.github.com/users/ghalliday/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ghalliday/HPCC-Platform","description":"HPCC Systems big data solution, community edition","fork":true,"url":"https://api.github.com/repos/ghalliday/HPCC-Platform","forks_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/forks","keys_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/teams","hooks_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/hooks","issue_events_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/issues/events{/number}","events_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/events","assignees_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/assignees{/user}","branches_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/branches{/branch}","tags_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/tags","blobs_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/git/refs{/sha}","trees_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/statuses/{sha}","languages_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/languages","stargazers_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/stargazers","contributors_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/contributors","subscribers_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/subscribers","subscription_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/subscription","commits_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/commits{/sha}","git_commits_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/git/commits{/sha}","comments_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/comments{/number}","issue_comment_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/issues/comments{/number}","contents_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/contents/{+path}","compare_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/merges","archive_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/downloads","issues_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/issues{/number}","pulls_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/pulls{/number}","milestones_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/milestones{/number}","notifications_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/labels{/name}","releases_url":"https://api.github.com/repos/ghalliday/HPCC-Platform/releases{/id}","created_at":"2011-07-12T08:05:04Z","updated_at":"2015-05-08T12:57:14Z","pushed_at":"2015-10-22T09:29:36Z","git_url":"git://github.com/ghalliday/HPCC-Platform.git","ssh_url":"git@github.com:ghalliday/HPCC-Platform.git","clone_url":"https://github.com/ghalliday/HPCC-Platform.git","svn_url":"https://github.com/ghalliday/HPCC-Platform","homepage":"www.hpccsystems.com","size":163046,"stargazers_count":2,"watchers_count":2,"language":"C++","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":2,"default_branch":"master"}},"base":{"label":"hpcc-systems:master","ref":"master","sha":"6d928b931ac851aed1265a271ef61d2257381113","user":{"login":"hpcc-systems","id":900867,"avatar_url":"https://avatars.githubusercontent.com/u/900867?v=3","gravatar_id":"","url":"https://api.github.com/users/hpcc-systems","html_url":"https://github.com/hpcc-systems","followers_url":"https://api.github.com/users/hpcc-systems/followers","following_url":"https://api.github.com/users/hpcc-systems/following{/other_user}","gists_url":"https://api.github.com/users/hpcc-systems/gists{/gist_id}","starred_url":"https://api.github.com/users/hpcc-systems/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hpcc-systems/subscriptions","organizations_url":"https://api.github.com/users/hpcc-systems/orgs","repos_url":"https://api.github.com/users/hpcc-systems/repos","events_url":"https://api.github.com/users/hpcc-systems/events{/privacy}","received_events_url":"https://api.github.com/users/hpcc-systems/received_events","type":"Organization","site_admin":false},"repo":{"id":2030681,"name":"HPCC-Platform","full_name":"hpcc-systems/HPCC-Platform","owner":{"login":"hpcc-systems","id":900867,"avatar_url":"https://avatars.githubusercontent.com/u/900867?v=3","gravatar_id":"","url":"https://api.github.com/users/hpcc-systems","html_url":"https://github.com/hpcc-systems","followers_url":"https://api.github.com/users/hpcc-systems/followers","following_url":"https://api.github.com/users/hpcc-systems/following{/other_user}","gists_url":"https://api.github.com/users/hpcc-systems/gists{/gist_id}","starred_url":"https://api.github.com/users/hpcc-systems/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hpcc-systems/subscriptions","organizations_url":"https://api.github.com/users/hpcc-systems/orgs","repos_url":"https://api.github.com/users/hpcc-systems/repos","events_url":"https://api.github.com/users/hpcc-systems/events{/privacy}","received_events_url":"https://api.github.com/users/hpcc-systems/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hpcc-systems/HPCC-Platform","description":"IMPORTANT - Issue tracking has been migrated to JIRA - use your github ID and password reset mechanism to log in","fork":false,"url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform","forks_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/forks","keys_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/teams","hooks_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/hooks","issue_events_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/events{/number}","events_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/events","assignees_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/assignees{/user}","branches_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/branches{/branch}","tags_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/tags","blobs_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/git/refs{/sha}","trees_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/statuses/{sha}","languages_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/languages","stargazers_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/stargazers","contributors_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/contributors","subscribers_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/subscribers","subscription_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/subscription","commits_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/commits{/sha}","git_commits_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/git/commits{/sha}","comments_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/comments{/number}","issue_comment_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/comments{/number}","contents_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/contents/{+path}","compare_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/merges","archive_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/downloads","issues_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues{/number}","pulls_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls{/number}","milestones_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/milestones{/number}","notifications_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/labels{/name}","releases_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/releases{/id}","created_at":"2011-07-11T14:29:51Z","updated_at":"2015-10-14T13:56:56Z","pushed_at":"2015-10-22T09:24:59Z","git_url":"git://github.com/hpcc-systems/HPCC-Platform.git","ssh_url":"git@github.com:hpcc-systems/HPCC-Platform.git","clone_url":"https://github.com/hpcc-systems/HPCC-Platform.git","svn_url":"https://github.com/hpcc-systems/HPCC-Platform","homepage":"track.hpccsystems.com","size":346823,"stargazers_count":257,"watchers_count":257,"language":"C++","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":123,"mirror_url":null,"open_issues_count":431,"forks":123,"open_issues":431,"watchers":257,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7594"},"html":{"href":"https://github.com/hpcc-systems/HPCC-Platform/pull/7594"},"issue":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/7594"},"comments":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/7594/comments"},"review_comments":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7594/comments"},"review_comment":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7594/commits"},"statuses":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/statuses/2edefb3177ef66f3e5689ef54ee2c5665e0e0499"}}}}