{"action":"created","comment":{"url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/comments/35309739","id":35309739,"diff_hunk":"@@ -400,14 +384,25 @@ class CSortMaster: public IThorSorterMaster, public CSimpleInterface\n     void SortSetup(IRowInterfaces *_rowif,ICompare *_icompare,ISortKeySerializer *_keyserializer,bool cosort,bool needconnect,const char *_cosortfilenames,IRowInterfaces *_auxrowif)\n     {\n         ActPrintLog(activity, \"Sort setup cosort=%s, needconnect=%s %s\",cosort?\"true\":\"false\",needconnect?\"true\":\"false\",_keyserializer?\"has key serializer\":\"\");\n+        assertex(_icompare);\n         rowif.set(_rowif);\n         if (_auxrowif&&_auxrowif->queryRowMetaData())\n             auxrowif.set(_auxrowif);\n         else\n             auxrowif.set(_rowif);\n         synchronized proc(slavemutex);\n-        icompare = _icompare;\n         keyserializer = _keyserializer;\n+        if (keyserializer)\n+        {\n+            keyIf.setown(createRowInterfaces(keyserializer->queryRecordSize(), activity->queryContainer().queryId(), activity->queryCodeContext()));\n+            keyIf.set(keyIf);","path":"thorlcr/msort/tsortm.cpp","position":101,"original_position":101,"commit_id":"fac432e9712324900af300331889df46689dac43","original_commit_id":"fac432e9712324900af300331889df46689dac43","user":{"login":"jakesmith","id":902700,"avatar_url":"https://avatars.githubusercontent.com/u/902700?v=3","gravatar_id":"","url":"https://api.github.com/users/jakesmith","html_url":"https://github.com/jakesmith","followers_url":"https://api.github.com/users/jakesmith/followers","following_url":"https://api.github.com/users/jakesmith/following{/other_user}","gists_url":"https://api.github.com/users/jakesmith/gists{/gist_id}","starred_url":"https://api.github.com/users/jakesmith/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jakesmith/subscriptions","organizations_url":"https://api.github.com/users/jakesmith/orgs","repos_url":"https://api.github.com/users/jakesmith/repos","events_url":"https://api.github.com/users/jakesmith/events{/privacy}","received_events_url":"https://api.github.com/users/jakesmith/received_events","type":"User","site_admin":false},"body":"Will remove","created_at":"2015-07-23T10:47:06Z","updated_at":"2015-07-23T10:47:06Z","html_url":"https://github.com/hpcc-systems/HPCC-Platform/pull/7549#discussion_r35309739","pull_request_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7549","_links":{"self":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/comments/35309739"},"html":{"href":"https://github.com/hpcc-systems/HPCC-Platform/pull/7549#discussion_r35309739"},"pull_request":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7549"}}},"pull_request":{"url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7549","id":39700263,"html_url":"https://github.com/hpcc-systems/HPCC-Platform/pull/7549","diff_url":"https://github.com/hpcc-systems/HPCC-Platform/pull/7549.diff","patch_url":"https://github.com/hpcc-systems/HPCC-Platform/pull/7549.patch","issue_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/7549","number":7549,"state":"open","locked":false,"title":"HPCC-13791 Use keys in sort partitioning","user":{"login":"jakesmith","id":902700,"avatar_url":"https://avatars.githubusercontent.com/u/902700?v=3","gravatar_id":"","url":"https://api.github.com/users/jakesmith","html_url":"https://github.com/jakesmith","followers_url":"https://api.github.com/users/jakesmith/followers","following_url":"https://api.github.com/users/jakesmith/following{/other_user}","gists_url":"https://api.github.com/users/jakesmith/gists{/gist_id}","starred_url":"https://api.github.com/users/jakesmith/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jakesmith/subscriptions","organizations_url":"https://api.github.com/users/jakesmith/orgs","repos_url":"https://api.github.com/users/jakesmith/repos","events_url":"https://api.github.com/users/jakesmith/events{/privacy}","received_events_url":"https://api.github.com/users/jakesmith/received_events","type":"User","site_admin":false},"body":"Global sort/join prior to this change, serialized whole rows to and from the master to perform the partitioning calculation. This is particularly expensive on large clusters with large rows.\r\nIt's also very memory hungry, which has meant the global sorts of large rows have caused OOM's on the master whilst handling the partition row data.\r\nThis change serializes the sort keys only to and from master to slaves.\r\n","created_at":"2015-07-10T16:21:33Z","updated_at":"2015-07-23T10:47:06Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f0f21f61846854c6a6bdb4372ce6c4dff25145a8","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7549/commits","review_comments_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7549/comments","review_comment_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/7549/comments","statuses_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/statuses/fac432e9712324900af300331889df46689dac43","head":{"label":"jakesmith:hpcc-13791","ref":"hpcc-13791","sha":"fac432e9712324900af300331889df46689dac43","user":{"login":"jakesmith","id":902700,"avatar_url":"https://avatars.githubusercontent.com/u/902700?v=3","gravatar_id":"","url":"https://api.github.com/users/jakesmith","html_url":"https://github.com/jakesmith","followers_url":"https://api.github.com/users/jakesmith/followers","following_url":"https://api.github.com/users/jakesmith/following{/other_user}","gists_url":"https://api.github.com/users/jakesmith/gists{/gist_id}","starred_url":"https://api.github.com/users/jakesmith/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jakesmith/subscriptions","organizations_url":"https://api.github.com/users/jakesmith/orgs","repos_url":"https://api.github.com/users/jakesmith/repos","events_url":"https://api.github.com/users/jakesmith/events{/privacy}","received_events_url":"https://api.github.com/users/jakesmith/received_events","type":"User","site_admin":false},"repo":{"id":2041370,"name":"HPCC-Platform","full_name":"jakesmith/HPCC-Platform","owner":{"login":"jakesmith","id":902700,"avatar_url":"https://avatars.githubusercontent.com/u/902700?v=3","gravatar_id":"","url":"https://api.github.com/users/jakesmith","html_url":"https://github.com/jakesmith","followers_url":"https://api.github.com/users/jakesmith/followers","following_url":"https://api.github.com/users/jakesmith/following{/other_user}","gists_url":"https://api.github.com/users/jakesmith/gists{/gist_id}","starred_url":"https://api.github.com/users/jakesmith/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jakesmith/subscriptions","organizations_url":"https://api.github.com/users/jakesmith/orgs","repos_url":"https://api.github.com/users/jakesmith/repos","events_url":"https://api.github.com/users/jakesmith/events{/privacy}","received_events_url":"https://api.github.com/users/jakesmith/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jakesmith/HPCC-Platform","description":"HPCC Systems big data solution, community edition","fork":true,"url":"https://api.github.com/repos/jakesmith/HPCC-Platform","forks_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/forks","keys_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/teams","hooks_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/hooks","issue_events_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/issues/events{/number}","events_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/events","assignees_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/assignees{/user}","branches_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/branches{/branch}","tags_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/tags","blobs_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/git/refs{/sha}","trees_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/statuses/{sha}","languages_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/languages","stargazers_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/stargazers","contributors_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/contributors","subscribers_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/subscribers","subscription_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/subscription","commits_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/commits{/sha}","git_commits_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/git/commits{/sha}","comments_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/comments{/number}","issue_comment_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/issues/comments{/number}","contents_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/contents/{+path}","compare_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/merges","archive_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/downloads","issues_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/issues{/number}","pulls_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/pulls{/number}","milestones_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/milestones{/number}","notifications_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/labels{/name}","releases_url":"https://api.github.com/repos/jakesmith/HPCC-Platform/releases{/id}","created_at":"2011-07-13T10:47:52Z","updated_at":"2014-09-16T18:05:11Z","pushed_at":"2015-07-23T10:34:32Z","git_url":"git://github.com/jakesmith/HPCC-Platform.git","ssh_url":"git@github.com:jakesmith/HPCC-Platform.git","clone_url":"https://github.com/jakesmith/HPCC-Platform.git","svn_url":"https://github.com/jakesmith/HPCC-Platform","homepage":"www.hpccsystems.com","size":155971,"stargazers_count":1,"watchers_count":1,"language":"C++","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":5,"forks":0,"open_issues":5,"watchers":1,"default_branch":"master"}},"base":{"label":"hpcc-systems:master","ref":"master","sha":"58f419cd3b453a3cf132c9472f482135b3e27fa0","user":{"login":"hpcc-systems","id":900867,"avatar_url":"https://avatars.githubusercontent.com/u/900867?v=3","gravatar_id":"","url":"https://api.github.com/users/hpcc-systems","html_url":"https://github.com/hpcc-systems","followers_url":"https://api.github.com/users/hpcc-systems/followers","following_url":"https://api.github.com/users/hpcc-systems/following{/other_user}","gists_url":"https://api.github.com/users/hpcc-systems/gists{/gist_id}","starred_url":"https://api.github.com/users/hpcc-systems/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hpcc-systems/subscriptions","organizations_url":"https://api.github.com/users/hpcc-systems/orgs","repos_url":"https://api.github.com/users/hpcc-systems/repos","events_url":"https://api.github.com/users/hpcc-systems/events{/privacy}","received_events_url":"https://api.github.com/users/hpcc-systems/received_events","type":"Organization","site_admin":false},"repo":{"id":2030681,"name":"HPCC-Platform","full_name":"hpcc-systems/HPCC-Platform","owner":{"login":"hpcc-systems","id":900867,"avatar_url":"https://avatars.githubusercontent.com/u/900867?v=3","gravatar_id":"","url":"https://api.github.com/users/hpcc-systems","html_url":"https://github.com/hpcc-systems","followers_url":"https://api.github.com/users/hpcc-systems/followers","following_url":"https://api.github.com/users/hpcc-systems/following{/other_user}","gists_url":"https://api.github.com/users/hpcc-systems/gists{/gist_id}","starred_url":"https://api.github.com/users/hpcc-systems/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hpcc-systems/subscriptions","organizations_url":"https://api.github.com/users/hpcc-systems/orgs","repos_url":"https://api.github.com/users/hpcc-systems/repos","events_url":"https://api.github.com/users/hpcc-systems/events{/privacy}","received_events_url":"https://api.github.com/users/hpcc-systems/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hpcc-systems/HPCC-Platform","description":"IMPORTANT - Issue tracking has been migrated to JIRA - use your github ID and password reset mechanism to log in","fork":false,"url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform","forks_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/forks","keys_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/teams","hooks_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/hooks","issue_events_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/events{/number}","events_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/events","assignees_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/assignees{/user}","branches_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/branches{/branch}","tags_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/tags","blobs_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/git/refs{/sha}","trees_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/statuses/{sha}","languages_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/languages","stargazers_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/stargazers","contributors_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/contributors","subscribers_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/subscribers","subscription_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/subscription","commits_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/commits{/sha}","git_commits_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/git/commits{/sha}","comments_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/comments{/number}","issue_comment_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/comments{/number}","contents_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/contents/{+path}","compare_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/merges","archive_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/downloads","issues_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues{/number}","pulls_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls{/number}","milestones_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/milestones{/number}","notifications_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/labels{/name}","releases_url":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/releases{/id}","created_at":"2011-07-11T14:29:51Z","updated_at":"2015-07-21T10:02:19Z","pushed_at":"2015-07-23T10:35:42Z","git_url":"git://github.com/hpcc-systems/HPCC-Platform.git","ssh_url":"git@github.com:hpcc-systems/HPCC-Platform.git","clone_url":"https://github.com/hpcc-systems/HPCC-Platform.git","svn_url":"https://github.com/hpcc-systems/HPCC-Platform","homepage":"track.hpccsystems.com","size":339812,"stargazers_count":253,"watchers_count":253,"language":"C++","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":118,"mirror_url":null,"open_issues_count":425,"forks":118,"open_issues":425,"watchers":253,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7549"},"html":{"href":"https://github.com/hpcc-systems/HPCC-Platform/pull/7549"},"issue":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/7549"},"comments":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/issues/7549/comments"},"review_comments":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7549/comments"},"review_comment":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/pulls/7549/commits"},"statuses":{"href":"https://api.github.com/repos/hpcc-systems/HPCC-Platform/statuses/fac432e9712324900af300331889df46689dac43"}}}}