{"action":"created","comment":{"url":"https://api.github.com/repos/cloudera/quince/pulls/comments/40688032","id":40688032,"diff_hunk":"@@ -0,0 +1,126 @@\n+/*\n+ * Copyright (c) 2015, Cloudera, Inc. All Rights Reserved.\n+ *\n+ * Cloudera, Inc. licenses this file to you under the Apache License,\n+ * Version 2.0 (the \"License\"). You may not use this file except in\n+ * compliance with the License. You may obtain a copy of the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * This software is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n+ * CONDITIONS OF ANY KIND, either express or implied. See the License for\n+ * the specific language governing permissions and limitations under the\n+ * License.\n+ */\n+\n+package com.cloudera.science.quince;\n+\n+import org.apache.crunch.DoFn;\n+import org.apache.crunch.Emitter;\n+import org.apache.crunch.PCollection;\n+import org.apache.crunch.PTable;\n+import org.apache.crunch.Pair;\n+import org.apache.crunch.lib.SecondarySort;\n+\n+import org.apache.crunch.types.avro.Avros;\n+import org.ga4gh.models.Call;\n+import org.ga4gh.models.FlatVariantCall;\n+import org.ga4gh.models.Variant;\n+\n+import static com.cloudera.science.quince.FlattenVariantFn.flatten;\n+import static org.apache.crunch.types.avro.Avros.*;\n+\n+public final class CrunchUtils {\n+\n+  private CrunchUtils() {\n+  }\n+\n+  public static PTable<String, FlatVariantCall> partitionAndSort(\n+      PCollection<Variant> records, long segmentSize, String sampleGroup) {\n+    // flatten variants\n+    PCollection<FlatVariantCall> flatRecords = records.parallelDo(\n+        new FlattenVariantFn(), Avros.specifics(FlatVariantCall.class));\n+    // group by partition key (table key), then prepare for sorting by sample ID (first\n+    // element in value pair)\n+    PTable<String, Pair<String, FlatVariantCall>> partitionedRecords =\n+        flatRecords.parallelDo(new ExtractPartitionKeyFn(segmentSize, sampleGroup),\n+            tableOf(strings(), pairs(strings(), flatRecords.getPType())));\n+    // do the sort, and extract the partition key and full record\n+    PTable<String, FlatVariantCall> partitionedAndSortedRecords =\n+        SecondarySort.sortAndApply(partitionedRecords, new ExtractEntityFn(),\n+            tableOf(strings(), Avros.specifics(FlatVariantCall.class)));\n+    return partitionedAndSortedRecords;\n+  }\n+\n+  public static String extractPartitionKey(FlatVariantCall variant, long segmentSize,\n+      String sampleGroup) {\n+    StringBuilder sb = new StringBuilder();\n+    sb.append(\"chr=\").append(variant.getReferenceName());\n+    sb.append(\"/pos=\").append(getRangeStart(segmentSize, variant.getStart()));\n+    sb.append(\"/sample_group=\").append(sampleGroup);\n+    return sb.toString();\n+  }\n+\n+  public static long getRangeStart(long size, long value) {\n+    return Math.round(Math.floor(value / ((double) size))) * size;\n+  }\n+\n+  /*\n+   * Turns a variant call into a (partition key, (sample id, variant call)) pair.\n+   */\n+  private static final class ExtractPartitionKeyFn\n+      extends DoFn<FlatVariantCall, Pair<String, Pair<String, FlatVariantCall>>> {\n+    private long segmentSize;\n+    private String sampleGroup;\n+\n+    private ExtractPartitionKeyFn(long segmentSize, String sampleGroup) {\n+      this.segmentSize = segmentSize;\n+      this.sampleGroup = sampleGroup;\n+    }\n+\n+    @Override\n+    public void process(FlatVariantCall input,\n+        Emitter<Pair<String, Pair<String, FlatVariantCall>>> emitter) {\n+      String partitionKey = extractPartitionKey(input, segmentSize, sampleGroup);\n+      emitter.emit(Pair.of(partitionKey, Pair.of(input.getCallSetId().toString(), input)));","path":"src/main/java/com/cloudera/science/quince/CrunchUtils.java","position":85,"original_position":85,"commit_id":"b79e15b9ea41912f3942654f8efc2025b3513845","original_commit_id":"b79e15b9ea41912f3942654f8efc2025b3513845","user":{"login":"laserson","id":118141,"avatar_url":"https://avatars.githubusercontent.com/u/118141?v=3","gravatar_id":"","url":"https://api.github.com/users/laserson","html_url":"https://github.com/laserson","followers_url":"https://api.github.com/users/laserson/followers","following_url":"https://api.github.com/users/laserson/following{/other_user}","gists_url":"https://api.github.com/users/laserson/gists{/gist_id}","starred_url":"https://api.github.com/users/laserson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/laserson/subscriptions","organizations_url":"https://api.github.com/users/laserson/orgs","repos_url":"https://api.github.com/users/laserson/repos","events_url":"https://api.github.com/users/laserson/events{/privacy}","received_events_url":"https://api.github.com/users/laserson/received_events","type":"User","site_admin":false},"body":"In the interest of maximally homogenizing the Parquet pages, does it makes sense to do a secondary sort on `(sample_id_id, chr, pos)` instead of just `(sample_id,)`?","created_at":"2015-09-29T15:37:58Z","updated_at":"2015-09-29T15:37:58Z","html_url":"https://github.com/cloudera/quince/pull/17#discussion_r40688032","pull_request_url":"https://api.github.com/repos/cloudera/quince/pulls/17","_links":{"self":{"href":"https://api.github.com/repos/cloudera/quince/pulls/comments/40688032"},"html":{"href":"https://github.com/cloudera/quince/pull/17#discussion_r40688032"},"pull_request":{"href":"https://api.github.com/repos/cloudera/quince/pulls/17"}}},"pull_request":{"url":"https://api.github.com/repos/cloudera/quince/pulls/17","id":45763824,"html_url":"https://github.com/cloudera/quince/pull/17","diff_url":"https://github.com/cloudera/quince/pull/17.diff","patch_url":"https://github.com/cloudera/quince/pull/17.patch","issue_url":"https://api.github.com/repos/cloudera/quince/issues/17","number":17,"state":"open","locked":false,"title":"Do partitioning and sorting in Crunch","user":{"login":"tomwhite","id":85085,"avatar_url":"https://avatars.githubusercontent.com/u/85085?v=3","gravatar_id":"","url":"https://api.github.com/users/tomwhite","html_url":"https://github.com/tomwhite","followers_url":"https://api.github.com/users/tomwhite/followers","following_url":"https://api.github.com/users/tomwhite/following{/other_user}","gists_url":"https://api.github.com/users/tomwhite/gists{/gist_id}","starred_url":"https://api.github.com/users/tomwhite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tomwhite/subscriptions","organizations_url":"https://api.github.com/users/tomwhite/orgs","repos_url":"https://api.github.com/users/tomwhite/repos","events_url":"https://api.github.com/users/tomwhite/events{/privacy}","received_events_url":"https://api.github.com/users/tomwhite/received_events","type":"User","site_admin":false},"body":"This change gives a marked performance increase (e.g. from 28 to 14 minutes on ALL.chr22.phase1_release_v3.20101123.snps_indels_svs.genotypes.vcf).\r\n\r\nIt relies on an extension to Crunch that produces one file per partition of a PTable<String, T>, where the key is the file name. This already exists in Crunch for Avro files, but not for Parquet, so I've added that in https://issues.apache.org/jira/browse/CRUNCH-562. Until this is released in Crunch I've included the files in this PR too.\r\n\r\nThis change also fixes #11 by using Hive commands to create the table and add partitions.","created_at":"2015-09-23T10:28:29Z","updated_at":"2015-09-29T15:37:58Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0e8edb9391492b62f67c3a8d0b8afe5719297742","assignee":{"login":"tomwhite","id":85085,"avatar_url":"https://avatars.githubusercontent.com/u/85085?v=3","gravatar_id":"","url":"https://api.github.com/users/tomwhite","html_url":"https://github.com/tomwhite","followers_url":"https://api.github.com/users/tomwhite/followers","following_url":"https://api.github.com/users/tomwhite/following{/other_user}","gists_url":"https://api.github.com/users/tomwhite/gists{/gist_id}","starred_url":"https://api.github.com/users/tomwhite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tomwhite/subscriptions","organizations_url":"https://api.github.com/users/tomwhite/orgs","repos_url":"https://api.github.com/users/tomwhite/repos","events_url":"https://api.github.com/users/tomwhite/events{/privacy}","received_events_url":"https://api.github.com/users/tomwhite/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/cloudera/quince/pulls/17/commits","review_comments_url":"https://api.github.com/repos/cloudera/quince/pulls/17/comments","review_comment_url":"https://api.github.com/repos/cloudera/quince/pulls/comments{/number}","comments_url":"https://api.github.com/repos/cloudera/quince/issues/17/comments","statuses_url":"https://api.github.com/repos/cloudera/quince/statuses/b79e15b9ea41912f3942654f8efc2025b3513845","head":{"label":"cloudera:perf","ref":"perf","sha":"b79e15b9ea41912f3942654f8efc2025b3513845","user":{"login":"cloudera","id":87383,"avatar_url":"https://avatars.githubusercontent.com/u/87383?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudera","html_url":"https://github.com/cloudera","followers_url":"https://api.github.com/users/cloudera/followers","following_url":"https://api.github.com/users/cloudera/following{/other_user}","gists_url":"https://api.github.com/users/cloudera/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudera/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudera/subscriptions","organizations_url":"https://api.github.com/users/cloudera/orgs","repos_url":"https://api.github.com/users/cloudera/repos","events_url":"https://api.github.com/users/cloudera/events{/privacy}","received_events_url":"https://api.github.com/users/cloudera/received_events","type":"Organization","site_admin":false},"repo":{"id":39590591,"name":"quince","full_name":"cloudera/quince","owner":{"login":"cloudera","id":87383,"avatar_url":"https://avatars.githubusercontent.com/u/87383?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudera","html_url":"https://github.com/cloudera","followers_url":"https://api.github.com/users/cloudera/followers","following_url":"https://api.github.com/users/cloudera/following{/other_user}","gists_url":"https://api.github.com/users/cloudera/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudera/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudera/subscriptions","organizations_url":"https://api.github.com/users/cloudera/orgs","repos_url":"https://api.github.com/users/cloudera/repos","events_url":"https://api.github.com/users/cloudera/events{/privacy}","received_events_url":"https://api.github.com/users/cloudera/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cloudera/quince","description":"Scalable genomics variant store and analytics","fork":false,"url":"https://api.github.com/repos/cloudera/quince","forks_url":"https://api.github.com/repos/cloudera/quince/forks","keys_url":"https://api.github.com/repos/cloudera/quince/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cloudera/quince/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cloudera/quince/teams","hooks_url":"https://api.github.com/repos/cloudera/quince/hooks","issue_events_url":"https://api.github.com/repos/cloudera/quince/issues/events{/number}","events_url":"https://api.github.com/repos/cloudera/quince/events","assignees_url":"https://api.github.com/repos/cloudera/quince/assignees{/user}","branches_url":"https://api.github.com/repos/cloudera/quince/branches{/branch}","tags_url":"https://api.github.com/repos/cloudera/quince/tags","blobs_url":"https://api.github.com/repos/cloudera/quince/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cloudera/quince/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cloudera/quince/git/refs{/sha}","trees_url":"https://api.github.com/repos/cloudera/quince/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cloudera/quince/statuses/{sha}","languages_url":"https://api.github.com/repos/cloudera/quince/languages","stargazers_url":"https://api.github.com/repos/cloudera/quince/stargazers","contributors_url":"https://api.github.com/repos/cloudera/quince/contributors","subscribers_url":"https://api.github.com/repos/cloudera/quince/subscribers","subscription_url":"https://api.github.com/repos/cloudera/quince/subscription","commits_url":"https://api.github.com/repos/cloudera/quince/commits{/sha}","git_commits_url":"https://api.github.com/repos/cloudera/quince/git/commits{/sha}","comments_url":"https://api.github.com/repos/cloudera/quince/comments{/number}","issue_comment_url":"https://api.github.com/repos/cloudera/quince/issues/comments{/number}","contents_url":"https://api.github.com/repos/cloudera/quince/contents/{+path}","compare_url":"https://api.github.com/repos/cloudera/quince/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cloudera/quince/merges","archive_url":"https://api.github.com/repos/cloudera/quince/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cloudera/quince/downloads","issues_url":"https://api.github.com/repos/cloudera/quince/issues{/number}","pulls_url":"https://api.github.com/repos/cloudera/quince/pulls{/number}","milestones_url":"https://api.github.com/repos/cloudera/quince/milestones{/number}","notifications_url":"https://api.github.com/repos/cloudera/quince/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cloudera/quince/labels{/name}","releases_url":"https://api.github.com/repos/cloudera/quince/releases{/id}","created_at":"2015-07-23T20:26:58Z","updated_at":"2015-09-27T18:04:17Z","pushed_at":"2015-09-27T23:18:50Z","git_url":"git://github.com/cloudera/quince.git","ssh_url":"git@github.com:cloudera/quince.git","clone_url":"https://github.com/cloudera/quince.git","svn_url":"https://github.com/cloudera/quince","homepage":null,"size":314,"stargazers_count":9,"watchers_count":9,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":9,"forks":5,"open_issues":9,"watchers":9,"default_branch":"master"}},"base":{"label":"cloudera:master","ref":"master","sha":"0fad83efc649be4662b275d51fcb79b9f81d7f17","user":{"login":"cloudera","id":87383,"avatar_url":"https://avatars.githubusercontent.com/u/87383?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudera","html_url":"https://github.com/cloudera","followers_url":"https://api.github.com/users/cloudera/followers","following_url":"https://api.github.com/users/cloudera/following{/other_user}","gists_url":"https://api.github.com/users/cloudera/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudera/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudera/subscriptions","organizations_url":"https://api.github.com/users/cloudera/orgs","repos_url":"https://api.github.com/users/cloudera/repos","events_url":"https://api.github.com/users/cloudera/events{/privacy}","received_events_url":"https://api.github.com/users/cloudera/received_events","type":"Organization","site_admin":false},"repo":{"id":39590591,"name":"quince","full_name":"cloudera/quince","owner":{"login":"cloudera","id":87383,"avatar_url":"https://avatars.githubusercontent.com/u/87383?v=3","gravatar_id":"","url":"https://api.github.com/users/cloudera","html_url":"https://github.com/cloudera","followers_url":"https://api.github.com/users/cloudera/followers","following_url":"https://api.github.com/users/cloudera/following{/other_user}","gists_url":"https://api.github.com/users/cloudera/gists{/gist_id}","starred_url":"https://api.github.com/users/cloudera/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cloudera/subscriptions","organizations_url":"https://api.github.com/users/cloudera/orgs","repos_url":"https://api.github.com/users/cloudera/repos","events_url":"https://api.github.com/users/cloudera/events{/privacy}","received_events_url":"https://api.github.com/users/cloudera/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cloudera/quince","description":"Scalable genomics variant store and analytics","fork":false,"url":"https://api.github.com/repos/cloudera/quince","forks_url":"https://api.github.com/repos/cloudera/quince/forks","keys_url":"https://api.github.com/repos/cloudera/quince/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cloudera/quince/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cloudera/quince/teams","hooks_url":"https://api.github.com/repos/cloudera/quince/hooks","issue_events_url":"https://api.github.com/repos/cloudera/quince/issues/events{/number}","events_url":"https://api.github.com/repos/cloudera/quince/events","assignees_url":"https://api.github.com/repos/cloudera/quince/assignees{/user}","branches_url":"https://api.github.com/repos/cloudera/quince/branches{/branch}","tags_url":"https://api.github.com/repos/cloudera/quince/tags","blobs_url":"https://api.github.com/repos/cloudera/quince/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cloudera/quince/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cloudera/quince/git/refs{/sha}","trees_url":"https://api.github.com/repos/cloudera/quince/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cloudera/quince/statuses/{sha}","languages_url":"https://api.github.com/repos/cloudera/quince/languages","stargazers_url":"https://api.github.com/repos/cloudera/quince/stargazers","contributors_url":"https://api.github.com/repos/cloudera/quince/contributors","subscribers_url":"https://api.github.com/repos/cloudera/quince/subscribers","subscription_url":"https://api.github.com/repos/cloudera/quince/subscription","commits_url":"https://api.github.com/repos/cloudera/quince/commits{/sha}","git_commits_url":"https://api.github.com/repos/cloudera/quince/git/commits{/sha}","comments_url":"https://api.github.com/repos/cloudera/quince/comments{/number}","issue_comment_url":"https://api.github.com/repos/cloudera/quince/issues/comments{/number}","contents_url":"https://api.github.com/repos/cloudera/quince/contents/{+path}","compare_url":"https://api.github.com/repos/cloudera/quince/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cloudera/quince/merges","archive_url":"https://api.github.com/repos/cloudera/quince/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cloudera/quince/downloads","issues_url":"https://api.github.com/repos/cloudera/quince/issues{/number}","pulls_url":"https://api.github.com/repos/cloudera/quince/pulls{/number}","milestones_url":"https://api.github.com/repos/cloudera/quince/milestones{/number}","notifications_url":"https://api.github.com/repos/cloudera/quince/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cloudera/quince/labels{/name}","releases_url":"https://api.github.com/repos/cloudera/quince/releases{/id}","created_at":"2015-07-23T20:26:58Z","updated_at":"2015-09-27T18:04:17Z","pushed_at":"2015-09-27T23:18:50Z","git_url":"git://github.com/cloudera/quince.git","ssh_url":"git@github.com:cloudera/quince.git","clone_url":"https://github.com/cloudera/quince.git","svn_url":"https://github.com/cloudera/quince","homepage":null,"size":314,"stargazers_count":9,"watchers_count":9,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":9,"forks":5,"open_issues":9,"watchers":9,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/cloudera/quince/pulls/17"},"html":{"href":"https://github.com/cloudera/quince/pull/17"},"issue":{"href":"https://api.github.com/repos/cloudera/quince/issues/17"},"comments":{"href":"https://api.github.com/repos/cloudera/quince/issues/17/comments"},"review_comments":{"href":"https://api.github.com/repos/cloudera/quince/pulls/17/comments"},"review_comment":{"href":"https://api.github.com/repos/cloudera/quince/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/cloudera/quince/pulls/17/commits"},"statuses":{"href":"https://api.github.com/repos/cloudera/quince/statuses/b79e15b9ea41912f3942654f8efc2025b3513845"}}}}