{"action":"created","comment":{"url":"https://api.github.com/repos/broadinstitute/hellbender/pulls/comments/37742000","id":37742000,"diff_hunk":"@@ -0,0 +1,90 @@\n+package org.broadinstitute.hellbender.engine.spark.datasources;\n+\n+import htsjdk.samtools.*;\n+import org.apache.commons.collections4.iterators.IteratorIterable;\n+import org.apache.hadoop.conf.Configuration;\n+import org.apache.hadoop.fs.FileSystem;\n+import org.apache.hadoop.fs.Path;\n+import org.apache.hadoop.io.NullWritable;\n+import org.apache.hadoop.mapreduce.RecordWriter;\n+import org.apache.hadoop.mapreduce.TaskAttemptContext;\n+import org.apache.spark.api.java.JavaPairRDD;\n+import org.apache.spark.api.java.JavaRDD;\n+import org.apache.spark.api.java.JavaSparkContext;\n+import org.apache.spark.broadcast.Broadcast;\n+import org.broadinstitute.hellbender.utils.read.GATKRead;\n+import org.broadinstitute.hellbender.utils.read.ReadCoordinateComparator;\n+import org.seqdoop.hadoop_bam.KeyIgnoringBAMOutputFormat;\n+import org.seqdoop.hadoop_bam.SAMRecordWritable;\n+import scala.Tuple2;\n+\n+import java.io.IOException;\n+\n+/**\n+ * ReadsSparkSink writes GATKReads to a file. This code lifts from the HadoopGenomics/Hadoop-BAM\n+ * read writing code as well as from bigdatagenomics/adam.\n+ */\n+public class ReadsSparkSink {\n+\n+    // We need an output format for saveAsNewAPIHadoopFile.\n+    public static class MyOutputFormat extends KeyIgnoringBAMOutputFormat<NullWritable> {\n+        public static SAMFileHeader bamHeader = null;\n+\n+        public static void setHeader(final SAMFileHeader header) {\n+            bamHeader = header;\n+        }\n+\n+        @Override\n+        public RecordWriter<NullWritable, SAMRecordWritable> getRecordWriter(TaskAttemptContext ctx) throws IOException {\n+            setSAMHeader(bamHeader);\n+            return super.getRecordWriter(ctx);\n+        }\n+    }\n+\n+    /**\n+     * writeParallelReads writes rddReads to outputFile with header as the file header.\n+     * @param ctx the JavaSparkContext to write.\n+     * @param outputFile path to the output bam.\n+     * @param rddReads reads to write.\n+     * @param header the header to put at the top of the file.s\n+     */\n+    public static void writeParallelReads(\n+            final JavaSparkContext ctx, final String outputFile, final JavaRDD<GATKRead> rddReads,\n+            final SAMFileHeader header) throws IOException {\n+        // Set the header on the main thread.\n+        MyOutputFormat.setHeader(header);\n+\n+        // The expected format for writing is JavaPairRDD where the key is ignored and the value is a\n+        // SAMRecordWritable. We use GATKRead as the key, so we can sort the reads before writing them to a file.\n+        JavaPairRDD<GATKRead, SAMRecordWritable> rddSamRecordWriteable = rddReads.mapToPair(gatkRead -> {\n+            SAMRecord samRecord = gatkRead.convertToSAMRecord(header);\n+            SAMRecordWritable samRecordWritable = new SAMRecordWritable();\n+            samRecordWritable.set(samRecord);\n+            return new Tuple2<>(gatkRead, samRecordWritable);\n+        });\n+\n+        final JavaPairRDD<GATKRead, SAMRecordWritable> out =\n+                rddSamRecordWriteable.sortByKey(new ReadCoordinateComparator(header)).coalesce(1); // needed?","path":"src/main/java/org/broadinstitute/hellbender/engine/spark/datasources/ReadsSparkSink.java","position":67,"original_position":67,"commit_id":"2310062d70e1531ee2dc79556797d39ea1be1807","original_commit_id":"2310062d70e1531ee2dc79556797d39ea1be1807","user":{"login":"tomwhite","id":85085,"avatar_url":"https://avatars.githubusercontent.com/u/85085?v=3","gravatar_id":"","url":"https://api.github.com/users/tomwhite","html_url":"https://github.com/tomwhite","followers_url":"https://api.github.com/users/tomwhite/followers","following_url":"https://api.github.com/users/tomwhite/following{/other_user}","gists_url":"https://api.github.com/users/tomwhite/gists{/gist_id}","starred_url":"https://api.github.com/users/tomwhite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tomwhite/subscriptions","organizations_url":"https://api.github.com/users/tomwhite/orgs","repos_url":"https://api.github.com/users/tomwhite/repos","events_url":"https://api.github.com/users/tomwhite/events{/privacy}","received_events_url":"https://api.github.com/users/tomwhite/received_events","type":"User","site_admin":false},"body":"The coalesce is needed since we're writing a single file. This can be relaxed later (to have sharded files, for larger BAMs), but it's OK for testing. Maybe update the comment to make this clearer.","created_at":"2015-08-24T11:34:02Z","updated_at":"2015-08-24T11:34:02Z","html_url":"https://github.com/broadinstitute/hellbender/pull/850#discussion_r37742000","pull_request_url":"https://api.github.com/repos/broadinstitute/hellbender/pulls/850","_links":{"self":{"href":"https://api.github.com/repos/broadinstitute/hellbender/pulls/comments/37742000"},"html":{"href":"https://github.com/broadinstitute/hellbender/pull/850#discussion_r37742000"},"pull_request":{"href":"https://api.github.com/repos/broadinstitute/hellbender/pulls/850"}}},"pull_request":{"url":"https://api.github.com/repos/broadinstitute/hellbender/pulls/850","id":43057494,"html_url":"https://github.com/broadinstitute/hellbender/pull/850","diff_url":"https://github.com/broadinstitute/hellbender/pull/850.diff","patch_url":"https://github.com/broadinstitute/hellbender/pull/850.patch","issue_url":"https://api.github.com/repos/broadinstitute/hellbender/issues/850","number":850,"state":"open","locked":false,"title":"Initial Spark dataflow commit, basic skeleton with some tests.","user":{"login":"davidadamsphd","id":12084283,"avatar_url":"https://avatars.githubusercontent.com/u/12084283?v=3","gravatar_id":"","url":"https://api.github.com/users/davidadamsphd","html_url":"https://github.com/davidadamsphd","followers_url":"https://api.github.com/users/davidadamsphd/followers","following_url":"https://api.github.com/users/davidadamsphd/following{/other_user}","gists_url":"https://api.github.com/users/davidadamsphd/gists{/gist_id}","starred_url":"https://api.github.com/users/davidadamsphd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davidadamsphd/subscriptions","organizations_url":"https://api.github.com/users/davidadamsphd/orgs","repos_url":"https://api.github.com/users/davidadamsphd/repos","events_url":"https://api.github.com/users/davidadamsphd/events{/privacy}","received_events_url":"https://api.github.com/users/davidadamsphd/received_events","type":"User","site_admin":false},"body":"A few things worth mentioning.\r\n\r\n1) This skeleton is mostly a direct port of the existing Dataflow pipeline.\r\n2) I had to modify some test data because there is a (masked) bug in the Dataflow code, see #795.\r\n3) Serialization was a slight pain and I had to bump the kryo version to the latest 2.x, as well as add two custom Serializers. If there's a cleaner way to handle any of that I'm all ears.\r\n4) I'm using Hadoop-bam for reading and writing reads and variants.\r\n5) There are unit tests for all code except for the skeleton itself. I've run it on a cluster on my machine successfully, but haven't written the tests (which will take a little while to design and get right).\r\n","created_at":"2015-08-21T16:45:48Z","updated_at":"2015-08-24T11:34:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2c867f3918a4ad97581315ea7a00e58c221dc648","assignee":{"login":"tomwhite","id":85085,"avatar_url":"https://avatars.githubusercontent.com/u/85085?v=3","gravatar_id":"","url":"https://api.github.com/users/tomwhite","html_url":"https://github.com/tomwhite","followers_url":"https://api.github.com/users/tomwhite/followers","following_url":"https://api.github.com/users/tomwhite/following{/other_user}","gists_url":"https://api.github.com/users/tomwhite/gists{/gist_id}","starred_url":"https://api.github.com/users/tomwhite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tomwhite/subscriptions","organizations_url":"https://api.github.com/users/tomwhite/orgs","repos_url":"https://api.github.com/users/tomwhite/repos","events_url":"https://api.github.com/users/tomwhite/events{/privacy}","received_events_url":"https://api.github.com/users/tomwhite/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/broadinstitute/hellbender/pulls/850/commits","review_comments_url":"https://api.github.com/repos/broadinstitute/hellbender/pulls/850/comments","review_comment_url":"https://api.github.com/repos/broadinstitute/hellbender/pulls/comments{/number}","comments_url":"https://api.github.com/repos/broadinstitute/hellbender/issues/850/comments","statuses_url":"https://api.github.com/repos/broadinstitute/hellbender/statuses/2310062d70e1531ee2dc79556797d39ea1be1807","head":{"label":"broadinstitute:da_spark_skeleton","ref":"da_spark_skeleton","sha":"2310062d70e1531ee2dc79556797d39ea1be1807","user":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"repo":{"id":27452807,"name":"hellbender","full_name":"broadinstitute/hellbender","owner":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/broadinstitute/hellbender","description":"GATK4 development","fork":false,"url":"https://api.github.com/repos/broadinstitute/hellbender","forks_url":"https://api.github.com/repos/broadinstitute/hellbender/forks","keys_url":"https://api.github.com/repos/broadinstitute/hellbender/keys{/key_id}","collaborators_url":"https://api.github.com/repos/broadinstitute/hellbender/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/broadinstitute/hellbender/teams","hooks_url":"https://api.github.com/repos/broadinstitute/hellbender/hooks","issue_events_url":"https://api.github.com/repos/broadinstitute/hellbender/issues/events{/number}","events_url":"https://api.github.com/repos/broadinstitute/hellbender/events","assignees_url":"https://api.github.com/repos/broadinstitute/hellbender/assignees{/user}","branches_url":"https://api.github.com/repos/broadinstitute/hellbender/branches{/branch}","tags_url":"https://api.github.com/repos/broadinstitute/hellbender/tags","blobs_url":"https://api.github.com/repos/broadinstitute/hellbender/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/broadinstitute/hellbender/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/broadinstitute/hellbender/git/refs{/sha}","trees_url":"https://api.github.com/repos/broadinstitute/hellbender/git/trees{/sha}","statuses_url":"https://api.github.com/repos/broadinstitute/hellbender/statuses/{sha}","languages_url":"https://api.github.com/repos/broadinstitute/hellbender/languages","stargazers_url":"https://api.github.com/repos/broadinstitute/hellbender/stargazers","contributors_url":"https://api.github.com/repos/broadinstitute/hellbender/contributors","subscribers_url":"https://api.github.com/repos/broadinstitute/hellbender/subscribers","subscription_url":"https://api.github.com/repos/broadinstitute/hellbender/subscription","commits_url":"https://api.github.com/repos/broadinstitute/hellbender/commits{/sha}","git_commits_url":"https://api.github.com/repos/broadinstitute/hellbender/git/commits{/sha}","comments_url":"https://api.github.com/repos/broadinstitute/hellbender/comments{/number}","issue_comment_url":"https://api.github.com/repos/broadinstitute/hellbender/issues/comments{/number}","contents_url":"https://api.github.com/repos/broadinstitute/hellbender/contents/{+path}","compare_url":"https://api.github.com/repos/broadinstitute/hellbender/compare/{base}...{head}","merges_url":"https://api.github.com/repos/broadinstitute/hellbender/merges","archive_url":"https://api.github.com/repos/broadinstitute/hellbender/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/broadinstitute/hellbender/downloads","issues_url":"https://api.github.com/repos/broadinstitute/hellbender/issues{/number}","pulls_url":"https://api.github.com/repos/broadinstitute/hellbender/pulls{/number}","milestones_url":"https://api.github.com/repos/broadinstitute/hellbender/milestones{/number}","notifications_url":"https://api.github.com/repos/broadinstitute/hellbender/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/broadinstitute/hellbender/labels{/name}","releases_url":"https://api.github.com/repos/broadinstitute/hellbender/releases{/id}","created_at":"2014-12-02T20:50:36Z","updated_at":"2015-08-12T03:14:08Z","pushed_at":"2015-08-21T23:31:31Z","git_url":"git://github.com/broadinstitute/hellbender.git","ssh_url":"git@github.com:broadinstitute/hellbender.git","clone_url":"https://github.com/broadinstitute/hellbender.git","svn_url":"https://github.com/broadinstitute/hellbender","homepage":"","size":39106,"stargazers_count":29,"watchers_count":29,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":8,"mirror_url":null,"open_issues_count":240,"forks":8,"open_issues":240,"watchers":29,"default_branch":"master"}},"base":{"label":"broadinstitute:master","ref":"master","sha":"921e730af17d0461ee7f04512f91906a9235165c","user":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"repo":{"id":27452807,"name":"hellbender","full_name":"broadinstitute/hellbender","owner":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/broadinstitute/hellbender","description":"GATK4 development","fork":false,"url":"https://api.github.com/repos/broadinstitute/hellbender","forks_url":"https://api.github.com/repos/broadinstitute/hellbender/forks","keys_url":"https://api.github.com/repos/broadinstitute/hellbender/keys{/key_id}","collaborators_url":"https://api.github.com/repos/broadinstitute/hellbender/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/broadinstitute/hellbender/teams","hooks_url":"https://api.github.com/repos/broadinstitute/hellbender/hooks","issue_events_url":"https://api.github.com/repos/broadinstitute/hellbender/issues/events{/number}","events_url":"https://api.github.com/repos/broadinstitute/hellbender/events","assignees_url":"https://api.github.com/repos/broadinstitute/hellbender/assignees{/user}","branches_url":"https://api.github.com/repos/broadinstitute/hellbender/branches{/branch}","tags_url":"https://api.github.com/repos/broadinstitute/hellbender/tags","blobs_url":"https://api.github.com/repos/broadinstitute/hellbender/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/broadinstitute/hellbender/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/broadinstitute/hellbender/git/refs{/sha}","trees_url":"https://api.github.com/repos/broadinstitute/hellbender/git/trees{/sha}","statuses_url":"https://api.github.com/repos/broadinstitute/hellbender/statuses/{sha}","languages_url":"https://api.github.com/repos/broadinstitute/hellbender/languages","stargazers_url":"https://api.github.com/repos/broadinstitute/hellbender/stargazers","contributors_url":"https://api.github.com/repos/broadinstitute/hellbender/contributors","subscribers_url":"https://api.github.com/repos/broadinstitute/hellbender/subscribers","subscription_url":"https://api.github.com/repos/broadinstitute/hellbender/subscription","commits_url":"https://api.github.com/repos/broadinstitute/hellbender/commits{/sha}","git_commits_url":"https://api.github.com/repos/broadinstitute/hellbender/git/commits{/sha}","comments_url":"https://api.github.com/repos/broadinstitute/hellbender/comments{/number}","issue_comment_url":"https://api.github.com/repos/broadinstitute/hellbender/issues/comments{/number}","contents_url":"https://api.github.com/repos/broadinstitute/hellbender/contents/{+path}","compare_url":"https://api.github.com/repos/broadinstitute/hellbender/compare/{base}...{head}","merges_url":"https://api.github.com/repos/broadinstitute/hellbender/merges","archive_url":"https://api.github.com/repos/broadinstitute/hellbender/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/broadinstitute/hellbender/downloads","issues_url":"https://api.github.com/repos/broadinstitute/hellbender/issues{/number}","pulls_url":"https://api.github.com/repos/broadinstitute/hellbender/pulls{/number}","milestones_url":"https://api.github.com/repos/broadinstitute/hellbender/milestones{/number}","notifications_url":"https://api.github.com/repos/broadinstitute/hellbender/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/broadinstitute/hellbender/labels{/name}","releases_url":"https://api.github.com/repos/broadinstitute/hellbender/releases{/id}","created_at":"2014-12-02T20:50:36Z","updated_at":"2015-08-12T03:14:08Z","pushed_at":"2015-08-21T23:31:31Z","git_url":"git://github.com/broadinstitute/hellbender.git","ssh_url":"git@github.com:broadinstitute/hellbender.git","clone_url":"https://github.com/broadinstitute/hellbender.git","svn_url":"https://github.com/broadinstitute/hellbender","homepage":"","size":39106,"stargazers_count":29,"watchers_count":29,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":8,"mirror_url":null,"open_issues_count":240,"forks":8,"open_issues":240,"watchers":29,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/broadinstitute/hellbender/pulls/850"},"html":{"href":"https://github.com/broadinstitute/hellbender/pull/850"},"issue":{"href":"https://api.github.com/repos/broadinstitute/hellbender/issues/850"},"comments":{"href":"https://api.github.com/repos/broadinstitute/hellbender/issues/850/comments"},"review_comments":{"href":"https://api.github.com/repos/broadinstitute/hellbender/pulls/850/comments"},"review_comment":{"href":"https://api.github.com/repos/broadinstitute/hellbender/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/broadinstitute/hellbender/pulls/850/commits"},"statuses":{"href":"https://api.github.com/repos/broadinstitute/hellbender/statuses/2310062d70e1531ee2dc79556797d39ea1be1807"}}}}