{"action":"created","comment":{"url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/comments/40041402","id":40041402,"diff_hunk":"@@ -0,0 +1,189 @@\n+package com.google.cloud.genomics.dataflow.functions;\n+\n+import com.google.api.services.storage.model.ComposeRequest;\n+import com.google.api.services.storage.model.ComposeRequest.SourceObjects;\n+import com.google.api.services.storage.model.StorageObject;\n+import com.google.api.services.storage.Storage;\n+import com.google.api.services.storage.Storage.Objects.Compose;\n+\n+import com.google.cloud.dataflow.sdk.transforms.DoFn;\n+import com.google.cloud.dataflow.sdk.util.GcsUtil;\n+import com.google.cloud.dataflow.sdk.util.gcsfs.GcsPath;\n+import com.google.cloud.dataflow.sdk.util.Transport;\n+import com.google.cloud.dataflow.sdk.values.PCollectionView;\n+import com.google.cloud.genomics.dataflow.readers.bam.BAMIO;\n+import com.google.cloud.genomics.dataflow.utils.GCSOptions;\n+import com.google.cloud.genomics.dataflow.utils.ShardedBAMWritingOptions;\n+\n+import com.google.common.base.Stopwatch;\n+import com.google.common.collect.Lists;\n+\n+import htsjdk.samtools.BAMIndexer;\n+import htsjdk.samtools.SAMRecord;\n+import htsjdk.samtools.SamReader;\n+import htsjdk.samtools.ValidationStringency;\n+import htsjdk.samtools.util.BlockCompressedStreamConstants;\n+\n+import java.io.IOException;\n+import java.io.OutputStream;\n+import java.nio.channels.Channels;\n+import java.util.ArrayList;\n+import java.util.Collections;\n+import java.util.List;\n+import java.util.concurrent.TimeUnit;\n+import java.util.logging.Logger;\n+\n+public class CombineShardsFn extends DoFn<String, String> {","path":"src/main/java/com/google/cloud/genomics/dataflow/functions/CombineShardsFn.java","position":36,"original_position":36,"commit_id":"000601c2e28aa48b0d197df26265691c96e68bb6","original_commit_id":"000601c2e28aa48b0d197df26265691c96e68bb6","user":{"login":"deflaux","id":1395500,"avatar_url":"https://avatars.githubusercontent.com/u/1395500?v=3","gravatar_id":"","url":"https://api.github.com/users/deflaux","html_url":"https://github.com/deflaux","followers_url":"https://api.github.com/users/deflaux/followers","following_url":"https://api.github.com/users/deflaux/following{/other_user}","gists_url":"https://api.github.com/users/deflaux/gists{/gist_id}","starred_url":"https://api.github.com/users/deflaux/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/deflaux/subscriptions","organizations_url":"https://api.github.com/users/deflaux/orgs","repos_url":"https://api.github.com/users/deflaux/repos","events_url":"https://api.github.com/users/deflaux/events{/privacy}","received_events_url":"https://api.github.com/users/deflaux/received_events","type":"User","site_admin":false},"body":"Please add a comment here to give the overview of why someone might want to use this DoFn in their pipeline, thanks!","created_at":"2015-09-21T23:50:35Z","updated_at":"2015-09-21T23:50:35Z","html_url":"https://github.com/googlegenomics/dataflow-java/pull/143#discussion_r40041402","pull_request_url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/143","_links":{"self":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/comments/40041402"},"html":{"href":"https://github.com/googlegenomics/dataflow-java/pull/143#discussion_r40041402"},"pull_request":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/143"}}},"pull_request":{"url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/143","id":45174114,"html_url":"https://github.com/googlegenomics/dataflow-java/pull/143","diff_url":"https://github.com/googlegenomics/dataflow-java/pull/143.diff","patch_url":"https://github.com/googlegenomics/dataflow-java/pull/143.patch","issue_url":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/143","number":143,"state":"open","locked":false,"title":"Sharded bam writer","user":{"login":"jakeakopp","id":1377517,"avatar_url":"https://avatars.githubusercontent.com/u/1377517?v=3","gravatar_id":"","url":"https://api.github.com/users/jakeakopp","html_url":"https://github.com/jakeakopp","followers_url":"https://api.github.com/users/jakeakopp/followers","following_url":"https://api.github.com/users/jakeakopp/following{/other_user}","gists_url":"https://api.github.com/users/jakeakopp/gists{/gist_id}","starred_url":"https://api.github.com/users/jakeakopp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jakeakopp/subscriptions","organizations_url":"https://api.github.com/users/jakeakopp/orgs","repos_url":"https://api.github.com/users/jakeakopp/repos","events_url":"https://api.github.com/users/jakeakopp/events{/privacy}","received_events_url":"https://api.github.com/users/jakeakopp/received_events","type":"User","site_admin":false},"body":"Break out ShardedBAMWriting.java into multiple files.","created_at":"2015-09-16T13:55:13Z","updated_at":"2015-09-21T23:50:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d9b28a945a3aea61fcd333dfe1338d7cbf47f42f","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/143/commits","review_comments_url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/143/comments","review_comment_url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/comments{/number}","comments_url":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/143/comments","statuses_url":"https://api.github.com/repos/googlegenomics/dataflow-java/statuses/000601c2e28aa48b0d197df26265691c96e68bb6","head":{"label":"jakeakopp:sharded-bam-writer","ref":"sharded-bam-writer","sha":"000601c2e28aa48b0d197df26265691c96e68bb6","user":{"login":"jakeakopp","id":1377517,"avatar_url":"https://avatars.githubusercontent.com/u/1377517?v=3","gravatar_id":"","url":"https://api.github.com/users/jakeakopp","html_url":"https://github.com/jakeakopp","followers_url":"https://api.github.com/users/jakeakopp/followers","following_url":"https://api.github.com/users/jakeakopp/following{/other_user}","gists_url":"https://api.github.com/users/jakeakopp/gists{/gist_id}","starred_url":"https://api.github.com/users/jakeakopp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jakeakopp/subscriptions","organizations_url":"https://api.github.com/users/jakeakopp/orgs","repos_url":"https://api.github.com/users/jakeakopp/repos","events_url":"https://api.github.com/users/jakeakopp/events{/privacy}","received_events_url":"https://api.github.com/users/jakeakopp/received_events","type":"User","site_admin":false},"repo":{"id":35048954,"name":"dataflow-java","full_name":"jakeakopp/dataflow-java","owner":{"login":"jakeakopp","id":1377517,"avatar_url":"https://avatars.githubusercontent.com/u/1377517?v=3","gravatar_id":"","url":"https://api.github.com/users/jakeakopp","html_url":"https://github.com/jakeakopp","followers_url":"https://api.github.com/users/jakeakopp/followers","following_url":"https://api.github.com/users/jakeakopp/following{/other_user}","gists_url":"https://api.github.com/users/jakeakopp/gists{/gist_id}","starred_url":"https://api.github.com/users/jakeakopp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jakeakopp/subscriptions","organizations_url":"https://api.github.com/users/jakeakopp/orgs","repos_url":"https://api.github.com/users/jakeakopp/repos","events_url":"https://api.github.com/users/jakeakopp/events{/privacy}","received_events_url":"https://api.github.com/users/jakeakopp/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jakeakopp/dataflow-java","description":"Google Cloud Dataflow pipelines such as Identity-By-State as well as useful utility classes.","fork":true,"url":"https://api.github.com/repos/jakeakopp/dataflow-java","forks_url":"https://api.github.com/repos/jakeakopp/dataflow-java/forks","keys_url":"https://api.github.com/repos/jakeakopp/dataflow-java/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jakeakopp/dataflow-java/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jakeakopp/dataflow-java/teams","hooks_url":"https://api.github.com/repos/jakeakopp/dataflow-java/hooks","issue_events_url":"https://api.github.com/repos/jakeakopp/dataflow-java/issues/events{/number}","events_url":"https://api.github.com/repos/jakeakopp/dataflow-java/events","assignees_url":"https://api.github.com/repos/jakeakopp/dataflow-java/assignees{/user}","branches_url":"https://api.github.com/repos/jakeakopp/dataflow-java/branches{/branch}","tags_url":"https://api.github.com/repos/jakeakopp/dataflow-java/tags","blobs_url":"https://api.github.com/repos/jakeakopp/dataflow-java/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jakeakopp/dataflow-java/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jakeakopp/dataflow-java/git/refs{/sha}","trees_url":"https://api.github.com/repos/jakeakopp/dataflow-java/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jakeakopp/dataflow-java/statuses/{sha}","languages_url":"https://api.github.com/repos/jakeakopp/dataflow-java/languages","stargazers_url":"https://api.github.com/repos/jakeakopp/dataflow-java/stargazers","contributors_url":"https://api.github.com/repos/jakeakopp/dataflow-java/contributors","subscribers_url":"https://api.github.com/repos/jakeakopp/dataflow-java/subscribers","subscription_url":"https://api.github.com/repos/jakeakopp/dataflow-java/subscription","commits_url":"https://api.github.com/repos/jakeakopp/dataflow-java/commits{/sha}","git_commits_url":"https://api.github.com/repos/jakeakopp/dataflow-java/git/commits{/sha}","comments_url":"https://api.github.com/repos/jakeakopp/dataflow-java/comments{/number}","issue_comment_url":"https://api.github.com/repos/jakeakopp/dataflow-java/issues/comments{/number}","contents_url":"https://api.github.com/repos/jakeakopp/dataflow-java/contents/{+path}","compare_url":"https://api.github.com/repos/jakeakopp/dataflow-java/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jakeakopp/dataflow-java/merges","archive_url":"https://api.github.com/repos/jakeakopp/dataflow-java/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jakeakopp/dataflow-java/downloads","issues_url":"https://api.github.com/repos/jakeakopp/dataflow-java/issues{/number}","pulls_url":"https://api.github.com/repos/jakeakopp/dataflow-java/pulls{/number}","milestones_url":"https://api.github.com/repos/jakeakopp/dataflow-java/milestones{/number}","notifications_url":"https://api.github.com/repos/jakeakopp/dataflow-java/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jakeakopp/dataflow-java/labels{/name}","releases_url":"https://api.github.com/repos/jakeakopp/dataflow-java/releases{/id}","created_at":"2015-05-04T17:30:08Z","updated_at":"2015-05-04T17:30:09Z","pushed_at":"2015-09-16T16:11:20Z","git_url":"git://github.com/jakeakopp/dataflow-java.git","ssh_url":"git@github.com:jakeakopp/dataflow-java.git","clone_url":"https://github.com/jakeakopp/dataflow-java.git","svn_url":"https://github.com/jakeakopp/dataflow-java","homepage":"","size":124331,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"googlegenomics:master","ref":"master","sha":"3206a7b905a7ef76aa93609df96d4a4ae69af854","user":{"login":"googlegenomics","id":7434120,"avatar_url":"https://avatars.githubusercontent.com/u/7434120?v=3","gravatar_id":"","url":"https://api.github.com/users/googlegenomics","html_url":"https://github.com/googlegenomics","followers_url":"https://api.github.com/users/googlegenomics/followers","following_url":"https://api.github.com/users/googlegenomics/following{/other_user}","gists_url":"https://api.github.com/users/googlegenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/googlegenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/googlegenomics/subscriptions","organizations_url":"https://api.github.com/users/googlegenomics/orgs","repos_url":"https://api.github.com/users/googlegenomics/repos","events_url":"https://api.github.com/users/googlegenomics/events{/privacy}","received_events_url":"https://api.github.com/users/googlegenomics/received_events","type":"Organization","site_admin":false},"repo":{"id":20532587,"name":"dataflow-java","full_name":"googlegenomics/dataflow-java","owner":{"login":"googlegenomics","id":7434120,"avatar_url":"https://avatars.githubusercontent.com/u/7434120?v=3","gravatar_id":"","url":"https://api.github.com/users/googlegenomics","html_url":"https://github.com/googlegenomics","followers_url":"https://api.github.com/users/googlegenomics/followers","following_url":"https://api.github.com/users/googlegenomics/following{/other_user}","gists_url":"https://api.github.com/users/googlegenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/googlegenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/googlegenomics/subscriptions","organizations_url":"https://api.github.com/users/googlegenomics/orgs","repos_url":"https://api.github.com/users/googlegenomics/repos","events_url":"https://api.github.com/users/googlegenomics/events{/privacy}","received_events_url":"https://api.github.com/users/googlegenomics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/googlegenomics/dataflow-java","description":"Google Cloud Dataflow pipelines such as Identity-By-State as well as useful utility classes.","fork":false,"url":"https://api.github.com/repos/googlegenomics/dataflow-java","forks_url":"https://api.github.com/repos/googlegenomics/dataflow-java/forks","keys_url":"https://api.github.com/repos/googlegenomics/dataflow-java/keys{/key_id}","collaborators_url":"https://api.github.com/repos/googlegenomics/dataflow-java/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/googlegenomics/dataflow-java/teams","hooks_url":"https://api.github.com/repos/googlegenomics/dataflow-java/hooks","issue_events_url":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/events{/number}","events_url":"https://api.github.com/repos/googlegenomics/dataflow-java/events","assignees_url":"https://api.github.com/repos/googlegenomics/dataflow-java/assignees{/user}","branches_url":"https://api.github.com/repos/googlegenomics/dataflow-java/branches{/branch}","tags_url":"https://api.github.com/repos/googlegenomics/dataflow-java/tags","blobs_url":"https://api.github.com/repos/googlegenomics/dataflow-java/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/googlegenomics/dataflow-java/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/googlegenomics/dataflow-java/git/refs{/sha}","trees_url":"https://api.github.com/repos/googlegenomics/dataflow-java/git/trees{/sha}","statuses_url":"https://api.github.com/repos/googlegenomics/dataflow-java/statuses/{sha}","languages_url":"https://api.github.com/repos/googlegenomics/dataflow-java/languages","stargazers_url":"https://api.github.com/repos/googlegenomics/dataflow-java/stargazers","contributors_url":"https://api.github.com/repos/googlegenomics/dataflow-java/contributors","subscribers_url":"https://api.github.com/repos/googlegenomics/dataflow-java/subscribers","subscription_url":"https://api.github.com/repos/googlegenomics/dataflow-java/subscription","commits_url":"https://api.github.com/repos/googlegenomics/dataflow-java/commits{/sha}","git_commits_url":"https://api.github.com/repos/googlegenomics/dataflow-java/git/commits{/sha}","comments_url":"https://api.github.com/repos/googlegenomics/dataflow-java/comments{/number}","issue_comment_url":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/comments{/number}","contents_url":"https://api.github.com/repos/googlegenomics/dataflow-java/contents/{+path}","compare_url":"https://api.github.com/repos/googlegenomics/dataflow-java/compare/{base}...{head}","merges_url":"https://api.github.com/repos/googlegenomics/dataflow-java/merges","archive_url":"https://api.github.com/repos/googlegenomics/dataflow-java/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/googlegenomics/dataflow-java/downloads","issues_url":"https://api.github.com/repos/googlegenomics/dataflow-java/issues{/number}","pulls_url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls{/number}","milestones_url":"https://api.github.com/repos/googlegenomics/dataflow-java/milestones{/number}","notifications_url":"https://api.github.com/repos/googlegenomics/dataflow-java/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/googlegenomics/dataflow-java/labels{/name}","releases_url":"https://api.github.com/repos/googlegenomics/dataflow-java/releases{/id}","created_at":"2014-06-05T16:27:31Z","updated_at":"2015-06-19T07:36:04Z","pushed_at":"2015-09-21T23:25:44Z","git_url":"git://github.com/googlegenomics/dataflow-java.git","ssh_url":"git@github.com:googlegenomics/dataflow-java.git","clone_url":"https://github.com/googlegenomics/dataflow-java.git","svn_url":"https://github.com/googlegenomics/dataflow-java","homepage":"","size":125750,"stargazers_count":11,"watchers_count":11,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":13,"mirror_url":null,"open_issues_count":13,"forks":13,"open_issues":13,"watchers":11,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/143"},"html":{"href":"https://github.com/googlegenomics/dataflow-java/pull/143"},"issue":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/143"},"comments":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/143/comments"},"review_comments":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/143/comments"},"review_comment":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/143/commits"},"statuses":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/statuses/000601c2e28aa48b0d197df26265691c96e68bb6"}}}}