{"action":"created","comment":{"url":"https://api.github.com/repos/druid-io/druid/pulls/comments/26235763","id":26235763,"diff_hunk":"@@ -0,0 +1,65 @@\n+/*\n+ * Druid - a distributed column store.\n+ * Copyright 2012 - 2015 Metamarkets Group Inc.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\");\n+ * you may not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *     http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package io.druid.storage.cassandra;\n+\n+import com.fasterxml.jackson.annotation.JacksonInject;\n+import com.fasterxml.jackson.annotation.JsonCreator;\n+import com.fasterxml.jackson.annotation.JsonProperty;\n+import com.fasterxml.jackson.annotation.JsonTypeName;\n+import com.metamx.common.ISE;\n+import io.druid.segment.loading.DataSegmentPuller;\n+import io.druid.segment.loading.LoadSpec;\n+import io.druid.segment.loading.SegmentLoadingException;\n+\n+import java.io.File;\n+import java.util.Map;\n+\n+/**\n+ *\n+ */\n+@JsonTypeName(CassandraDataSegmentPuller.SCHEME)\n+public class CassandraLoadSpec implements LoadSpec\n+{\n+  @JsonProperty\n+  private final String key;\n+  private final CassandraDataSegmentPuller puller;\n+  @JsonCreator\n+  public CassandraLoadSpec(\n+      @JacksonInject Map<String, DataSegmentPuller> pullers,\n+      @JsonProperty(\"key\") String key\n+  ){\n+    final DataSegmentPuller puller = pullers.get(CassandraDataSegmentPuller.SCHEME);\n+    if(puller == null){\n+      throw new ISE(\"Puller for SCHEME [%s] not found\", CassandraDataSegmentPuller.SCHEME);\n+    }\n+    if(!(puller instanceof CassandraDataSegmentPuller)){\n+      throw new ISE(\"Puller for SCHEME [%s] not instance of [%s]. Instead found [%s]\",\n+                    CassandraDataSegmentPuller.SCHEME,\n+                    CassandraDataSegmentPuller.class.getCanonicalName(),\n+                    puller.getClass().getCanonicalName());\n+    }\n+    this.puller = (CassandraDataSegmentPuller) puller;\n+    this.key = key;\n+  }\n+\n+  @Override\n+  public Long putSegment(File outDir) throws SegmentLoadingException","path":"extensions/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraLoadSpec.java","position":61,"original_position":61,"commit_id":"58c4df8351a58a9bbf66e4bfa738756e27dbb3da","original_commit_id":"58c4df8351a58a9bbf66e4bfa738756e27dbb3da","user":{"login":"drcrallen","id":8213081,"avatar_url":"https://avatars.githubusercontent.com/u/8213081?v=3","gravatar_id":"","url":"https://api.github.com/users/drcrallen","html_url":"https://github.com/drcrallen","followers_url":"https://api.github.com/users/drcrallen/followers","following_url":"https://api.github.com/users/drcrallen/following{/other_user}","gists_url":"https://api.github.com/users/drcrallen/gists{/gist_id}","starred_url":"https://api.github.com/users/drcrallen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/drcrallen/subscriptions","organizations_url":"https://api.github.com/users/drcrallen/orgs","repos_url":"https://api.github.com/users/drcrallen/repos","events_url":"https://api.github.com/users/drcrallen/events{/privacy}","received_events_url":"https://api.github.com/users/drcrallen/received_events","type":"User","site_admin":false},"body":"I'm fine with that. Will change on next pass","created_at":"2015-03-11T17:39:47Z","updated_at":"2015-03-11T17:39:47Z","html_url":"https://github.com/druid-io/druid/pull/1132#discussion_r26235763","pull_request_url":"https://api.github.com/repos/druid-io/druid/pulls/1132","_links":{"self":{"href":"https://api.github.com/repos/druid-io/druid/pulls/comments/26235763"},"html":{"href":"https://github.com/druid-io/druid/pull/1132#discussion_r26235763"},"pull_request":{"href":"https://api.github.com/repos/druid-io/druid/pulls/1132"}}},"pull_request":{"url":"https://api.github.com/repos/druid-io/druid/pulls/1132","id":29494629,"html_url":"https://github.com/druid-io/druid/pull/1132","diff_url":"https://github.com/druid-io/druid/pull/1132.diff","patch_url":"https://github.com/druid-io/druid/pull/1132.patch","issue_url":"https://api.github.com/repos/druid-io/druid/issues/1132","number":1132,"state":"open","locked":false,"title":"Overhaul of SegmentPullers to add consistency and retries","user":{"login":"drcrallen","id":8213081,"avatar_url":"https://avatars.githubusercontent.com/u/8213081?v=3","gravatar_id":"","url":"https://api.github.com/users/drcrallen","html_url":"https://github.com/drcrallen","followers_url":"https://api.github.com/users/drcrallen/followers","following_url":"https://api.github.com/users/drcrallen/following{/other_user}","gists_url":"https://api.github.com/users/drcrallen/gists{/gist_id}","starred_url":"https://api.github.com/users/drcrallen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/drcrallen/subscriptions","organizations_url":"https://api.github.com/users/drcrallen/orgs","repos_url":"https://api.github.com/users/drcrallen/repos","events_url":"https://api.github.com/users/drcrallen/events{/privacy}","received_events_url":"https://api.github.com/users/drcrallen/received_events","type":"User","site_admin":false},"body":"The Goal of this is to make the segment pullers a little more general use. Specifically, this uses URIs when possible. URIs have a standard defined format (unlike the free-form loadSpec used previously). This also unifies a lot of the way segments are pulled, so that any datasource should be able to support a zip or gz request in a more standard manner.\r\n\r\nGeneral workflow goes like this:\r\n\r\n1. LoadSpec makes sure the correct Puller is called with the correct parameters. This offloads the \"standard\" interface from the puller to the LoadSpec. The only thing segment information was being used for in the puller was to get the loadSpec, so I simply moved where the guaranteed interface resides since we have an actual LoadSpec now.\r\n2. The Puller sets up general information like how to make an InputStream, how to find a file name (for .gz files for example), and when to retry. \r\n3. CompressionUtils does most of the heavy lifting when it can\r\n\r\n* Requires https://github.com/druid-io/druid-api/pull/37\r\n* Requires https://github.com/metamx/java-util/pull/22\r\n\r\nI'll need to update the POM once the above has been published.\r\n\r\n~~I also want to try this on some S3 segments just to make sure it didn't break anything unexpected. But would like feedback on the approach first.~~ Tested, seems to be fine.\r\n\r\n~~This *MIGHT* be subject to http://bugs.java.com/bugdatabase/view_bug.do?bug_id=7036144  Added [unit tests](https://github.com/metamx/java-util/blob/CompressionFileUtils/src/test/java/com/metamx/common/CompressionUtilsTest.java) Adding more to double-check.~~ Fixed with workaround in java-util\r\n","created_at":"2015-02-18T01:31:25Z","updated_at":"2015-03-11T17:39:47Z","closed_at":null,"merged_at":null,"merge_commit_sha":"38c96916a29230bb51e60e00049b16ca0ede925b","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/druid-io/druid/pulls/1132/commits","review_comments_url":"https://api.github.com/repos/druid-io/druid/pulls/1132/comments","review_comment_url":"https://api.github.com/repos/druid-io/druid/pulls/comments{/number}","comments_url":"https://api.github.com/repos/druid-io/druid/issues/1132/comments","statuses_url":"https://api.github.com/repos/druid-io/druid/statuses/58c4df8351a58a9bbf66e4bfa738756e27dbb3da","head":{"label":"metamx:uriSegmentLoaders","ref":"uriSegmentLoaders","sha":"58c4df8351a58a9bbf66e4bfa738756e27dbb3da","user":{"login":"metamx","id":299329,"avatar_url":"https://avatars.githubusercontent.com/u/299329?v=3","gravatar_id":"","url":"https://api.github.com/users/metamx","html_url":"https://github.com/metamx","followers_url":"https://api.github.com/users/metamx/followers","following_url":"https://api.github.com/users/metamx/following{/other_user}","gists_url":"https://api.github.com/users/metamx/gists{/gist_id}","starred_url":"https://api.github.com/users/metamx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/metamx/subscriptions","organizations_url":"https://api.github.com/users/metamx/orgs","repos_url":"https://api.github.com/users/metamx/repos","events_url":"https://api.github.com/users/metamx/events{/privacy}","received_events_url":"https://api.github.com/users/metamx/received_events","type":"Organization","site_admin":false},"repo":{"id":28248872,"name":"druid","full_name":"metamx/druid","owner":{"login":"metamx","id":299329,"avatar_url":"https://avatars.githubusercontent.com/u/299329?v=3","gravatar_id":"","url":"https://api.github.com/users/metamx","html_url":"https://github.com/metamx","followers_url":"https://api.github.com/users/metamx/followers","following_url":"https://api.github.com/users/metamx/following{/other_user}","gists_url":"https://api.github.com/users/metamx/gists{/gist_id}","starred_url":"https://api.github.com/users/metamx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/metamx/subscriptions","organizations_url":"https://api.github.com/users/metamx/orgs","repos_url":"https://api.github.com/users/metamx/repos","events_url":"https://api.github.com/users/metamx/events{/privacy}","received_events_url":"https://api.github.com/users/metamx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/metamx/druid","description":"Real²time Exploratory Analytics on Large Datasets","fork":true,"url":"https://api.github.com/repos/metamx/druid","forks_url":"https://api.github.com/repos/metamx/druid/forks","keys_url":"https://api.github.com/repos/metamx/druid/keys{/key_id}","collaborators_url":"https://api.github.com/repos/metamx/druid/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/metamx/druid/teams","hooks_url":"https://api.github.com/repos/metamx/druid/hooks","issue_events_url":"https://api.github.com/repos/metamx/druid/issues/events{/number}","events_url":"https://api.github.com/repos/metamx/druid/events","assignees_url":"https://api.github.com/repos/metamx/druid/assignees{/user}","branches_url":"https://api.github.com/repos/metamx/druid/branches{/branch}","tags_url":"https://api.github.com/repos/metamx/druid/tags","blobs_url":"https://api.github.com/repos/metamx/druid/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/metamx/druid/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/metamx/druid/git/refs{/sha}","trees_url":"https://api.github.com/repos/metamx/druid/git/trees{/sha}","statuses_url":"https://api.github.com/repos/metamx/druid/statuses/{sha}","languages_url":"https://api.github.com/repos/metamx/druid/languages","stargazers_url":"https://api.github.com/repos/metamx/druid/stargazers","contributors_url":"https://api.github.com/repos/metamx/druid/contributors","subscribers_url":"https://api.github.com/repos/metamx/druid/subscribers","subscription_url":"https://api.github.com/repos/metamx/druid/subscription","commits_url":"https://api.github.com/repos/metamx/druid/commits{/sha}","git_commits_url":"https://api.github.com/repos/metamx/druid/git/commits{/sha}","comments_url":"https://api.github.com/repos/metamx/druid/comments{/number}","issue_comment_url":"https://api.github.com/repos/metamx/druid/issues/comments{/number}","contents_url":"https://api.github.com/repos/metamx/druid/contents/{+path}","compare_url":"https://api.github.com/repos/metamx/druid/compare/{base}...{head}","merges_url":"https://api.github.com/repos/metamx/druid/merges","archive_url":"https://api.github.com/repos/metamx/druid/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/metamx/druid/downloads","issues_url":"https://api.github.com/repos/metamx/druid/issues{/number}","pulls_url":"https://api.github.com/repos/metamx/druid/pulls{/number}","milestones_url":"https://api.github.com/repos/metamx/druid/milestones{/number}","notifications_url":"https://api.github.com/repos/metamx/druid/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/metamx/druid/labels{/name}","releases_url":"https://api.github.com/repos/metamx/druid/releases{/id}","created_at":"2014-12-19T22:30:25Z","updated_at":"2015-03-11T00:56:58Z","pushed_at":"2015-03-11T17:23:05Z","git_url":"git://github.com/metamx/druid.git","ssh_url":"git@github.com:metamx/druid.git","clone_url":"https://github.com/metamx/druid.git","svn_url":"https://github.com/metamx/druid","homepage":"http://druid.io","size":92459,"stargazers_count":16,"watchers_count":16,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":9,"mirror_url":null,"open_issues_count":0,"forks":9,"open_issues":0,"watchers":16,"default_branch":"master"}},"base":{"label":"druid-io:master","ref":"master","sha":"2abdce1dc067f03e9005c899676db3e192f4a854","user":{"login":"druid-io","id":4203474,"avatar_url":"https://avatars.githubusercontent.com/u/4203474?v=3","gravatar_id":"","url":"https://api.github.com/users/druid-io","html_url":"https://github.com/druid-io","followers_url":"https://api.github.com/users/druid-io/followers","following_url":"https://api.github.com/users/druid-io/following{/other_user}","gists_url":"https://api.github.com/users/druid-io/gists{/gist_id}","starred_url":"https://api.github.com/users/druid-io/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/druid-io/subscriptions","organizations_url":"https://api.github.com/users/druid-io/orgs","repos_url":"https://api.github.com/users/druid-io/repos","events_url":"https://api.github.com/users/druid-io/events{/privacy}","received_events_url":"https://api.github.com/users/druid-io/received_events","type":"Organization","site_admin":false},"repo":{"id":6358188,"name":"druid","full_name":"druid-io/druid","owner":{"login":"druid-io","id":4203474,"avatar_url":"https://avatars.githubusercontent.com/u/4203474?v=3","gravatar_id":"","url":"https://api.github.com/users/druid-io","html_url":"https://github.com/druid-io","followers_url":"https://api.github.com/users/druid-io/followers","following_url":"https://api.github.com/users/druid-io/following{/other_user}","gists_url":"https://api.github.com/users/druid-io/gists{/gist_id}","starred_url":"https://api.github.com/users/druid-io/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/druid-io/subscriptions","organizations_url":"https://api.github.com/users/druid-io/orgs","repos_url":"https://api.github.com/users/druid-io/repos","events_url":"https://api.github.com/users/druid-io/events{/privacy}","received_events_url":"https://api.github.com/users/druid-io/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/druid-io/druid","description":"Real²time Exploratory Analytics on Large Datasets","fork":false,"url":"https://api.github.com/repos/druid-io/druid","forks_url":"https://api.github.com/repos/druid-io/druid/forks","keys_url":"https://api.github.com/repos/druid-io/druid/keys{/key_id}","collaborators_url":"https://api.github.com/repos/druid-io/druid/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/druid-io/druid/teams","hooks_url":"https://api.github.com/repos/druid-io/druid/hooks","issue_events_url":"https://api.github.com/repos/druid-io/druid/issues/events{/number}","events_url":"https://api.github.com/repos/druid-io/druid/events","assignees_url":"https://api.github.com/repos/druid-io/druid/assignees{/user}","branches_url":"https://api.github.com/repos/druid-io/druid/branches{/branch}","tags_url":"https://api.github.com/repos/druid-io/druid/tags","blobs_url":"https://api.github.com/repos/druid-io/druid/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/druid-io/druid/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/druid-io/druid/git/refs{/sha}","trees_url":"https://api.github.com/repos/druid-io/druid/git/trees{/sha}","statuses_url":"https://api.github.com/repos/druid-io/druid/statuses/{sha}","languages_url":"https://api.github.com/repos/druid-io/druid/languages","stargazers_url":"https://api.github.com/repos/druid-io/druid/stargazers","contributors_url":"https://api.github.com/repos/druid-io/druid/contributors","subscribers_url":"https://api.github.com/repos/druid-io/druid/subscribers","subscription_url":"https://api.github.com/repos/druid-io/druid/subscription","commits_url":"https://api.github.com/repos/druid-io/druid/commits{/sha}","git_commits_url":"https://api.github.com/repos/druid-io/druid/git/commits{/sha}","comments_url":"https://api.github.com/repos/druid-io/druid/comments{/number}","issue_comment_url":"https://api.github.com/repos/druid-io/druid/issues/comments{/number}","contents_url":"https://api.github.com/repos/druid-io/druid/contents/{+path}","compare_url":"https://api.github.com/repos/druid-io/druid/compare/{base}...{head}","merges_url":"https://api.github.com/repos/druid-io/druid/merges","archive_url":"https://api.github.com/repos/druid-io/druid/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/druid-io/druid/downloads","issues_url":"https://api.github.com/repos/druid-io/druid/issues{/number}","pulls_url":"https://api.github.com/repos/druid-io/druid/pulls{/number}","milestones_url":"https://api.github.com/repos/druid-io/druid/milestones{/number}","notifications_url":"https://api.github.com/repos/druid-io/druid/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/druid-io/druid/labels{/name}","releases_url":"https://api.github.com/repos/druid-io/druid/releases{/id}","created_at":"2012-10-23T19:08:07Z","updated_at":"2015-03-11T17:31:30Z","pushed_at":"2015-03-11T03:25:28Z","git_url":"git://github.com/druid-io/druid.git","ssh_url":"git@github.com:druid-io/druid.git","clone_url":"https://github.com/druid-io/druid.git","svn_url":"https://github.com/druid-io/druid","homepage":"http://druid.io","size":157647,"stargazers_count":1446,"watchers_count":1446,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":313,"mirror_url":null,"open_issues_count":122,"forks":313,"open_issues":122,"watchers":1446,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/druid-io/druid/pulls/1132"},"html":{"href":"https://github.com/druid-io/druid/pull/1132"},"issue":{"href":"https://api.github.com/repos/druid-io/druid/issues/1132"},"comments":{"href":"https://api.github.com/repos/druid-io/druid/issues/1132/comments"},"review_comments":{"href":"https://api.github.com/repos/druid-io/druid/pulls/1132/comments"},"review_comment":{"href":"https://api.github.com/repos/druid-io/druid/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/druid-io/druid/pulls/1132/commits"},"statuses":{"href":"https://api.github.com/repos/druid-io/druid/statuses/58c4df8351a58a9bbf66e4bfa738756e27dbb3da"}}}}