{"action":"created","comment":{"url":"https://api.github.com/repos/apache/accumulo/pulls/comments/38815475","id":38815475,"diff_hunk":"@@ -0,0 +1,497 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *     http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.accumulo.test;\n+\n+import java.io.IOException;\n+import java.util.ArrayList;\n+import java.util.Arrays;\n+import java.util.Collection;\n+import java.util.Collections;\n+import java.util.Iterator;\n+import java.util.List;\n+import java.util.Map;\n+import java.util.Map.Entry;\n+import java.util.Set;\n+import java.util.TreeMap;\n+\n+import org.apache.accumulo.core.client.AccumuloException;\n+import org.apache.accumulo.core.client.AccumuloSecurityException;\n+import org.apache.accumulo.core.client.BatchScanner;\n+import org.apache.accumulo.core.client.BatchWriter;\n+import org.apache.accumulo.core.client.BatchWriterConfig;\n+import org.apache.accumulo.core.client.ClientSideIteratorScanner;\n+import org.apache.accumulo.core.client.Connector;\n+import org.apache.accumulo.core.client.IsolatedScanner;\n+import org.apache.accumulo.core.client.IteratorSetting;\n+import org.apache.accumulo.core.client.MutationsRejectedException;\n+import org.apache.accumulo.core.client.SampleNotPresentException;\n+import org.apache.accumulo.core.client.Scanner;\n+import org.apache.accumulo.core.client.ScannerBase;\n+import org.apache.accumulo.core.client.TableNotFoundException;\n+import org.apache.accumulo.core.client.admin.CompactionConfig;\n+import org.apache.accumulo.core.client.admin.NewTableConfiguration;\n+import org.apache.accumulo.core.client.admin.SamplerConfiguration;\n+import org.apache.accumulo.core.client.impl.Credentials;\n+import org.apache.accumulo.core.client.impl.OfflineScanner;\n+import org.apache.accumulo.core.data.ByteSequence;\n+import org.apache.accumulo.core.data.Key;\n+import org.apache.accumulo.core.data.Mutation;\n+import org.apache.accumulo.core.data.Range;\n+import org.apache.accumulo.core.data.Value;\n+import org.apache.accumulo.core.iterators.IteratorEnvironment;\n+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;\n+import org.apache.accumulo.core.iterators.WrappingIterator;\n+import org.apache.accumulo.core.sample.RowSampler;\n+import org.apache.accumulo.core.security.Authorizations;\n+import org.apache.accumulo.harness.AccumuloClusterHarness;\n+import org.junit.Assert;\n+import org.junit.Test;\n+\n+import com.google.common.collect.ImmutableMap;\n+import com.google.common.collect.Iterables;\n+\n+public class SampleIT extends AccumuloClusterHarness {\n+\n+  private static final Map<String,String> OPTIONS_1 = ImmutableMap.of(\"hasher\", \"murmur3_32\", \"modulus\", \"1009\");\n+  private static final Map<String,String> OPTIONS_2 = ImmutableMap.of(\"hasher\", \"murmur3_32\", \"modulus\", \"997\");\n+\n+  private static final SamplerConfiguration SC1 = new SamplerConfiguration(RowSampler.class.getName()).setOptions(OPTIONS_1);\n+  private static final SamplerConfiguration SC2 = new SamplerConfiguration(RowSampler.class.getName()).setOptions(OPTIONS_2);\n+\n+  public static class IteratorThatUsesSample extends WrappingIterator {\n+    private SortedKeyValueIterator<Key,Value> sampleDC;\n+    private boolean hasTop;\n+\n+    @Override\n+    public boolean hasTop() {\n+      return hasTop && super.hasTop();\n+    }\n+\n+    @Override\n+    public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {\n+\n+      int sampleCount = 0;\n+      sampleDC.seek(range, columnFamilies, inclusive);\n+\n+      while (sampleDC.hasTop()) {\n+        sampleCount++;\n+        sampleDC.next();\n+      }\n+\n+      if (sampleCount < 10) {\n+        hasTop = true;\n+        super.seek(range, columnFamilies, inclusive);\n+      } else {\n+        // its too much data\n+        hasTop = false;\n+      }\n+    }\n+\n+    @Override\n+    public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {\n+      super.init(source, options, env);\n+\n+      IteratorEnvironment sampleEnv = env.newIEWithSamplingEnabled();\n+\n+      sampleDC = source.deepCopy(sampleEnv);\n+    }\n+  }\n+\n+  @Test\n+  public void testBasic() throws Exception {\n+\n+    Connector conn = getConnector();\n+    String tableName = getUniqueNames(1)[0];\n+    String clone = tableName + \"_clone\";\n+\n+    conn.tableOperations().create(tableName, new NewTableConfiguration().enableSampling(SC1));\n+\n+    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());\n+\n+    TreeMap<Key,Value> expected = new TreeMap<Key,Value>();\n+    String someRow = writeData(bw, SC1, expected);\n+\n+    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);","path":"test/src/main/java/org/apache/accumulo/test/SampleIT.java","position":129,"original_position":129,"commit_id":"7654fa1161aa4dd99960a030aee2a35c6e274c11","original_commit_id":"7654fa1161aa4dd99960a030aee2a35c6e274c11","user":{"login":"joshelser","id":67863,"avatar_url":"https://avatars.githubusercontent.com/u/67863?v=3","gravatar_id":"","url":"https://api.github.com/users/joshelser","html_url":"https://github.com/joshelser","followers_url":"https://api.github.com/users/joshelser/followers","following_url":"https://api.github.com/users/joshelser/following{/other_user}","gists_url":"https://api.github.com/users/joshelser/gists{/gist_id}","starred_url":"https://api.github.com/users/joshelser/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joshelser/subscriptions","organizations_url":"https://api.github.com/users/joshelser/orgs","repos_url":"https://api.github.com/users/joshelser/repos","events_url":"https://api.github.com/users/joshelser/events{/privacy}","received_events_url":"https://api.github.com/users/joshelser/received_events","type":"User","site_admin":false},"body":"Seeing this block of code makes me think more that we should have a ScannerConfig to reduce the amount method calls over and over again. Or, some basic scan information stored in the connector itself (e.g. always enable sampling).","created_at":"2015-09-05T22:40:17Z","updated_at":"2015-09-05T22:40:17Z","html_url":"https://github.com/apache/accumulo/pull/46#discussion_r38815475","pull_request_url":"https://api.github.com/repos/apache/accumulo/pulls/46","_links":{"self":{"href":"https://api.github.com/repos/apache/accumulo/pulls/comments/38815475"},"html":{"href":"https://github.com/apache/accumulo/pull/46#discussion_r38815475"},"pull_request":{"href":"https://api.github.com/repos/apache/accumulo/pulls/46"}}},"pull_request":{"url":"https://api.github.com/repos/apache/accumulo/pulls/46","id":43835205,"html_url":"https://github.com/apache/accumulo/pull/46","diff_url":"https://github.com/apache/accumulo/pull/46.diff","patch_url":"https://github.com/apache/accumulo/pull/46.patch","issue_url":"https://api.github.com/repos/apache/accumulo/issues/46","number":46,"state":"open","locked":false,"title":"ACCUMULO-3913 Added per table sampling","user":{"login":"keith-turner","id":1268739,"avatar_url":"https://avatars.githubusercontent.com/u/1268739?v=3","gravatar_id":"","url":"https://api.github.com/users/keith-turner","html_url":"https://github.com/keith-turner","followers_url":"https://api.github.com/users/keith-turner/followers","following_url":"https://api.github.com/users/keith-turner/following{/other_user}","gists_url":"https://api.github.com/users/keith-turner/gists{/gist_id}","starred_url":"https://api.github.com/users/keith-turner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/keith-turner/subscriptions","organizations_url":"https://api.github.com/users/keith-turner/orgs","repos_url":"https://api.github.com/users/keith-turner/repos","events_url":"https://api.github.com/users/keith-turner/events{/privacy}","received_events_url":"https://api.github.com/users/keith-turner/received_events","type":"User","site_admin":false},"body":"This a large PR, if you don't have time to look at the impl, a review of the API changes, docs, and test would be very helpful.\r\n\r\nI took Readme.sample from this PR and made a [Gist](https://gist.github.com/keith-turner/1a073eb032e4d8c448cb) out of it so it would render nicely for reading.","created_at":"2015-09-01T03:58:07Z","updated_at":"2015-09-05T22:40:17Z","closed_at":null,"merged_at":null,"merge_commit_sha":"08269054ab475c2ed41d7d4ae697542617c4907b","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/accumulo/pulls/46/commits","review_comments_url":"https://api.github.com/repos/apache/accumulo/pulls/46/comments","review_comment_url":"https://api.github.com/repos/apache/accumulo/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/accumulo/issues/46/comments","statuses_url":"https://api.github.com/repos/apache/accumulo/statuses/7654fa1161aa4dd99960a030aee2a35c6e274c11","head":{"label":"keith-turner:ACCUMULO-3913","ref":"ACCUMULO-3913","sha":"7654fa1161aa4dd99960a030aee2a35c6e274c11","user":{"login":"keith-turner","id":1268739,"avatar_url":"https://avatars.githubusercontent.com/u/1268739?v=3","gravatar_id":"","url":"https://api.github.com/users/keith-turner","html_url":"https://github.com/keith-turner","followers_url":"https://api.github.com/users/keith-turner/followers","following_url":"https://api.github.com/users/keith-turner/following{/other_user}","gists_url":"https://api.github.com/users/keith-turner/gists{/gist_id}","starred_url":"https://api.github.com/users/keith-turner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/keith-turner/subscriptions","organizations_url":"https://api.github.com/users/keith-turner/orgs","repos_url":"https://api.github.com/users/keith-turner/repos","events_url":"https://api.github.com/users/keith-turner/events{/privacy}","received_events_url":"https://api.github.com/users/keith-turner/received_events","type":"User","site_admin":false},"repo":{"id":4228808,"name":"accumulo","full_name":"keith-turner/accumulo","owner":{"login":"keith-turner","id":1268739,"avatar_url":"https://avatars.githubusercontent.com/u/1268739?v=3","gravatar_id":"","url":"https://api.github.com/users/keith-turner","html_url":"https://github.com/keith-turner","followers_url":"https://api.github.com/users/keith-turner/followers","following_url":"https://api.github.com/users/keith-turner/following{/other_user}","gists_url":"https://api.github.com/users/keith-turner/gists{/gist_id}","starred_url":"https://api.github.com/users/keith-turner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/keith-turner/subscriptions","organizations_url":"https://api.github.com/users/keith-turner/orgs","repos_url":"https://api.github.com/users/keith-turner/repos","events_url":"https://api.github.com/users/keith-turner/events{/privacy}","received_events_url":"https://api.github.com/users/keith-turner/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/keith-turner/accumulo","description":"Mirror of Apache Accumulo","fork":true,"url":"https://api.github.com/repos/keith-turner/accumulo","forks_url":"https://api.github.com/repos/keith-turner/accumulo/forks","keys_url":"https://api.github.com/repos/keith-turner/accumulo/keys{/key_id}","collaborators_url":"https://api.github.com/repos/keith-turner/accumulo/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/keith-turner/accumulo/teams","hooks_url":"https://api.github.com/repos/keith-turner/accumulo/hooks","issue_events_url":"https://api.github.com/repos/keith-turner/accumulo/issues/events{/number}","events_url":"https://api.github.com/repos/keith-turner/accumulo/events","assignees_url":"https://api.github.com/repos/keith-turner/accumulo/assignees{/user}","branches_url":"https://api.github.com/repos/keith-turner/accumulo/branches{/branch}","tags_url":"https://api.github.com/repos/keith-turner/accumulo/tags","blobs_url":"https://api.github.com/repos/keith-turner/accumulo/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/keith-turner/accumulo/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/keith-turner/accumulo/git/refs{/sha}","trees_url":"https://api.github.com/repos/keith-turner/accumulo/git/trees{/sha}","statuses_url":"https://api.github.com/repos/keith-turner/accumulo/statuses/{sha}","languages_url":"https://api.github.com/repos/keith-turner/accumulo/languages","stargazers_url":"https://api.github.com/repos/keith-turner/accumulo/stargazers","contributors_url":"https://api.github.com/repos/keith-turner/accumulo/contributors","subscribers_url":"https://api.github.com/repos/keith-turner/accumulo/subscribers","subscription_url":"https://api.github.com/repos/keith-turner/accumulo/subscription","commits_url":"https://api.github.com/repos/keith-turner/accumulo/commits{/sha}","git_commits_url":"https://api.github.com/repos/keith-turner/accumulo/git/commits{/sha}","comments_url":"https://api.github.com/repos/keith-turner/accumulo/comments{/number}","issue_comment_url":"https://api.github.com/repos/keith-turner/accumulo/issues/comments{/number}","contents_url":"https://api.github.com/repos/keith-turner/accumulo/contents/{+path}","compare_url":"https://api.github.com/repos/keith-turner/accumulo/compare/{base}...{head}","merges_url":"https://api.github.com/repos/keith-turner/accumulo/merges","archive_url":"https://api.github.com/repos/keith-turner/accumulo/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/keith-turner/accumulo/downloads","issues_url":"https://api.github.com/repos/keith-turner/accumulo/issues{/number}","pulls_url":"https://api.github.com/repos/keith-turner/accumulo/pulls{/number}","milestones_url":"https://api.github.com/repos/keith-turner/accumulo/milestones{/number}","notifications_url":"https://api.github.com/repos/keith-turner/accumulo/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/keith-turner/accumulo/labels{/name}","releases_url":"https://api.github.com/repos/keith-turner/accumulo/releases{/id}","created_at":"2012-05-04T20:22:00Z","updated_at":"2015-08-28T21:58:49Z","pushed_at":"2015-09-01T03:47:05Z","git_url":"git://github.com/keith-turner/accumulo.git","ssh_url":"git@github.com:keith-turner/accumulo.git","clone_url":"https://github.com/keith-turner/accumulo.git","svn_url":"https://github.com/keith-turner/accumulo","homepage":"","size":45462,"stargazers_count":1,"watchers_count":1,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"9d25381b7a0fa1f728eda13158db0888cb4b6b7d","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":2524488,"name":"accumulo","full_name":"apache/accumulo","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/accumulo","description":"Mirror of Apache Accumulo","fork":false,"url":"https://api.github.com/repos/apache/accumulo","forks_url":"https://api.github.com/repos/apache/accumulo/forks","keys_url":"https://api.github.com/repos/apache/accumulo/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/accumulo/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/accumulo/teams","hooks_url":"https://api.github.com/repos/apache/accumulo/hooks","issue_events_url":"https://api.github.com/repos/apache/accumulo/issues/events{/number}","events_url":"https://api.github.com/repos/apache/accumulo/events","assignees_url":"https://api.github.com/repos/apache/accumulo/assignees{/user}","branches_url":"https://api.github.com/repos/apache/accumulo/branches{/branch}","tags_url":"https://api.github.com/repos/apache/accumulo/tags","blobs_url":"https://api.github.com/repos/apache/accumulo/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/accumulo/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/accumulo/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/accumulo/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/accumulo/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/accumulo/languages","stargazers_url":"https://api.github.com/repos/apache/accumulo/stargazers","contributors_url":"https://api.github.com/repos/apache/accumulo/contributors","subscribers_url":"https://api.github.com/repos/apache/accumulo/subscribers","subscription_url":"https://api.github.com/repos/apache/accumulo/subscription","commits_url":"https://api.github.com/repos/apache/accumulo/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/accumulo/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/accumulo/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/accumulo/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/accumulo/contents/{+path}","compare_url":"https://api.github.com/repos/apache/accumulo/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/accumulo/merges","archive_url":"https://api.github.com/repos/apache/accumulo/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/accumulo/downloads","issues_url":"https://api.github.com/repos/apache/accumulo/issues{/number}","pulls_url":"https://api.github.com/repos/apache/accumulo/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/accumulo/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/accumulo/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/accumulo/labels{/name}","releases_url":"https://api.github.com/repos/apache/accumulo/releases{/id}","created_at":"2011-10-06T07:00:09Z","updated_at":"2015-08-25T04:04:15Z","pushed_at":"2015-09-05T21:26:07Z","git_url":"git://github.com/apache/accumulo.git","ssh_url":"git@github.com:apache/accumulo.git","clone_url":"https://github.com/apache/accumulo.git","svn_url":"https://github.com/apache/accumulo","homepage":"","size":82284,"stargazers_count":174,"watchers_count":174,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":95,"mirror_url":"git://git.apache.org/accumulo.git","open_issues_count":3,"forks":95,"open_issues":3,"watchers":174,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/accumulo/pulls/46"},"html":{"href":"https://github.com/apache/accumulo/pull/46"},"issue":{"href":"https://api.github.com/repos/apache/accumulo/issues/46"},"comments":{"href":"https://api.github.com/repos/apache/accumulo/issues/46/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/accumulo/pulls/46/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/accumulo/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/accumulo/pulls/46/commits"},"statuses":{"href":"https://api.github.com/repos/apache/accumulo/statuses/7654fa1161aa4dd99960a030aee2a35c6e274c11"}}}}