{"action":"created","comment":{"url":"https://api.github.com/repos/caskdata/cdap/pulls/comments/29176646","id":29176646,"diff_hunk":"@@ -0,0 +1,308 @@\n+/*\n+ * Copyright © 2015 Cask Data, Inc.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n+ * use this file except in compliance with the License. You may obtain a copy of\n+ * the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n+ * License for the specific language governing permissions and limitations under\n+ * the License.\n+ */\n+\n+package co.cask.cdap.data2.transaction.queue.hbase;\n+\n+import co.cask.cdap.api.common.Bytes;\n+import co.cask.cdap.common.guice.ConfigModule;\n+import co.cask.cdap.common.guice.DiscoveryRuntimeModule;\n+import co.cask.cdap.common.guice.LocationRuntimeModule;\n+import co.cask.cdap.common.guice.ZKClientModule;\n+import co.cask.cdap.common.queue.QueueName;\n+import co.cask.cdap.data.runtime.DataSetsModules;\n+import co.cask.cdap.data2.queue.ConsumerConfig;\n+import co.cask.cdap.data2.queue.ConsumerGroupConfig;\n+import co.cask.cdap.data2.queue.QueueClientFactory;\n+import co.cask.cdap.data2.transaction.queue.ConsumerEntryState;\n+import co.cask.cdap.data2.transaction.queue.QueueAdmin;\n+import co.cask.cdap.data2.transaction.queue.QueueConstants;\n+import co.cask.cdap.data2.transaction.queue.QueueEntryRow;\n+import co.cask.cdap.data2.util.TableId;\n+import co.cask.cdap.data2.util.hbase.HBaseTableUtil;\n+import co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory;\n+import co.cask.tephra.Transaction;\n+import co.cask.tephra.distributed.TransactionServiceClient;\n+import co.cask.tephra.runtime.TransactionClientModule;\n+import co.cask.tephra.runtime.TransactionModules;\n+import com.google.common.base.Function;\n+import com.google.common.collect.Multimap;\n+import com.google.common.util.concurrent.AbstractIdleService;\n+import com.google.gson.Gson;\n+import com.google.inject.AbstractModule;\n+import com.google.inject.Guice;\n+import com.google.inject.Inject;\n+import com.google.inject.Injector;\n+import com.google.inject.Singleton;\n+import org.apache.hadoop.hbase.client.HTable;\n+import org.apache.hadoop.hbase.client.Result;\n+import org.apache.hadoop.hbase.client.ResultScanner;\n+import org.apache.hadoop.hbase.client.Scan;\n+import org.apache.twill.zookeeper.ZKClientService;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+import java.io.IOException;\n+import java.net.URI;\n+import java.util.Collection;\n+import java.util.Iterator;\n+import java.util.NavigableMap;\n+import javax.annotation.Nullable;\n+\n+/**\n+ * Debugging tool for queues in hbase.\n+ */\n+public class HBaseQueueDebugger extends AbstractIdleService {\n+\n+  private static final Logger LOG = LoggerFactory.getLogger(HBaseQueueDebugger.class);\n+  private static final Gson GSON = new Gson();\n+\n+  private final HBaseQueueAdmin queueAdmin;\n+  private final TransactionServiceClient txClient;\n+  private final ZKClientService zkClientService;\n+  private final HBaseQueueClientFactory queueClientFactory;\n+\n+  @Inject\n+  public HBaseQueueDebugger(HBaseQueueAdmin queueAdmin,\n+                            HBaseQueueClientFactory queueClientFactory,\n+                            TransactionServiceClient txClient,\n+                            ZKClientService zkClientService) {\n+    this.queueAdmin = queueAdmin;\n+    this.queueClientFactory = queueClientFactory;\n+    this.txClient = txClient;\n+    this.zkClientService = zkClientService;\n+  }\n+\n+  @Override\n+  protected void startUp() throws Exception {\n+    zkClientService.startAndWait();\n+  }\n+\n+  @Override\n+  protected void shutDown() throws Exception {\n+    zkClientService.stopAndWait();\n+  }\n+\n+  /**\n+   * Only works for {@link co.cask.cdap.data2.transaction.queue.hbase.ShardedHBaseQueueStrategy}.\n+   */\n+  private void scanQueue(URI queueNameURI) throws Exception {\n+    QueueName queueName = QueueName.from(queueNameURI);\n+    HBaseConsumerStateStore stateStore = queueAdmin.getConsumerStateStore(queueName);\n+\n+    Transaction tx = txClient.startLong();\n+    stateStore.startTx(tx);\n+\n+    Multimap<Long, QueueBarrier> barriers = stateStore.getAllBarriers();\n+    LOG.info(\"Got {} barriers\", barriers.size());\n+\n+    QueueStatistics stats = new QueueStatistics();\n+    int currentSection = 1;\n+\n+    for (Long groupId : barriers.keySet()) {\n+      LOG.info(\"Scanning barriers for group {}\", groupId);\n+      Collection<QueueBarrier> groupBarriers = barriers.get(groupId);\n+      Iterator<QueueBarrier> barrierIterator = groupBarriers.iterator();\n+\n+      QueueBarrier previous;\n+      QueueBarrier current = null;\n+      while (barrierIterator.hasNext()) {\n+        previous = current;\n+        current = barrierIterator.next();\n+\n+        if (previous != null) {\n+          // after first iteration\n+          LOG.info(\"Scanning section {}/{}...\", currentSection, barriers.size());\n+          QueueStatistics sectionStats = scanQueue(tx, queueName, previous, current);\n+          LOG.info(\"Section results: {}\", sectionStats.getReport());\n+\n+          stats = QueueStatistics.of(stats, sectionStats);\n+          currentSection++;\n+        }\n+      }\n+      // handle last barrier\n+      LOG.info(\"Scanning section {}/{}...\", currentSection, barriers.size());\n+      stats = QueueStatistics.of(stats, scanQueue(tx, queueName, current, null));\n+      LOG.info(\"Scanning complete\", currentSection, barriers.size());\n+    }\n+\n+    LOG.info(\"Total results: {}\", stats.getReport());\n+    stateStore.commitTx();\n+  }\n+\n+  private QueueStatistics scanQueue(Transaction tx, QueueName queueName, QueueBarrier start,\n+                                    @Nullable QueueBarrier end) throws IOException {\n+\n+    byte[] queueRowPrefix = QueueEntryRow.getQueueRowPrefix(queueName);\n+\n+    ConsumerGroupConfig consumerConfig = start.getGroupConfig();\n+    LOG.info(\"Got consumer group config: {}\", consumerConfig);\n+\n+    HBaseQueueAdmin admin = queueClientFactory.ensureTableExists(queueName);","path":"cdap-data-fabric/src/main/java/co/cask/cdap/data2/transaction/queue/hbase/HBaseQueueDebugger.java","position":153,"original_position":153,"commit_id":"0892fd411a3422daefcdc6010213c6ebabd717d8","original_commit_id":"0892fd411a3422daefcdc6010213c6ebabd717d8","user":{"login":"chtyim","id":885032,"avatar_url":"https://avatars.githubusercontent.com/u/885032?v=3","gravatar_id":"","url":"https://api.github.com/users/chtyim","html_url":"https://github.com/chtyim","followers_url":"https://api.github.com/users/chtyim/followers","following_url":"https://api.github.com/users/chtyim/following{/other_user}","gists_url":"https://api.github.com/users/chtyim/gists{/gist_id}","starred_url":"https://api.github.com/users/chtyim/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chtyim/subscriptions","organizations_url":"https://api.github.com/users/chtyim/orgs","repos_url":"https://api.github.com/users/chtyim/repos","events_url":"https://api.github.com/users/chtyim/events{/privacy}","received_events_url":"https://api.github.com/users/chtyim/received_events","type":"User","site_admin":false},"body":"If the queue table doesn't exists, this tool shouldn't creating it. It should just return (as if there is no queue entry).","created_at":"2015-04-27T18:38:36Z","updated_at":"2015-04-27T18:38:36Z","html_url":"https://github.com/caskdata/cdap/pull/2290#discussion_r29176646","pull_request_url":"https://api.github.com/repos/caskdata/cdap/pulls/2290","_links":{"self":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/comments/29176646"},"html":{"href":"https://github.com/caskdata/cdap/pull/2290#discussion_r29176646"},"pull_request":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/2290"}}},"pull_request":{"url":"https://api.github.com/repos/caskdata/cdap/pulls/2290","id":34019058,"html_url":"https://github.com/caskdata/cdap/pull/2290","diff_url":"https://github.com/caskdata/cdap/pull/2290.diff","patch_url":"https://github.com/caskdata/cdap/pull/2290.patch","issue_url":"https://api.github.com/repos/caskdata/cdap/issues/2290","number":2290,"state":"open","locked":false,"title":"CDAP-2105: Queue introspection","user":{"login":"jwang47","id":1084704,"avatar_url":"https://avatars.githubusercontent.com/u/1084704?v=3","gravatar_id":"","url":"https://api.github.com/users/jwang47","html_url":"https://github.com/jwang47","followers_url":"https://api.github.com/users/jwang47/followers","following_url":"https://api.github.com/users/jwang47/following{/other_user}","gists_url":"https://api.github.com/users/jwang47/gists{/gist_id}","starred_url":"https://api.github.com/users/jwang47/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jwang47/subscriptions","organizations_url":"https://api.github.com/users/jwang47/orgs","repos_url":"https://api.github.com/users/jwang47/repos","events_url":"https://api.github.com/users/jwang47/events{/privacy}","received_events_url":"https://api.github.com/users/jwang47/received_events","type":"User","site_admin":false},"body":null,"created_at":"2015-04-24T05:04:39Z","updated_at":"2015-04-27T18:38:36Z","closed_at":null,"merged_at":null,"merge_commit_sha":"369bf30be1bc4d9f66684a4eb13051ec9efb580f","assignee":null,"milestone":{"url":"https://api.github.com/repos/caskdata/cdap/milestones/5","html_url":"https://github.com/caskdata/cdap/milestones/2.8.1","labels_url":"https://api.github.com/repos/caskdata/cdap/milestones/5/labels","id":1081686,"number":5,"title":"2.8.1","description":null,"creator":{"login":"jwang47","id":1084704,"avatar_url":"https://avatars.githubusercontent.com/u/1084704?v=3","gravatar_id":"","url":"https://api.github.com/users/jwang47","html_url":"https://github.com/jwang47","followers_url":"https://api.github.com/users/jwang47/followers","following_url":"https://api.github.com/users/jwang47/following{/other_user}","gists_url":"https://api.github.com/users/jwang47/gists{/gist_id}","starred_url":"https://api.github.com/users/jwang47/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jwang47/subscriptions","organizations_url":"https://api.github.com/users/jwang47/orgs","repos_url":"https://api.github.com/users/jwang47/repos","events_url":"https://api.github.com/users/jwang47/events{/privacy}","received_events_url":"https://api.github.com/users/jwang47/received_events","type":"User","site_admin":false},"open_issues":1,"closed_issues":0,"state":"open","created_at":"2015-04-24T05:06:00Z","updated_at":"2015-04-24T05:06:00Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/caskdata/cdap/pulls/2290/commits","review_comments_url":"https://api.github.com/repos/caskdata/cdap/pulls/2290/comments","review_comment_url":"https://api.github.com/repos/caskdata/cdap/pulls/comments{/number}","comments_url":"https://api.github.com/repos/caskdata/cdap/issues/2290/comments","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/0892fd411a3422daefcdc6010213c6ebabd717d8","head":{"label":"caskdata:feature/q-introspection-hf","ref":"feature/q-introspection-hf","sha":"0892fd411a3422daefcdc6010213c6ebabd717d8","user":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"repo":{"id":22542759,"name":"cdap","full_name":"caskdata/cdap","owner":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/caskdata/cdap","description":"Cask Data Application Platform (CDAP)","fork":false,"url":"https://api.github.com/repos/caskdata/cdap","forks_url":"https://api.github.com/repos/caskdata/cdap/forks","keys_url":"https://api.github.com/repos/caskdata/cdap/keys{/key_id}","collaborators_url":"https://api.github.com/repos/caskdata/cdap/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/caskdata/cdap/teams","hooks_url":"https://api.github.com/repos/caskdata/cdap/hooks","issue_events_url":"https://api.github.com/repos/caskdata/cdap/issues/events{/number}","events_url":"https://api.github.com/repos/caskdata/cdap/events","assignees_url":"https://api.github.com/repos/caskdata/cdap/assignees{/user}","branches_url":"https://api.github.com/repos/caskdata/cdap/branches{/branch}","tags_url":"https://api.github.com/repos/caskdata/cdap/tags","blobs_url":"https://api.github.com/repos/caskdata/cdap/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/caskdata/cdap/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/caskdata/cdap/git/refs{/sha}","trees_url":"https://api.github.com/repos/caskdata/cdap/git/trees{/sha}","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/{sha}","languages_url":"https://api.github.com/repos/caskdata/cdap/languages","stargazers_url":"https://api.github.com/repos/caskdata/cdap/stargazers","contributors_url":"https://api.github.com/repos/caskdata/cdap/contributors","subscribers_url":"https://api.github.com/repos/caskdata/cdap/subscribers","subscription_url":"https://api.github.com/repos/caskdata/cdap/subscription","commits_url":"https://api.github.com/repos/caskdata/cdap/commits{/sha}","git_commits_url":"https://api.github.com/repos/caskdata/cdap/git/commits{/sha}","comments_url":"https://api.github.com/repos/caskdata/cdap/comments{/number}","issue_comment_url":"https://api.github.com/repos/caskdata/cdap/issues/comments{/number}","contents_url":"https://api.github.com/repos/caskdata/cdap/contents/{+path}","compare_url":"https://api.github.com/repos/caskdata/cdap/compare/{base}...{head}","merges_url":"https://api.github.com/repos/caskdata/cdap/merges","archive_url":"https://api.github.com/repos/caskdata/cdap/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/caskdata/cdap/downloads","issues_url":"https://api.github.com/repos/caskdata/cdap/issues{/number}","pulls_url":"https://api.github.com/repos/caskdata/cdap/pulls{/number}","milestones_url":"https://api.github.com/repos/caskdata/cdap/milestones{/number}","notifications_url":"https://api.github.com/repos/caskdata/cdap/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/caskdata/cdap/labels{/name}","releases_url":"https://api.github.com/repos/caskdata/cdap/releases{/id}","created_at":"2014-08-02T09:00:10Z","updated_at":"2015-04-27T17:12:37Z","pushed_at":"2015-04-27T18:35:34Z","git_url":"git://github.com/caskdata/cdap.git","ssh_url":"git@github.com:caskdata/cdap.git","clone_url":"https://github.com/caskdata/cdap.git","svn_url":"https://github.com/caskdata/cdap","homepage":"http://cdap.io","size":859094,"stargazers_count":41,"watchers_count":41,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":16,"mirror_url":null,"open_issues_count":17,"forks":16,"open_issues":17,"watchers":41,"default_branch":"develop"}},"base":{"label":"caskdata:hotfix/temp-fix-flow-change-instances","ref":"hotfix/temp-fix-flow-change-instances","sha":"0b24a6fe98b4d940e6c4e45f351fe715dc911228","user":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"repo":{"id":22542759,"name":"cdap","full_name":"caskdata/cdap","owner":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/caskdata/cdap","description":"Cask Data Application Platform (CDAP)","fork":false,"url":"https://api.github.com/repos/caskdata/cdap","forks_url":"https://api.github.com/repos/caskdata/cdap/forks","keys_url":"https://api.github.com/repos/caskdata/cdap/keys{/key_id}","collaborators_url":"https://api.github.com/repos/caskdata/cdap/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/caskdata/cdap/teams","hooks_url":"https://api.github.com/repos/caskdata/cdap/hooks","issue_events_url":"https://api.github.com/repos/caskdata/cdap/issues/events{/number}","events_url":"https://api.github.com/repos/caskdata/cdap/events","assignees_url":"https://api.github.com/repos/caskdata/cdap/assignees{/user}","branches_url":"https://api.github.com/repos/caskdata/cdap/branches{/branch}","tags_url":"https://api.github.com/repos/caskdata/cdap/tags","blobs_url":"https://api.github.com/repos/caskdata/cdap/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/caskdata/cdap/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/caskdata/cdap/git/refs{/sha}","trees_url":"https://api.github.com/repos/caskdata/cdap/git/trees{/sha}","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/{sha}","languages_url":"https://api.github.com/repos/caskdata/cdap/languages","stargazers_url":"https://api.github.com/repos/caskdata/cdap/stargazers","contributors_url":"https://api.github.com/repos/caskdata/cdap/contributors","subscribers_url":"https://api.github.com/repos/caskdata/cdap/subscribers","subscription_url":"https://api.github.com/repos/caskdata/cdap/subscription","commits_url":"https://api.github.com/repos/caskdata/cdap/commits{/sha}","git_commits_url":"https://api.github.com/repos/caskdata/cdap/git/commits{/sha}","comments_url":"https://api.github.com/repos/caskdata/cdap/comments{/number}","issue_comment_url":"https://api.github.com/repos/caskdata/cdap/issues/comments{/number}","contents_url":"https://api.github.com/repos/caskdata/cdap/contents/{+path}","compare_url":"https://api.github.com/repos/caskdata/cdap/compare/{base}...{head}","merges_url":"https://api.github.com/repos/caskdata/cdap/merges","archive_url":"https://api.github.com/repos/caskdata/cdap/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/caskdata/cdap/downloads","issues_url":"https://api.github.com/repos/caskdata/cdap/issues{/number}","pulls_url":"https://api.github.com/repos/caskdata/cdap/pulls{/number}","milestones_url":"https://api.github.com/repos/caskdata/cdap/milestones{/number}","notifications_url":"https://api.github.com/repos/caskdata/cdap/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/caskdata/cdap/labels{/name}","releases_url":"https://api.github.com/repos/caskdata/cdap/releases{/id}","created_at":"2014-08-02T09:00:10Z","updated_at":"2015-04-27T17:12:37Z","pushed_at":"2015-04-27T18:35:34Z","git_url":"git://github.com/caskdata/cdap.git","ssh_url":"git@github.com:caskdata/cdap.git","clone_url":"https://github.com/caskdata/cdap.git","svn_url":"https://github.com/caskdata/cdap","homepage":"http://cdap.io","size":859094,"stargazers_count":41,"watchers_count":41,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":16,"mirror_url":null,"open_issues_count":17,"forks":16,"open_issues":17,"watchers":41,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/2290"},"html":{"href":"https://github.com/caskdata/cdap/pull/2290"},"issue":{"href":"https://api.github.com/repos/caskdata/cdap/issues/2290"},"comments":{"href":"https://api.github.com/repos/caskdata/cdap/issues/2290/comments"},"review_comments":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/2290/comments"},"review_comment":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/2290/commits"},"statuses":{"href":"https://api.github.com/repos/caskdata/cdap/statuses/0892fd411a3422daefcdc6010213c6ebabd717d8"}}}}