{"action":"created","comment":{"url":"https://api.github.com/repos/VoltDB/voltdb/pulls/comments/34159740","id":34159740,"diff_hunk":"@@ -0,0 +1,596 @@\n+/* This file is part of VoltDB.\n+ * Copyright (C) 2008-2015 VoltDB Inc.\n+ *\n+ * This program is free software: you can redistribute it and/or modify\n+ * it under the terms of the GNU Affero General Public License as\n+ * published by the Free Software Foundation, either version 3 of the\n+ * License, or (at your option) any later version.\n+ *\n+ * This program is distributed in the hope that it will be useful,\n+ * but WITHOUT ANY WARRANTY; without even the implied warranty of\n+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+ * GNU Affero General Public License for more details.\n+ *\n+ * You should have received a copy of the GNU Affero General Public License\n+ * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.\n+ */\n+package org.voltdb.importclient.kafka;\n+\n+import java.net.URI;\n+import java.net.URISyntaxException;\n+import java.nio.ByteBuffer;\n+import java.util.ArrayList;\n+import kafka.api.FetchRequestBuilder;\n+\n+import java.util.Arrays;\n+import java.util.Collections;\n+import java.util.List;\n+import kafka.javaapi.consumer.SimpleConsumer;\n+import java.util.HashMap;\n+import java.util.Map;\n+import java.util.Properties;\n+import java.util.Set;\n+import java.util.TreeSet;\n+import java.util.concurrent.ExecutorService;\n+import java.util.concurrent.Executors;\n+import java.util.concurrent.Semaphore;\n+import java.util.concurrent.TimeUnit;\n+import java.util.concurrent.atomic.AtomicLong;\n+import kafka.api.FetchRequest;\n+import kafka.api.PartitionOffsetRequestInfo;\n+import kafka.common.ErrorMapping;\n+import kafka.common.OffsetAndMetadata;\n+import kafka.common.OffsetMetadataAndError;\n+import kafka.common.TopicAndPartition;\n+import kafka.javaapi.FetchResponse;\n+import kafka.javaapi.OffsetCommitRequest;\n+import kafka.javaapi.OffsetCommitResponse;\n+import kafka.javaapi.OffsetFetchRequest;\n+import kafka.javaapi.OffsetFetchResponse;\n+import kafka.javaapi.OffsetRequest;\n+import kafka.javaapi.OffsetResponse;\n+import kafka.javaapi.PartitionMetadata;\n+import kafka.javaapi.TopicMetadata;\n+import kafka.javaapi.TopicMetadataRequest;\n+import kafka.message.MessageAndOffset;\n+\n+import org.osgi.framework.BundleActivator;\n+import org.osgi.framework.BundleContext;\n+import org.voltdb.client.ClientResponse;\n+import org.voltdb.client.ProcedureCallback;\n+import org.voltdb.importer.CSVInvocation;\n+import org.voltdb.importer.ImportHandlerProxy;\n+\n+/**\n+ * Based on SimpleConsumer Implement a BundleActivator interface and extend ImportHandlerProxy.\n+ * For each partition for a topic a thread is launched to pull data and submit transactions.\n+ *\n+ * @author akhanzode\n+ */\n+public class KafkaStreamImporter extends ImportHandlerProxy implements BundleActivator {\n+\n+    //Properties for the importer\n+    private Properties m_properties;\n+    //Procedure to be invoked with params.\n+    private String m_procedure;\n+    //List of topics form comma seperated list.\n+    private List<String> m_topicList;\n+    //List of brokers.\n+    private final List<HostAndPort> m_brokerList = new ArrayList<HostAndPort>();\n+    //kafka properties which has defaults\n+    private int m_fetchSize = 65536;\n+    private int m_consumerSocketTimeout = 30000; //In milliseconds\n+\n+    private static final String GROUP_ID = \"voltdb\";\n+    private static final String CLIENT_ID = \"voltdb-importer\";\n+    private static final int KAFKA_DEFAULT_BROKER_PORT = 9092;\n+    //readyForData is waiting for this released by shutdown\n+    private final Semaphore m_done = new Semaphore(0);\n+\n+    //topic partition metadata\n+    private final Map<String, List<TopicMetadata>> m_topicPartitionMetaData = new HashMap<String, List<TopicMetadata>>();\n+    //Topic partitions\n+    private final Map<String, List<Integer>> m_topicPartitions = new HashMap<String, List<Integer>>();\n+    //topic partition leader\n+    private final Map<String, String> m_topicPartitionLeader = new HashMap<String, String>();\n+    private final Map<String, Integer> m_topicPartitionLeaderPort = new HashMap<String, Integer>();\n+    private final Map<String, TopicPartitionFetcher> m_fetchers = new HashMap<String, TopicPartitionFetcher>();\n+\n+    private ExecutorService m_es = null;\n+\n+    //Simple Host and Port abstraction....dont want to use our big stuff here orgi bundle import nastiness.\n+    public static class HostAndPort {\n+\n+        public final String host;\n+        public final int port;\n+\n+        public HostAndPort(String h, int p) {\n+            host = h;\n+            port = p;\n+        }\n+\n+        public static HostAndPort fromString(String hap) {\n+            String s[] = hap.split(\":\");\n+            int p = KAFKA_DEFAULT_BROKER_PORT;\n+            if (s.length > 1 && s[1] != null && s[1].length() > 0) {\n+                p = Integer.parseInt(s[1]);\n+            }\n+            return new HostAndPort(s[0], p);\n+        }\n+    }\n+\n+    // Register ImportHandlerProxy service.\n+    @Override\n+    public void start(BundleContext context) throws Exception {\n+        context.registerService(this.getClass().getName(), this, null);","path":"src/frontend/org/voltdb/importclient/kafka/KafkaStreamImporter.java","position":129,"original_position":125,"commit_id":"9ed0535ff26b9a1a0ddbdc8933b14e06bfd00a23","original_commit_id":"232e2f69813626c03cfba49d4cdf1a9cff28f857","user":{"login":"akhanzode","id":1675698,"avatar_url":"https://avatars.githubusercontent.com/u/1675698?v=3","gravatar_id":"","url":"https://api.github.com/users/akhanzode","html_url":"https://github.com/akhanzode","followers_url":"https://api.github.com/users/akhanzode/followers","following_url":"https://api.github.com/users/akhanzode/following{/other_user}","gists_url":"https://api.github.com/users/akhanzode/gists{/gist_id}","starred_url":"https://api.github.com/users/akhanzode/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/akhanzode/subscriptions","organizations_url":"https://api.github.com/users/akhanzode/orgs","repos_url":"https://api.github.com/users/akhanzode/repos","events_url":"https://api.github.com/users/akhanzode/events{/privacy}","received_events_url":"https://api.github.com/users/akhanzode/received_events","type":"User","site_admin":false},"body":"I believe thats through XML Don't know of any auto magic.","created_at":"2015-07-08T15:18:54Z","updated_at":"2015-07-08T15:18:54Z","html_url":"https://github.com/VoltDB/voltdb/pull/2513#discussion_r34159740","pull_request_url":"https://api.github.com/repos/VoltDB/voltdb/pulls/2513","_links":{"self":{"href":"https://api.github.com/repos/VoltDB/voltdb/pulls/comments/34159740"},"html":{"href":"https://github.com/VoltDB/voltdb/pull/2513#discussion_r34159740"},"pull_request":{"href":"https://api.github.com/repos/VoltDB/voltdb/pulls/2513"}}},"pull_request":{"url":"https://api.github.com/repos/VoltDB/voltdb/pulls/2513","id":39290485,"html_url":"https://github.com/VoltDB/voltdb/pull/2513","diff_url":"https://github.com/VoltDB/voltdb/pull/2513.diff","patch_url":"https://github.com/VoltDB/voltdb/pull/2513.patch","issue_url":"https://api.github.com/repos/VoltDB/voltdb/issues/2513","number":2513,"state":"open","locked":false,"title":"ENG-8000 - Kafka Importer","user":{"login":"akhanzode","id":1675698,"avatar_url":"https://avatars.githubusercontent.com/u/1675698?v=3","gravatar_id":"","url":"https://api.github.com/users/akhanzode","html_url":"https://github.com/akhanzode","followers_url":"https://api.github.com/users/akhanzode/followers","following_url":"https://api.github.com/users/akhanzode/following{/other_user}","gists_url":"https://api.github.com/users/akhanzode/gists{/gist_id}","starred_url":"https://api.github.com/users/akhanzode/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/akhanzode/subscriptions","organizations_url":"https://api.github.com/users/akhanzode/orgs","repos_url":"https://api.github.com/users/akhanzode/repos","events_url":"https://api.github.com/users/akhanzode/events{/privacy}","received_events_url":"https://api.github.com/users/akhanzode/received_events","type":"User","site_admin":false},"body":"Do not merge. No changes on pro.","created_at":"2015-07-06T17:22:26Z","updated_at":"2015-07-08T15:18:54Z","closed_at":null,"merged_at":null,"merge_commit_sha":"8f7398860e2b37a54090293a87b6da277e77f2fb","assignee":{"login":"vtkstef","id":662716,"avatar_url":"https://avatars.githubusercontent.com/u/662716?v=3","gravatar_id":"","url":"https://api.github.com/users/vtkstef","html_url":"https://github.com/vtkstef","followers_url":"https://api.github.com/users/vtkstef/followers","following_url":"https://api.github.com/users/vtkstef/following{/other_user}","gists_url":"https://api.github.com/users/vtkstef/gists{/gist_id}","starred_url":"https://api.github.com/users/vtkstef/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vtkstef/subscriptions","organizations_url":"https://api.github.com/users/vtkstef/orgs","repos_url":"https://api.github.com/users/vtkstef/repos","events_url":"https://api.github.com/users/vtkstef/events{/privacy}","received_events_url":"https://api.github.com/users/vtkstef/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/VoltDB/voltdb/pulls/2513/commits","review_comments_url":"https://api.github.com/repos/VoltDB/voltdb/pulls/2513/comments","review_comment_url":"https://api.github.com/repos/VoltDB/voltdb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/VoltDB/voltdb/issues/2513/comments","statuses_url":"https://api.github.com/repos/VoltDB/voltdb/statuses/9ed0535ff26b9a1a0ddbdc8933b14e06bfd00a23","head":{"label":"VoltDB:ENG-8000","ref":"ENG-8000","sha":"9ed0535ff26b9a1a0ddbdc8933b14e06bfd00a23","user":{"login":"VoltDB","id":703910,"avatar_url":"https://avatars.githubusercontent.com/u/703910?v=3","gravatar_id":"","url":"https://api.github.com/users/VoltDB","html_url":"https://github.com/VoltDB","followers_url":"https://api.github.com/users/VoltDB/followers","following_url":"https://api.github.com/users/VoltDB/following{/other_user}","gists_url":"https://api.github.com/users/VoltDB/gists{/gist_id}","starred_url":"https://api.github.com/users/VoltDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VoltDB/subscriptions","organizations_url":"https://api.github.com/users/VoltDB/orgs","repos_url":"https://api.github.com/users/VoltDB/repos","events_url":"https://api.github.com/users/VoltDB/events{/privacy}","received_events_url":"https://api.github.com/users/VoltDB/received_events","type":"Organization","site_admin":false},"repo":{"id":2445970,"name":"voltdb","full_name":"VoltDB/voltdb","owner":{"login":"VoltDB","id":703910,"avatar_url":"https://avatars.githubusercontent.com/u/703910?v=3","gravatar_id":"","url":"https://api.github.com/users/VoltDB","html_url":"https://github.com/VoltDB","followers_url":"https://api.github.com/users/VoltDB/followers","following_url":"https://api.github.com/users/VoltDB/following{/other_user}","gists_url":"https://api.github.com/users/VoltDB/gists{/gist_id}","starred_url":"https://api.github.com/users/VoltDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VoltDB/subscriptions","organizations_url":"https://api.github.com/users/VoltDB/orgs","repos_url":"https://api.github.com/users/VoltDB/repos","events_url":"https://api.github.com/users/VoltDB/events{/privacy}","received_events_url":"https://api.github.com/users/VoltDB/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/VoltDB/voltdb","description":"VoltDB","fork":false,"url":"https://api.github.com/repos/VoltDB/voltdb","forks_url":"https://api.github.com/repos/VoltDB/voltdb/forks","keys_url":"https://api.github.com/repos/VoltDB/voltdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/VoltDB/voltdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/VoltDB/voltdb/teams","hooks_url":"https://api.github.com/repos/VoltDB/voltdb/hooks","issue_events_url":"https://api.github.com/repos/VoltDB/voltdb/issues/events{/number}","events_url":"https://api.github.com/repos/VoltDB/voltdb/events","assignees_url":"https://api.github.com/repos/VoltDB/voltdb/assignees{/user}","branches_url":"https://api.github.com/repos/VoltDB/voltdb/branches{/branch}","tags_url":"https://api.github.com/repos/VoltDB/voltdb/tags","blobs_url":"https://api.github.com/repos/VoltDB/voltdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/VoltDB/voltdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/VoltDB/voltdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/VoltDB/voltdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/VoltDB/voltdb/statuses/{sha}","languages_url":"https://api.github.com/repos/VoltDB/voltdb/languages","stargazers_url":"https://api.github.com/repos/VoltDB/voltdb/stargazers","contributors_url":"https://api.github.com/repos/VoltDB/voltdb/contributors","subscribers_url":"https://api.github.com/repos/VoltDB/voltdb/subscribers","subscription_url":"https://api.github.com/repos/VoltDB/voltdb/subscription","commits_url":"https://api.github.com/repos/VoltDB/voltdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/VoltDB/voltdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/VoltDB/voltdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/VoltDB/voltdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/VoltDB/voltdb/contents/{+path}","compare_url":"https://api.github.com/repos/VoltDB/voltdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/VoltDB/voltdb/merges","archive_url":"https://api.github.com/repos/VoltDB/voltdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/VoltDB/voltdb/downloads","issues_url":"https://api.github.com/repos/VoltDB/voltdb/issues{/number}","pulls_url":"https://api.github.com/repos/VoltDB/voltdb/pulls{/number}","milestones_url":"https://api.github.com/repos/VoltDB/voltdb/milestones{/number}","notifications_url":"https://api.github.com/repos/VoltDB/voltdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/VoltDB/voltdb/labels{/name}","releases_url":"https://api.github.com/repos/VoltDB/voltdb/releases{/id}","created_at":"2011-09-23T18:46:12Z","updated_at":"2015-07-03T07:17:53Z","pushed_at":"2015-07-08T15:08:41Z","git_url":"git://github.com/VoltDB/voltdb.git","ssh_url":"git@github.com:VoltDB/voltdb.git","clone_url":"https://github.com/VoltDB/voltdb.git","svn_url":"https://github.com/VoltDB/voltdb","homepage":"http://community.voltdb.com/","size":714192,"stargazers_count":588,"watchers_count":588,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":173,"mirror_url":null,"open_issues_count":32,"forks":173,"open_issues":32,"watchers":588,"default_branch":"master"}},"base":{"label":"VoltDB:master","ref":"master","sha":"2008db17e65cace2bf571e56032392a06e6906c0","user":{"login":"VoltDB","id":703910,"avatar_url":"https://avatars.githubusercontent.com/u/703910?v=3","gravatar_id":"","url":"https://api.github.com/users/VoltDB","html_url":"https://github.com/VoltDB","followers_url":"https://api.github.com/users/VoltDB/followers","following_url":"https://api.github.com/users/VoltDB/following{/other_user}","gists_url":"https://api.github.com/users/VoltDB/gists{/gist_id}","starred_url":"https://api.github.com/users/VoltDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VoltDB/subscriptions","organizations_url":"https://api.github.com/users/VoltDB/orgs","repos_url":"https://api.github.com/users/VoltDB/repos","events_url":"https://api.github.com/users/VoltDB/events{/privacy}","received_events_url":"https://api.github.com/users/VoltDB/received_events","type":"Organization","site_admin":false},"repo":{"id":2445970,"name":"voltdb","full_name":"VoltDB/voltdb","owner":{"login":"VoltDB","id":703910,"avatar_url":"https://avatars.githubusercontent.com/u/703910?v=3","gravatar_id":"","url":"https://api.github.com/users/VoltDB","html_url":"https://github.com/VoltDB","followers_url":"https://api.github.com/users/VoltDB/followers","following_url":"https://api.github.com/users/VoltDB/following{/other_user}","gists_url":"https://api.github.com/users/VoltDB/gists{/gist_id}","starred_url":"https://api.github.com/users/VoltDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VoltDB/subscriptions","organizations_url":"https://api.github.com/users/VoltDB/orgs","repos_url":"https://api.github.com/users/VoltDB/repos","events_url":"https://api.github.com/users/VoltDB/events{/privacy}","received_events_url":"https://api.github.com/users/VoltDB/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/VoltDB/voltdb","description":"VoltDB","fork":false,"url":"https://api.github.com/repos/VoltDB/voltdb","forks_url":"https://api.github.com/repos/VoltDB/voltdb/forks","keys_url":"https://api.github.com/repos/VoltDB/voltdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/VoltDB/voltdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/VoltDB/voltdb/teams","hooks_url":"https://api.github.com/repos/VoltDB/voltdb/hooks","issue_events_url":"https://api.github.com/repos/VoltDB/voltdb/issues/events{/number}","events_url":"https://api.github.com/repos/VoltDB/voltdb/events","assignees_url":"https://api.github.com/repos/VoltDB/voltdb/assignees{/user}","branches_url":"https://api.github.com/repos/VoltDB/voltdb/branches{/branch}","tags_url":"https://api.github.com/repos/VoltDB/voltdb/tags","blobs_url":"https://api.github.com/repos/VoltDB/voltdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/VoltDB/voltdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/VoltDB/voltdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/VoltDB/voltdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/VoltDB/voltdb/statuses/{sha}","languages_url":"https://api.github.com/repos/VoltDB/voltdb/languages","stargazers_url":"https://api.github.com/repos/VoltDB/voltdb/stargazers","contributors_url":"https://api.github.com/repos/VoltDB/voltdb/contributors","subscribers_url":"https://api.github.com/repos/VoltDB/voltdb/subscribers","subscription_url":"https://api.github.com/repos/VoltDB/voltdb/subscription","commits_url":"https://api.github.com/repos/VoltDB/voltdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/VoltDB/voltdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/VoltDB/voltdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/VoltDB/voltdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/VoltDB/voltdb/contents/{+path}","compare_url":"https://api.github.com/repos/VoltDB/voltdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/VoltDB/voltdb/merges","archive_url":"https://api.github.com/repos/VoltDB/voltdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/VoltDB/voltdb/downloads","issues_url":"https://api.github.com/repos/VoltDB/voltdb/issues{/number}","pulls_url":"https://api.github.com/repos/VoltDB/voltdb/pulls{/number}","milestones_url":"https://api.github.com/repos/VoltDB/voltdb/milestones{/number}","notifications_url":"https://api.github.com/repos/VoltDB/voltdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/VoltDB/voltdb/labels{/name}","releases_url":"https://api.github.com/repos/VoltDB/voltdb/releases{/id}","created_at":"2011-09-23T18:46:12Z","updated_at":"2015-07-03T07:17:53Z","pushed_at":"2015-07-08T15:08:41Z","git_url":"git://github.com/VoltDB/voltdb.git","ssh_url":"git@github.com:VoltDB/voltdb.git","clone_url":"https://github.com/VoltDB/voltdb.git","svn_url":"https://github.com/VoltDB/voltdb","homepage":"http://community.voltdb.com/","size":714192,"stargazers_count":588,"watchers_count":588,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":173,"mirror_url":null,"open_issues_count":32,"forks":173,"open_issues":32,"watchers":588,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/VoltDB/voltdb/pulls/2513"},"html":{"href":"https://github.com/VoltDB/voltdb/pull/2513"},"issue":{"href":"https://api.github.com/repos/VoltDB/voltdb/issues/2513"},"comments":{"href":"https://api.github.com/repos/VoltDB/voltdb/issues/2513/comments"},"review_comments":{"href":"https://api.github.com/repos/VoltDB/voltdb/pulls/2513/comments"},"review_comment":{"href":"https://api.github.com/repos/VoltDB/voltdb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/VoltDB/voltdb/pulls/2513/commits"},"statuses":{"href":"https://api.github.com/repos/VoltDB/voltdb/statuses/9ed0535ff26b9a1a0ddbdc8933b14e06bfd00a23"}}}}