{"action":"created","comment":{"url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/comments/42836635","id":42836635,"diff_hunk":"@@ -0,0 +1,103 @@\n+package com.splunk.modinput.kafka;\n+\n+import java.lang.IllegalStateException;\n+import java.lang.Number;\n+import java.lang.NumberFormatException;\n+import java.nio.charset.Charset;\n+\n+import java.time.Instant;\n+import java.time.OffsetDateTime;\n+import java.time.format.DateTimeFormatter;\n+import java.time.format.DateTimeParseException;\n+import java.util.Map;\n+\n+import org.apache.log4j.Logger;\n+\n+import org.json.JSONObject;\n+\n+import com.splunk.modinput.kafka.KafkaModularInput.MessageReceiver;\n+\n+public class JSONBodyWithFieldExtraction extends AbstractMessageHandler {\n+\n+    protected static Logger logger = Logger.getLogger(JSONBodyWithFieldExtraction.class);\n+\n+    String charset = Charset.defaultCharset().name();\n+    String timefield = \"\";\n+    String hostfield = \"\";\n+    DateTimeFormatter timeformatPattern = null;\n+    int timeformatDivisor = 1;\n+\n+    @Override\n+    public void handleMessage(String topic, byte[] messageContents, MessageReceiver context)\n+            throws Exception {\n+\n+        String text = getMessageBody(messageContents, charset);\n+\n+        JSONObject json = new JSONObject(text);\n+\n+        Instant time = null;\n+        if (timefield.length() > 0) {\n+            String timeStr = json.getString(timefield);\n+            if (timeformatPattern != null) {\n+                try {\n+                    time = OffsetDateTime.parse(timeStr, timeformatPattern).toInstant();\n+                } catch (DateTimeParseException e) {\n+                    logger.warn(\"Unable to parse time format: \" + timeStr);\n+                    // fall back to arrival time\n+                }\n+            } else {\n+                try {\n+                    long timeNum = Long.parseLong(timeStr);\n+                    switch (timeformatDivisor) {\n+                        case 1:\n+                            time = Instant.ofEpochSecond(timeNum);\n+                            break;\n+                        case 1000:\n+                            time = Instant.ofEpochMilli(timeNum);\n+                            break;\n+                        default:\n+                            throw new IllegalStateException(\"Unsupported time format divisor: \" + timeformatDivisor);\n+                    }\n+                } catch (NumberFormatException e) {\n+                    logger.warn(\"Unable to parse time number: \" + timeStr);\n+                    // fall back to arrival time\n+                }\n+            }\n+        }\n+\n+        String host = \"\";\n+        if (hostfield.length() > 0) {\n+            host = json.getString(hostfield);\n+        }\n+\n+        transportMessage(\n+                text,\n+                (time != null)\n+                        ? time.getEpochSecond() + \".\" + String.format(\"%09d\", time.getNano())\n+                        : \"\",\n+                host,\n+                context.getStanzaName() + \":\" + topic);","path":"kafka/src/com/splunk/modinput/kafka/JSONBodyWithFieldExtraction.java","position":79,"original_position":79,"commit_id":"198e9711f08568b672478fe6e9e62611c1af15ed","original_commit_id":"198e9711f08568b672478fe6e9e62611c1af15ed","user":{"login":"llamahunter","id":843173,"avatar_url":"https://avatars.githubusercontent.com/u/843173?v=3","gravatar_id":"","url":"https://api.github.com/users/llamahunter","html_url":"https://github.com/llamahunter","followers_url":"https://api.github.com/users/llamahunter/followers","following_url":"https://api.github.com/users/llamahunter/following{/other_user}","gists_url":"https://api.github.com/users/llamahunter/gists{/gist_id}","starred_url":"https://api.github.com/users/llamahunter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/llamahunter/subscriptions","organizations_url":"https://api.github.com/users/llamahunter/orgs","repos_url":"https://api.github.com/users/llamahunter/repos","events_url":"https://api.github.com/users/llamahunter/events{/privacy}","received_events_url":"https://api.github.com/users/llamahunter/received_events","type":"User","site_admin":false},"body":"Hmm... thinking about this more, this seems possibly problematic, no?  If the topic is highly partitioned, is it going to get events from just one partition on that topic?  ","created_at":"2015-10-23T06:32:21Z","updated_at":"2015-10-23T06:32:21Z","html_url":"https://github.com/damiendallimore/SplunkModularInputsJavaFramework/pull/11#discussion_r42836635","pull_request_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/11","_links":{"self":{"href":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/comments/42836635"},"html":{"href":"https://github.com/damiendallimore/SplunkModularInputsJavaFramework/pull/11#discussion_r42836635"},"pull_request":{"href":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/11"}}},"pull_request":{"url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/11","id":48544314,"html_url":"https://github.com/damiendallimore/SplunkModularInputsJavaFramework/pull/11","diff_url":"https://github.com/damiendallimore/SplunkModularInputsJavaFramework/pull/11.diff","patch_url":"https://github.com/damiendallimore/SplunkModularInputsJavaFramework/pull/11.patch","issue_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/issues/11","number":11,"state":"open","locked":false,"title":"Improvements to json and kafka handling.","user":{"login":"llamahunter","id":843173,"avatar_url":"https://avatars.githubusercontent.com/u/843173?v=3","gravatar_id":"","url":"https://api.github.com/users/llamahunter","html_url":"https://github.com/llamahunter","followers_url":"https://api.github.com/users/llamahunter/followers","following_url":"https://api.github.com/users/llamahunter/following{/other_user}","gists_url":"https://api.github.com/users/llamahunter/gists{/gist_id}","starred_url":"https://api.github.com/users/llamahunter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/llamahunter/subscriptions","organizations_url":"https://api.github.com/users/llamahunter/orgs","repos_url":"https://api.github.com/users/llamahunter/repos","events_url":"https://api.github.com/users/llamahunter/events{/privacy}","received_events_url":"https://api.github.com/users/llamahunter/received_events","type":"User","site_admin":false},"body":"Improve escaping of kafka payloads.  Improve time parsing. Add host parsing. Add topic as source.","created_at":"2015-10-23T05:48:24Z","updated_at":"2015-10-23T06:32:21Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bab81613dc4d04a74f93b4632381ac2bf07b8785","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/11/commits","review_comments_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/11/comments","review_comment_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/comments{/number}","comments_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/issues/11/comments","statuses_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/statuses/198e9711f08568b672478fe6e9e62611c1af15ed","head":{"label":"TiVo:json-handling","ref":"json-handling","sha":"198e9711f08568b672478fe6e9e62611c1af15ed","user":{"login":"TiVo","id":4993382,"avatar_url":"https://avatars.githubusercontent.com/u/4993382?v=3","gravatar_id":"","url":"https://api.github.com/users/TiVo","html_url":"https://github.com/TiVo","followers_url":"https://api.github.com/users/TiVo/followers","following_url":"https://api.github.com/users/TiVo/following{/other_user}","gists_url":"https://api.github.com/users/TiVo/gists{/gist_id}","starred_url":"https://api.github.com/users/TiVo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TiVo/subscriptions","organizations_url":"https://api.github.com/users/TiVo/orgs","repos_url":"https://api.github.com/users/TiVo/repos","events_url":"https://api.github.com/users/TiVo/events{/privacy}","received_events_url":"https://api.github.com/users/TiVo/received_events","type":"Organization","site_admin":false},"repo":{"id":44778071,"name":"SplunkModularInputsJavaFramework","full_name":"TiVo/SplunkModularInputsJavaFramework","owner":{"login":"TiVo","id":4993382,"avatar_url":"https://avatars.githubusercontent.com/u/4993382?v=3","gravatar_id":"","url":"https://api.github.com/users/TiVo","html_url":"https://github.com/TiVo","followers_url":"https://api.github.com/users/TiVo/followers","following_url":"https://api.github.com/users/TiVo/following{/other_user}","gists_url":"https://api.github.com/users/TiVo/gists{/gist_id}","starred_url":"https://api.github.com/users/TiVo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TiVo/subscriptions","organizations_url":"https://api.github.com/users/TiVo/orgs","repos_url":"https://api.github.com/users/TiVo/repos","events_url":"https://api.github.com/users/TiVo/events{/privacy}","received_events_url":"https://api.github.com/users/TiVo/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/TiVo/SplunkModularInputsJavaFramework","description":"This is a framework for building Splunk Modular Inputs in Java","fork":true,"url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework","forks_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/forks","keys_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/keys{/key_id}","collaborators_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/teams","hooks_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/hooks","issue_events_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/issues/events{/number}","events_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/events","assignees_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/assignees{/user}","branches_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/branches{/branch}","tags_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/tags","blobs_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/git/refs{/sha}","trees_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/git/trees{/sha}","statuses_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/statuses/{sha}","languages_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/languages","stargazers_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/stargazers","contributors_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/contributors","subscribers_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/subscribers","subscription_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/subscription","commits_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/commits{/sha}","git_commits_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/git/commits{/sha}","comments_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/comments{/number}","issue_comment_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/issues/comments{/number}","contents_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/contents/{+path}","compare_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/compare/{base}...{head}","merges_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/merges","archive_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/downloads","issues_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/issues{/number}","pulls_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/pulls{/number}","milestones_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/milestones{/number}","notifications_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/labels{/name}","releases_url":"https://api.github.com/repos/TiVo/SplunkModularInputsJavaFramework/releases{/id}","created_at":"2015-10-22T22:57:21Z","updated_at":"2015-10-22T22:57:36Z","pushed_at":"2015-10-23T05:48:04Z","git_url":"git://github.com/TiVo/SplunkModularInputsJavaFramework.git","ssh_url":"git@github.com:TiVo/SplunkModularInputsJavaFramework.git","clone_url":"https://github.com/TiVo/SplunkModularInputsJavaFramework.git","svn_url":"https://github.com/TiVo/SplunkModularInputsJavaFramework","homepage":null,"size":898169,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"damiendallimore:master","ref":"master","sha":"33e3e3253f90883c3b9642cd883b713e5af71395","user":{"login":"damiendallimore","id":843258,"avatar_url":"https://avatars.githubusercontent.com/u/843258?v=3","gravatar_id":"","url":"https://api.github.com/users/damiendallimore","html_url":"https://github.com/damiendallimore","followers_url":"https://api.github.com/users/damiendallimore/followers","following_url":"https://api.github.com/users/damiendallimore/following{/other_user}","gists_url":"https://api.github.com/users/damiendallimore/gists{/gist_id}","starred_url":"https://api.github.com/users/damiendallimore/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/damiendallimore/subscriptions","organizations_url":"https://api.github.com/users/damiendallimore/orgs","repos_url":"https://api.github.com/users/damiendallimore/repos","events_url":"https://api.github.com/users/damiendallimore/events{/privacy}","received_events_url":"https://api.github.com/users/damiendallimore/received_events","type":"User","site_admin":false},"repo":{"id":6913286,"name":"SplunkModularInputsJavaFramework","full_name":"damiendallimore/SplunkModularInputsJavaFramework","owner":{"login":"damiendallimore","id":843258,"avatar_url":"https://avatars.githubusercontent.com/u/843258?v=3","gravatar_id":"","url":"https://api.github.com/users/damiendallimore","html_url":"https://github.com/damiendallimore","followers_url":"https://api.github.com/users/damiendallimore/followers","following_url":"https://api.github.com/users/damiendallimore/following{/other_user}","gists_url":"https://api.github.com/users/damiendallimore/gists{/gist_id}","starred_url":"https://api.github.com/users/damiendallimore/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/damiendallimore/subscriptions","organizations_url":"https://api.github.com/users/damiendallimore/orgs","repos_url":"https://api.github.com/users/damiendallimore/repos","events_url":"https://api.github.com/users/damiendallimore/events{/privacy}","received_events_url":"https://api.github.com/users/damiendallimore/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/damiendallimore/SplunkModularInputsJavaFramework","description":"This is a framework for building Splunk Modular Inputs in Java","fork":false,"url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework","forks_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/forks","keys_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/keys{/key_id}","collaborators_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/teams","hooks_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/hooks","issue_events_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/issues/events{/number}","events_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/events","assignees_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/assignees{/user}","branches_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/branches{/branch}","tags_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/tags","blobs_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/git/refs{/sha}","trees_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/git/trees{/sha}","statuses_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/statuses/{sha}","languages_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/languages","stargazers_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/stargazers","contributors_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/contributors","subscribers_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/subscribers","subscription_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/subscription","commits_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/commits{/sha}","git_commits_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/git/commits{/sha}","comments_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/comments{/number}","issue_comment_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/issues/comments{/number}","contents_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/contents/{+path}","compare_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/compare/{base}...{head}","merges_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/merges","archive_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/downloads","issues_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/issues{/number}","pulls_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls{/number}","milestones_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/milestones{/number}","notifications_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/labels{/name}","releases_url":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/releases{/id}","created_at":"2012-11-29T01:34:11Z","updated_at":"2015-10-08T05:25:57Z","pushed_at":"2015-10-23T05:48:24Z","git_url":"git://github.com/damiendallimore/SplunkModularInputsJavaFramework.git","ssh_url":"git@github.com:damiendallimore/SplunkModularInputsJavaFramework.git","clone_url":"https://github.com/damiendallimore/SplunkModularInputsJavaFramework.git","svn_url":"https://github.com/damiendallimore/SplunkModularInputsJavaFramework","homepage":null,"size":898169,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":10,"forks":2,"open_issues":10,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/11"},"html":{"href":"https://github.com/damiendallimore/SplunkModularInputsJavaFramework/pull/11"},"issue":{"href":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/issues/11"},"comments":{"href":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/issues/11/comments"},"review_comments":{"href":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/11/comments"},"review_comment":{"href":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/pulls/11/commits"},"statuses":{"href":"https://api.github.com/repos/damiendallimore/SplunkModularInputsJavaFramework/statuses/198e9711f08568b672478fe6e9e62611c1af15ed"}}}}