{"action":"created","comment":{"url":"https://api.github.com/repos/mozilla-services/heka/pulls/comments/38379450","id":38379450,"diff_hunk":"@@ -0,0 +1,170 @@\n+-- This Source Code Form is subject to the terms of the Mozilla Public\n+-- License, v. 2.0. If a copy of the MPL was not distributed with this\n+-- file, You can obtain one at http://mozilla.org/MPL/2.0/.\n+\n+--[[\n+Parses a payload containing JSON.\n+\n+Config:\n+\n+- type (string, optional, default \"json\"):\n+    Sets the message 'Type' header to the specified value,\n+    will be overridden if Type config option is specified.\n+\n+- payload_keep (bool, optional, default false)\n+    Whether to preserve the original log line in the message payload.\n+\n+- map_fields (bool, optional, default false)\n+    Enables mapping of json fields to heka message fields.\n+\n+- Uuid (string, optional, default nil)\n+    string specifying json field to map to message Uuid,\n+    expects field value to be a string.\n+\n+- Type (string, optional, default nil)\n+    string specifying json field to map to to message Type,\n+    expects field value to be a string.\n+\n+- Logger (string, optional, default nil)\n+    string specifying json field to map to message Logger,\n+    expects field value to be a string.\n+\n+- Hostname (string, optional, default nil)\n+    string specifying json field to map to message Hostname,\n+    expects field value to be a string.\n+\n+- Severity (string, optional, default nil)\n+    string specifying json field to map to message Severity,\n+    expects field value to be numeric.\n+\n+- EnvVersion (string, optional, default nil)\n+    string specifying json field to map to message EnvVersion,\n+    expects field value to be numeric.\n+\n+- Pid (string, optional, default nil)\n+    string specifying json field to map to message Pid,\n+    expects field value to be numeric\n+\n+- Timestamp (string, optional, default nil)\n+    string specifying json field to map to message Timestamp,\n+    expects field value to be unix epoch timestamp in seconds.\n+\n+.. code-block:: javascript\n+\n+    {\n+      \"msg\": \"Start Request\",\n+      \"event\": \"artemis.web.ensure-running\",\n+      \"extra\": {\n+        \"workspace-id\": \"cN907xLngi\"\n+      },\n+      \"time\": \"2015-05-06T20:40:05.509926234Z\",\n+      \"severity\": 1\n+    }\n+\n+*Example Heka Configuration*\n+\n+.. code-block:: ini\n+\n+    [ArtemisLogInput]\n+    type = \"LogstreamerInput\"\n+    log_directory = \"/srv/artemis/current/logs\"\n+    file_match = 'artemis\\.log'\n+    decoder = \"JsonDecoder\"\n+\n+    [JsonDecoder]\n+    type = \"SandboxDecoder\"\n+    filename = \"lua_decoders/json.lua\"\n+\n+        [JsonDecoder.config]\n+        type = \"artemis\"\n+        payload_keep = true\n+        map_fields = true\n+        Severity = \"severity\"\n+\n+*Example Heka Message*\n+\n+:Timestamp: 2015-05-06 20:40:05 -0800 PST\n+:Type: artemis\n+:Hostname: test.example.com\n+:Pid: 0\n+:UUID: 8e414f01-9d7f-4a48-a5e1-ae92e5954df5\n+:Payload:\n+:EnvVersion:\n+:Severity: 1\n+:Fields:\n+    | name:\"msg\" value_type:STRING value_string:\"Start Request\"\n+    | name:\"event\" value_type:STRING value_string:\"artemis.web.ensure-running\"\n+    | name:\"extra.workspace-id\" value_type:STRING value_string:\"cN907xLngi\"\n+    | name:\"time\" value_type:STRING value_string:\"2015-05-06T20:40:05.509926234Z\"\n+--]]\n+\n+require \"cjson\"\n+local util = require(\"util\")\n+\n+local msg_type     = read_config(\"type\") or \"json\"\n+local payload_keep = read_config(\"payload_keep\")\n+local map_fields   = read_config(\"map_fields\")\n+\n+local field_map = {\n+    Uuid       = read_config(\"Uuid\"),\n+    Type       = read_config(\"Type\"),\n+    Logger     = read_config(\"Logger\"),\n+    Hostname   = read_config(\"Hostname\"),\n+    Severity   = read_config(\"Severity\"),\n+    EnvVersion = read_config(\"EnvVersion\"),\n+    Pid        = read_config(\"Pid\"),\n+    Timestamp  = read_config(\"Timestamp\")\n+}\n+\n+local field_type_map = {\n+    Uuid       = \"string\",\n+    Type       = \"string\",\n+    Logger     = \"string\",\n+    Hostname   = \"string\",\n+    Severity   = \"number\",\n+    EnvVersion = \"number\",\n+    Pid        = \"number\",\n+    Timestamp  = \"number\"\n+}\n+\n+local msg = {\n+    Type     = msg_type,\n+    Payload  = nil,\n+    Fields   = nil,\n+    Severity = nil\n+}\n+\n+function process_message()\n+    local ok, json = pcall(cjson.decode, read_message(\"Payload\"))\n+    if not ok then\n+        return -1\n+    end\n+\n+    -- keep payload, or not\n+    if payload_keep then\n+        msg.Payload = read_message(\"Payload\")\n+    end\n+\n+    -- map fields\n+    if map_fields then\n+        for F, f in pairs(field_map) do\n+            if type(json[f]) == field_type_map[F] then\n+                if F == \"Timestamp\" then\n+                    msg[F] = json[f] * 1e9\n+                else\n+                    msg[F] = json[f]\n+                end\n+                json[f] = nil\n+            end\n+        end\n+    end","path":"sandbox/lua/decoders/json.lua","position":null,"original_position":160,"commit_id":"a44a7f1f1055c7505ee91f617515bcdee2fede7f","original_commit_id":"c103b538c613d64c27e8b2f3fdfe0802f4595f2c","user":{"login":"rafrombrc","id":151364,"avatar_url":"https://avatars.githubusercontent.com/u/151364?v=3","gravatar_id":"","url":"https://api.github.com/users/rafrombrc","html_url":"https://github.com/rafrombrc","followers_url":"https://api.github.com/users/rafrombrc/followers","following_url":"https://api.github.com/users/rafrombrc/following{/other_user}","gists_url":"https://api.github.com/users/rafrombrc/gists{/gist_id}","starred_url":"https://api.github.com/users/rafrombrc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rafrombrc/subscriptions","organizations_url":"https://api.github.com/users/rafrombrc/orgs","repos_url":"https://api.github.com/users/rafrombrc/repos","events_url":"https://api.github.com/users/rafrombrc/events{/privacy}","received_events_url":"https://api.github.com/users/rafrombrc/received_events","type":"User","site_admin":false},"body":"If you really want to default to `type` when `Type` fails to return a value then it could be special cased in the loop, would only cost one string compare per field that is set. I don't feel strongly about what the behaviour should be here.","created_at":"2015-09-01T01:56:53Z","updated_at":"2015-09-01T01:56:53Z","html_url":"https://github.com/mozilla-services/heka/pull/1687#discussion_r38379450","pull_request_url":"https://api.github.com/repos/mozilla-services/heka/pulls/1687","_links":{"self":{"href":"https://api.github.com/repos/mozilla-services/heka/pulls/comments/38379450"},"html":{"href":"https://github.com/mozilla-services/heka/pull/1687#discussion_r38379450"},"pull_request":{"href":"https://api.github.com/repos/mozilla-services/heka/pulls/1687"}}},"pull_request":{"url":"https://api.github.com/repos/mozilla-services/heka/pulls/1687","id":42893636,"html_url":"https://github.com/mozilla-services/heka/pull/1687","diff_url":"https://github.com/mozilla-services/heka/pull/1687.diff","patch_url":"https://github.com/mozilla-services/heka/pull/1687.patch","issue_url":"https://api.github.com/repos/mozilla-services/heka/issues/1687","number":1687,"state":"open","locked":false,"title":"add generic json sandbox lua decoder (fixes #1651)","user":{"login":"nathwill","id":371597,"avatar_url":"https://avatars.githubusercontent.com/u/371597?v=3","gravatar_id":"","url":"https://api.github.com/users/nathwill","html_url":"https://github.com/nathwill","followers_url":"https://api.github.com/users/nathwill/followers","following_url":"https://api.github.com/users/nathwill/following{/other_user}","gists_url":"https://api.github.com/users/nathwill/gists{/gist_id}","starred_url":"https://api.github.com/users/nathwill/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nathwill/subscriptions","organizations_url":"https://api.github.com/users/nathwill/orgs","repos_url":"https://api.github.com/users/nathwill/repos","events_url":"https://api.github.com/users/nathwill/events{/privacy}","received_events_url":"https://api.github.com/users/nathwill/received_events","type":"User","site_admin":false},"body":"submitting new PR to clean up the git history. see #1653 for prior discussion. addresses #1651 ","created_at":"2015-08-20T05:47:13Z","updated_at":"2015-09-01T01:56:53Z","closed_at":null,"merged_at":null,"merge_commit_sha":"53cfe4915faa2aa4b4ed5ebd5a180adbf878641d","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/mozilla-services/heka/pulls/1687/commits","review_comments_url":"https://api.github.com/repos/mozilla-services/heka/pulls/1687/comments","review_comment_url":"https://api.github.com/repos/mozilla-services/heka/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mozilla-services/heka/issues/1687/comments","statuses_url":"https://api.github.com/repos/mozilla-services/heka/statuses/a44a7f1f1055c7505ee91f617515bcdee2fede7f","head":{"label":"nathwill:0.10-add-generic-json-decoder","ref":"0.10-add-generic-json-decoder","sha":"a44a7f1f1055c7505ee91f617515bcdee2fede7f","user":{"login":"nathwill","id":371597,"avatar_url":"https://avatars.githubusercontent.com/u/371597?v=3","gravatar_id":"","url":"https://api.github.com/users/nathwill","html_url":"https://github.com/nathwill","followers_url":"https://api.github.com/users/nathwill/followers","following_url":"https://api.github.com/users/nathwill/following{/other_user}","gists_url":"https://api.github.com/users/nathwill/gists{/gist_id}","starred_url":"https://api.github.com/users/nathwill/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nathwill/subscriptions","organizations_url":"https://api.github.com/users/nathwill/orgs","repos_url":"https://api.github.com/users/nathwill/repos","events_url":"https://api.github.com/users/nathwill/events{/privacy}","received_events_url":"https://api.github.com/users/nathwill/received_events","type":"User","site_admin":false},"repo":{"id":41077721,"name":"heka","full_name":"nathwill/heka","owner":{"login":"nathwill","id":371597,"avatar_url":"https://avatars.githubusercontent.com/u/371597?v=3","gravatar_id":"","url":"https://api.github.com/users/nathwill","html_url":"https://github.com/nathwill","followers_url":"https://api.github.com/users/nathwill/followers","following_url":"https://api.github.com/users/nathwill/following{/other_user}","gists_url":"https://api.github.com/users/nathwill/gists{/gist_id}","starred_url":"https://api.github.com/users/nathwill/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nathwill/subscriptions","organizations_url":"https://api.github.com/users/nathwill/orgs","repos_url":"https://api.github.com/users/nathwill/repos","events_url":"https://api.github.com/users/nathwill/events{/privacy}","received_events_url":"https://api.github.com/users/nathwill/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/nathwill/heka","description":"Data collection and processing made easy.","fork":true,"url":"https://api.github.com/repos/nathwill/heka","forks_url":"https://api.github.com/repos/nathwill/heka/forks","keys_url":"https://api.github.com/repos/nathwill/heka/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nathwill/heka/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nathwill/heka/teams","hooks_url":"https://api.github.com/repos/nathwill/heka/hooks","issue_events_url":"https://api.github.com/repos/nathwill/heka/issues/events{/number}","events_url":"https://api.github.com/repos/nathwill/heka/events","assignees_url":"https://api.github.com/repos/nathwill/heka/assignees{/user}","branches_url":"https://api.github.com/repos/nathwill/heka/branches{/branch}","tags_url":"https://api.github.com/repos/nathwill/heka/tags","blobs_url":"https://api.github.com/repos/nathwill/heka/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nathwill/heka/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nathwill/heka/git/refs{/sha}","trees_url":"https://api.github.com/repos/nathwill/heka/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nathwill/heka/statuses/{sha}","languages_url":"https://api.github.com/repos/nathwill/heka/languages","stargazers_url":"https://api.github.com/repos/nathwill/heka/stargazers","contributors_url":"https://api.github.com/repos/nathwill/heka/contributors","subscribers_url":"https://api.github.com/repos/nathwill/heka/subscribers","subscription_url":"https://api.github.com/repos/nathwill/heka/subscription","commits_url":"https://api.github.com/repos/nathwill/heka/commits{/sha}","git_commits_url":"https://api.github.com/repos/nathwill/heka/git/commits{/sha}","comments_url":"https://api.github.com/repos/nathwill/heka/comments{/number}","issue_comment_url":"https://api.github.com/repos/nathwill/heka/issues/comments{/number}","contents_url":"https://api.github.com/repos/nathwill/heka/contents/{+path}","compare_url":"https://api.github.com/repos/nathwill/heka/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nathwill/heka/merges","archive_url":"https://api.github.com/repos/nathwill/heka/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nathwill/heka/downloads","issues_url":"https://api.github.com/repos/nathwill/heka/issues{/number}","pulls_url":"https://api.github.com/repos/nathwill/heka/pulls{/number}","milestones_url":"https://api.github.com/repos/nathwill/heka/milestones{/number}","notifications_url":"https://api.github.com/repos/nathwill/heka/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nathwill/heka/labels{/name}","releases_url":"https://api.github.com/repos/nathwill/heka/releases{/id}","created_at":"2015-08-20T05:43:17Z","updated_at":"2015-08-20T05:43:18Z","pushed_at":"2015-09-01T01:56:01Z","git_url":"git://github.com/nathwill/heka.git","ssh_url":"git@github.com:nathwill/heka.git","clone_url":"https://github.com/nathwill/heka.git","svn_url":"https://github.com/nathwill/heka","homepage":"http://hekad.readthedocs.org/","size":11368,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"dev"}},"base":{"label":"mozilla-services:versions/0.10","ref":"versions/0.10","sha":"b381f02eab9aa635f36ae67061263c424868f627","user":{"login":"mozilla-services","id":1066228,"avatar_url":"https://avatars.githubusercontent.com/u/1066228?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla-services","html_url":"https://github.com/mozilla-services","followers_url":"https://api.github.com/users/mozilla-services/followers","following_url":"https://api.github.com/users/mozilla-services/following{/other_user}","gists_url":"https://api.github.com/users/mozilla-services/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla-services/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla-services/subscriptions","organizations_url":"https://api.github.com/users/mozilla-services/orgs","repos_url":"https://api.github.com/users/mozilla-services/repos","events_url":"https://api.github.com/users/mozilla-services/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla-services/received_events","type":"Organization","site_admin":false},"repo":{"id":6248212,"name":"heka","full_name":"mozilla-services/heka","owner":{"login":"mozilla-services","id":1066228,"avatar_url":"https://avatars.githubusercontent.com/u/1066228?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla-services","html_url":"https://github.com/mozilla-services","followers_url":"https://api.github.com/users/mozilla-services/followers","following_url":"https://api.github.com/users/mozilla-services/following{/other_user}","gists_url":"https://api.github.com/users/mozilla-services/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla-services/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla-services/subscriptions","organizations_url":"https://api.github.com/users/mozilla-services/orgs","repos_url":"https://api.github.com/users/mozilla-services/repos","events_url":"https://api.github.com/users/mozilla-services/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla-services/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla-services/heka","description":"Data collection and processing made easy.","fork":false,"url":"https://api.github.com/repos/mozilla-services/heka","forks_url":"https://api.github.com/repos/mozilla-services/heka/forks","keys_url":"https://api.github.com/repos/mozilla-services/heka/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla-services/heka/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla-services/heka/teams","hooks_url":"https://api.github.com/repos/mozilla-services/heka/hooks","issue_events_url":"https://api.github.com/repos/mozilla-services/heka/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla-services/heka/events","assignees_url":"https://api.github.com/repos/mozilla-services/heka/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla-services/heka/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla-services/heka/tags","blobs_url":"https://api.github.com/repos/mozilla-services/heka/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla-services/heka/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla-services/heka/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla-services/heka/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla-services/heka/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla-services/heka/languages","stargazers_url":"https://api.github.com/repos/mozilla-services/heka/stargazers","contributors_url":"https://api.github.com/repos/mozilla-services/heka/contributors","subscribers_url":"https://api.github.com/repos/mozilla-services/heka/subscribers","subscription_url":"https://api.github.com/repos/mozilla-services/heka/subscription","commits_url":"https://api.github.com/repos/mozilla-services/heka/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla-services/heka/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla-services/heka/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla-services/heka/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla-services/heka/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla-services/heka/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla-services/heka/merges","archive_url":"https://api.github.com/repos/mozilla-services/heka/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla-services/heka/downloads","issues_url":"https://api.github.com/repos/mozilla-services/heka/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla-services/heka/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla-services/heka/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla-services/heka/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla-services/heka/labels{/name}","releases_url":"https://api.github.com/repos/mozilla-services/heka/releases{/id}","created_at":"2012-10-16T17:20:51Z","updated_at":"2015-08-31T22:30:56Z","pushed_at":"2015-09-01T01:56:02Z","git_url":"git://github.com/mozilla-services/heka.git","ssh_url":"git@github.com:mozilla-services/heka.git","clone_url":"https://github.com/mozilla-services/heka.git","svn_url":"https://github.com/mozilla-services/heka","homepage":"http://hekad.readthedocs.org/","size":20672,"stargazers_count":2521,"watchers_count":2521,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":343,"mirror_url":null,"open_issues_count":161,"forks":343,"open_issues":161,"watchers":2521,"default_branch":"dev"}},"_links":{"self":{"href":"https://api.github.com/repos/mozilla-services/heka/pulls/1687"},"html":{"href":"https://github.com/mozilla-services/heka/pull/1687"},"issue":{"href":"https://api.github.com/repos/mozilla-services/heka/issues/1687"},"comments":{"href":"https://api.github.com/repos/mozilla-services/heka/issues/1687/comments"},"review_comments":{"href":"https://api.github.com/repos/mozilla-services/heka/pulls/1687/comments"},"review_comment":{"href":"https://api.github.com/repos/mozilla-services/heka/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mozilla-services/heka/pulls/1687/commits"},"statuses":{"href":"https://api.github.com/repos/mozilla-services/heka/statuses/a44a7f1f1055c7505ee91f617515bcdee2fede7f"}}}}