{"action":"created","comment":{"url":"https://api.github.com/repos/rcbops/rpc-extras/pulls/comments/31355250","id":31355250,"diff_hunk":"@@ -1,72 +1,8 @@\n filter {\n-  #---------------------------------------------------------------------------\n-  # Parse and tag openstack logs\n-  #\n-  grok {\n-    match => { \"@message\" => \"%{TIMESTAMP_ISO8601:os_timestamp} %{POSINT:os_program_pid} %{AUDITLOGLEVEL:os_level} %{OPENSTACK_FUNCTION:openstack_function} %{OPENSTACK_CODE:openstack_code} %{GREEDYDATA:openstack_message}\" }\n-    remove_tag => [ \"_grokparsefailure\" ]\n-    add_tag => [ \"openstack\" ]\n-  }\n-  #---------------------------------------------------------------------------\n-  # Parse and tag syslog logs\n-  # i.e.:\n-  # 1) Nov 10 23:18:55 nosthame proxy-server: User: test uses token AUTH_tkc9512a6c98bb4ccf84e17e453347ebf8 (trans_id tx7872ea1e11eb48d989735-00546147df)\n-  # 2) Jan  8 11:31:15 aio1_swift_proxy_container-cfcbc099 proxy-server: 172.29.237.150 172.29.236.100 08/Jan/2015/11/31/15 GET /v1/AUTH_648fa418490d41c6ab9b45ad1ca7918d%3Fformat%3Djson%26limit%3D1001 HTTP/1.0 200 - python-swiftclient-2.2.0 5cf068b1df774293... - 2 - tx5e26af272b6e4ce3950de-0054ae6a83 - 0.1823 - - 1420716675.014713049 1420716675.197021008\n-  #\n-  grok {\n-    match => { \"@message\" => \"^%{SYSLOGTIMESTAMP:os_timestamp} (?:%{SYSLOGFACILITY} )?%{DATA:@source_host} %{DATA:@fields.program}(?:\\[%{POSINT:@fields.processid}\\])?: %{GREEDYDATA:openstack_message}\" }\n-    add_tag => [ \"syslog\" ]\n-    overwrite => [ \"os_timestamp\", \"@source_host\", \"@fields[program]\", \"@fields[processid]\" ]\n-    remove_tag => [ \"_grokparsefailure\" ]\n-    tag_on_failure => []\n-  }\n-  if \"syslog\" in [tags] {\n-    mutate {\n-      add_field => {\n-         \"os_program\" => \"%{@source_host}_%{@fields.program}\"\n-         \"os_level\" => \"%{@fields[severity]}\"\n-      }\n-    }\n-  }\n-  #---------------------------------------------------------------------------\n-  # Populate the os_program field using the container name, if applicable.\n-  #\n-  grok {\n-    match => { \"@fields[program]\" => \"%{CONTAINER_STRIP:os_program}\" }\n-    overwrite => [ \"os_program\" ]\n-    remove_tag => [ \"_grokparsefailure\" ]\n-    tag_on_failure => []\n-  }\n-  # cater for nova-compute hosts\n-  if \"openstack\" in [tags] and ![os_program] {\n-    mutate {\n-      add_field => {\n-        \"os_program\" => \"%{@fields[program]}\"\n-      }\n-    }\n-  }\n-  #---------------------------------------------------------------------------\n-  # If none of the above worked, treat it generically\n-  #\n-  if \"_grokparsefailure\" in [tags] {\n-    mutate {\n-      add_field => {\n-        \"os_program\" => \"%{@source_host}_%{@fields[program]}\"\n-        \"openstack_message\" => \"%{@message}\"\n-        \"os_level\" => \"%{@fields[severity]}\"\n-      }\n-      add_tag => [ \"infrastructure\" ]\n-      remove_tag => [ \"_grokparsefailure\" ]\n-    }\n-  }\n-  #---------------------------------------------------------------------------\n-  # If it's a message from rsyslog specifically, override some of the data\n-  # as rsyslog tends to report program names like rsyslog0 and rsyslog-2359.\n-  # FIXME (this doesn't seem to do the os_program mutation properly for some reason)\n-  if [os_program] =~ /rsyslogd/ {\n-    mutate {\n-      replace => [ \"os_program\", \"%{@source_host}_rsyslogd\" ]\n-      replace => [ \"os_level\", \"%{@fields[severity]}\" ]\n+  if \"oslofmt\" in [tags] {","path":"playbooks/roles/logstash/templates/02-general.conf","position":70,"original_position":70,"commit_id":"bb4952ddc7965e309dd7b3a8020fcbcd35146aa6","original_commit_id":"bb4952ddc7965e309dd7b3a8020fcbcd35146aa6","user":{"login":"d34dh0r53","id":248299,"avatar_url":"https://avatars.githubusercontent.com/u/248299?v=3","gravatar_id":"","url":"https://api.github.com/users/d34dh0r53","html_url":"https://github.com/d34dh0r53","followers_url":"https://api.github.com/users/d34dh0r53/followers","following_url":"https://api.github.com/users/d34dh0r53/following{/other_user}","gists_url":"https://api.github.com/users/d34dh0r53/gists{/gist_id}","starred_url":"https://api.github.com/users/d34dh0r53/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/d34dh0r53/subscriptions","organizations_url":"https://api.github.com/users/d34dh0r53/orgs","repos_url":"https://api.github.com/users/d34dh0r53/repos","events_url":"https://api.github.com/users/d34dh0r53/events{/privacy}","received_events_url":"https://api.github.com/users/d34dh0r53/received_events","type":"User","site_admin":false},"body":"This makes logstash parse the oslo portion of the log message properly.","created_at":"2015-05-29T19:07:43Z","updated_at":"2015-05-29T19:07:43Z","html_url":"https://github.com/rcbops/rpc-extras/pull/123#discussion_r31355250","pull_request_url":"https://api.github.com/repos/rcbops/rpc-extras/pulls/123","_links":{"self":{"href":"https://api.github.com/repos/rcbops/rpc-extras/pulls/comments/31355250"},"html":{"href":"https://github.com/rcbops/rpc-extras/pull/123#discussion_r31355250"},"pull_request":{"href":"https://api.github.com/repos/rcbops/rpc-extras/pulls/123"}}},"pull_request":{"url":"https://api.github.com/repos/rcbops/rpc-extras/pulls/123","id":36521149,"html_url":"https://github.com/rcbops/rpc-extras/pull/123","diff_url":"https://github.com/rcbops/rpc-extras/pull/123.diff","patch_url":"https://github.com/rcbops/rpc-extras/pull/123.patch","issue_url":"https://api.github.com/repos/rcbops/rpc-extras/issues/123","number":123,"state":"open","locked":false,"title":"Add distributed, multi-line log shipping","user":{"login":"odyssey4me","id":2152106,"avatar_url":"https://avatars.githubusercontent.com/u/2152106?v=3","gravatar_id":"","url":"https://api.github.com/users/odyssey4me","html_url":"https://github.com/odyssey4me","followers_url":"https://api.github.com/users/odyssey4me/followers","following_url":"https://api.github.com/users/odyssey4me/following{/other_user}","gists_url":"https://api.github.com/users/odyssey4me/gists{/gist_id}","starred_url":"https://api.github.com/users/odyssey4me/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/odyssey4me/subscriptions","organizations_url":"https://api.github.com/users/odyssey4me/orgs","repos_url":"https://api.github.com/users/odyssey4me/repos","events_url":"https://api.github.com/users/odyssey4me/events{/privacy}","received_events_url":"https://api.github.com/users/odyssey4me/received_events","type":"User","site_admin":false},"body":"This patch adds the beaver role and playbook to rpc-extras. Beaver is a log\r\nshipper which allows log shipping to be configured to squash multi-line log\r\nentries into one event prior to shipping them to logstash. It also enables\r\nthe addition of tags and fields at the source. Each of these capabilities\r\ndistributes some of the processing currently handled by logstash to the\r\nclients, improving the ability for RPC to handle logging at a greater scale.\r\n\r\nBeaver is now set to be the log shipper used by default for RPC and is\r\nexecuted within the setup-logging playbook.\r\n\r\nThis patch revises the logstash role to be a lot simpler, primarily to improve\r\nlog processing performance but also to be more standard with similar efforts\r\nelsewhere in the OpenStack community.\r\n\r\nTo ensure that the Kibana dashboard reflects the changes in the logstash\r\nfilters, it is also adjusted. The default event window has also been changed\r\nfrom 5m to 15m.\r\n\r\nChanges affecting upgrades from previous versions:\r\n - Logstash now listens for normal syslog messages on port 5544 instead of\r\n   json formatted syslog messages.\r\n - The user variable logstash_port is now logstash_syslog_port in order to\r\n   differentiate it from the tcp port which beaver uses.\r\n - For easier troubleshooting any messages coming into logstash via syslog\r\n   or tcp are tagged as logstash-input-syslog and logstash-input-tcp\r\n   respectively.\r\n - Logstash field changes to standardise with other OpenStack Logstash\r\n   efforts made in the community:\r\n    - httpversion > http_ver\r\n    - @message > message\r\n    - openstack_code > (removed)\r\n    - openstack_function > module\r\n    - os_level > loglevel\r\n    - os_program_path > (removed)\r\n    - os_program_pid > pid\r\n    - os_timestamp > @timestamp / req_data & req_time\r\n    - request > url_path\r\n    - request_id > requestid\r\n    - request_user_id > user_id\r\n    - request_tenant_id > tenant\r\n - New log files processed:\r\n    - auth\r\n    - beaver\r\n    - elasticsearch\r\n    - logstash\r\n    - mysql\r\n    - rabbitmq\r\n - Multline processing enabled for:\r\n    - elasticsearch\r\n    - logstash\r\n    - mysql-slow log\r\n    - OpenStack TRACE log lines\r\n    - rabbitmq","created_at":"2015-05-29T16:17:00Z","updated_at":"2015-05-29T19:07:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ece006bd58e7e5cd149143ec7e1c504de0151a6d","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/rcbops/rpc-extras/pulls/123/commits","review_comments_url":"https://api.github.com/repos/rcbops/rpc-extras/pulls/123/comments","review_comment_url":"https://api.github.com/repos/rcbops/rpc-extras/pulls/comments{/number}","comments_url":"https://api.github.com/repos/rcbops/rpc-extras/issues/123/comments","statuses_url":"https://api.github.com/repos/rcbops/rpc-extras/statuses/bb4952ddc7965e309dd7b3a8020fcbcd35146aa6","head":{"label":"odyssey4me:log_shipping","ref":"log_shipping","sha":"bb4952ddc7965e309dd7b3a8020fcbcd35146aa6","user":{"login":"odyssey4me","id":2152106,"avatar_url":"https://avatars.githubusercontent.com/u/2152106?v=3","gravatar_id":"","url":"https://api.github.com/users/odyssey4me","html_url":"https://github.com/odyssey4me","followers_url":"https://api.github.com/users/odyssey4me/followers","following_url":"https://api.github.com/users/odyssey4me/following{/other_user}","gists_url":"https://api.github.com/users/odyssey4me/gists{/gist_id}","starred_url":"https://api.github.com/users/odyssey4me/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/odyssey4me/subscriptions","organizations_url":"https://api.github.com/users/odyssey4me/orgs","repos_url":"https://api.github.com/users/odyssey4me/repos","events_url":"https://api.github.com/users/odyssey4me/events{/privacy}","received_events_url":"https://api.github.com/users/odyssey4me/received_events","type":"User","site_admin":false},"repo":{"id":33598997,"name":"rpc-extras","full_name":"odyssey4me/rpc-extras","owner":{"login":"odyssey4me","id":2152106,"avatar_url":"https://avatars.githubusercontent.com/u/2152106?v=3","gravatar_id":"","url":"https://api.github.com/users/odyssey4me","html_url":"https://github.com/odyssey4me","followers_url":"https://api.github.com/users/odyssey4me/followers","following_url":"https://api.github.com/users/odyssey4me/following{/other_user}","gists_url":"https://api.github.com/users/odyssey4me/gists{/gist_id}","starred_url":"https://api.github.com/users/odyssey4me/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/odyssey4me/subscriptions","organizations_url":"https://api.github.com/users/odyssey4me/orgs","repos_url":"https://api.github.com/users/odyssey4me/repos","events_url":"https://api.github.com/users/odyssey4me/events{/privacy}","received_events_url":"https://api.github.com/users/odyssey4me/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/odyssey4me/rpc-extras","description":"Optional add-ons for Rackspace Private Cloud","fork":true,"url":"https://api.github.com/repos/odyssey4me/rpc-extras","forks_url":"https://api.github.com/repos/odyssey4me/rpc-extras/forks","keys_url":"https://api.github.com/repos/odyssey4me/rpc-extras/keys{/key_id}","collaborators_url":"https://api.github.com/repos/odyssey4me/rpc-extras/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/odyssey4me/rpc-extras/teams","hooks_url":"https://api.github.com/repos/odyssey4me/rpc-extras/hooks","issue_events_url":"https://api.github.com/repos/odyssey4me/rpc-extras/issues/events{/number}","events_url":"https://api.github.com/repos/odyssey4me/rpc-extras/events","assignees_url":"https://api.github.com/repos/odyssey4me/rpc-extras/assignees{/user}","branches_url":"https://api.github.com/repos/odyssey4me/rpc-extras/branches{/branch}","tags_url":"https://api.github.com/repos/odyssey4me/rpc-extras/tags","blobs_url":"https://api.github.com/repos/odyssey4me/rpc-extras/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/odyssey4me/rpc-extras/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/odyssey4me/rpc-extras/git/refs{/sha}","trees_url":"https://api.github.com/repos/odyssey4me/rpc-extras/git/trees{/sha}","statuses_url":"https://api.github.com/repos/odyssey4me/rpc-extras/statuses/{sha}","languages_url":"https://api.github.com/repos/odyssey4me/rpc-extras/languages","stargazers_url":"https://api.github.com/repos/odyssey4me/rpc-extras/stargazers","contributors_url":"https://api.github.com/repos/odyssey4me/rpc-extras/contributors","subscribers_url":"https://api.github.com/repos/odyssey4me/rpc-extras/subscribers","subscription_url":"https://api.github.com/repos/odyssey4me/rpc-extras/subscription","commits_url":"https://api.github.com/repos/odyssey4me/rpc-extras/commits{/sha}","git_commits_url":"https://api.github.com/repos/odyssey4me/rpc-extras/git/commits{/sha}","comments_url":"https://api.github.com/repos/odyssey4me/rpc-extras/comments{/number}","issue_comment_url":"https://api.github.com/repos/odyssey4me/rpc-extras/issues/comments{/number}","contents_url":"https://api.github.com/repos/odyssey4me/rpc-extras/contents/{+path}","compare_url":"https://api.github.com/repos/odyssey4me/rpc-extras/compare/{base}...{head}","merges_url":"https://api.github.com/repos/odyssey4me/rpc-extras/merges","archive_url":"https://api.github.com/repos/odyssey4me/rpc-extras/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/odyssey4me/rpc-extras/downloads","issues_url":"https://api.github.com/repos/odyssey4me/rpc-extras/issues{/number}","pulls_url":"https://api.github.com/repos/odyssey4me/rpc-extras/pulls{/number}","milestones_url":"https://api.github.com/repos/odyssey4me/rpc-extras/milestones{/number}","notifications_url":"https://api.github.com/repos/odyssey4me/rpc-extras/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/odyssey4me/rpc-extras/labels{/name}","releases_url":"https://api.github.com/repos/odyssey4me/rpc-extras/releases{/id}","created_at":"2015-04-08T10:01:05Z","updated_at":"2015-05-28T20:13:18Z","pushed_at":"2015-05-29T16:16:30Z","git_url":"git://github.com/odyssey4me/rpc-extras.git","ssh_url":"git@github.com:odyssey4me/rpc-extras.git","clone_url":"https://github.com/odyssey4me/rpc-extras.git","svn_url":"https://github.com/odyssey4me/rpc-extras","homepage":null,"size":621,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"rcbops:master","ref":"master","sha":"ef5881399e43e74072eb83e78425229237447a91","user":{"login":"rcbops","id":1153548,"avatar_url":"https://avatars.githubusercontent.com/u/1153548?v=3","gravatar_id":"","url":"https://api.github.com/users/rcbops","html_url":"https://github.com/rcbops","followers_url":"https://api.github.com/users/rcbops/followers","following_url":"https://api.github.com/users/rcbops/following{/other_user}","gists_url":"https://api.github.com/users/rcbops/gists{/gist_id}","starred_url":"https://api.github.com/users/rcbops/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rcbops/subscriptions","organizations_url":"https://api.github.com/users/rcbops/orgs","repos_url":"https://api.github.com/users/rcbops/repos","events_url":"https://api.github.com/users/rcbops/events{/privacy}","received_events_url":"https://api.github.com/users/rcbops/received_events","type":"Organization","site_admin":false},"repo":{"id":30100099,"name":"rpc-extras","full_name":"rcbops/rpc-extras","owner":{"login":"rcbops","id":1153548,"avatar_url":"https://avatars.githubusercontent.com/u/1153548?v=3","gravatar_id":"","url":"https://api.github.com/users/rcbops","html_url":"https://github.com/rcbops","followers_url":"https://api.github.com/users/rcbops/followers","following_url":"https://api.github.com/users/rcbops/following{/other_user}","gists_url":"https://api.github.com/users/rcbops/gists{/gist_id}","starred_url":"https://api.github.com/users/rcbops/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rcbops/subscriptions","organizations_url":"https://api.github.com/users/rcbops/orgs","repos_url":"https://api.github.com/users/rcbops/repos","events_url":"https://api.github.com/users/rcbops/events{/privacy}","received_events_url":"https://api.github.com/users/rcbops/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/rcbops/rpc-extras","description":"Optional add-ons for Rackspace Private Cloud","fork":false,"url":"https://api.github.com/repos/rcbops/rpc-extras","forks_url":"https://api.github.com/repos/rcbops/rpc-extras/forks","keys_url":"https://api.github.com/repos/rcbops/rpc-extras/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rcbops/rpc-extras/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rcbops/rpc-extras/teams","hooks_url":"https://api.github.com/repos/rcbops/rpc-extras/hooks","issue_events_url":"https://api.github.com/repos/rcbops/rpc-extras/issues/events{/number}","events_url":"https://api.github.com/repos/rcbops/rpc-extras/events","assignees_url":"https://api.github.com/repos/rcbops/rpc-extras/assignees{/user}","branches_url":"https://api.github.com/repos/rcbops/rpc-extras/branches{/branch}","tags_url":"https://api.github.com/repos/rcbops/rpc-extras/tags","blobs_url":"https://api.github.com/repos/rcbops/rpc-extras/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rcbops/rpc-extras/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rcbops/rpc-extras/git/refs{/sha}","trees_url":"https://api.github.com/repos/rcbops/rpc-extras/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rcbops/rpc-extras/statuses/{sha}","languages_url":"https://api.github.com/repos/rcbops/rpc-extras/languages","stargazers_url":"https://api.github.com/repos/rcbops/rpc-extras/stargazers","contributors_url":"https://api.github.com/repos/rcbops/rpc-extras/contributors","subscribers_url":"https://api.github.com/repos/rcbops/rpc-extras/subscribers","subscription_url":"https://api.github.com/repos/rcbops/rpc-extras/subscription","commits_url":"https://api.github.com/repos/rcbops/rpc-extras/commits{/sha}","git_commits_url":"https://api.github.com/repos/rcbops/rpc-extras/git/commits{/sha}","comments_url":"https://api.github.com/repos/rcbops/rpc-extras/comments{/number}","issue_comment_url":"https://api.github.com/repos/rcbops/rpc-extras/issues/comments{/number}","contents_url":"https://api.github.com/repos/rcbops/rpc-extras/contents/{+path}","compare_url":"https://api.github.com/repos/rcbops/rpc-extras/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rcbops/rpc-extras/merges","archive_url":"https://api.github.com/repos/rcbops/rpc-extras/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rcbops/rpc-extras/downloads","issues_url":"https://api.github.com/repos/rcbops/rpc-extras/issues{/number}","pulls_url":"https://api.github.com/repos/rcbops/rpc-extras/pulls{/number}","milestones_url":"https://api.github.com/repos/rcbops/rpc-extras/milestones{/number}","notifications_url":"https://api.github.com/repos/rcbops/rpc-extras/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rcbops/rpc-extras/labels{/name}","releases_url":"https://api.github.com/repos/rcbops/rpc-extras/releases{/id}","created_at":"2015-01-31T02:51:12Z","updated_at":"2015-05-26T23:42:29Z","pushed_at":"2015-05-29T18:32:24Z","git_url":"git://github.com/rcbops/rpc-extras.git","ssh_url":"git@github.com:rcbops/rpc-extras.git","clone_url":"https://github.com/rcbops/rpc-extras.git","svn_url":"https://github.com/rcbops/rpc-extras","homepage":null,"size":1894,"stargazers_count":3,"watchers_count":3,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":17,"mirror_url":null,"open_issues_count":31,"forks":17,"open_issues":31,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/rcbops/rpc-extras/pulls/123"},"html":{"href":"https://github.com/rcbops/rpc-extras/pull/123"},"issue":{"href":"https://api.github.com/repos/rcbops/rpc-extras/issues/123"},"comments":{"href":"https://api.github.com/repos/rcbops/rpc-extras/issues/123/comments"},"review_comments":{"href":"https://api.github.com/repos/rcbops/rpc-extras/pulls/123/comments"},"review_comment":{"href":"https://api.github.com/repos/rcbops/rpc-extras/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/rcbops/rpc-extras/pulls/123/commits"},"statuses":{"href":"https://api.github.com/repos/rcbops/rpc-extras/statuses/bb4952ddc7965e309dd7b3a8020fcbcd35146aa6"}}}}