Logstash workers sky rocket to 100% CPU
<p>I have a problem on <code>logstash</code> where my workers jump to 100% CPU while activating 1 specific filter. Im currently using this input + filter (on which grok parses ok):</p>

<pre><code>input {
    redis {
        key =&gt; "logs_services_dyna_customer.lnxdev01"
        data_type =&gt; ['list']
        db =&gt; 8
        host =&gt; "kibana.test.local"
        type =&gt; "D_Customer"
        tags =&gt; ["d_local"]
        add_field =&gt; {
            server =&gt; "lnxdev01"
        }
        codec =&gt; "plain"
    }
}
filter {
    if [type] in ["D_Customer"] {
        mutate {
            gsub =&gt; ["message", "\r", ""]
        }
        grok {
          patterns_dir =&gt; "/etc/logstash/patterns/"
          match =&gt; ["message", "^Request Headers:\n(%{WORD:request_method} %{DATA:service_url} %{DATA}\n)?(Host: %{HOSTNAME:request_host}\n)?(Connection: %{WORD:request_connection}\n)?(User-Agent: %{DATA:user_agent}\n)?(Content-Type: %{DATA:content_type}\n)?(SOAPAction: \"%{DATA:request_action}\"\n)?(Content-Length: %{NUMBER:request_content_length}\n)?(Cookie: %{DATA}\n)?(:?X-START-DATE: %{APACHE_DATETIME:request_start_date})?\n+Request Body:\n%{GREEDIERDATA:request_body}\n+Response Headers:\n(HTTP/%{DATA}(?&lt;response_code&gt;\d{3}) %{DATA:response_message}\n)?(Cache-Control: %{DATA}\n)?(Content-Length: %{NUMBER:response_content_length}\n)?(Content-Type: %{DATA:response_content_type}\n)?(Server: %{DATA:response_server}\n)?(X-AspNet-Version: %{DATA}\n)?(X-Powered-By: %{DATA}\n)?(Date: %{APACHE_DATETIME:response_date}\n)?(Connection: %{WORD:response_connection}\n)?\n+Response Body:\n%{GREEDIERDATA:response_body}$"]
          add_field =&gt; ["category", "Service"]
        }
        date {
          match =&gt; ["request_start_date", "EEE, dd MMMM YYYY HH:mm:ss z"]
        }
        date {
          match =&gt; ["request_start_date", "EEE, dd MMMM YYYY HH:mm:ss z"]
          target =&gt; "request_start_date"
        }
        date {
          match =&gt; ["response_date", "EEE, dd MMMM YYYY HH:mm:ss z"]
          target =&gt; "response_date"
        }
    }
}
</code></pre>

<p>And below the log itself:</p>

<pre><code>Request Headers:
POST /Middleware.OIL.I01_SearchCustomer.Wcf/WcfService_Middleware_OIL_I01_SearchCustomer_App.svc HTTP/1.1
Host: w12vfdbiz101-t.dy.nl
Connection: close
User-Agent: PHP-SOAP/7.0.9-1+deb.sury.org~trusty+1
Content-Type: text/xml; charset=utf-8
SOAPAction: "SearchCustomer"
Content-Length: 1628
X-START-DATE: Mon, 07 Nov 2016 13:27:03 UTC



Request Body:
&lt;?xml version="1.0" encoding="UTF-8"?&gt;
&lt;SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns1="http://www.dy.eu/Customer/V1-0"&gt;&lt;SOAP-ENV:Body&gt;&lt;v1:SearchCustomerRequest xmlns:v1="http://www.dy.eu/Customer/V1-0" xmlns:v2="http://dy/CoreComponents/Schemas/dy/V2-0" xmlns:urn="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" xmlns:urn1="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" xmlns:v21="http://dy/CoreComponents/Schemas/dy/V2-0"&gt;
  &lt;v2:HeaderInfo&gt;
    &lt;v2:TransactionId&gt;58208127a1179&lt;/v2:TransactionId&gt;
    &lt;v2:UserId&gt;1&lt;/v2:UserId&gt;
    &lt;v2:ApplicationId&gt;Omnius&lt;/v2:ApplicationId&gt;
    &lt;v2:PartyName&gt;OGW&lt;/v2:PartyName&gt;
  &lt;/v2:HeaderInfo&gt;
  &lt;v1:SearchedAccount&gt;
    &lt;urn:PartyName&gt;
      &lt;urn1:Name/&gt;
    &lt;/urn:PartyName&gt;
  &lt;/v1:SearchedAccount&gt;
  &lt;v1:SearchedContact&gt;
    &lt;v21:Role&gt;
      &lt;v21:Person&gt;
        &lt;urn1:ID/&gt;
        &lt;urn1:FirstName&gt;Friedrich&lt;/urn1:FirstName&gt;
        &lt;urn1:FamilyName&gt;Muller&lt;/urn1:FamilyName&gt;
        &lt;urn:Contact&gt;
          &lt;urn1:Telephone/&gt;
          &lt;urn1:ElectronicMail/&gt;
        &lt;/urn:Contact&gt;
        &lt;urn:ResidenceAddress&gt;
          &lt;urn1:StreetName/&gt;
          &lt;urn1:BuildingNumber/&gt;
          &lt;urn1:CityName/&gt;
          &lt;urn1:PostalZone/&gt;
        &lt;/urn:ResidenceAddress&gt;
      &lt;/v21:Person&gt;
    &lt;/v21:Role&gt;
  &lt;/v1:SearchedContact&gt;
  &lt;v1:SearchedEquipment&gt;
    &lt;urn:ManufacturersItemIdentification&gt;
      &lt;urn1:ID/&gt;
    &lt;/urn:ManufacturersItemIdentification&gt;
  &lt;/v1:SearchedEquipment&gt;
&lt;/v1:SearchCustomerRequest&gt;&lt;/SOAP-ENV:Body&gt;&lt;/SOAP-ENV:Envelope&gt;



Response Headers:
HTTP/1.1 200 OK
Cache-Control: private
Content-Length: 2379
Content-Type: text/xml; charset=utf-8
Server: Microsoft-IIS/8.5
X-AspNet-Version: 4.0.30319
X-Powered-By: ASP.NET
Date: Mon, 07 Nov 2016 13:27:10 GMT
Connection: close


Response Body:
&lt;s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/"&gt;&lt;s:Body&gt;&lt;ns0:SearchCustomerResponse xmlns:ns0="http://www.dy.eu/Customer/V1-0" xmlns:ns1="http://uri.etsi.org/01903/v1.3.2#" xmlns:ns2="urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2" xmlns:ns3="urn:oasis:names:specification:ubl:schema:xsd:SignatureBasicComponents-2" xmlns:ns4="http://www.w3.org/2000/09/xmldsig#" xmlns:ns5="urn:oasis:names:specification:ubl:schema:xsd:QualifiedDataTypes-2" xmlns:ns6="urn:oasis:names:specification:ubl:schema:xsd:UnqualifiedDataTypes-2" xmlns:ns7="urn:un:unece:uncefact:data:specification:CoreComponentTypeSchemaModule:2" xmlns:ns8="urn:oasis:names:specification:ubl:schema:xsd:SignatureAggregateComponents-2" xmlns:ns9="http://uri.etsi.org/01903/v1.4.1#" xmlns:ns10="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" xmlns:ns11="http://Dy/CoreComponents/Schemas/Dy/V2-0" xmlns:ns12="urn:oasis:names:specification:ubl:schema:xsd:CommonSignatureComponents-2" xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"&gt;&lt;cbc:ReferenceID&gt;8&lt;/cbc:ReferenceID&gt;&lt;ns10:Status xsi:type="ns10:StatusType"&gt;&lt;cbc:StatusReasonCode/&gt;&lt;cbc:StatusReason/&gt;&lt;/ns10:Status&gt;&lt;ns11:Success&gt;true&lt;/ns11:Success&gt;&lt;ns0:ReturnedNumberOfRecords&gt;1&lt;/ns0:ReturnedNumberOfRecords&gt;&lt;ns11:Account&gt;&lt;ns10:PartyName&gt;&lt;cbc:Name&gt;CompanyName_0&lt;/cbc:Name&gt;&lt;/ns10:PartyName&gt;&lt;ns10:PostalAddress xsi:type="ns10:AddressType"&gt;&lt;cbc:StreetName&gt;Street0&lt;/cbc:StreetName&gt;&lt;cbc:BuildingNumber&gt;1&lt;/cbc:BuildingNumber&gt;&lt;cbc:CityName&gt;City0&lt;/cbc:CityName&gt;&lt;cbc:PostalZone&gt;12345&lt;/cbc:PostalZone&gt;&lt;/ns10:PostalAddress&gt;&lt;ns10:Contact xsi:type="ns10:ContactType"&gt;&lt;cbc:ID&gt;KS_12366&lt;/cbc:ID&gt;&lt;cbc:ElectronicMail/&gt;&lt;/ns10:Contact&gt;&lt;ns11:AccountCategory&gt;KD&lt;/ns11:AccountCategory&gt;&lt;ns11:AccountClass&gt;P&lt;/ns11:AccountClass&gt;&lt;ns11:ParentAccountNumber&gt;&lt;cbc:ID&gt;555&lt;/cbc:ID&gt;&lt;/ns11:ParentAccountNumber&gt;&lt;ns11:BankAccountNumber&gt;1234&lt;/ns11:BankAccountNumber&gt;&lt;ns11:Role&gt;&lt;ns11:Person&gt;&lt;cbc:FirstName&gt;Friedrich&lt;/cbc:FirstName&gt;&lt;cbc:FamilyName&gt;Müller&lt;/cbc:FamilyName&gt;&lt;cbc:BirthDate&gt;19700101&lt;/cbc:BirthDate&gt;&lt;ns11:ContactPhoneNumber&gt;&lt;ns11:PhoneNumber&gt;0612387777&lt;/ns11:PhoneNumber&gt;&lt;ns11:Type&gt;Mobile&lt;/ns11:Type&gt;&lt;/ns11:ContactPhoneNumber&gt;&lt;/ns11:Person&gt;&lt;/ns11:Role&gt;&lt;/ns11:Account&gt;&lt;/ns0:SearchCustomerResponse&gt;&lt;/s:Body&gt;&lt;/s:Envelope&gt;
</code></pre>

<p>So the question is, how come that logstash is spiking to 100% CPU and is this because of below filter? Should it be different or can it be more efficient?</p>

<p>Thanks
Pim</p>