Issue with Multiple grok patterns in single log ingest pipeline

[Jason_Paralta]

Hi All,

I am trying to read different log paths, this logs are of different patterns. I have single elastic template for this and have settings of default pipeline configured on same template. Now in log ingest pipeline while giving different patterns only one pattern is getting parsed and other is not.

Below is the pipeline:

PUT _ingest/pipeline/abc-ingest-pipeline
{
  "processors": [
    {
      "grok": {
        "field": "message",
        "patterns": [
          "%{TIMESTAMP_ISO8601:@timestamp:date}-%{WORD:host.name}-%{WORD:app.name}-%{WORD:request.id}-%{WORD:log.level}( *)-%{GREEDYDATA:log.logger}:%{NUMBER:log.origin.file.line:long} - (?<statement>(.|\n|\n)*)",
		  """%{TIMESTAMP_ISO8601:timestamp}-%{WORD:server_name}-%{DATA:python_module}-%{DATA:req_id}-%{LOGLEVEL:log_level}-%{DATA:method_name}:%{NUMBER:line_number} - %{WORD:usecase.uniqueId}\|%{WORD:abc.runningHost}\|%{WORD:abc.useCaseName}\|%{NUMBER:abc.pid}\|%{UNIXPATH:abc.logfile}\|%{TIMESTAMP_ISO8601:abc.startTime}\|%{TIMESTAMP_ISO8601:abc.endTime}\|%{DATA:abc.duration}\|%{TIMESTAMP_ISO8601:abc.nextEvent}\|%{NUMBER:abc.cpuUsage}\|%{NUMBER:abc.memoryUsage}\|%{NUMBER:abc.VM}\|%{NUMBER:abc.RM}\|%{NUMBER:abc.SM}\|%{DATA:abc.command}\|%{GREEDYDATA:abc.pythonPath}""" 
        ],
        "on_failure": [
          {
            "set": {
              "field": "error.message",
              "value": "{{ _ingest.on_failure_message }}"
            }
          }
        ]
      }
    },
    {
      "trim": {
        "field": "statement"
      }
    },
    {
      "set": {
        "field": "log.original",
        "value": "{{message}}"
      }
    },
    {
      "remove": {
        "field": [
          "message"
        ]
      }
    },
    {
      "rename": {
        "field": "host.hostname",
        "target_field": "agent.hostname",
		"ignore_failure": true
      }
    },
    {
      "rename": {
        "field": "host.ip",
        "target_field": "agent.ip",
		"ignore_failure": true
      }
    }
  ]
}

Can someone suggest how can I configure above pipeline correctly to parse different log patterns for different log files executed by filebeat.

Note: I am not using logstash for this.