Elastic Ingest with multiple grok processors

Elastic Stack Elasticsearch
1

Happy New Year everyone!

I am trying to configure an ingest pipeline and I want to apply grok to two distinct fields. I couldn't find a way to do this. Does anyone have a suggestion on how to accomplish this.

Thank you!

2

You can add 2 grok processors (one for each field) in the pipeline.

3

That was my first try.

I am using version 5.1 and it only takes into account the last grok.

"processors": [
    {
      "grok": {
        "field": "message",
        "patterns": [
          "%{STATUS:status};\\s+%{WORD:service};\\s+%{MSG:message}\\|\\s+%{WORD:key01}=%{NUMBER:value01}.*",
          "%{STATUS:status};\\s+%{WORD:service};\\s+%{GREEDYDATA:message}"
        ],
        "pattern_definitions": {
          "STATUS": "\\d+",
          "MSG": ".+?"
        }
      },
      "grok": {
          "field": "source",
          "patterns": [".+?%{TIMESTAMP:timestamp}.+"],
          "pattern_definitions" : {
            "TIMESTAMP" : "[0-9]+"}
        }
      ,
      "date" : {
        "field" : "timestamp",
        "formats" : ["UNIX_MS"]
      },
      "remove": {
        "field": "timestamp"
      }
    }
  ]
4

Can you check with verbose parameter if only one grok is actually applied?

See https://www.elastic.co/guide/en/elasticsearch/reference/current/simulate-pipeline-api.html#ingest-verbose-param

5

My processors:

"processors": [
    {
      "grok": {
        "field": "message",
        "patterns": [
          "%{STATUS:status};\\s+%{WORD:service};\\s+%{MSG:message}\\|\\s+%{WORD:key01}=%{NUMBER:value01}.*",
          "%{STATUS:status};\\s+%{WORD:service};\\s+%{GREEDYDATA:message}"
        ],
        "pattern_definitions": {
          "STATUS": "\\d+",
          "MSG": ".+?"
        }
      },
      "grok": {
        "field": "source",
        "patterns": [
          ".+?%{TIMESTAMP:timestamp}.+"
        ],
        "pattern_definitions": {
          "TIMESTAMP": "[0-9]+"
        }
      },
      "date": {
        "field": "timestamp",
        "formats": [
          "UNIX_MS"
        ]
      }
    }
  ]

My Doc:

  "docs": [
    {
      "_index": "filebeat-test-2017.01.03",
      "_type": "log",
      "_id": "AVlkvVA78rJBRllM9MQ-",
      "_source": {
        "source": "/opt/shared-nagios/nagios_file_1483453815014.txt",
        "message": "0; SERVICE_A; online | time=1s",
        "@timestamp": "2017-01-03T14:30:15.311Z"
        }
      ,
      "fields": {
        "@timestamp": [
          1483453815311
        ]
      },
      "sort": [
        1483453815311
      ]
    }
  ]

My result:

{
  "docs": [
    {
      "processor_results": [
        {
          "doc": {
            "_id": "AVlkvVA78rJBRllM9MQ-",
            "_index": "filebeat-test-2017.01.03",
            "_type": "log",
            "_source": {
              "@timestamp": "2017-01-03T14:30:15.311Z",
              "source": "/opt/shared-nagios/nagios_file_1483453815014.txt",
              "message": "0; SERVICE_A; online | time=1s",
              "timestamp": "1483453815014"
            },
            "_ingest": {
              "timestamp": "2017-01-03T18:10:23.455+0000"
            }
          }
        },
        {
          "doc": {
            "_id": "AVlkvVA78rJBRllM9MQ-",
            "_index": "filebeat-test-2017.01.03",
            "_type": "log",
            "_source": {
              "@timestamp": "2017-01-03T14:30:15.014Z",
              "source": "/opt/shared-nagios/nagios_file_1483453815014.txt",
              "message": "0; SERVICE_A; online | time=1s",
              "timestamp": "1483453815014"
            },
            "_ingest": {
              "timestamp": "2017-01-03T18:10:23.455+0000"
            }
          }
        }
      ]
    }
  ]
}
6

I think it should be:

{
   "processors":[
      {
         "grok":{
            "field":"message",
            "patterns":[
               "%{STATUS:status};\\s+%{WORD:service};\\s+%{MSG:message}\\|\\s+%{WORD:key01}=%{NUMBER:value01}.*",
               "%{STATUS:status};\\s+%{WORD:service};\\s+%{GREEDYDATA:message}"
            ],
            "pattern_definitions":{
               "STATUS":"\\d+",
               "MSG":".+?"
            }
         }
      },
      {
         "grok":{
            "field":"source",
            "patterns":[
               ".+?%{TIMESTAMP:timestamp}.+"
            ],
            "pattern_definitions":{
               "TIMESTAMP":"[0-9]+"
            }
         }
      },
      {
         "date":{
            "field":"timestamp",
            "formats":[
               "UNIX_MS"
            ]
         }
      }
   ]
}
7

OMG... I am such a noob....

Sorry for that!

Thank you!

8

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.