Hello elastic - Multiple Pipelines

Elastic Stack Elasticsearch
,
1 
DELETE index
PUT index/_doc/1
{
  "foo": "bar"
}
GET index/_search
{
  "query": {
    "match": {
      "foo": "bar"
    }
  }
}
2

G'Day. What can we help with?

3

Hello warkolm

Can multiple pipelines be configured for one log message in Filebeat. yml

5

No only a single pipeline per input type.

Here are some thoughts on a solution.

6

hi
How do I write multiple Groks to parse a log file

7

You should really read this Page / Doc entirely Here

See the Docs Here

You can put multiple Groks in a single grok processor

Example

POST _ingest/pipeline/_simulate
{
  "pipeline": {
    "description": "...",
    "processors": [
      {
        "grok": {
          "field": "message",
          "patterns": [
            "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes:int} %{NUMBER:duration:double}",
            "%{IP:client} - %{WORD:level} - %{NUMBER:bytes:int} %{DATA:message_detail}"
          ]
        }
      }
    ]
  },
  "docs": [
    {
      "_source": {
        "message": "55.3.244.1 GET /index.html 15824 0.043"
      }
    },
    {
      "_source": {
        "message": "55.3.244.1 - WARN - 932 - Can not connect to server "
      }
    }
  ]
}

result

{
  "docs" : [
    {
      "doc" : {
        "_index" : "_index",
        "_id" : "_id",
        "_source" : {
          "duration" : 0.043,
          "request" : "/index.html",
          "method" : "GET",
          "bytes" : 15824,
          "client" : "55.3.244.1",
          "message" : "55.3.244.1 GET /index.html 15824 0.043"
        },
        "_ingest" : {
          "timestamp" : "2022-03-02T02:28:54.1600216Z"
        }
      }
    },
    {
      "doc" : {
        "_index" : "_index",
        "_id" : "_id",
        "_source" : {
          "level" : "WARN",
          "bytes" : 932,
          "client" : "55.3.244.1",
          "message" : "55.3.244.1 - WARN - 932 - Can not connect to server ",
          "message_detail" : ""
        },
        "_ingest" : {
          "timestamp" : "2022-03-02T02:28:54.1600448Z"
        }
      }
    }
  ]
}

Or you can create something like this if you have a tag or field you can use see here

PUT _ingest/pipeline/log-type1
{
  "description": "...",
  "processors": [
    {
      "grok": {
        "field": "message",
        "patterns": [
          "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes:int} %{NUMBER:duration:double}"
        ]
      }
    }
  ]
}


PUT _ingest/pipeline/log-type2
{
  "description": "...",
  "processors": [
    {
      "grok": {
        "field": "message",
        "patterns": [
          "%{IP:client} - %{WORD:level} - %{NUMBER:bytes:int} %{DATA:message_detail}"
        ]
      }
    }
  ]
}


PUT _ingest/pipeline/top-level-log-pipeline
{
  "description": "...",
  "processors": [
    {
      "pipeline": {
        "description": "If logtype web-log",
        "if": "ctx.type == 'web-log'",
        "name": "log-type1"
      }
    },
    {
      "pipeline": {
        "description": "If logtype app-log",
        "if": "ctx.type == 'app-log'",
        "name": "log-type2"
      }
    }
  ]
}

POST _ingest/pipeline/top-level-log-pipeline/_simulate
{
  "docs": [
    {
      "_source": {
        "type" : "web-log",
        "message": "55.3.244.1 GET /index.html 15824 0.043"
      }
    },
    {
      "_source": {
        "type" : "app-log",
        "message": "55.3.244.1 - WARN - 932 - Can not connect to server "
      }
    }
  ]
}

results

{
  "docs" : [
    {
      "doc" : {
        "_index" : "_index",
        "_id" : "_id",
        "_source" : {
          "duration" : 0.043,
          "request" : "/index.html",
          "method" : "GET",
          "bytes" : 15824,
          "client" : "55.3.244.1",
          "type" : "web-log",
          "message" : "55.3.244.1 GET /index.html 15824 0.043"
        },
        "_ingest" : {
          "timestamp" : "2022-03-02T02:55:50.6633169Z"
        }
      }
    },
    {
      "doc" : {
        "_index" : "_index",
        "_id" : "_id",
        "_source" : {
          "level" : "WARN",
          "bytes" : 932,
          "client" : "55.3.244.1",
          "type" : "app-log",
          "message" : "55.3.244.1 - WARN - 932 - Can not connect to server ",
          "message_detail" : ""
        },
        "_ingest" : {
          "timestamp" : "2022-03-02T02:55:50.6633322Z"
        }
      }
    }
  ]
}
8

Thank you very much

1 Like
9

hi
Sorry to bother you
The problem

10

No easy solution you are going to need to write a grok for each pattern...and put them all the grok processor patterns array like in the first example above.

You should define the most common first and then next most common and continue to iterate

11

Ok, thanks for your help

12

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.