Ingest Processor Conditional

Elastic Stack Elasticsearch
1

I want to process data in the message field from a source if fields.log_type is combinedlogs...but it's not working. Can someone help me there?
Thanks in Advance.

Ingest Pipeline:

PUT _ingest/pipeline/pipeline_combined_logs
{
  "description" : "Process Combined Logs",
  "processors" : [
    {
  "dissect": {
    "if": "['fields']['log_type'] =='combinedlogs'",
    "field": "message",
    "pattern" : "%{eventdate}\t%{etime}\t%{logtype}\t%{loglevel}\t%{}\t%{host}\t%{user}|%{pid}|%{service}\t%{component_message}"
   }
},
    {
      "set": {
        "field": "edate",
        "value": "{{eventdate}} {{etime}}"
      }
    },
    {
      "date" : {
        "field" : "edate",
        "target_field" : "eventdate",
        "formats" : ["yyyy-MM-dd HH:mm:ss"]
      }
    },
    {
  "remove": {
    "field": "message"
  }
},
    {
      "date_index_name" : {
        "field" : "eventdate",
        "index_name_prefix" : "combined-",
        "date_rounding" : "M"
      }
    }
  ]
}

LogData:

PUT /combined/_doc/1?pipeline=pipeline_combined_logs
{
  "fields" : {"log_type":"combinedlogs"},
  "message": "Log-DATA-Here"
}

Error I get:

{
  "error" : {
    "root_cause" : [
      {
        "type" : "script_exception",
        "reason" : "compile error",
        "script_stack" : [
          "['fields']['log_type'] =='combinedlo ...",
          "           ^---- HERE"
        ],
        "script" : "['fields']['log_type'] =='combinedlogs'",
        "lang" : "painless",
        "position" : {
          "offset" : 11,
          "start" : 0,
          "end" : 36
        }
      }
    ],
    "type" : "script_exception",
    "reason" : "compile error",
    "script_stack" : [
      "['fields']['log_type'] =='combinedlo ...",
      "           ^---- HERE"
    ],
    "script" : "['fields']['log_type'] =='combinedlogs'",
    "lang" : "painless",
    "position" : {
      "offset" : 11,
      "start" : 0,
      "end" : 36
    },
    "caused_by" : {
      "type" : "class_cast_exception",
      "reason" : "Cannot cast from [java.lang.String] to [int]."
    }
  },
  "status" : 400
}
2

I figured it out how to do that...
I just need to change pipeline definition as follows

PUT _ingest/pipeline/pipeline_combined_logs
{
  "description" : "Process Combined Logs",
  "processors" : [
    {
  "dissect": {
    "if": "ctx.fields.log_type =='combinedlogs'",
    "field": "message",
    "pattern" : "%{eventdate}\t%{etime}\t%{logtype}\t%{loglevel}\t%{}\t%{host}\t%{user}|%{pid}|%{service}\t%{component_message}"
   }
},
    {
      "set": {
        "field": "edate",
        "value": "{{eventdate}} {{etime}}"
      }
    },
    {
      "date" : {
        "field" : "edate",
        "target_field" : "eventdate",
        "formats" : ["yyyy-MM-dd HH:mm:ss"]
      }
    },
    {
  "remove": {
    "field": "message"
  }
},
    {
      "date_index_name" : {
        "field" : "eventdate",
        "index_name_prefix" : "combined-",
        "date_rounding" : "M"
      }
    }
  ]
}

Instead of using

"if": "['fields']['log_type'] =='combinedlogs'"

Use

 "if": "ctx.fields.log_type =='combinedlogs'"
3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.