Disapearing fields during kv parsing

Elastic Stack Elasticsearch
1

Hi!

I'm trying to parse the following log record
__policy_id_tag:"product=APP1 & APP2[db_tag={ABCDEF}; mgmt=abcd; date=123456789; policy_name=foo]"; product:"APP1 & APP2"; mgmt:"abcd"; event:"Canceled"

and when I use the filter

filter {
  grok {
    match => { "message" => "%{GREEDYDATA:[@metadata][message]}" }
  }

  kv {
    source => "[@metadata][message]"
    field_split => ";"
    value_split => ":"
    trim_key => " _"
    target => "fws"
  }

  grok {
    match => { "[fws][policy_id_tag]" => "%{GREEDYDATA:[@metadata][policy_id_tag]}" }
  }
}

I see the output

"@timestamp" => 2021-12-01T10:52:13.353Z,
      "@version" => "1",
      "sequence" => 0,
          "host" => "localhost.localdomain",
       "message" => "__policy_id_tag:\"product=APP1 & APP2[db_tag={ABCDEF}; mgmt=abcd; date=123456789; policy_name=foo]\"; product:\"APP1 & APP2\"; mgmt:\"abcd\"; event:\"Canceled\"",
           "fws" => {
              "product" => "APP1 & APP2",
                 "mgmt" => "abcd",
                "event" => "Canceled",
        "policy_id_tag" => "product=APP1 & APP2[db_tag={ABCDEF}; mgmt=abcd; date=123456789; policy_name=foo]"

It looks OK.

Then I add next kv section to parse policy_id_tag field

filter {
  grok {
    match => { "message" => "%{GREEDYDATA:[@metadata][message]}" }
  }

  kv {
    source => "[@metadata][message]"
    field_split => ";"
    value_split => ":"
    trim_key => " _"
    target => "fws"
  }

  grok {
    match => { "[fws][policy_id_tag]" => "%{GREEDYDATA:[@metadata][policy_id_tag]}" }
  }

  kv {
    source => "[@metadata][policy_id_tag]"
    field_split => "\[;"
    trim_key => " "
    remove_char_value => "\]"
    target => "fws"
  }
}

What I see the event and policy_id_tag fields dissapeared

"@version" => "1",
    "@timestamp" => 2021-12-01T10:54:05.935Z,
       "message" => "__policy_id_tag:\"product=APP1 & APP2[db_tag={ABCDEF}; mgmt=abcd; date=123456789; policy_name=foo]\"; product:\"APP1 & APP2\"; mgmt:\"abcd\"; event:\"Canceled\"",
          "host" => "localhost.localdomain",
           "fws" => {
               "mgmt" => "abcd",
        "policy_name" => "foo",
            "product" => "APP1 & APP2",
               "date" => "123456789",
             "db_tag" => "{ABCDEF}"
    },
      "sequence" => 0

What could be problem and how to solve it?

Note: without target => "fws" it is parsed well

"@version" => "1",
             "host" => "localhost.localdomain",
       "@timestamp" => 2021-12-01T11:34:15.475Z,
          "message" => "__policy_id_tag:\"product=APP1 & APP2[db_tag={ABCDEF}; mgmt=abcd; date=123456789; policy_name=foo]\"; product:\"APP1 & APP2\"; mgmt:\"abcd\"; event:\"Canceled\"",
         "sequence" => 0,
      "policy_name" => "foo",
          "product" => "APP1 & APP2",
            "event" => "Canceled",
    "policy_id_tag" => "product=APP1 & APP2[db_tag={ABCDEF}; mgmt=abcd; date=123456789; policy_name=foo]",
             "mgmt" => "abcd",
             "date" => "123456789",
           "db_tag" => "{ABCDEF}"
2

Well, the problem is kv rewrites fws field.

filter {
  grok {
    match => { "message" => "%{GREEDYDATA:[@metadata][message]}" }
  }

  kv {
    source => "[@metadata][message]"
    field_split => ";"
    value_split => ":"
    trim_key => " _"
    target => "fws"
  }

  grok {
    match => { "[fws][policy_id_tag]" => "%{GREEDYDATA:[@metadata][policy_id_tag]}" }
  }

  kv {
    source => "[@metadata][policy_id_tag]"
    field_split => "\[;"
    trim_key => " "
    remove_char_value => "\]"
    target => "fws_tmp"
  }

 mutate {
    merge => { "fws" =>  "fws_tmp" }
  }
}

solved the problem

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.