How to parse multiple json using logstash which can occure anywhere in the log

Elastic Stack Logstash
1

I want to parse this log into ES which has multiple JSON string in it, the position of JSON is not fixed!

2020-03-30 17:42:15,672 INFO [DefaultMessageListenerContainer-4] (MeetingServiceImpl.getMeetingParticipants:270) - {"a": 123, "b": { "b1": 234 } } some text here {"c":"567","d":"789"}

I have tried this logstash filter:

filter{
    grok {
        match => { "message" => "%{TIMESTAMP_ISO8601:time} %{LOGLEVEL:logLevel}\s*\[(?<thread>([\w\-]+|[\w\s]+))\] (\(%{DATA:className}\.%{DATA:methodName}:%{NUMBER:lineNumber}\)) - %{GREEDYDATA:message}"}
        overwrite => [ "message" ]
    }
ruby {
    code => "
        json1 = event.get('message').match(\{.*?\})[1]
        event.set('json1',json1)
        "
    }

    json { 
        source => "json1"
        target => "payload"
    }

    if "TRACE" in [logLevel]{
      drop { }
    }
    date{
        match => ["time","ISO8601"]
        target => "time"
    }
    mutate{
        convert => { "lineNumber" => "integer" }
    }
    mutate{
        remove_field => ["@version","offset","tags","agent","ecs"]
    }
    mutate {
      gsub => ["message","\(", "=("]
    }
    kv {
      source => "message"
      recursive => "true"
      field_split => ",\s\(\)"
      value_split => "="
      trim_key => "\s"
      target => "payload"
    }
    if "_grokparsefailure" in [tags] {
      drop { }
    }

    ruby {
        code => "
                  hash = event.to_hash
                  hash.each { |key,value|
                  if value != nil
                      str = value.to_s
                        if str.blank?
                          event.remove(key)
                        end
                  end
                }
              "
    }
}

But got this exception: java.lang.IllegalStateException: Logstash stopped processing because of an error: (SyntaxError) (ruby filter code):3: syntax error, unexpected null json1 = event.get('message').match({.*?})[1] ^

Expected Output:
{
"logLevel" => "INFO",
"lineNumber" => 270,
"methodName" => "getMeetingParticipants",
"payload" => {
"b" => {
"b1" => 234
},
"a" => 123,
"c" => 567,
"d" => 789
},
"@timestamp" => 2020-04-13T09:51:48.333Z,
"host" => "ThinkPad-E470",
"time" => 2020-03-30T12:12:15.672Z,
"message" => "{"a": 123, "b": { "b1": 234 } } some text here {"c":567,"d":789}",
"className" => "MeetingServiceImpl",
"path" => "/logstashworks/logs/malogs.log",
"thread" => "DefaultMessageListenerContainer-4"
}

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.