Logstash parse logfiles

Hello Team,

kindly advise how I can parse the following logfile entry on Kibana/logstash?

{"log":"LOG:  aborting any active transactions\n","stream":"stderr","time":"2022-05-17T11:04:05.353977974Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361875035Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361890357Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361893247Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361895596Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361897842Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361900013Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361902075Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361904287Z"}
{"log":"LOG:  autovacuum launcher shutting down\n","stream":"stderr","time":"2022-05-17T11:04:05.361906394Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361908559Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361910661Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T11:04:05.361912695Z"}
{"log":"LOG:  shutting down\n","stream":"stderr","time":"2022-05-17T11:04:05.361914754Z"}
{"log":"LOG:  database system is shut down\n","stream":"stderr","time":"2022-05-17T11:04:05.391072854Z"}
{"log":"\n","stream":"stdout","time":"2022-05-17T11:05:00.211634392Z"}
{"log":"PostgreSQL Database directory appears to contain a database; Skipping initialization\n","stream":"stdout","time":"2022-05-17T11:05:00.211659394Z"}
{"log":"\n","stream":"stdout","time":"2022-05-17T11:05:00.211663616Z"}
{"log":"LOG:  database system was shut down at 2022-05-17 11:04:05 UTC\n","stream":"stderr","time":"2022-05-17T11:05:00.268897853Z"}
{"log":"LOG:  MultiXact member wraparound protections are now enabled\n","stream":"stderr","time":"2022-05-17T11:05:00.277738915Z"}
{"log":"LOG:  autovacuum launcher started\n","stream":"stderr","time":"2022-05-17T11:05:00.282189849Z"}
{"log":"LOG:  database system is ready to accept connections\n","stream":"stderr","time":"2022-05-17T11:05:00.28233607Z"}
{"log":"LOG:  received fast shutdown request\n","stream":"stderr","time":"2022-05-17T14:33:58.086125541Z"}
{"log":"LOG:  aborting any active transactions\n","stream":"stderr","time":"2022-05-17T14:33:58.086157723Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T14:33:58.086257808Z"}
{"log":"LOG:  autovacuum launcher shutting down\n","stream":"stderr","time":"2022-05-17T14:33:58.086282083Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T14:33:58.086300595Z"}
{"log":"FATAL:  terminating connection due to administrator command\n","stream":"stderr","time":"2022-05-17T14:33:58.087540174Z"}
{"log":"LOG:  shutting down\n","stream":"stderr","time":"2022-05-17T14:33:58.089711765Z"}
{"log":"LOG:  database system is shut down\n","stream":"stderr","time":"2022-05-17T14:33:58.114704185Z"}
{"log":"\n","stream":"stdout","time":"2022-05-17T14:47:03.38454473Z"}
{"log":"PostgreSQL Database directory appears to contain a database; Skipping initialization\n","stream":"stdout","time":"2022-05-17T14:47:03.384594052Z"}
{"log":"\n","stream":"stdout","time":"2022-05-17T14:47:03.384608294Z"}
{"log":"LOG:  database system was shut down at 2022-05-17 14:33:58 UTC\n","stream":"stderr","time":"2022-05-17T14:47:03.451569109Z"}
{"log":"LOG:  MultiXact member wraparound protections are now enabled\n","stream":"stderr","time":"2022-05-17T14:47:03.461784035Z"}
{"log":"LOG:  autovacuum launcher started\n","stream":"stderr","time":"2022-05-17T14:47:03.465356014Z"}
{"log":"LOG:  database system is ready to accept connections\n","stream":"stderr","time":"2022-05-17T14:47:03.465396933Z"}

Thanks,
Roshan

input {
	file {
      path => "/location/file.json"
      start_position => beginning
      sincedb_path => ["/dev/null"]
      codec => json
	}
} # input

filter {
     date {
      match => [ "time", "ISO8601"]
	  target=> "@timestamp"
     }    
	 
	 mutate {split  => { "log" => ':' } }

	 mutate {
	   rename  => { "[log][1]" => 'msg' }
	   strip => ["msg"]
	 }
     mutate { rename  => { "[log][0]" => 'level' } }
     mutate { remove_field => ["log", "time" ] }
	 
}

output {
 elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "mylog"
    user => "elastic"
    password => "pass"
   } 
 stdout {
        codec => rubydebug{}
    }
}

Result:

{
        "stream" => "stderr",
          "path" => "/location/file.json",
    "@timestamp" => 2022-05-17T11:04:05.361Z,
      "@version" => "1",
       "log_msg" => "terminating connection due to administrator command",
         "level" => "FATAL"
}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.