How to remove full_message or message from log under filter

[tharu85 (Tharaka)]

I need to remove full_message or message field using logstash filter. Keep both these values increase the disk capacity.

I tried following syntax under logstash filter.

filter {
    grok {
		remove_field => ["full_message"]
	}
}

Above will not remove none of mentioned filed

Then I tried following syntax:

filter {
    mutate {
           remove_field => ["full_message"]
    }
}

Above syntax drop all data and nothing sent to elasticsearch ( no data)

So my question is, is it possible to remove full_message or message fields in a log using logstash filter ?

[raihan]

mutate {
                remove_field => [
                        "@timestamp",
                        "@version",
                        "author",
                        "tags"
                ]
                }

This works for me, change the names accordingly as per your needs

[tharu85 (Tharaka)]

@raihan

I tried your code before and just now copied and replace according to my filed name as well.

But it doesn't work. Still the full_message and message fields appeared in the parser log

[raihan]

@tharu85
Please put the full config, just wanna make sure if you are not renaming the fields.

[tharu85 (Tharaka)]

@raihan

Refer below logstash config file

input {
   udp {
	  port => 30000 
      type => "fortigate-syslog"
	  codec => plain {
        charset => "ISO-8859-1"
    }
}

filter {
   if [type] == "fortigate-syslog" 
   {
     mutate {
		gsub => [ 
			"message", "<.*>", ""
		]
	 }
	 grok 
	 {
		match => [ "message", "(?<msg>.*)" ]
	 }
	   
	kv { source => "msg" }
	   
	mutate {
	    add_field => ["log_timestamp", "%{date} %{time}"]
		rename => [ "crlevel" , "severity_level" ]
		rename => [ "devname" , "devicename" ]
		rename => [ "devid" , "deviceid" ]
		rename => [ "srcintf" , "srcinterface" ]
		rename => [ "dstintf" , "dstinterface" ]
		rename => [ "proto" , "protocol" ]
		rename => [ "appcat" , "appcategory" ]
		rename => [ "inintf" , "srcinterface" ]
		rename => [ "outintf" , "dstinterface" ]
		rename => [ "locport" , "srcport" ]
		rename => [ "remport" , "dstport" ]
		rename => [ "locip" , "srcip" ]
		rename => [ "remip" , "dstip" ]
		rename => [ "logdesc" , "log_description" ]
		rename => [ "peer_notif" , "peer_notification" ]
		rename => [ "dir" , "direction" ]
		rename => [ "type" , "data_type" ]
		rename => [ "log_type", "type" ]
		rename => [ "level", "log_level" ]
		rename => [ "sessionid", "session_id" ]
		rename => [ "action", "action_type" ]
		rename => [ "policyid", "policy_id" ]
		rename => [ "sentbyte", "sent_bytes" ]
		rename => [ "rcvdbyte", "input_bytes" ]
		rename => [ "sentpkt", "sent_packets" ]
		rename => [ "rcvdpkt", "input_packets" ]
		
		convert => [ "srcport", "integer"]
		convert => [ "dstport", "integer"]
		convert => [ "protocol", "integer"]
		convert => [ "duration", "integer"]
		convert => [ "policy_id", "integer"]
		convert => [ "session_id", "integer"]
		convert => [ "sent_bytes", "integer"]
		convert => [ "input_bytes", "integer"]
		convert => [ "sent_packets", "integer"]
		convert => [ "input_packets", "integer"]
		
		remove_field => ["msg", "full_message"]
	}
			
	date {
		match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd HH:mm:ss.ZZZ" ]
		target => "log_timestamp"
	}		   
  }			
}


output {
    gelf {
        host => "192.168.1.100"
        port => 5000
    }
}