Logstash configuration

Elastic Stack Logstash
41

I tried this filter to skip mongo logs other than "command"

filter {

    grok {

      match => { message =>

"%{TIMESTAMP_ISO8601:@timestamp} %{MONGO3_SEVERITY:severity}

%{MONGO3_COMPONENT:component}%{SPACE}(?:[%{DATA:context}])? %{GREEDYDATA:content}" }

      }

 if  [ component ] != " COMMAND "  {
         drop { }
  }

logstash is start successfully but it not showing any output

42

command hitesh.restaurants command: drop { drop: "restaurants" } keyUpdates:0 writeConflicts:0 numYields:0 reslen:81 locks:{ Global: { acquireCount: { r: 1, w: 1 } }, Database: { acquireCount: { W: 1 } } } protocol:op_query 39ms

this is my output of logstash config file.
now can i seperate only query time that is 39ms???
I tried this using dissect filter.But the space between every field is given problem.
it gives output like below

{
"severity" => "I",
"numYields" => "drop:",
"query" => "command",
"pid" => "",
"collection" => "writeConflicts:0",
"message" => "2017-03-14T17:29:54.037+0530 I COMMAND [conn4] command hitesh.restaurants command: drop { drop: "restaurants" } keyUpdates:0 writeConflicts:0 numYields:0 reslen:81 locks:{ Global: { acquireCount: { r: 1, w: 1 } }, Database: { acquireCount: { W: 1 } } } protocol:op_query 39ms\r",
"locks" => "}",
"content" => "numYields:0 reslen:81 locks:{ Global: { acquireCount: { r: 1, w: 1 } }, Database: { acquireCount: { W: 1 } } } protocol:op_query 39ms\r",
"command" => "hitesh.restaurants",
"path" => "C:/data/log/mongo2.log",
"writeConflicts" => "{",
"component" => "COMMAND",
"database" => "keyUpdates:0",
"protocol" => "",
"@timestamp" => 2017-03-16T10:12:23.196Z,
"ninserted" => "command:",
"keyUpdates" => "drop",
"@version" => "1",
"host" => "DESKTOP-PKMSR1Q",
"context" => "conn4",
"reslen" => ""restaurants""
}
please help

43

What does your configuration look like? I've never used the dissect filter but maybe I can help anyway.

44

following is my filter

dissect {
"content" => "%{query} %{command} %{ninserted} %{keyUpdates} %{writeConflicts} %{numYields} %{reslen} %{locks} %{database} %{collection} %{protocol} [%{pid}]: %{content}"}

45

input {
file {
path => "C:/Data/log/mongo97.log"
start_position => "beginning" }
}
filter {
grok {

      match => { message => "%{TIMESTAMP_ISO8601:timestamp} %{MONGO3_SEVERITY:severity} %{MONGO3_COMPONENT:component}%{SPACE}(?:\[%{DATA:context}\])? %{GREEDYDATA:content}"}
      
     }
	 
	 
	 mutate {
	       remove_field => ["message" ,"@timestamp" , "tags" , "@version" ]
		   }

  if  [component] != "COMMAND"  {
  drop { }
       }
     
  dissect {

"content" => "%{query} %{command} %{ninserted} %{keyUpdates} %{writeConflicts} %{numYields} %{reslen} %{locks} %{database} %{collection} %{protocol} [%{pid}]: %{content}"}
}

output {
elasticsearch{ hosts => ["localhost:9000"] index => "logs1" }
stdout {codec => "rubydebug" }
}

this is config file

46

This is hard to follow since you appear to be overwriting the input content field with the content field captured by the dissect filter. Please use different fields and try again.

47

thanks @magnusbaeck.
I got the expected result by writing grok expression as follow

grok {

      match => { message =>

"%{TIMESTAMP_ISO8601:@timestamp} %{MONGO3_SEVERITY:severity} %{MONGO3_COMPONENT:component}%{SPACE}(?:[%{DATA:context}])? %{GREEDYDATA:content}.*%{NUMBER:duration}" }
}

thanks

48

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.