Adding 1 day to the date

HRG · April 25, 2018, 2:40pm

#Date comparision
date
{
if "LogEventTimeStamp >= "BusinesDateTimeConverted" {
mutate {
add_field => {
"LateFileStatus" => "LateArrival"
}
}
} else {
mutate {
add_field => {
"LateFileStatus" => "On-Time"
}
}
}
}
#End of Date Comparision

HRG · April 25, 2018, 2:41pm

here is the complete code

filter {

#Match patterns based on source log origin

if [type] == "xxx" {		
	grok {
	match => ["message", "%{TIMESTAMP_ISO8601:LogEventTimeStamp}%{SPACE}%{LOGLEVEL:EventStatus}%{SPACE}%{INT:EventId}%{SPACE}%{NOTSPACE:dotchars}%{SPACE}%{NOTSPACE:SourceLogModule}%{SPACE}%{NOTSPACE:ProcessName}%{SPACE}%{NOTSPACE}%{SPACE}Get response data to file%{SPACE}%{GREEDYDATA:FilePath}/%{GREEDYDATA:file}"]
	overwrite => ["message"]
	}      

    if "_grokparsefailure" in [tags] {		
		mutate {				
			remove_tag => ["_grokparsefailure"]						
		}				
	}
}			
# End of xxx

#Extract Business date from file

grok {
	match => ["file", "%{WORD}.%{WORD:BusinessDate}.%{GREEDYDATA:FileLastPart}"]		
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]		
	}				
}

#Parse Business date
	
grok {
	match => ["BusinessDate", "%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}"]
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]					
	}				
}

# Set Timestamp

mutate {
	add_field => {
		ExpectedTime => "03:00:00.000"
	}
}


mutate {
	add_field => {
		BusinesDateTime => "%{year}-%{month}-%{day} %{ExpectedTime}"
	}
}
#Not required ExptectedTime in output
mutate {
	remove_field => [ExpectedTime]
}


#Convert to Date
date
{
	match => ["BusinesDateTime", "YYYY-MM-DD HH:mm:ss.SSS"]		
	target => "BusinesDateTime"
}

#Adding one day to BusinessDateTime

ruby {
	code => 'event.set("BusinesDateTimeConverted", LogStash::Timestamp.new(Time.at(event.get("BusinesDateTime").to_f+86400)))'				
}
	
#Remove Milliseconds	
mutate {
	gsub => ["LogEventTimeStamp", "\.\d{3}$", ""]
}


#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["BusinesDateTimeConverted", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "BusinesDateTimeConverted"		
}

#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["LogEventTimeStamp", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "LogEventTimeStamp"		
}

#Date comparision
date
{
if "LogEventTimeStamp >= "BusinesDateTimeConverted" {
mutate {
add_field => {
"LateFileStatus" => "LateArrival"
}
}
} else {
mutate {
add_field => {
"LateFileStatus" => "On-Time"
}
}
}
}
#End of Date Comparision

}

Jenni · April 25, 2018, 3:02pm

Aaaaaah. I should have seen it earlier. You have an if condition in a filter. That is not possible. Besides the date filter doesn't make any sense there.

HRG · April 25, 2018, 3:23pm

Where should i keep the "if" condition". Should i write outside of " date filter" plugin?

HRG · April 25, 2018, 3:25pm

If i remove below Date filter, the code is running as expected. Can't we have "if" condition inside date filter?

#Date comparision
date
{
if "LogEventTimeStamp >= "BusinesDateTimeConverted" {
mutate {
add_field => {
"LateFileStatus" => "LateArrival"
}
}
} else {
mutate {
add_field => {
"LateFileStatus" => "On-Time"
}
}
}
}
#End of Date Comparision

Jenni · April 25, 2018, 3:42pm

As I said, conditions must be outside of filters.

HRG · April 25, 2018, 3:51pm

umm....somehow outside it does not like look like. It expecting some filter

[2018-04-25T11:48:27,347][ERROR][logstash.agent ] Failed to execute action {:action=>LogS
or", :message=>"Expected one of #, input, filter, output at line 116, column 5 (byte 2652) after ",

Code

filter {

#Match patterns based on source log origin

if [type] == "xxx" {		
	grok {
	match => ["message", "%{TIMESTAMP_ISO8601:LogEventTimeStamp}%{SPACE}%{LOGLEVEL:EventStatus}%{SPACE}%{INT:EventId}%{SPACE}%{NOTSPACE:dotchars}%{SPACE}%{NOTSPACE:SourceLogModule}%{SPACE}%{NOTSPACE:ProcessName}%{SPACE}%{NOTSPACE}%{SPACE}Get response data to file%{SPACE}%{GREEDYDATA:FilePath}/%{GREEDYDATA:file}"]
	overwrite => ["message"]
	}      

    if "_grokparsefailure" in [tags] {		
		mutate {				
			remove_tag => ["_grokparsefailure"]						
		}				
	}
}			
# End of xxx	

#Extract Business date from file

grok {
	match => ["file", "%{WORD}.%{WORD:BusinessDate}.%{GREEDYDATA:FileLastPart}"]		
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]		
	}				
}

#Parse Business date
	
grok {
	match => ["BusinessDate", "%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}"]
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]					
	}				
}

# Set Timestamp

mutate {
	add_field => {
		ExpectedTime => "03:00:00.000"
	}
}


mutate {
	add_field => {
		BusinesDateTime => "%{year}-%{month}-%{day} %{ExpectedTime}"
	}
}
#Not required ExptectedTime in output
mutate {
	remove_field => [ExpectedTime]
}


#Convert to Date
date
{
	match => ["BusinesDateTime", "YYYY-MM-DD HH:mm:ss.SSS"]		
	target => "BusinesDateTime"
}

#Adding one day to BusinessDateTime

ruby {
	code => 'event.set("BusinesDateTimeConverted", LogStash::Timestamp.new(Time.at(event.get("BusinesDateTime").to_f+86400)))'				
}
	
#Remove Milliseconds	
mutate {
	gsub => ["LogEventTimeStamp", "\.\d{3}$", ""]
}


#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["BusinesDateTimeConverted", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "BusinesDateTimeConverted"		
}

#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["LogEventTimeStamp", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "LogEventTimeStamp"		
}

} #End of Main Filter

#Date Comparision

if "LogEventTimeStamp" >= "BusinesDateTimeConverted" {
	mutate {
		add_field => {
			"LateFileStatus" => "LateArrival"
	    }	
	}	
} else {
	mutate {
		add_field => {
			"LateFileStatus" => "On-Time"
	    }	
	} 	
}

#End of Date Comparision

Jenni · April 25, 2018, 3:54pm

Outside of filter objects, not the complete filter{} block

HRG · April 25, 2018, 3:55pm

Would you please mind correct the code i sent?. It looks i am missing something.

HRG · April 25, 2018, 3:58pm

Got it. Worked.

HRG · April 25, 2018, 4:01pm

The "if" condition worked. But the date comparision is not giving correct result. It looks the dates are not in the same format.

Jenni · April 25, 2018, 4:03pm

The structure is always
--- input/filter/output
------ conditions
---------inputs/filters/outputs

You can nest multiple conditions. But you can't put anything inside of mutate{}, date{}, etc, but their own options (add_field =>..., match =>..., etc)

Did you make the changes to the condition I had suggested?

HRG · April 25, 2018, 4:07pm

Yeah..i made the change. I moved "if" condition inside "filter" plug-in. It worked. Thank you for the solution.

Somehow , the date comparision result is giving wrong. It my case "LogEventTimeStamp" is smaller than "BusinesDateTimeConverted". The expected outcome is "on-time" but it is giving "LateArrival"

In the Kibana, the dates are showing as below

BusinesDateTimeConverted: 2018-01-20T08:00:00.000Z (This is logstash date format)

LogEventTimeStamp : April 19th 2018, 13:15:09.000

I am comparing both of the above two dates. The result should be "on-Time" but it showing "LateArrival"

Anything format issue?

Jenni · April 25, 2018, 4:15pm

I'm a bit irritated that Kibana doesn't display both of them in the same way. Do they have the same type in ES?

For the condition I had suggested square brackets. Did you make that change?

HRG · April 25, 2018, 4:22pm

Somehow square brackers are not working. Its throwing error

In the logstash, both are same type(timestamp).

Code

if [LogEventTimeStamp] >= [BusinesDateTimeConverted] {
mutate {
add_field => {
"LateFileStatus" => "LateArrival"
}
}
} else {
mutate {
add_field => {
"LateFileStatus" => "On-Time"
}
}
}

Error

[2018-04-25T12:17:09,633][ERROR][logstash.pipeline ] Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart L
ogstash. {:pipeline_id=>"main", "exception"=>"undefined method >=' for nil:NilClass", "backtrace"=>["(eval):977:inblock in initialize'", "org/jruby/RubyArray.java:1734:in each'", "( eval):975:inblock in initialize'", "(eval):558:in block in filter_func'", "C:/Projects/logstash-6.2.3/logstash-6.2.3/logstash-core/lib/logstash/pipeline.rb:447:infilter_batch'", "C
:/Projects/logstash-6.2.3/logstash-6.2.3/logstash-core/lib/logstash/pipeline.rb:426:in worker_loop'", "C:/Projects/logstash-6.2.3/logstash-6.2.3/logstash-core/lib/logstash/pipeline.rb: 385:inblock in start_workers'"], :thread=>"#<Thread:0x1d15c1c sleep>"}

Jenni · April 25, 2018, 4:45pm

I'm not sure what's wrong. Could you comment the error causing part out, run the configuration with a rubydebug output and post it here?

HRG · April 25, 2018, 4:54pm

{
"tags"←[0;37m => ←[0m[
←[1;37m[0] ←[0m←[0;33m"_dateparsefailure"←[0m
],
"BusinesDateTime"←[0;37m => ←[0m←[0;33m"%{year}-%{month}-%{day} 03:00:00.000"←[0m,
"BusinesDateTimeConverted"←[0;37m => ←[0m1970-01-02T00:00:00.000Z,
"sourceSystem"←[0;37m => ←[0m←[0;33m"xxx"←[0m,
"LateFileStatus"←[0;37m => ←[0m←[0;33m"LateArrival"←[0m,
"message"←[0;37m => ←[0m←[0;33m"2018-04-19 00:18:42.394 INFO 4506994501 --- [http-nio-8080-exec-2] c.c.d.m.f.RequestLoggingFilter : [start-request]:
client-ip=10.90.16.15, client-host=testserver, request-method=GET, url=http://10.90.106.48:8080/health, payload="←[0m,
"@version"←[0;37m => ←[0m←[0;33m"1"←[0m,
"host"←[0;37m => ←[0m←[0;33m"dt-x"←[0m,
"type"←[0;37m => ←[0m←[0;33m"xxx"←[0m,
"path"←[0;37m => ←[0m←[0;33m"C:/Projects/data/xxx/xxx.log"←[0m,
"@timestamp"←[0;37m => ←[0m2018-04-25T16:52:34.207Z

In the kibana, it is showing 1 day added to the "BusinesDateTimeConverted" but when i did rubudebug, its not showing.

It looks this is the reason its not giving me "on-time" value when i compared dates.

Jenni · April 25, 2018, 5:10pm

What's this part of the configuration for? It should already be a Timestamp at this point, shouldn't it?

#Setting up Log timestamp to LogEventTimeStamp for the given formats. date { 	match => ["BusinesDateTimeConverted", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		 	target => "BusinesDateTimeConverted"		 }

HRG · April 25, 2018, 6:23pm

"LogEventTimeStamp"←[0;37m => ←[0m2018-04-19T04:15:08.790Z
"BusinesDateTimeConverted"←[0;37m => ←[0m2018-04-20T07:00:00.000Z

When output showing is " "LateFileStatus"←[0;37m => ←[0m←[0;33m"LateArrival"←[0m"

The expected outcome is "OnTime".

Here is logic

if "LogEventTimeStamp" >= "BusinesDateTimeConverted" {
		mutate {
			add_field => {
				"LateFileStatus" => "LateArrival"
		    }	
		}	
    } else {
		mutate {
			add_field => {
				"LateFileStatus" => "On-Time"
		    }	
		} 	
}

Badger · April 25, 2018, 6:38pm

That is doing a string comparison. Try

if [LogEventTimeStamp] >= [BusinesDateTimeConverted] {