Adding 1 day to the date

I'm afraid I don't really understand the question. Do you want to convert logEventTimeStamp to a Time object?

I would like to see both "BusinessDateTimeConverted" and "LogEventTImeStamp" to "YYYY-MM-DD HH:MI;SS.SSS" format.Currently it is showing as "2018-01-19T08:00:00.000Z" format.

Business Datetime Converted doesn't really have a fixed format as it is not a string, but a Timestamp object. 2018-01-19T08:00:00.000Z is Logstash's way of displaying it. You could build a formatted string from that Timestamp again. But wouldn't Timestamp objects be more useful for comparisons?

Understood. Thank you.

Is this the way we compare dates to add new filed? . Somehow its throwing error.

date {
if "LogEventTimeStamp" >= "BusinesDateTimeConverted" {
mutate {
add_field => {
LateFileStatus => "LateArrival"
}
}
} else {
mutate {
add_field => {
LateFileStatus => "On-Time"
}
}
}
}

Error

[2018-04-25T09:46:11,762][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_i
or", :message=>"Expected one of #, => at line 114, column 8 (byte 2728) after filter {\n\t\t\n\t#Match patterns based on source log origin

Line 114 is above "if" condition.

Could you post more of your configuration? The text snippet in those syntax error logs usually ends right before the error. So I'm not sure if the wrong part is even included in the code you posted.

"LogEventTimeStamp" and "BusinesDateTimeConverted" are strings. You want to compare the field values [LogEventTimeStamp] and [BusinesDateTimeConverted] (if these are both timestamps)

Please see below

#Extract Business date from file

grok {
	match => ["file", "%{WORD}.%{WORD:BusinessDate}.%{GREEDYDATA:FileLastPart}"]		
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]		
	}				
}

#Parse Business date
	
grok {
	match => ["BusinessDate", "%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}"]
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]					
	}				
}

# Set Timestamp

mutate {
	add_field => {
		ExpectedTime => "03:00:00.000"
	}
}


mutate {
	add_field => {
		BusinesDateTime => "%{year}-%{month}-%{day} %{ExpectedTime}"
	}
}
#Not required ExptectedTime in output
mutate {
	remove_field => [ExpectedTime]
}


#Convert to Date
date
{
	match => ["BusinesDateTime", "YYYY-MM-DD HH:mm:ss.SSS"]		
	target => "BusinesDateTime"
}

#Adding one day to BusinessDateTime

ruby {
	code => 'event.set("BusinesDateTimeConverted", LogStash::Timestamp.new(Time.at(event.get("BusinesDateTime").to_f+86400)))'				
}
	
#Remove Milliseconds	
mutate {
	gsub => ["LogEventTimeStamp", "\.\d{3}$", ""]
}


#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["BusinesDateTimeConverted", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "BusinesDateTimeConverted"		
}

#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["LogEventTimeStamp", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "LogEventTimeStamp"		
}

#Date comparision
date
{
if "LogEventTimeStamp >= "BusinesDateTimeConverted" {
mutate {
add_field => {
"LateFileStatus" => "LateArrival"
}
}
} else {
mutate {
add_field => {
"LateFileStatus" => "On-Time"
}
}
}
}
#End of Date Comparision

yeah, i wanted to compare both timestamps.

Where is the line that says "#Match patterns based ...."? The error has to be after that.

Try if [LogEventTimeStamp] >= [BusinesDateTimeConverted] { for the comparison.

#Extract Business date from file

grok {
	match => ["file", "%{WORD}.%{WORD:BusinessDate}.%{GREEDYDATA:FileLastPart}"]		
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]		
	}				
}

#Parse Business date
	
grok {
	match => ["BusinessDate", "%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}"]
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]					
	}				
}

# Set Timestamp

mutate {
	add_field => {
		ExpectedTime => "03:00:00.000"
	}
}


mutate {
	add_field => {
		BusinesDateTime => "%{year}-%{month}-%{day} %{ExpectedTime}"
	}
}
#Not required ExptectedTime in output
mutate {
	remove_field => [ExpectedTime]
}


#Convert to Date
date
{
	match => ["BusinesDateTime", "YYYY-MM-DD HH:mm:ss.SSS"]		
	target => "BusinesDateTime"
}

#Adding one day to BusinessDateTime

ruby {
	code => 'event.set("BusinesDateTimeConverted", LogStash::Timestamp.new(Time.at(event.get("BusinesDateTime").to_f+86400)))'				
}
	
#Remove Milliseconds	
mutate {
	gsub => ["LogEventTimeStamp", "\.\d{3}$", ""]
}


#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["BusinesDateTimeConverted", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "BusinesDateTimeConverted"		
}

#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["LogEventTimeStamp", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "LogEventTimeStamp"		
}

#Date comparision
date
{
if "LogEventTimeStamp >= "BusinesDateTimeConverted" {
mutate {
add_field => {
"LateFileStatus" => "LateArrival"
}
}
} else {
mutate {
add_field => {
"LateFileStatus" => "On-Time"
}
}
}
}
#End of Date Comparision

#Date comparision
date
{
if "LogEventTimeStamp >= "BusinesDateTimeConverted" {
mutate {
add_field => {
"LateFileStatus" => "LateArrival"
}
}
} else {
mutate {
add_field => {
"LateFileStatus" => "On-Time"
}
}
}
}
#End of Date Comparision

here is the complete code

filter {

#Match patterns based on source log origin

if [type] == "xxx" {		
	grok {
	match => ["message", "%{TIMESTAMP_ISO8601:LogEventTimeStamp}%{SPACE}%{LOGLEVEL:EventStatus}%{SPACE}%{INT:EventId}%{SPACE}%{NOTSPACE:dotchars}%{SPACE}%{NOTSPACE:SourceLogModule}%{SPACE}%{NOTSPACE:ProcessName}%{SPACE}%{NOTSPACE}%{SPACE}Get response data to file%{SPACE}%{GREEDYDATA:FilePath}/%{GREEDYDATA:file}"]
	overwrite => ["message"]
	}      

    if "_grokparsefailure" in [tags] {		
		mutate {				
			remove_tag => ["_grokparsefailure"]						
		}				
	}
}			
# End of xxx

#Extract Business date from file

grok {
	match => ["file", "%{WORD}.%{WORD:BusinessDate}.%{GREEDYDATA:FileLastPart}"]		
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]		
	}				
}

#Parse Business date
	
grok {
	match => ["BusinessDate", "%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}"]
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]					
	}				
}

# Set Timestamp

mutate {
	add_field => {
		ExpectedTime => "03:00:00.000"
	}
}


mutate {
	add_field => {
		BusinesDateTime => "%{year}-%{month}-%{day} %{ExpectedTime}"
	}
}
#Not required ExptectedTime in output
mutate {
	remove_field => [ExpectedTime]
}


#Convert to Date
date
{
	match => ["BusinesDateTime", "YYYY-MM-DD HH:mm:ss.SSS"]		
	target => "BusinesDateTime"
}

#Adding one day to BusinessDateTime

ruby {
	code => 'event.set("BusinesDateTimeConverted", LogStash::Timestamp.new(Time.at(event.get("BusinesDateTime").to_f+86400)))'				
}
	
#Remove Milliseconds	
mutate {
	gsub => ["LogEventTimeStamp", "\.\d{3}$", ""]
}


#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["BusinesDateTimeConverted", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "BusinesDateTimeConverted"		
}

#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["LogEventTimeStamp", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "LogEventTimeStamp"		
}

#Date comparision
date
{
if "LogEventTimeStamp >= "BusinesDateTimeConverted" {
mutate {
add_field => {
"LateFileStatus" => "LateArrival"
}
}
} else {
mutate {
add_field => {
"LateFileStatus" => "On-Time"
}
}
}
}
#End of Date Comparision

}

Aaaaaah. I should have seen it earlier. You have an if condition in a filter. That is not possible. Besides the date filter doesn't make any sense there.

Where should i keep the "if" condition". Should i write outside of " date filter" plugin?

If i remove below Date filter, the code is running as expected. Can't we have "if" condition inside date filter?

#Date comparision
date
{
if "LogEventTimeStamp >= "BusinesDateTimeConverted" {
mutate {
add_field => {
"LateFileStatus" => "LateArrival"
}
}
} else {
mutate {
add_field => {
"LateFileStatus" => "On-Time"
}
}
}
}
#End of Date Comparision

As I said, conditions must be outside of filters.

umm....somehow outside it does not like look like. It expecting some filter

[2018-04-25T11:48:27,347][ERROR][logstash.agent ] Failed to execute action {:action=>LogS
or", :message=>"Expected one of #, input, filter, output at line 116, column 5 (byte 2652) after ",

Code

filter {

#Match patterns based on source log origin

if [type] == "xxx" {		
	grok {
	match => ["message", "%{TIMESTAMP_ISO8601:LogEventTimeStamp}%{SPACE}%{LOGLEVEL:EventStatus}%{SPACE}%{INT:EventId}%{SPACE}%{NOTSPACE:dotchars}%{SPACE}%{NOTSPACE:SourceLogModule}%{SPACE}%{NOTSPACE:ProcessName}%{SPACE}%{NOTSPACE}%{SPACE}Get response data to file%{SPACE}%{GREEDYDATA:FilePath}/%{GREEDYDATA:file}"]
	overwrite => ["message"]
	}      

    if "_grokparsefailure" in [tags] {		
		mutate {				
			remove_tag => ["_grokparsefailure"]						
		}				
	}
}			
# End of xxx	

#Extract Business date from file

grok {
	match => ["file", "%{WORD}.%{WORD:BusinessDate}.%{GREEDYDATA:FileLastPart}"]		
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]		
	}				
}

#Parse Business date
	
grok {
	match => ["BusinessDate", "%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}"]
}      

if "_grokparsefailure" in [tags] {		
	mutate {				
			remove_tag => ["_grokparsefailure"]					
	}				
}

# Set Timestamp

mutate {
	add_field => {
		ExpectedTime => "03:00:00.000"
	}
}


mutate {
	add_field => {
		BusinesDateTime => "%{year}-%{month}-%{day} %{ExpectedTime}"
	}
}
#Not required ExptectedTime in output
mutate {
	remove_field => [ExpectedTime]
}


#Convert to Date
date
{
	match => ["BusinesDateTime", "YYYY-MM-DD HH:mm:ss.SSS"]		
	target => "BusinesDateTime"
}

#Adding one day to BusinessDateTime

ruby {
	code => 'event.set("BusinesDateTimeConverted", LogStash::Timestamp.new(Time.at(event.get("BusinesDateTime").to_f+86400)))'				
}
	
#Remove Milliseconds	
mutate {
	gsub => ["LogEventTimeStamp", "\.\d{3}$", ""]
}


#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["BusinesDateTimeConverted", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "BusinesDateTimeConverted"		
}

#Setting up Log timestamp to LogEventTimeStamp for the given formats.
date
{
	match => ["LogEventTimeStamp", "yyyy-MM-dd HH:mm:ss", "MMM dd, yyyy HH:mm:ss", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss.SSS"]		
	target => "LogEventTimeStamp"		
}

} #End of Main Filter

#Date Comparision

if "LogEventTimeStamp" >= "BusinesDateTimeConverted" {
	mutate {
		add_field => {
			"LateFileStatus" => "LateArrival"
	    }	
	}	
} else {
	mutate {
		add_field => {
			"LateFileStatus" => "On-Time"
	    }	
	} 	
}	

#End of Date Comparision

Outside of filter objects, not the complete filter{} block

Would you please mind correct the code i sent?. It looks i am missing something.

Got it. Worked.

The "if" condition worked. But the date comparision is not giving correct result. It looks the dates are not in the same format.