Unable to view logs on kibana


(shruti) #1

I am able to parse logs on logstash but the logs are not visible on kibana dashboard.

error thrown is:
[2018-11-06T12:29:41,731][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"elkrft-ind-2018.11.06", :_type=>"doc", :_routing=>nil}, #LogStash::Event:0xf115a54], :response=>{"index"=>{"_index"=>"elkrft-ind-2018.11.06", "_type"=>"doc", "_id"=>"RuTR52YB8vPDK4j72lID", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [host]", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:20638"}}}}}

Logstash is :
input {
beats {
port => 5044
}
file {
path => "D:\USICE ELK Testing\SP-SAP/*.txt"
start_position => beginning
add_field => [ "log_type","SP-SAP" ]
}
}

The filter part of this file is commented out to indicate that it is

optional.

filter {
if [fields][log_type] == "LA-SAG" {
if "SOURCE MESSAGEJMS Message In" in [message] {
dissect {
mapping => {
message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{Filler1}##%{Correlation_Id}##%{Filler2}##START SERVICE MESSAGE SERVICE NAME : %{StartServiceMessage}##%{Filler3}##%{Filler4}##%{Filler5}##%{Filler6}##%{ServiceContextId}##%{UserName}##%{Filler7}##SOURCE MESSAGEJMS Message In : %{sourcexmlfile}" }
}
xml {
source => "sourcexmlfile"
store_xml => false
xpath => ["/ZWBBDLD05_S01/IDOC/EDI_DC40/DOCNUM/text()", "idoc_no"]
xpath => ["/ZWBBDLD05_S01/IDOC/EDI_DC40/RCVPRN/text()", "store_no"]
}
mutate {
remove_field => [ "sourcexmlfile" ]
}
}
if "TARGET MESSAGEJMS Message Out" in [message] {
dissect {
mapping => {
message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{IDOC}##%{Correlation_Id}##%{Filler1}##%{SuccessMessageID}##%{Filler2}##%{Filler3}##%{Filler4}##%{Filler5}##%{ServiceContextId}##%{UserName}##%{Filler6}##%{Filler7}" }
}

	}
	
	if "Validation is success" in [message] {
		dissect {
			mapping => { 
				message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{IDOC}##%{Correlation_Id}##%{Filler1}##%{SourceValidationMessage}##%{Filler2}##%{Filler3}##%{Filler4}##%{Filler5}##%{ServiceContextId}##%{UserName}##%{Filler6}##%{Filler7}" }
		}
					
	}
	if "##ERROR CODE" in [message] {
		dissect {
			mapping => {
				message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{Filler1}##%{Correlation_Id}##%{Filler2}##END SERVICE MESSAGE - SERVICE NAME : %{EndServiceMessage}##%{Filler3}##%{Filler4}##%{Filler5}##%{Filler6}##%{ServiceContextId}##%{UserName}##%{Filler7}##%{ErrorMessage}"
			}
		}
	}
}

	if [fields][log_type] == "SP-SAG" {
	if "JMS Message In" in [message] {
		dissect {
			mapping => { 
				message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{Filler1}##%{Correlation_Id}##%{Filler2}##Service Started : %{StartServiceMessage}##%{Filler3}##%{Filler4}##%{Filler5}##%{Filler6}##%{ServiceContextId}##%{UserName}##%{Filler7}##JMS Message In : %{SourceMessage}" }
		}
		xml {
			source => "SourceMessage"
			store_xml => false
			xpath => ["/ZWP_PLU03/IDOC/EDI_DC40/DOCNUM/text()", "idoc_no"]
			xpath => ["/ZWP_PLU03/IDOC/E1WPA01[@SEGMENT='1']/FILIALE/text()", "store_no"]				
		}
		mutate { 
			remove_field => [ "SourceMessage" ]
		}
	}
	if "TARGET MESSAGEJMS Message Out" in [message] {
		dissect {
			mapping => { 
				message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{IDOC}##%{Correlation_Id}##%{Filler1}##END SERVICE MESSAGE - SERVICE NAME : %{EndServiceMessage}##%{Filler2}##%{Filler3}##%{Filler4}##%{Filler5}##%{ServiceContextId}##%{UserName}##%{Filler6}##TARGET MESSAGEJMS Message Out : %{targetxmlfile}" }

		}
		xml {
			source => "targetxmlfile"
			store_xml => false
			xpath => ["/FILE/@batchNo", "target_idoc_no"]
			xpath => ["/FILE/TRANSACTION/PRICE/PRICES/PRODUCT_PRICE/@price_band", "store_no"]
		}
		mutate { 
			remove_field => [ "targetxmlfile" ]
		}
		
	}
	if "##ERROR CODE" in [message] {
		dissect {
			mapping => {
				message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{Filler1}##%{Correlation_Id}##%{Filler2}##END SERVICE MESSAGE - SERVICE NAME : %{EndServiceMessage}##%{Filler3}##%{Filler4}##%{Filler5}##%{Filler6}##%{ServiceContextId}##%{UserName}##%{Filler7}##%{ERRORMESSAGE}"
			}
		}
	}
	
	if "##Idoc   :  IDOC :" in [message] {
		dissect {
			mapping => {
				message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{IDOC}##%{Correlation_Id}##%{Filler1}##%{Filler2}##%{Filler3}##%{Filler4}##%{Filler5}##%{Filler6}##%{ServiceContextId}##%{UserName}##%{Filler7}##%{IDOC}"
			}
		}
}	
}

}

output {
elasticsearch {
hosts => "localhost:9200"
manage_template => false
index => "elkrft-ind-%{+YYYY.MM.dd}"
}
stdout { codec => rubydebug }
}


(system) #3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.