I am able to parse logs on logstash but the logs are not visible on kibana dashboard.
error thrown is:
[2018-11-06T12:29:41,731][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"elkrft-ind-2018.11.06", :_type=>"doc", :_routing=>nil}, #LogStash::Event:0xf115a54], :response=>{"index"=>{"_index"=>"elkrft-ind-2018.11.06", "_type"=>"doc", "_id"=>"RuTR52YB8vPDK4j72lID", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [host]", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:20638"}}}}}
Logstash is :
input {
beats {
port => 5044
}
file {
path => "D:\USICE ELK Testing\SP-SAP/*.txt"
start_position => beginning
add_field => [ "log_type","SP-SAP" ]
}
}
The filter part of this file is commented out to indicate that it is
optional.
filter {
if [fields][log_type] == "LA-SAG" {
if "SOURCE MESSAGEJMS Message In" in [message] {
dissect {
mapping => {
message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{Filler1}##%{Correlation_Id}##%{Filler2}##START SERVICE MESSAGE SERVICE NAME : %{StartServiceMessage}##%{Filler3}##%{Filler4}##%{Filler5}##%{Filler6}##%{ServiceContextId}##%{UserName}##%{Filler7}##SOURCE MESSAGEJMS Message In : %{sourcexmlfile}" }
}
xml {
source => "sourcexmlfile"
store_xml => false
xpath => ["/ZWBBDLD05_S01/IDOC/EDI_DC40/DOCNUM/text()", "idoc_no"]
xpath => ["/ZWBBDLD05_S01/IDOC/EDI_DC40/RCVPRN/text()", "store_no"]
}
mutate {
remove_field => [ "sourcexmlfile" ]
}
}
if "TARGET MESSAGEJMS Message Out" in [message] {
dissect {
mapping => {
message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{IDOC}##%{Correlation_Id}##%{Filler1}##%{SuccessMessageID}##%{Filler2}##%{Filler3}##%{Filler4}##%{Filler5}##%{ServiceContextId}##%{UserName}##%{Filler6}##%{Filler7}" }
}
}
if "Validation is success" in [message] {
dissect {
mapping => {
message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{IDOC}##%{Correlation_Id}##%{Filler1}##%{SourceValidationMessage}##%{Filler2}##%{Filler3}##%{Filler4}##%{Filler5}##%{ServiceContextId}##%{UserName}##%{Filler6}##%{Filler7}" }
}
}
if "##ERROR CODE" in [message] {
dissect {
mapping => {
message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{Filler1}##%{Correlation_Id}##%{Filler2}##END SERVICE MESSAGE - SERVICE NAME : %{EndServiceMessage}##%{Filler3}##%{Filler4}##%{Filler5}##%{Filler6}##%{ServiceContextId}##%{UserName}##%{Filler7}##%{ErrorMessage}"
}
}
}
}
if [fields][log_type] == "SP-SAG" {
if "JMS Message In" in [message] {
dissect {
mapping => {
message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{Filler1}##%{Correlation_Id}##%{Filler2}##Service Started : %{StartServiceMessage}##%{Filler3}##%{Filler4}##%{Filler5}##%{Filler6}##%{ServiceContextId}##%{UserName}##%{Filler7}##JMS Message In : %{SourceMessage}" }
}
xml {
source => "SourceMessage"
store_xml => false
xpath => ["/ZWP_PLU03/IDOC/EDI_DC40/DOCNUM/text()", "idoc_no"]
xpath => ["/ZWP_PLU03/IDOC/E1WPA01[@SEGMENT='1']/FILIALE/text()", "store_no"]
}
mutate {
remove_field => [ "SourceMessage" ]
}
}
if "TARGET MESSAGEJMS Message Out" in [message] {
dissect {
mapping => {
message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{IDOC}##%{Correlation_Id}##%{Filler1}##END SERVICE MESSAGE - SERVICE NAME : %{EndServiceMessage}##%{Filler2}##%{Filler3}##%{Filler4}##%{Filler5}##%{ServiceContextId}##%{UserName}##%{Filler6}##TARGET MESSAGEJMS Message Out : %{targetxmlfile}" }
}
xml {
source => "targetxmlfile"
store_xml => false
xpath => ["/FILE/@batchNo", "target_idoc_no"]
xpath => ["/FILE/TRANSACTION/PRICE/PRICES/PRODUCT_PRICE/@price_band", "store_no"]
}
mutate {
remove_field => [ "targetxmlfile" ]
}
}
if "##ERROR CODE" in [message] {
dissect {
mapping => {
message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{Filler1}##%{Correlation_Id}##%{Filler2}##END SERVICE MESSAGE - SERVICE NAME : %{EndServiceMessage}##%{Filler3}##%{Filler4}##%{Filler5}##%{Filler6}##%{ServiceContextId}##%{UserName}##%{Filler7}##%{ERRORMESSAGE}"
}
}
}
if "##Idoc : IDOC :" in [message] {
dissect {
mapping => {
message => "%{Timestamp} ##%{HostName}##%{Service_Name}##%{RICEF}##%{IDOC}##%{Correlation_Id}##%{Filler1}##%{Filler2}##%{Filler3}##%{Filler4}##%{Filler5}##%{Filler6}##%{ServiceContextId}##%{UserName}##%{Filler7}##%{IDOC}"
}
}
}
}
}
output {
elasticsearch {
hosts => "localhost:9200"
manage_template => false
index => "elkrft-ind-%{+YYYY.MM.dd}"
}
stdout { codec => rubydebug }
}