I am having this json file for which I am using .conf file to parse it into elasticsearch indices but I am getting error.
My .conf file is :
input {
file {
codec => json
path => ["/home/2017.09.04.json"]
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter {
json {
source => "message"
}
}
output {
stdout { codec => rubydebug}
elasticsearch
{ hosts => ["http://192.168.0.178:9200"]
index => "syslog"
# codec => "json_lines"
}
}
and I am getting an error saying
"[WARN ] 2019-09-11 15:39:24.960 [[main]>worker8] elasticsearch - Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"syslog", :_type=>"_doc", :routing=>nil}, #LogStash::Event:0x58a57c7b], :response=>{"index"=>{"_index"=>"syslog", "_type"=>"_doc", "_id"=>"jWPMH20BWRLyyJYol9Ib", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"Field [_type] is a metadata field and cannot be added inside a document. Use the index API request parameters."}}}}"
Although when I use stdout => rubydebug
I get an output saying:
{
"_index" => "live-2017.09.04",
"_type" => "live-2017.09.04",
"@version" => "1",
"path" => "/home/authbase/WORK/as/live-2017.09.04.json",
"_id" => "AV5M9WBNRm1hJaJ0kotK",
"@timestamp" => 2019-09-11T10:15:03.519Z,
"_source" => {
"deviceCustomNumber2Label" => "ICMP Code",
"categoryBehavior" => "/Access",
"type" => "CEF",
"agentZoneURI" => "/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 172.16.0.0-172.31.255.255",
"deviceCustomString5Label" => "Connection Type",
"deviceDirection" => "1",
"agentId" => "3EtTQoV0BABCKibaHdmbfnw==",
"agentTimeZone" => "America/Los_Angeles",
"deviceCustomNumber1Label" => "ICMP Type",
"deviceCustomString4Label" => "Order",
"deviceSeverity" => "6",
"deviceCustomIPv6Address4" => "fe80:0:0:0:b0cb:dba4:af5d:24ad",
"destinationAddress" => "168.95.192.14",
"deviceAddress" => "172.22.0.250",
"agentHost" => "IN-KOLCONPWV001.pwccpc.in",
"sourceZoneURI" => "/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 172.16.0.0-172.31.255.255",
"categoryObject" => "/Host/Application/Service",
"destinationTranslatedPort" => "53",
"transportProtocol" => "UDP",
"sourceTranslatedPort" => "60720",
"deviceCustomString2Label" => "Unit",
"categoryOutcome" => "/Success",
"agentAddress" => "172.22.2.63",
"sourcePort" => "60720",
"sourceTranslatedZoneURI" => "/All Zones/ArcSight System/Public Address Space Zones/APNIC/218.0.0.0-223.255.255.255 (APNIC)",
"name" => "Built outbound UDP connection",
"@version" => "1",
"destinationZoneURI" => "/All Zones/ArcSight System/Public Address Space Zones/ARIN/164.0.0.0-169.253.255.255 (ARIN)",
"deviceProduct" => "ASA",
"destinationTranslatedZoneURI" => "/All Zones/ArcSight System/Public Address Space Zones/ARIN/164.0.0.0-169.253.255.255 (ARIN)",
"deviceEventClassId" => "302015",
"categorySignificance" => "/Normal",
"destinationPort" => "53",
"deviceCustomIPv6Address4Label" => "Agent IPv6 Address",
"agentType" => "syslog",
"categoryDeviceGroup" => "/Firewall",
"categoryDeviceType" => "Firewall",
"deviceZoneURI" => "/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 172.16.0.0-172.31.255.255",
"deviceCustomNumber3Label" => "DurationInSeconds",
"destinationTimeZone" => "America/Los_Angeles",
"host" => "172.22.2.63",
"deviceCustomString1Label" => "ACL",
"agentVersion" => "7.3.0.7886.0",
"severity" => "Low",
"deviceCustomString3Label" => "TCP Flags",
"sourceTranslatedAddress" => "219.65.72.154",
"cefVersion" => "0.1",
"eventId" => "135058887",
"deviceOutboundInterface" => "outside",
"deviceReceiptTime" => "1504573267000",
"sourceAddress" => "172.22.2.5",
"externalId" => "145055176",
"destinationTranslatedAddress" => "168.95.192.14",
"deviceVersion" => "",
"deviceCustomString6Label" => "Duration",
"agentReceiptTime" => "1504529702732",
"@timestamp" => "2017-09-04T12:54:46.529Z",
"deviceInboundInterface" => "inside",
"deviceVendor" => "CISCO"
},
"_score" => 1,
"host" => "authbase1"
}