Hello there !
I've read lots of doc and forum topics but can't succeed to find a solution
Winlogbeats are pushed to Kafka, then Logstash consume these logs
I have the following (valid) json pulled on kafka by Logstash :
{"@timestamp":"2020-02-18T15:14:02.352Z","@metadata":{"beat":"winlogbeat","type":"_doc","version":"7.5.2"},"agent":{"ephemeral_id":"363b6102-b019-482b-a868-df0455cb707b","hostname":"WIN7SPLUNKPC","id":"89c9d4d6-5c59-4db1-9cdc-5cdfc0521a02","version":"7.5.2","type":"winlogbeat"},"ecs":{"version":"1.1.0"},"message":"Le service Parental Controls est entré dans l’état : en cours d’exécution.","winlog":{"record_id":13383,"task":"","computer_name":"WIN7SPLUNKPC","event_id":7036,"event_data":{"param2":"en cours d’exécution","Binary":"5700500043005300760063002F0034000000","param1":"Parental Controls"},"provider_guid":"{555908d1-a6d7-4695-8e1e-26931d2012f4}","channel":"System","provider_name":"Service Control Manager","api":"wineventlog","process":{"thread":{"id":3388},"pid":436},"keywords":["Classique"]},"event":{"kind":"event","code":7036,"provider":"Service Control Manager","created":"2020-02-18T15:14:04.338Z"},"log":{"level":"information"},"host":{"hostname":"WIN7SPLUNKPC","architecture":"x86_64","os":{"build":"7601.24544","platform":"windows","version":"6.1","family":"windows","name":"Windows 7 Professional","kernel":"6.1.7601.24545 (win7sp1_ldr_escrow.200102-1707)"},"id":"a931f168-a8ce-4411-a3a3-6077d2830b48","name":"WIN7SPLUNKPC"}}
Here is the logstash conf :
input {
kafka {
bootstrap_servers => "192.168.250.102:9092"
topics => "winlogbeat"
codec => "json"
type => kafka
}
}
output {
elasticsearch {
ilm_enabled => true
hosts => ["http://coord01:9200","http://coord02:9200"]
index => "ims_linux_logs"
}
file {
path => "/usr/share/logstash/logs/logstash.log"
codec => rubydebug
}
}
I have the following error in logstash logs :
logstash01 | [WARN ] 2020-02-18 15:14:06.419 [[winlogbeat_kafka]>worker1] elasticsearch - Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"ims_linux_logs", :_type=>"_doc", :routing=>nil}, #LogStash::Event:0x4795a2e1], :response=>{"index"=>{"_index"=>"ims_linux_logs-000001", "_type"=>"_doc", "_id"=>"givdWHABmYntWF_FC0jK", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse field [host] of type [text] in document with id 'givdWHABmYntWF_FC0jK'. Preview of field's value: '{hostname=WIN7SPLUNKPC, os={build=7601.24544, kernel=6.1.7601.24545 (win7sp1_ldr_escrow.200102-1707), name=Windows 7 Professional, family=windows, version=6.1, platform=windows}, name=WIN7SPLUNKPC, id=a931f168-a8ce-4411-a3a3-6077d2830b48, architecture=x86_64}'", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:24"}}}}}
Ruby debug is like this :
{
"@version" => "1",
"host" => {
"id" => "a931f168-a8ce-4411-a3a3-6077d2830b48",
"name" => "WIN7SPLUNKPC",
"architecture" => "x86_64",
"os" => {
"platform" => "windows",
"family" => "windows",
"version" => "6.1",
"kernel" => "6.1.7601.24545 (win7sp1_ldr_escrow.200102-1707)",
"build" => "7601.24544",
"name" => "Windows 7 Professional"
},
"hostname" => "WIN7SPLUNKPC"
},
"type" => "kafka",
"event" => {
"kind" => "event",
"created" => "2020-02-18T15:14:04.338Z",
"provider" => "Service Control Manager",
"code" => 7036
},
"ecs" => {
"version" => "1.1.0"
},
"winlog" => {
"event_id" => 7036,
"computer_name" => "WIN7SPLUNKPC",
"channel" => "System",
"provider_name" => "Service Control Manager",
"api" => "wineventlog",
"keywords" => [
[0] "Classique"
],
"provider_guid" => "{555908d1-a6d7-4695-8e1e-26931d2012f4}",
"record_id" => 13383,
"event_data" => {
"param2" => "en cours d’exécution",
"Binary" => "5700500043005300760063002F0034000000",
"param1" => "Parental Controls"
},
"process" => {
"pid" => 436,
"thread" => {
"id" => 3388
}
},
"task" => ""
},
"@timestamp" => 2020-02-18T15:14:02.352Z,
"agent" => {
"id" => "89c9d4d6-5c59-4db1-9cdc-5cdfc0521a02",
"version" => "7.5.2",
"type" => "winlogbeat",
"ephemeral_id" => "363b6102-b019-482b-a868-df0455cb707b",
"hostname" => "WIN7SPLUNKPC"
},
"message" => "Le service Parental Controls est entré dans l’état : en cours d’exécution.",
"log" => {
"level" => "information"
}
}
Thanks for your help !