Hi there,
I'm trying to learn logstash. I have three inputs in logstash listening on different ports.
This is what the config looks like
input {
udp {
tags => ["netflow"]
type => "netflow"
port => 5046
codec => netflow {
versions => [9]
}
}
udp {
tags => ["syslog"]
type => "syslog"
port => 5045
}
beats {
port => 5044
tags => ["winlogbeat"]
type => "winlogbeat"
}
}
output {
if [type] == "netflow" {
elasticsearch { hosts => ["http://localhost:9200"] }
stdout { codec => rubydebug }
file { path => "/home/user/netflow.log" }
}
if [type] == "syslog" {
elasticsearch { hosts => ["http://localhost:9200"] }
stdout { codec => rubydebug }
file { path => "/home/user/syslog.log" }
}
if [type] == "winlogbeat" {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
stdout { codec => rubydebug }
file { path => "/home/user/winlogbeat.log" }
}
else {
file { path=> "/home/user/unknown.log" }
}
}
The problem that I have is winlogbeat doesnt get recognized properly and it goes to the unknown.log
The netflow and syslog appear to be logging into netflow.log and syslog.log which I've added for testing purposes
Can someone have a look at my syntax and tell me what could be wrong?
./logstash -f /etc/logstash/conf.d/elk.conf -log.level=debug
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs to console
10:57:55.591 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://localhost:9200/]}}
10:57:55.594 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://localhost:9200/, :path=>"/"}
log4j:WARN No appenders could be found for logger (org.apache.http.client.protocol.RequestAuthCache).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
10:57:55.660 [[main]-pipeline-manager] WARN logstash.outputs.elasticsearch - Restored connection to ES instance {:url=>#}
10:57:55.661 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Using mapping template from {:path=>nil}
10:57:55.800 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword"}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "include_in_all"=>false}, "@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
10:57:55.803 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>[#]}
10:57:55.809 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://localhost:9200/]}}
10:57:55.810 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://localhost:9200/, :path=>"/"}
10:57:55.812 [[main]-pipeline-manager] WARN logstash.outputs.elasticsearch - Restored connection to ES instance {:url=>#}
10:57:55.818 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Using mapping template from {:path=>nil}
10:57:55.821 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword"}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "include_in_all"=>false}, "@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
10:57:55.825 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>[#]}
10:57:55.826 [[main]-pipeline-manager] INFO logstash.pipeline - Starting pipeline {"id"=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>500}
10:57:56.182 [[main]-pipeline-manager] INFO logstash.inputs.beats - Beats inputs: Starting input listener {:address=>"0.0.0.0:5044"}
10:57:56.226 [[main]-pipeline-manager] INFO logstash.pipeline - Pipeline main started
10:57:56.234 [[main]"0.0.0.0:5045"}
10:57:56.235 [[main]"0.0.0.0:5046"}
10:57:56.264 [[main]"0.0.0.0:5046", :receive_buffer_bytes=>"106496", :queue_size=>"2000"}
10:57:56.267 [[main]"0.0.0.0:5045", :receive_buffer_bytes=>"106496", :queue_size=>"2000"}
10:57:56.330 [Api Webserver] INFO logstash.agent - Successfully started Logstash API endpoint {:port=>9600}
10:58:46.533 [worker1] INFO logstash.outputs.file - Opening file {:path=>"/home/user/unknown.log"}