Hello,
-
Kibana's Discover tab not showing the data comming from Filebeat on its corresponding Elasticsearch index pattern. Tried ingesting logs into a new index with different index pattern.
-
The logs are visible on Logstash stdout & via a querry in Dev Tools, indicating that index is getting populated.
-
Index Management shows that the particular index queried is present.
-
Logs shipping pipeline:
Metricbeat(Port 5044), Winlogbeat(Port 5044) & Filebeat(Port 5045) --> Logstash --> Elasticsearch <-- Kibana
- Logs from windows system index (ingested using Winlogbeat into different a index using the same Logstash pipeline) are visible on both stdout and in discover tab, indicating that Logstash is not the culprit.
Logstash Pipeline:
input
{
beats {
id => "internal"
port => 5045
}
beats {
id => "mylumberjack"
codec => json
port => 5044
ssl_certificate => "/etc/logstash/certs/logstash-vm0/instance.crt"
ssl_key => "/etc/logstash/certs/logstash-vm0/instance.pk8"
ssl => true
}
}
filter
{
if [metricset][module] == "windows"
{
mutate
{
add_field => { "index" => "metrics-windows-%{+YYYY.MM.dd}" }
add_field => { "pipeline" => "default" }
}
}
if [metricset][module] == "system"
{
mutate
{
add_field => { "index" => "metrics-system-%{+YYYY.MM.dd}" }
add_field => { "pipeline" => "default" }
}
}
if [type] == "wineventlog"
{
mutate
{
add_field => { "index" => "winlogbeat-%{[beat][version]}-%{+YYYY.MM.dd}" }
add_field => { "pipeline" => "default" }
}
}
if [fileset][module] == "apache2"
{
if [fileset][name] == "access"
{
mutate
{
add_field => { "index" => "logs-apache-%{+YYYY.MM.dd}" }
add_field => { "pipeline" => "logs-apache-access-ingest" }
}
}
else
{
mutate
{
add_field => { "index" => "logs-apache-%{+YYYY.MM.dd}" }
add_field => { "pipeline" => "logs-apache-error-ingest" }
}
}
}
if [fileset][module] == "mysql"
{
if [fileset][name] == "error"
{
mutate
{
add_field => { "index" => "logs-mysql-%{+YYYY.MM.dd}" }
add_field => { "pipeline" => "logs-mysql-error-ingest" }
}
}
else
{
mutate
{
add_field => { "index" => "logs-mysql-%{+YYYY.MM.dd}" }
add_field => { "pipeline" => "logs-mysql-slowlog-ingest" }
}
}
}
if [fileset][module] == "system"
{
if [fileset][name] == "auth"
{
mutate
{
add_field => { "index" => "logs-system-%{+YYYY.MM.dd}" }
add_field => { "pipeline" => "logs-system-auth-ingest" }
}
}
else
{
mutate
{
add_field => { "index" => "logs-system-%{+YYYY.MM.dd}" }
add_field => { "pipeline" => "logs-system-syslog-ingest" }
}
}
}
}
output
{
stdout { codec => rubydebug }
elasticsearch
{
hosts => [ "https://elasticsearch-ingest-vm0:9200" ]
ssl => true
ssl_certificate_verification => true
cacert => '/etc/logstash/certs/elasticsearch-master-vm0/ca.crt'
user => "logstash_writer"
password => "*********"
manage_template => false
pipeline => "%{pipeline}"
index => "%{index}"
}
}
Attached snapshot of Dev tools query and Discover tab.
Dev Tools
Discover Tab
