I confifured filebeat to send logs to elasticsearch. I am facing issue in kibana visualizations. I define a pattern in filebeat module and I want to see logs in Discover tab grouped by fields defined in the filebeat module pattern. My module pipeline[module/glassfish/server/ingest] configuration is:
{
"description": "Pipeline for parsing Glassfish3.2.2 server logs",
"processors": [
{
"grok": {
"field": "message",
"patterns": ["\\[#\\|%{TIMESTAMP_ISO8601:glassfish.server.timestamp}\\|%{LOGLEVEL:glassfish.server.loglevel}\\|%{DATA:glassfish.server.application}\\|%{GREEDYDATA:glassfish.server.component}\\|%{GLASSFISHTHREADS:glassfish.server.threadinfo}\\|%{LOGLEVEL:glassfish.server.app_log_level} %{GREEDYDATA:glassfish.server.app_correl_id} %{DATA:glassfish.server.app_class_name} - %{GREEDYDATA:glassfish.server.app_message}\\|#\\]","\\[#\\|%{TIMESTAMP_ISO8601:glassfish.server.timestamp}\\|%{LOGLEVEL:glassfish.server.loglevel}\\|%{DATA:glassfish.server.application}\\|%{GREEDYDATA:glassfish.server.component}\\|%{GLASSFISHTHREADS:glassfish.server.threadinfo}\\|%{GREEDYDATA:glassfish.server.app_timestamp}: \\[%{LOGLEVEL:glassfish.server.app_log_level}\\]: \\[%{GREEDYDATA:glassfish.server.app_correl_id}\\] Source Class = %{GREEDYDATA:glassfish.server.app_class_name} %{GREEDYDATA:glassfish.server.app_message}\\|#\\]"
],
"ignore_missing": true,
"pattern_definitions": {
"GLASSFISHTHREADS": "_ThreadID=%{NUMBER:glassfish.server.threadid};_ThreadName=Thread-%{NUMBER:glassfish.server.threadnumberinname};"
}
}
},
],
"on_failure" : [{
"set" : {
"field" : "error",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
}
My custom template configuration is:
{
"mappings": {
"_default_": {
"_all": {
"norms": false
},
"_meta": {
"version": "5.4.2"
},
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"glassfish": {
"properties": {
"server": {
"properties": {
"loglevel": {
"ignore_above": 1024,
"type": "keyword"
},
"application": {
"ignore_above": 1024,
"type": "keyword"
},
"component": {
"norms": false,
"type": "text"
},
"threadnumberinname": {
"type": "long"
},
"threadid": {
"type": "long"
},
"app_log_level": {
"ignore_above": 1024,
"type": "keyword"
},
"app_correl_id": {
"type": "keyword"
},
"app_class_name": {
"ignore_above": 1024,
"type": "keyword"
},
"app_message": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"beat": {
"properties": {
"hostname": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"ignore_above": 1024,
"type": "keyword"
},
"fileset": {
"properties": {
"module": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"input_type": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"norms": false,
"type": "text"
},
"meta": {
"properties": {
"cloud": {
"properties": {
"availability_zone": {
"ignore_above": 1024,
"type": "keyword"
},
"instance_id": {
"ignore_above": 1024,
"type": "keyword"
},
"machine_type": {
"ignore_above": 1024,
"type": "keyword"
},
"project_id": {
"ignore_above": 1024,
"type": "keyword"
},
"provider": {
"ignore_above": 1024,
"type": "keyword"
},
"region": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"offset": {
"type": "long"
},
"read_timestamp": {
"ignore_above": 1024,
"type": "keyword"
},
"source": {
"ignore_above": 2048,
"type": "keyword"
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
},
"order": 0,
"settings": {
"index.mapping.total_fields.limit": 10000,
"index.refresh_interval": "5s"
},
"template": "filebeat-*"
}
I am using
.\filebeat.exe -c filebeat.yml -e -v -modules=glassfish
command to start filebeat. When I load the index pattern in kibana, I could not see any data are present in the defined fields. It is by default reside in either of below fields 1. _search
2. _message. But I want to visualize in individual columns to give better visibility for readers. I don't know where I am missing? Please help me to configure kibana based on my need.
