Hello, newby here,
My task is to take multiple (8) logs with different formats and using filebeat send them directly to elastic. They should all end inside the same index and each log should be searchable using "level" field:
"log" : {
"level": "access log",
"message": "access message 1"
},
"log": {
"level": "error log",
"message": "error message 1"
}
The problem I am facing is, I am getting no fields inside kibana once filebeat harvests the data:
"hits" : [
{
"_index" : "tomcat_logs",
"_type" : "_doc",
"_id" : "XO8HEngBdkAWhdismvyE",
"_score" : 1.0,
"_source" : {
"@timestamp" : "2021-03-08T13:29:50.444Z",
"agent" : {
"version" : "7.10.2"
},
"fields" : {
"type" : "tomcat_logs"
}
}
}
My filebeat:
filebeat.inputs:
- type: log
enabled: true
paths:
- /opt/apache-tomcat-9.0.41/logs/catalina.*
fields:
codec: plain
type: tomcat_logs
processors:
- drop_event:
when:
regexp:
message: 'INFO|DEBUG'
- dissect:
tokenizer: '%{date} %{time} %{logLevel} [%{method}] %{class} %{message}'
field: "message"
target_prefix: 'log'
- type: log
enabled: true
paths:
- /opt/apache-tomcat-9.0.41/logs/localhost.*
fields:
codec: plain
type: tomcat_logs
processors:
- drop_event:
when:
regexp:
message: 'INFO|DEBUG'
- dissect:
tokenizer: '%{date} %{time} %{logLevel} [%{method}] %{class} %{listenerType}: %{listenerLog}'
field: "message"
target_prefix: 'log'
- type: log
enabled: true
paths:
- /opt/apache-tomcat-9.0.41/logs/localhost_access_log.*.txt
fields:
codec: plain
type: tomcat_logs
processors:
- drop_event:
when:
regexp:
message: 'INFO|DEBUG'
- dissect:
tokenizer: '%{hostName} %{userName} %{authenticatedUser} [%{dateTime}] "%{request}" %{response} %{bytes}'
field: "message"
target_prefix: 'log'
- type: log
enabled: true
paths:
- /var/log/httpd/error_log-*
fields:
codec: plain
type: tomcat_logs
processors:
- drop_event:
when:
not:
regexp:
message: 'auth_openidc:error'
- dissect:
tokenizer: '[%{dateTime}] [%{logLevel}] [pid %{pid|integer}] [client %{clientIP}] %{message}'
field: "message"
target_prefix: 'log'
- type: log
enabled: true
paths:
- /var/log/httpd/error_log-*
fields:
codec: plain
type: tomcat_logs
processors:
- drop_event:
when:
not:
regexp:
message: 'proxy:error'
- dissect:
tokenizer: '[%{dateTime}] [%{logLevel}] [pid %{pid|integer}] %{message}'
field: "message"
target_prefix: 'log'
- type: log
enabled: true
paths:
- /var/log/httpd/error_log-*
fields:
codec: plain
type: tomcat_logs
processors:
- drop_event:
when:
not:
regexp:
message: 'proxy_http:error'
- dissect:
tokenizer: '[%{dateTime}] [%{logLevel}] [pid %{pid|integer}] [client %{clientIP}] %{message}'
field: "message"
target_prefix: 'log'
- type: log
enabled: true
paths:
- /var/log/httpd/ssl_error_log-*
fields:
codec: plain
type: tomcat_logs
processors:
- dissect:
tokenizer: '[%{dateTime}] [%{logLevel}] [pid %{pid|integer}] %{message}'
field: "message"
target_prefix: 'log'
- type: log
enabled: true
paths:
- /var/log/httpd/access_log-*
fields:
codec: plain
type: tomcat_logs
processors:
- dissect:
tokenizer: '%{hostName} %{logicalUsername} %{authenticatedUser} [%{dateTime}] "%{request}" %{statusCode} %{bytes} "-" "%{header}"'
field: "message"
target_prefix: 'log'
- type: log
enabled: true
paths:
- /var/log/httpd/ssl_access_log-*
fields:
codec: plain
type: tomcat_logs
processors:
- dissect:
tokenizer: '%{hostName} %{logicalUsername} %{authenticatedUser} [%{dateTime}] "%{request}" %{statusCode} %{bytes}'
field: "message"
target_prefix: 'log'
- type: log
enabled: true
paths:
- /var/log/httpd/ssl_request_log-*
fields:
codec: plain
type: tomcat_logs
processors:
- dissect:
tokenizer: '[%{dateTime}] %{logLevel} %{sslProtocol} %{sslCipher} "%{request}" %{bytes}'
field: "message"
target_prefix: 'log'
# ======================= Elasticsearch template setting =======================
setup.template:
name: "tomcat_logs"
pattern: "tomcat_logs-*"
overwrite: true
enabled: false
setup.ilm.enabled: false
setup.template.settings:
index.number_of_shards: 1
#index.codec: best_compression
#_source.enabled: false
# ---------------------------- Elasticsearch Output ----------------------------
output.elasticsearch:
# Array of hosts to connect to.
hosts: ["xxxx:9200"]
index: "tomcat_logs"
Before, each log had its own index and it worked well, now I am completely stuck and would appreciate any help.