Hi all,
Rather new to ELK in general, I am trying to migrate from filebeat v5.6.4 to filebeat 7.17.3. The problem is that the messages from the new install are not being parsed correctly. I know the document_type as type was deprecated in v6, and have tried to adapt our configuration accordingly to use input type or some other field for filtering in logstash, but it does not seem to work.
The new version is sending events to logstash, but these are not parsed the same way the old version did and we are missing several necessary fields. The old version is for now still running in production/test/dev, while the new one is being tested in our playground/alpha, but the logs these different environments produce have the same format. We are indexing them based on environment, loglevel and year/part.
Both filebeats use the same logstash output configuration. e.g. sent to the same place and we use an IF condition to differentiate what filters events from old and new filebeat gets. In general, its the exact same config, apart from a few things
The main issue is that the log level is not being set unless i manually set it through logstash-indexer.conf, and that log lines/messages are not being parsed correctly into different fields like, pods, containers, namespaces etc. so its easily can be search for in ELK/Kibana.
As the old filebeat is REALLY OLD now, we would def. like to move to the more recent 7.17.3, but we cannot until this is sorted. Any and all suggestions are very welcome and most grateful for!
Below are some examples of events from the "new" filebeat as seen in ELK, as well as its config, followed by examples from the "old" filebeat as well as its config and a common logstash-indexer.conf at the end.
//fjkoz
event from "new" filebeat as seen in ELK/kibana
{
"_index": "logstash-alpha-%{level2}-2023.33",
"_type": "_doc",
"_id": "GIAz-YkBle9XL6NxCHIr",
"_version": 1,
"_score": 1,
"_source": {
"ecs": {
"version": "1.12.0"
},
"stream": "stderr",
"level": "UNDEFINED-LEVEL", #<< Manually set in logstash-indexer.conf with "if [level] else mutate add_field...."
"@timestamp": "2023-08-15T12:36:55.436Z",
"agent": {
"hostname": "host41",
"id": "0cdd806e-73b0-4256-ab6d-9ff14c084a63",
"ephemeral_id": "7f3a4dcb-a7d9-409e-869e-1c753f8f8bae",
"name": "host41",
"type": "filebeat",
"version": "7.17.3"
},
"environment": "alpha",
"time": "2023-08-15T12:36:55.030181603Z",
"host": {
"name": "host41"
},
"@version": "1",
"log": {
"offset": 6634937,
"file": {
"path": "/var/log/containers/kong-egress-kong-94d6f648d-9x4fk_kong-egress_proxy-f014bbc316fe21dbc02ce7d80685dabd8e5b14b200f2ab7a1792cfd2eed9287b.log"
}
},
"message": "2023/08/15 12:36:55 [warn] 1525#0: *41353 [lua] targets.lua:504: queryDns(): querying dns for echoserver.devdev.no failed: dns server error: 3 name error. Tried [\"(short)echoserver.devdev.no:(na) - cache-miss\",\"echoserver.devdev.no.kong-egress.svc.k8s-vmware-3:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.svc.k8s-vmware-3:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.k8s-vmware-3:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.kong-egress.svc.k8s-vmware-3:1 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.svc.k8s-vmware-3:1 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.k8s-vmware-3:1 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:1 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:1 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no:1 - cache-hit/stale/in progress (async)/dns client error: 101 empty record received\",\"echoserver.devdev.no.kong-egress.svc.k8s-vmware-3:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.svc.k8s-vmware-3:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.k8s-vmware-3:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\"], context: ngx.timer\n",
"tags": [
"beats_input_codec_plain_applied",
"_jsonparsefailure"
],
"input": {
"type": "filestream"
}
},
"fields": {
"agent.version.keyword": [
"7.17.3"
],
"environment.keyword": [
"alpha"
],
"stream.keyword": [
"stderr"
],
"input.type.keyword": [
"filestream"
],
"host.name.keyword": [
"host41"
],
"tags.keyword": [
"beats_input_codec_plain_applied",
"_jsonparsefailure"
],
"agent.hostname.keyword": [
"host41"
],
"agent.type": [
"filebeat"
],
"ecs.version.keyword": [
"1.12.0"
],
"stream": [
"stderr"
],
"@version": [
"1"
],
"agent.name": [
"host41"
],
"host.name": [
"host41"
],
"log.file.path.keyword": [
"/var/log/containers/kong-egress-kong-94d6f648d-9x4fk_kong-egress_proxy-f014bbc316fe21dbc02ce7d80685dabd8e5b14b200f2ab7a1792cfd2eed9287b.log"
],
"agent.type.keyword": [
"filebeat"
],
"agent.ephemeral_id.keyword": [
"7f3a4dcb-a7d9-409e-869e-1c753f8f8bae"
],
"level": [
"UNDEFINED-LEVEL"
],
"agent.name.keyword": [
"host41"
],
"agent.id.keyword": [
"0cdd806e-73b0-4256-ab6d-9ff14c084a63"
],
"input.type": [
"filestream"
],
"log.offset": [
6634937
],
"message": [
"2023/08/15 12:36:55 [warn] 1525#0: *41353 [lua] targets.lua:504: queryDns(): querying dns for echoserver.devdev.no failed: dns server error: 3 name error. Tried [\"(short)echoserver.devdev.no:(na) - cache-miss\",\"echoserver.devdev.no.kong-egress.svc.k8s-vmware-3:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.svc.k8s-vmware-3:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.k8s-vmware-3:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no:33 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.kong-egress.svc.k8s-vmware-3:1 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.svc.k8s-vmware-3:1 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.k8s-vmware-3:1 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:1 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:1 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no:1 - cache-hit/stale/in progress (async)/dns client error: 101 empty record received\",\"echoserver.devdev.no.kong-egress.svc.k8s-vmware-3:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.svc.k8s-vmware-3:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.k8s-vmware-3:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no.example.no:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\",\"echoserver.devdev.no:5 - cache-hit/stale/in progress (async)/dns server error: 3 name error\"], context: ngx.timer\n"
],
"agent.hostname": [
"host41"
],
"tags": [
"beats_input_codec_plain_applied",
"_jsonparsefailure"
],
"environment": [
"alpha"
],
"@timestamp": [
"2023-08-15T12:36:55.436Z"
],
"level.keyword": [
"UNDEFINED-LEVEL"
],
"agent.id": [
"0cdd806e-73b0-4256-ab6d-9ff14c084a63"
],
"ecs.version": [
"1.12.0"
],
"log.file.path": [
"/var/log/containers/kong-egress-kong-94d6f648d-9x4fk_kong-egress_proxy-f014bbc316fe21dbc02ce7d80685dabd8e5b14b200f2ab7a1792cfd2eed9287b.log"
],
"agent.ephemeral_id": [
"7f3a4dcb-a7d9-409e-869e-1c753f8f8bae"
],
"agent.version": [
"7.17.3"
],
"time": [
"2023-08-15T12:36:55.030Z"
]
}
}
filebeat config
filebeatConfig:
filebeat.yml: |-
filebeat.registry.path: /var/log/containers/registry
filebeat.inputs:
- type: filestream
enabled: true
id: kube-logs
paths:
- ${log_path_kong}
- ${log_path_common}
- ${log_path_ingress_nginx}
- ${log_path_flux}
processors:
- add_kubernetes_metadata:
host: "$${NODE_NAME}"
matchers:
- logs_path:
logs_path: "/var/log/containers/"
prospector.scanner.symlinks: true
parsers:
- container:
stream: all
fields:
environment: ${environment}
host: "$${HOSTNAME}"
fields_under_root: true
output.logstash:
hosts: ["applogs.example.com:5044"]
timeout: 15
logging.level: "$${LOG_LEVEL:debug}"
event from "old" filebeat as seen in ELK/Kibana
{
"_index": "logstash-prod-info-2023.33",
"_type": "_doc",
"_id": "4Rtq_okBHBzSRVfx5rDG",
"_version": 1,
"_score": 1,
"_source": {
"log": "127.0.0.1 - - [16/Aug/2023:12:56:03 +0000] \"GET /status HTTP/1.1\" 200 1344 \"-\" \"Go-http-client/1.1\"",
"environment": "prod",
"level2": "info",
"container_name": "proxy",
"pod_name": "kong-egress-kong-685dcdf858-7pr2p",
"container_id": "551d92cf9af17d08f2203af1a68c0f06ba0b38c2a575b17d8191d4a8e1094e43",
"namespace": "kong-egress",
"@timestamp": "2023-08-16T12:56:03.737Z",
"offset": 13537728,
"type": "kube-logs",
"@version": "1",
"tags": [
"beats_input_raw_event",
"_jsonparsefailure"
],
"host": "host100",
"level": "INFO"
},
"fields": {
"environment.keyword": [
"prod"
],
"log": [
"127.0.0.1 - - [16/Aug/2023:12:56:03 +0000] \"GET /status HTTP/1.1\" 200 1344 \"-\" \"Go-http-client/1.1\""
],
"tags.keyword": [
"beats_input_raw_event",
"_jsonparsefailure"
],
"level2.keyword": [
"info"
],
"container_id.keyword": [
"551d92cf9af17d08f2203af1a68c0f06ba0b38c2a575b17d8191d4a8e1094e43"
],
"type": [
"kube-logs"
],
"container_name.keyword": [
"proxy"
],
"type.keyword": [
"kube-logs"
],
"@version": [
"1"
],
"host": [
"host100"
],
"host.keyword": [
"host100"
],
"pod_name.keyword": [
"kong-egress-kong-685dcdf858-7pr2p"
],
"offset": [
13537728
],
"level": [
"INFO"
],
"namespace.keyword": [
"kong-egress"
],
"tags": [
"beats_input_raw_event",
"_jsonparsefailure"
],
"pod_name": [
"kong-egress-kong-685dcdf858-7pr2p"
],
"environment": [
"prod"
],
"@timestamp": [
"2023-08-16T12:56:03.737Z"
],
"level.keyword": [
"INFO"
],
"container_name": [
"proxy"
],
"log.keyword": [
"127.0.0.1 - - [16/Aug/2023:12:56:03 +0000] \"GET /status HTTP/1.1\" 200 1344 \"-\" \"Go-http-client/1.1\""
],
"namespace": [
"kong-egress"
],
"container_id": [
"551d92cf9af17d08f2203af1a68c0f06ba0b38c2a575b17d8191d4a8e1094e43"
],
"level2": [
"info"
]
}
}
Filebeat config
filebeat.yml: |-
filebeat.registry_file: /var/log/containers/filebeat_registry
filebeat.prospectors:
- input_type: log
paths:
- ${LOG_PATH_COMMON:null}
- ${LOG_PATH_KONG:null}
- ${LOG_PATH_INGRESS_NGINX:null}
- ${LOG_PATH_FLUX:null}
symlinks: true
json.message_key: log
json.keys_under_root: true
json.add_error_key: true
multiline.pattern: '^\s'
multiline.match: after
document_type: kube-logs
fields:
host: ${FILEBEAT_HOST:${HOSTNAME:null}}
environment: ${FILEBEAT_ENVIRONMENT:undefined-env}
fields_under_root: true
output.logstash:
hosts: ${LOGSTASH_HOSTS:null}
timeout: 15
logging.level: ${LOG_LEVEL:error}
logstash-indexer.conf for both old and new
input {
beats {
port => 5044
}
}
filter {
if [environment] == "alpha" {
mutate {
lowercase => [ "environment" ]
}
if [input][type] == "filestream" {
json {
source => "log"
}
mutate {
#rename => ["log", "message"]
remove_field => [ "fields", "beat", "input_type", "stream" ]
}
date {
match => ["time", "ISO8601"]
remove_field => ["time"]
}
grok {
match => { "source" => "/var/log/containers/%{DATA:pod_name}_%{DATA:namespace}_%{GREEDYDATA:container_name}-%{DATA:container_id}.log" }
remove_field => ["source"]
add_field => {"grok" => "MATCHED"}
}
}
# if [type] == "log" {
# mutate {
# #remove_field => [ "fields", "beat", "input_type" ]
# }
# json {
# source => "message"
# }
# }
if [level] {
} else {
mutate {
add_field => { "level" => "undefined-level" }
}
}
mutate {
uppercase => [ "level" ]
}
# drop debug level messages
if [level] == "TRACE" {
drop{}
}
if [level] == "DEBUG" {
mutate {
add_field => { "level2" => "debug" }
}
}
if [level] == "INFO" {
mutate {
add_field => { "level2" => "info" }
}
}
if [level] == "WARN" {
mutate {
add_field => { "level2" => "warn" }
}
}
if [level] == "WARNING" {
mutate {
add_field => { "level2" => "warning" }
}
}
if [level] == "ERR" {
mutate {
add_field => { "level2" => "err" }
}
}
if [level] == "ERROR" {
mutate {
add_field => { "level2" => "error" }
}
}
mutate {
lowercase => [ "level2" ]
}
}
else {
if [environment] {
} else {
mutate {
add_field => { "environment" => "undefined-env" }
}
}
# indexes in elastic search must be lowercase, so normalize
mutate {
lowercase => [ "environment" ]
}
# indexes in elastic search must be lowercase, so normalize
mutate {
lowercase => [ "environment" ]
}
if [environment] {
} else {
mutate {
add_field => { "environment" => "undefined-env" }
}
}
if [type] == "kube-logs" {
json {
source => "log"
}
mutate {
#rename => ["log", "message"]
remove_field => [ "fields", "beat", "input_type", "stream" ]
}
date {
match => ["time", "ISO8601"]
remove_field => ["time"]
}
grok {
match => { "source" => "/var/log/containers/%{DATA:pod_name}_%{DATA:namespace}_%{GREEDYDATA:container_name}-%{DATA:container_id}.log" }
remove_field => ["source"]
}
}
if [type] == "log" {
mutate {
remove_field => [ "fields", "beat", "input_type" ]
}
json {
source => "message"
}
}
if [level] {
} else {
mutate {
add_field => { "level" => "INFO" }
}
}
mutate {
uppercase => [ "level" ]
}
# drop debug level messages
if [level] == "TRACE" {
drop{}
}
if [level] == "DEBUG" {
mutate {
add_field => { "level2" => "debug" }
}
}
if [level] == "INFO" {
mutate {
add_field => { "level2" => "info" }
}
}
if [level] == "WARN" {
mutate {
add_field => { "level2" => "warn" }
}
}
if [level] == "WARNING" {
mutate {
add_field => { "level2" => "warning" }
}
}
if [level] == "ERR" {
mutate {
add_field => { "level2" => "err" }
}
}
if [level] == "ERROR" {
mutate {
add_field => { "level2" => "error" }
}
}
mutate {
lowercase => [ "level2" ]
}
}
}
output {
elasticsearch {
hosts => "logstashhost"
index => "logstash-%{environment}-%{level2}-%{+xxxx.ww}"
}
}
