i am using elasticsearch to store logs and metrics for our applications on kubernetes environment. so now there are two different containers whose logs are collected by fluentd agent and pushed to elasticsearch.
one of the log source has field kubernetes.labels.app as text/keyword data type value. whereas another log source has kubernetes.labels.app as object data type value and it has nested fields in there.
now as both logs sources are collected by fluentd and pushed to same index in elasticsearch, there is mapping conflict. i had defined kubernetes.labels.app as object and because of that logs from first log source can not be indexed as it has above field as concrete value.
i need help on how to solve this issue? as i can not use object data type in multi fields as well. i will really appreciate any help in fixing the issue.
below is the one of log i found in fluentd logs: which logs when elastic rejects the data:
2023-04-27 16:18:23 +0000 [warn]: #0 dump an error event: error_class=Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError error="400 - Rejected by Elasticsearch [error type]: mapper_parsing_exception [reason]: 'object mapping for [kubernetes.labels.app] tried to parse field [app] as object, but found a concrete value'" location=nil tag="kubernetes.var.log.containers.metricbeat-metricbeat-daemon-nix-zqnxd_monitoring_metricbeat-873e6e16f82b99d73353fe047a67acc83e21b5334b07aed392a877da1ffacb9f.log" time=2023-04-27 16:18:07.661200976 +0000 record={"log.level"=>"info", "@timestamp"=>"2023-04-27T16:18:07.190Z", "log.logger"=>"monitoring", "log.origin"=>{"file.name"=>"log/log.go", "file.line"=>185}, "message"=>"Non-zero metrics in the last 30s", "service.name"=>"metricbeat", "monitoring"=>{"metrics"=>{"beat"=>{"cgroup"=>{"cpuacct"=>{"total"=>{"ns"=>2589661030}}, "memory"=>{"mem"=>{"usage"=>{"bytes"=>4653056}}}}, "cpu"=>{"system"=>{"ticks"=>35770450, "time"=>{"ms"=>540}}, "total"=>{"ticks"=>146376870, "time"=>{"ms"=>1760}, "value"=>0}, "user"=>{"ticks"=>110606420, "time"=>{"ms"=>1220}}}, "handles"=>{"limit"=>{"hard"=>1048576, "soft"=>1048576}, "open"=>22}, "info"=>{"ephemeral_id"=>"f4298458-dd8b-4741-a3df-84f68dbda1a8", "uptime"=>{"ms"=>3714150253}, "version"=>"8.3.3"}, "memstats"=>{"gc_next"=>95918840, "memory_alloc"=>61532352, "memory_total"=>19705420836760, "rss"=>263163904}, "runtime"=>{"goroutines"=>609}}, "libbeat"=>{"config"=>{"module"=>{"running"=>0}}, "output"=>{"events"=>{"acked"=>2142, "active"=>26062, "batches"=>67, "duplicates"=>738, "total"=>2880}, "read"=>{"bytes"=>893725}, "write"=>{"bytes"=>4866694}}, "pipeline"=>{"clients"=>29, "events"=>{"active"=>123, "filtered"=>1, "published"=>2880, "total"=>2881}, "queue"=>{"acked"=>2880}}}, "metricbeat"=>{"elasticsearch"=>{"cluster_stats"=>{"events"=>3, "success"=>3}, "enrich"=>{"events"=>12, "success"=>12}, "index"=>{"events"=>369, "success"=>369}, "index_recovery"=>{"events"=>588, "success"=>588}, "index_summary"=>{"events"=>3, "success"=>3}, "node_stats"=>{"events"=>12, "success"=>12}, "shard"=>{"events"=>738, "success"=>738}}, "kubernetes"=>{"container"=>{"events"=>38, "success"=>38}, "event"=>{"events"=>1, "success"=>1}, "node"=>{"events"=>1, "success"=>1}, "pod"=>{"events"=>32, "success"=>32}, "state_container"=>{"events"=>366, "success"=>366}, "state_deployment"=>{"events"=>87, "success"=>87}, "state_node"=>{"events"=>12, "success"=>12}, "state_pod"=>{"events"=>282, "success"=>282}, "state_replicaset"=>{"events"=>231, "success"=>231}, "system"=>{"events"=>3, "success"=>3}, "volume"=>{"events"=>65, "success"=>65}}, "system"=>{"cpu"=>{"events"=>1, "success"=>1}, "filesystem"=>{"events"=>1, "success"=>1}, "fsstat"=>{"events"=>1, "success"=>1}, "load"=>{"events"=>1, "success"=>1}, "memory"=>{"events"=>1, "success"=>1}, "network"=>{"events"=>25, "success"=>25}, "process"=>{"events"=>7, "success"=>7}, "process_summary"=>{"events"=>1, "success"=>1}}}, "system"=>{"load"=>{"1"=>2.01, "15"=>1.33, "5"=>1.3, "norm"=>{"1"=>0.2512, "15"=>0.1663, "5"=>0.1625}}}}, "ecs.version"=>"1.6.0"}, "stream"=>"stderr", "logtag"=>"F", "docker"=>{"container_id"=>"873e6e16f82b99d73353fe047a67acc83e21b5334b07aed392a877da1ffacb9f"}, "kubernetes"=>{"container_name"=>"metricbeat", "namespace_name"=>"monitoring", "pod_name"=>"metricbeat-metricbeat-daemon-nix-zqnxd", "container_image"=>"<>.azurecr.io/metricbeat-rp-mb-fixes:2019474", "container_image_id"=>"<>.azurecr.io/metricbeat-rp-mb-fixes@sha256:7eb70c3a1d4fc53e073a720093b5596189d87dfa87b12d3642134173f88901fd", "pod_id"=>"5b7fead5-f880-47ce-b3f0-c14879949528", "pod_ip"=>"172.19.54.154", "host"=>"aks-nodepool1-30653362-vmss000003", "labels"=>{"app"=>"metricbeat-metricbeat", "chart"=>"metricbeat-1.0.2019474", "controller-revision-hash"=>"6c478b996b", "heritage"=>"Helm", "pod-template-generation"=>"2", "release"=>"metricbeat"}, "master_url"=>"https://10.0.0.1:443/api", "namespace_id"=>"4aa332b3-d7cc-4bc4-b8c5-bfa1f1f71f69", "namespace_labels"=>{"azure-key-vault-env-injection"=>"enabled", "kubernetes.io/metadata.name"=>"monitoring", "purpose"=>"monitoring"}}, "tag"=>"kubernetes.var.log.containers.metricbeat-metricbeat-daemon-nix-zqnxd_monitoring_metricbeat-873e6e16f82b99d73353fe047a67acc83e21b5334b07aed392a877da1ffacb9f.log"}
existing mapping:
"kubernetes": {
"properties": {
"container_image": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"container_image_id": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"container_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"host": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"labels": {
"properties": {
"aadpodidbinding": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"app": {
"properties": {
"kubernetes": {
"properties": {
"io/component": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"io/instance": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"io/managed-by": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"io/name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
},
"component": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},