Hi Team,
I recently upgraded my filebeat from 5.6.5 to 7.17.3 and found in 7.x fields are different from 5.6.5.
I want all the processor metadata should replaced with older keys but I still can't get rid of agent.ephemeral_id, agent.hostname, agent.id, agent.type, agent.version and ecs.version and log.offset.
Is there a way to change above fields with older values like beat.version,beat.hostname etc. Please find my sample Logstash filter configuration:-
# 23-parser-filter-preprocess-beats-generic.logstash.conf
filter {
if [agent][type] == "filebeat" and [agent][version] =~ "7.*" {
# beat.hostname = agent.hostname
if [hostname] in [agent] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_beat.hostname"
add_field => { "[beat][hostname]" => "%{[agent][hostname]}" }
}
}
# agent.version = beat.version
if [version] in [agent] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_agent.version"
add_field => { "[beat][version]" => "%{[agent][version]}" }
}
}
# meta.cloud.availability_zone = cloud.availability_zone
if [availability_zone] in [cloud] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_cloud.meta.cloud.availability_zone"
add_field => { "[meta][cloud][availability_zone]" => "%{[cloud][availability_zone]}" }
}
}
# meta.cloud.instance_id = cloud.instance.id
if [id] in [cloud][instance] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_meta.cloud.instance_id"
add_field => { "[meta][cloud][instance_id]" => "%{[cloud][instance][id]}" }
}
}
# meta.cloud.instance_name = cloud.instance.name
if [name] in [cloud][instance] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_meta.cloud.instance_name"
add_field => { "[meta][cloud][instance_name]" => "%{[cloud][instance][name]}" }
}
}
# meta.cloud.machine_type = cloud.machine.type
if [type] in [cloud][machine] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_meta.cloud.machine_type"
add_field => { "[meta][cloud][machine_type]" => "%{[cloud][machine][type]}" }
}
}
# meta.cloud.project_id = cloud.project.id
if [id] in [cloud][project] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_meta.cloud.project_id"
add_field => { "[meta][cloud][project_id]" => "%{[cloud][project][id]}" }
}
}
# meta.cloud.provider = cloud.provider
if [provider] in [cloud] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_meta.cloud.provider"
add_field => { "[meta][cloud][provider]" => "%{[cloud][provider]}" }
}
}
# meta.cloud.region = cloud.region
if [region] in [cloud] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_meta.cloud.region"
add_field => { "[meta][cloud][region]" => "%{[cloud][region]}" }
}
}
# docker.container.id = container.id
if [id] in [container] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_docker.container.id"
add_field => { "[docker][container][id]" => "%{[container][id]}" }
}
}
# docker.container.image = container.image.name
if [name] in [container][image] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_docker.container.image"
add_field => { "[docker][container][image]" => "%{[container][image][name]}" }
}
}
# docker.container.labels = container.labels
if [labels] in [container] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_docker.container.labels"
add_field => { "[docker][container][labels]" => "%{[container][labels]}" }
}
}
# docker.container.name = container.name
if [name] in [container] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_docker.container.name"
add_field => { "[docker][container][name]" => "%{[container][name]}" }
}
}
# read_timestamp = event.created
if [created] in [event] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_read_timestamp"
add_field => { "read_timestamp" => "%{[event][created]}" }
}
}
# http.response.elapsed_time = event.duration
if [duration] in [event] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_http.response.elapsed_time"
add_field => { "[http][response][elapsed_time]" => "%{[event][duration]}" }
}
}
# fileset.module = event.module
if [module] in [event] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_fileset.module"
add_field => { "[fileset][module]" => "%{[event][module]}" }
}
}
# beat.timezone = event.timezone
if [timezone] in [event] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_beat.timezone"
add_field => { "[beat][timezone]" => "%{[event][timezone]}" }
}
}
# beat.name = host.name
if [name] in [host] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_beat.name"
add_field => {"[beat][name]" => "%{[host][name]}" }
}
}
# http.response.content_length = http.response.body.bytes
if [bytes] in [http][response][body] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_http.response.content_length"
add_field => { "[http][response][content_length]" => "%{[http][response][body][bytes]}" }
}
}
# offset = log.offset
if [offset] in [log] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_offset"
add_field => { "[offset]" => "%{[log][offset]}" }
}
}
# process.exe = process.executable
if [executable] in [process] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_process.exe"
add_field => { "[process][exe]" => "%{[process][executable]}" }
}
}
# source_ecs.geo.city_name = source.geo.city_name
if [city_name] in [source][geo] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_source_ecs.geo.city_name"
add_field => { "[source_ecs][geo][city_name]" => "%{[source][geo][city_name]}" }
}
}
# source_ecs.geo.continent_name = source.geo.continent_name
if [continent_name] in [source][geo] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_source_ecs.geo.continent_name"
add_field => { "[source_ecs][geo][continent_name]" => "%{[source][geo][continent_name]}" }
}
}
# source_ecs.geo.country_iso_code = source.geo.country_iso_code
if [country_iso_code] in [source][geo] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_source_ecs.geo.country_iso_code"
add_field => { "[source_ecs][geo][country_iso_code]" => "%{[source][geo][country_iso_code]}" }
}
}
# source_ecs.geo.location = source.geo.location
if [location] in [source][geo] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_source_ecs.geo.location"
add_field => { "[source_ecs][geo][location]" => "%{[source][geo][location]}" }
}
}
# source_ecs.geo.region_iso_code = source.geo.region_iso_code
if [region_iso_code] in [source][geo] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_source_ecs.geo.region_iso_code"
add_field => { "[source_ecs][geo][region_iso_code]" => "%{[source][geo][region_iso_code]}" }
}
}
# source_ecs.geo.region_name = source.geo.region_name
if [region_name] in [source][geo] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_source_ecs.geo.region_name"
add_field => { "[source_ecs][geo][region_name]" => "%{[source][geo][region_name]}" }
}
}
# source_ecs.ip = source.ip
if [ip] in [source] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_source_ecs.ip"
add_field => { "[source_ecs][ip]" => "%{[source][ip]}" }
}
}
# source_ecs.port = source.port
if [port] in [source] {
mutate {
id => "23-parser-filter-preprocess-beats-generic.logstash.conf_mutate_source_ecs.port"
add_field => { "[source_ecs][port]" => "%{[source][port]}" }
}
}
}
}
thanks !!