I have added the output, the input line I use to test, and the full config I use to prove it's not creating a new field (in this case I went back to add_field at it's easiest to show).
Here is the output I get:
{
"metrictype" => [
[0] "DATA",
[1] "analytics"
],
"hw_rev" => "2",
"type" => "analytics",
"kafka2es-proxy" => "analytics-es54-scus-4",
"eventclick" => "%{EVENT_CLICK}",
"tid" => "a8d6c049-f751-412c-9152-745d691ac50c",
"host" => "orion-logf-1",
"msgid" => "fcdfc934-cbda-4044-a6b5-0fb1e0971eff",
"collection" => "analytics",
"hw_part_nbr" => "1_0",
"event_category" => "config_at_boot_time",
"product_name" => "system1",
"@timestamp" => 2017-08-14T20:48:00.626Z,
"kafka2es_timestamp" => "2017-08-14T20:48:00.673Z",
"sw_build" => 1000,
"EVENT_CLICK" => "button_home:id/welcomeFragment",
"category" => "INFO",
"did" => "4ea7ec19-78e3-43f2-994d-d24d06759f9f",
"sw_ver" => "0_9_0"
}
Here is my test line I am inputting (instead of rerunning from Kafka, results are the same):
{"metrictype" : "DATA", "hw_rev" : "2", "type" : "analytics", "tid" : "a8d6c049-f751-412c-9152-745d691ac50c", "msgid" : "fcdfc934-cbda-4044-a6b5-0fb1e0971eff", "hw_part_nbr" : "1.0", "collection" : "analytics", "event_category" : "config_at_boot_time", "product_name" : "system1", "arrivalTime" : "2017-07-12T19:33:32.630Z", "uploadTime" : "2017-07-12T19:33:32.630Z", "event_time" : "2017-07-12T19:33:32.630Z", "sw_build" : 1000, "category" : "INFO", "did" : "4ea7ec19-78e3-43f2-994d-d24d06759f9f", "sw_ver": "0.9.0", "EVENT_CLICK": "button_home:id/welcomeFragment" }
input {
stdin { add_field => { "collection" => "analytics" } }
}
filter {
if [collection] == "analytics" {
mutate { add_field => { "eventclick" => "%{EVENT_CLICK}" } }
mutate { add_field => { "btype" => "%{type}" } }
date { match => [ "arrivalTime", "ISO8601" ] target => "server-arrival-time" tag_on_failure => ["_arrivaltimefailure"]}
date { match => [ "uploadTime", "ISO8601" ] target => "client-upload-time" tag_on_failure => ["_uploadtimefailure"]}
date { match => [ "event_time", "ISO8601", "YYYY-MM-dd'T'HH:mm:ss'UTC'", "YYYY-MM-dd'T'HH:mm:ss'.GMT+8Z'"] tag_on_failure => ["_eventtimefailure"] }
mutate { lowercase => [ "tid" ] }
mutate { add_field => { "client-message" => "%{message}" } }
mutate { add_field => { "msgid" => "%{id}" } }
if [eventclick] and [eventclick] != "" and [eventclick] =~ /home/ {
grok {
match => [ "eventclick", 'button_home:id/(?<event_click_normalized>[^/]+)' ]
}
mutate { add_field => { "yesyes" => "grok checked" } }
}
if [client-message] != "%{message}" {
# remove any dots that might be present in field names
mutate {
gsub => [
"client-message", "\.", "_"
]
}
json { source => "client-message" }
} else {
mutate { remove_field => [ "client-message"] }
}
mutate { convert => { "value" => string "sw_build" => integer "id" => string } }
mutate { add_field => { "configvarvalue" => "%{config_var_value}" } }
if [configvarvalue] != "%{config_var_value}" {
json { source => "configvarvalue" }
} else {
mutate { remove_field => [ "configvarvalue"] }
}
# if [tid] == "" { mutate { replace => { "tid" => "no-tenant-id" } } }
if ![tid] { mutate { replace => { "tid" => "no-tenant-id" } } }
# check to see if something with a type field came in and overwrote the existing type specifier
if [type] {
if [type] != [btype] {
mutate { add_field => { "metrictype" => "%{type}" } }
mutate { replace => { "type" => "analytics" } }
}
} else {
mutate { add_field => { "type" => "%{collection}" } }
}
}
}
output {
stdout { codec => rubydebug }
}
}