Hi all,
Im working on moving cloudfront real-time logs from kinessis to our ELK stack using functionbeat.
I'm trying to convert logs timestamp (unix format) to regular date and use it in @timestemp field.
unfortunately, I cant get it to work.
here is my configuration:
if [type] == "api" {
mutate {
gsub => ["message", "\t", " "]
}
dissect {
mapping => {"message" => "%{timestamp} %{c-ip} %{time-to-first-byte} %{sc-status} %{sc-bytes} %{cs-method} %{cs-protocol} %{cs-host} %{cs-uri-stem} %{cs-bytes} %{x-edge-location} %{x-edge-request-id} %{x-host-header} %{time-taken} %{cs-protocol-version} %{c-ip-version} %{cs-user-agent} %{cs-referer} %{cs-cookie} %{cs-uri-query} %{x-edge-response-result-type} %{x-forwarded-for} %{ssl-protocol} %{ssl-cipher} %{x-edge-result-type} %{fle-encrypted-fields} %{fle-status} %{sc-content-type} %{sc-content-len} %{sc-range-start} %{sc-range-end} %{c-port} %{x-edge-detailed-result-type} %{c-country} %{cs-accept-encoding} %{cs-accept} %{cache-behavior-path-pattern} %{cs-headers} %{cs-header-names} %{cs-headers-count}"}
convert_datatype => {
"c-port" => "int"
"cs-bytes" => "int"
"sc-bytes" => "int"
"sc-status" => "int"
"time-taken" => "float"
"time-to-first-byte" => "float"
"timestamp" => "float"
}
add_field => {
"timestamp_edge" => "%{timestamp}"
"timestamp_elastic" => "%{@timestamp}"
}
}
date {
match => ["timestamp_edge", "MMM dd yyyy HH:mm:ss", "MMM d yyyy HH:mm:ss", "UNIX_MS"]
timezone => "UTC"
}
mutate {
remove_field => [ "timestamp", "message", "input", "agent", "event_id", "event_name", "event_source", "event_version", "kinesis_encryption_type", "kinesis_partition_key", "kinesis_schema_version", "kinesis_sequence_number", "aws_region", "cs-header-names","[host][name]", "[cloud][provider]", "[cloud][region]", "[ecs][version]", "[event][kind]" ]
gsub => ["agent", "%20", " "]
gsub => ["sc-content-type", "%20", " "]
gsub => ["cs-headers", "%20", " "]
gsub => ["cs-user-agent", "%20", " "]
gsub => ["cs-accept-encoding", "%20", " "]
gsub => ["cs-headers", "%0A", "; "]
gsub => ["cs-accept", "%20", " "]
gsub => ["cs-headers", "%22", ""]
}
}
I still get the elastic timestamp and not the edge timestamp.
here is timestamp format for example "1614712951.690"
can anyone help me figure out what am I doing wrong?
Thank you