Logstash convert string field to number

Elastic Stack Logstash
1

Hi I'm trying to ingest durationUs=786 into numeric value in elasticsearch but no luck tried using mutate
but the events still shows as string so any help would be great
if [type] == "pump" {
grok {
match => {
"message" => [
"%{NOTSPACE:timestamp} %{NOTSPACE:field_1} %{NOTSPACE:field_2} %
{GREEDYDATA:raw_data}", "%{NOTSPACE:timestamp},%{GREEDYDATA:raw_data}"
]
}
tag_on_failure => [ "fail_in_grok" ]
}
date {
match => [ "timestamp","yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ" ]
}
if [raw_data] {
kv {
field_split => ","
source => "raw_data"
transform_key => "lowercase"
}
}

   if "fail_in_grok" not in [tags] {
                                       mutate {
                          remove_field =>  ["timestamp"]
         }

    }

    mutate {
               convert => {
                           "durationus" => "integer"
             }
                     }

 }

Log Event : 2017-12-15T17:18:34.637368+00:00 ccdn-ats-tk-vbn-01 pump1[30512]: Level=Debug, subSystem=CONTENT, xmt: header in response trace-id=b0e44c2f-7050-45f7-8d39-afe870f14b0e;parent-id=0x15cf7c280bdb79b6;span-id=0x1137dafdbabfe9e8
2017-12-15T17:18:34.641027+00:00 ccdn-ats-tk-vbn-01 pump1[30512]: Level=Debug, subSystem=CONTENT, Event=Span_Success, span: trace-id=f8e41d0c-1f84-4112-b488-c0ee75530bef span-id=570631895973423548 parent-id=-7840287596547113655#012span-name=fetchRngAsync.http://mpeg4origin.sys..net/t6qam10/PFOX0022824020170718/1500331485928/Superhuman_105_HD_VOD8_AUTH_mezz_4QAM.ts.bytes=1270771712-1272119295 app-name=./pump1 start-time=1513358314 span-duration=23471 span-http-code=206 span-success=true

11 PM
Screen Shot 2017-12-15 at 12.20.11 PM.png1297×319 31 KB

2

I would lock the type in the elastic search mapping. In the properties section of the mapping template, I would put something like:

  ..."properties": {
    "durationus": {
      "type": "integer"
    },...
3

Thannks Very much will test tomrrow

4

Keep in mind that the mapping of an existing field can't be changed, so even if you've updated your Logstash configuration to produce documents with an integer durationus field that won't make a difference in ES if that field has already been mapped as a string.

5

Deleting old index is only option ?

1 Like
6

You can copy the data to a new index, delete the old index, and copy the data back.

1 Like
7

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.