Field type doesn't change in logstash

Elastic Stack Logstash
1

Hi. I want to extract hour from timestamp and I do it but its type is String and I'm trying to change the type of hour to integer but it doesn't effect. here is my logstash config:

input {
  beats {
    client_inactivity_timeout => 1200
    port => 5044
  }
}

filter {
  grok {
    match => {"message" => ["%{NOTSPACE:queueType}, (?<nothing>.{4})(?<part1>.{15})(?<nothing2>.{6}) %{NUMBER:part2}, %{INT:returnedCode}", "%{NOTSPACE:queueType}, (?<nothing>.{4})(?<part1>.{15})(?<nothing2>.{6}) %{NUMBER:part2}, %{INT:returnedCode}", "%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}  %{NOTSPACE:webService} @ %{NOTSPACE:function}, System:(?<systemName>.), User:%{NOTSPACE:userId}, StudentIDs:\[%{NUMBER:studentId}\], GroupID:%{GREEDYDATA:groupId}, DocType:%{NOTSPACE:docType}, returned %{INT:returnedCode}", "%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}  %{NOTSPACE:webService} @ batchAddOrUpdateStdMetadata, returned %{INT:returnedCode}", "%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{LOGLEVEL:loglevel}  %{NOTSPACE:webService} @ getInternalID, System:(?<systemName>.), resultCode:%{INT:resultCode}, returned %{INT:returnedCode}"]}
  }
  if "_grokparsefailure" in [tags] {
    drop { }
  }
  if [queueType]{
    mutate {
      add_tag => ["taskTerminated"]
      add_field => { 
        "timestamp" => "%{part1} %{part2}"
      }
      remove_field => ["part1", "part2", "nothing", "nothing2"]
    }
  }
  else{
    mutate{
      gsub => ["timestamp", "\,\d{3}$", ""]
      add_tag => ["taskStarted"]
    }
  }
  date {
    match => ["timestamp", "MMM dd HH:mm:ss YYYY", "MMM  d HH:mm:ss YYYY", "YYYY-MM-dd HH:mm:ss"]
    timezone => "Asia/Tehran"
    target => "@timestamp"
  }
  mutate {
      add_field => {"[hour]" => "%{+HH}"}
  }
  mutate {
    convert => {
      "hour" => "integer"
    }
  }
  elapsed {
    start_tag => "taskStarted"
    end_tag => "taskTerminated"
    unique_id_field => "returnedCode"
    timeout => 10000
    new_event_on_match => false
  }
}


output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
}

thanks for your help.

2

What does your output event/doc look like?

Use the stdout output with the rubydebug codec
stdout { codec => rubydebug }

3

here is one log line output:

{
            "tags" => [
        [0] "beats_input_codec_plain_applied",
        [1] "taskStarted"
    ],
       "timestamp" => "2019-07-24 23:10:00",
            "hour" => 18,
         "message" => "2019-07-24 23:10:00,395 INFO  ir.ac.ut.sdrwebservice.SDRWebService @ getInternalID, System:G, resultCode:1563993063971772, returned -1",
      "@timestamp" => 2019-07-24T18:40:00.000Z,
            "host" => {
        "name" => "localhost.localdomain"
    },
      "webService" => "ir.ac.ut.sdrwebservice.SDRWebService",
             "ecs" => {
        "version" => "1.0.0"
    },
           "agent" => {
            "hostname" => "localhost.localdomain",
                  "id" => "9c56df38-1f41-4972-8cad-e6b84a23b29e",
             "version" => "7.2.0",
                "type" => "filebeat",
        "ephemeral_id" => "2737fa09-4610-4e9d-b838-73c853b3613c"
    },
             "log" => {
          "file" => {
            "path" => "/home/mam23n/Desktop/elk/logs/sdrwebservice.log-2019-07-24"
        },
        "offset" => 1854777
    },
      "resultCode" => "1563993063971772",
        "@version" => "1",
        "loglevel" => "INFO",
    "returnedCode" => "-1",
           "input" => {
        "type" => "log"
    },
      "systemName" => "G"
}
4

There are no quotes around the value, so it is an integer in logstash, not a string. If it is a string in elasticsearch you need to modify the mapping.

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.