Data allways stack in field.keyword than field

Hi,

I try to install & configure elk 8.3.3 without security on the docker "host" and filebeat on another docker.

All works, i have the live logs, i imported my dashboards, but i have this:
image

and the problem is because all of my logs are in field.keyword and not field.

I checked my dataView (filebeat-*) and i got this warning:

Mapping conflict
A field is defined as several types (string, integer, etc) across the indices that match this pattern. You may still be able to use these conflict fields in parts of Kibana, but they will be unavailable for functions that require Kibana to know their type. Correcting this issue will require reindexing your data.

I go to http://192.168.66.214:9200/_all/_mapping for see if my index have other type than keyword

"log_data_direction":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"log_data_info1":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"log_data_info2":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"log_data_name":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"log_data_numerical":{"type":"float"},"log_data_value":{"type":"text","fields":{"keyword":{"type":"keyword"

And it's ok.

there is my .conf of my logstash:

input {
  beats {
    port => 5044
    id => "from_filebeat"
    ssl  => false
  }
}

output {
  elasticsearch {
    hosts => ["http://elasticsearch:9200"]
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    manage_template => false
    action => "create"
    }
    stdout { codec => rubydebug }
}

filter {
  dissect {
    mapping => {
      "message" => "%{log_time} %{log_absolutetime} %{log_model} %{log_zone} %{log_data_direction} %{log_data_name} %{log_data_value}"
    }
  }

  date {
    match => [ "log_time", "HH:mm:ss.SSS", "ISO8601" ]
  }

  ruby {
    init => "@current_car_mode = ''
             @transfer_car_mode = ''
             @start_driving_time = 0
             @current_weather = 'rain'
             @current_speed_limit = 300.0
             @current_speed = 0.0
             @start_over_speed_limit_time = 0
             @current_distraction = '0'
             @start_distraction_time = 0
             @start_reaction_time = 0
             @start_transfer_time = 0
             @current_driving_level = 'senior'
             #@test_reaction_time = 0
             #@test_transfer_time = 0
             @first_time = 0"
    code => "if @first_time == 0
               @first_time = 1
               new_event = event.clone()
               new_event.set('log_data_name', 'distraction_time');
               new_event.set('log_data_value', '0')
               new_event.set('log_data_info1', @current_weather)
               new_event.set('log_data_info2', @current_car_mode)
               new_event.set('log_data_numerical', 0)
               new_event_block.call(new_event)

               new_event = event.clone()
               new_event.set('log_data_name', 'over_speed_limit_time');
               new_event.set('log_data_value', '0')
               new_event.set('log_data_info1', @current_weather)
               new_event.set('log_data_info2', @current_car_mode)
               new_event.set('log_data_numerical', 0)
               new_event_block.call(new_event)

               @start_driving_time = event.get('log_absolutetime').to_f 
               @start_over_speed_limit_time = event.get('log_absolutetime').to_f
               @start_distraction_time = event.get('log_absolutetime').to_f
               @start_reaction_time = event.get('log_absolutetime').to_f
               @start_transfer_time = event.get('log_absolutetime').to_f
             end

             event.set('log_data_numerical', 0.0)
             if event.get('log_data_name') == 'car_mode' and event.get('log_model') == 'public_plugin_vehicle_data'
               if @current_car_mode == ''
                 @start_driving_time = event.get('log_absolutetime').to_f
               else
                 if event.get('log_data_value') != @current_car_mode
                   new_event = event.clone
                   new_event.set('log_data_name', 'driving_time');
                   new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_driving_time)
                   new_event.set('log_data_info1', @current_weather)
                   new_event.set('log_data_info2', @current_car_mode)
                   new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_driving_time)
                   new_event_block.call(new_event)
                   @start_driving_time = event.get('log_absolutetime').to_f
                 end
               end
               
               if @transfer_car_mode != ''
                 new_event = event.clone
                 new_event.set('log_data_name', 'transfer_time');
                 new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_transfer_time)
                 new_event.set('log_data_info1', @current_driving_level)
                 new_event.set('log_data_info2', @current_car_mode)
                 new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_transfer_time)
                 new_event_block.call(new_event)
                 @start_reaction_time = 0
                 @start_transfer_time = 0
                 @transfer_car_mode = ''
               end

               if event.get('log_data_value') == 'mrm'
                 new_event = event.clone
                 new_event.set('log_data_name', 'mrm_detection');
                 new_event.set('log_data_value', 'true')
                 new_event.set('log_data_info1', @current_driving_level)
                 new_event.set('log_data_info2', @current_car_mode)
                 new_event_block.call(new_event)
               end

               @current_car_mode = event.get('log_data_value')

             elsif event.get('log_data_name') == 'authority_transfer_request' and event.get('log_model') == 'public_plugin_smart_cabin'
               if event.get('log_data_value') == 'request_start'
                 @start_reaction_time = event.get('log_absolutetime').to_f
                 @start_transfer_time = event.get('log_absolutetime').to_f
                 @transfer_car_mode = ''
               elsif event.get('log_data_value') == 'request_taken_into_account'
                 if @start_reaction_time != 0 and @transfer_car_mode == ''
                   new_event = event.clone
                   new_event.set('log_data_name', 'reaction_time');
                   new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_reaction_time)
                   new_event.set('log_data_info1', @current_driving_level)
                   new_event.set('log_data_info2', @current_car_mode)
                   new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_reaction_time)
                   new_event_block.call(new_event)
                   @start_transfer_time = event.get('log_absolutetime').to_f
                   @transfer_car_mode = @current_car_mode
                 end
               elsif event.get('log_data_value') == 'request_cancel'
                   @start_reaction_time = 0
                   @start_transfer_time = 0
                   @transfer_car_mode = ''
               elsif event.get('log_data_value') == 'request_end'
                   #new_event = event.clone
                   #new_event.set('log_data_name', 'transfer_time');
                   #new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_transfer_time)
                   #new_event.set('log_data_info1', @current_driving_level)
                   #new_event.set('log_data_info2', @transfer_car_mode)
                   #new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_transfer_time)
                   #new_event_block.call(new_event)
                   #@start_transfer_time = 0
                   #@start_reaction_time = 0  
                   #@transfer_car_mode = ''
               end

             elsif event.get('log_data_name') == 'user.user_driving_level' and event.get('log_zone') == 'user_driver'
               @current_driving_level = event.get('log_data_value')
             
             elsif event.get('log_data_name') == 'weather' and event.get('log_model') == 'public_plugin_ros_driving_environment'
               if @current_car_mode != ''
                 new_event = event.clone
                 new_event.set('log_data_name', 'driving_time');
                 new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_driving_time)
                 new_event.set('log_data_info1', @current_weather)
                 new_event.set('log_data_info2', @current_car_mode)
                 new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_driving_time)
                 new_event_block.call(new_event)
                 @start_driving_time = event.get('log_absolutetime').to_f
               end
               if @current_distraction == '1'
                 new_event = event.clone
                 new_event.set('log_data_name', 'distraction_time');
                 new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_distraction_time)
                 new_event.set('log_data_info1', @current_weather)
                 new_event.set('log_data_info2', @current_car_mode)
                 new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_distraction_time)
                 new_event_block.call(new_event)
                 @start_distraction_time = event.get('log_absolutetime').to_f
               end
               if (@current_speed > @current_speed_limit)
                 new_event = event.clone
                 new_event.set('log_data_name', 'over_speed_limit_time');
                 new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
                 new_event.set('log_data_info1', @current_weather)
                 new_event.set('log_data_info2', @current_car_mode)
                 new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
                 new_event_block.call(new_event)
                 @start_over_speed_limit_time = event.get('log_absolutetime').to_f
               end
               @current_weather = event.get('log_data_value')

             elsif event.get('log_data_name') == 'distraction.distracted'
               if event.get('log_data_value') == '1'
                 if  @current_distraction == '0'
                   @start_distraction_time = event.get('log_absolutetime').to_f
                 end
                 event.set('log_data_numerical', '1');
                 @current_distraction = '1'

                 #new_event = event.clone
                 #new_event.set('log_data_name', 'reaction_time');
                 #new_event.set('log_data_value', @test_reaction_time)
                 #new_event.set('log_data_info1', @current_weather)
                 #new_event.set('log_data_info2', 'autonomous_driving')
                 #new_event.set('log_data_numerical', @test_reaction_time)
                 #@test_reaction_time = @test_reaction_time + 2
                 #new_event_block.call(new_event)

               else
                 if  @current_distraction == '1'
                   new_event = event.clone
                   new_event.set('log_data_name', 'distraction_time');
                   new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_distraction_time)
                   new_event.set('log_data_info1', @current_weather)
                   new_event.set('log_data_info2', @current_car_mode)
                   new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_distraction_time)
                   new_event_block.call(new_event)
                 end
                 event.set('log_data_numerical', '0');
                 @current_distraction = '0'

                 #new_event = event.clone
                 #new_event.set('log_data_name', 'transfer_time');
                 #new_event.set('log_data_value', @test_transfer_time)
                 #new_event.set('log_data_info1', @current_weather)
                 #new_event.set('log_data_info2', 'autonomous_driving')
                 #new_event.set('log_data_numerical', @test_transfer_time)
                 #@test_transfer_time = @test_transfer_time + 2
                 #new_event_block.call(new_event)

               end
             
             elsif event.get('log_data_name') == 'speed_limit' and event.get('log_model') == 'public_plugin_vehicle_data'
               if (@current_speed > @current_speed_limit)
                 @current_speed_limit = event.get('log_data_value').to_f
                 if @current_speed_limit >= @current_speed
                   new_event = event.clone
                   new_event.set('log_data_name', 'over_speed_limit_time');
                   new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
                   new_event.set('log_data_info1', @current_weather)
                   new_event.set('log_data_info2', @current_car_mode)
                   new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
                   new_event_block.call(new_event)
                 end
               else
                 @current_speed_limit = event.get('log_data_value').to_f
                 if @current_speed_limit < @current_speed
                   @start_over_speed_limit_time = event.get('log_absolutetime').to_f
                 end               
               end
             
             elsif event.get('log_data_name') == 'speed' and event.get('log_model') == 'public_plugin_vehicle_data'
               if @current_speed > @current_speed_limit
                 @current_speed = event.get('log_data_value').to_f
                 if @current_speed_limit >= @current_speed
                   new_event = event.clone
                   new_event.set('log_data_name', 'over_speed_limit_time');
                   new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
                   new_event.set('log_data_info1', @current_weather)
                   new_event.set('log_data_info2', @current_car_mode)
                   new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
                   new_event_block.call(new_event)
                 end
               else
                 @current_speed = event.get('log_data_value').to_f
                 if @current_speed_limit < @current_speed
                   @start_over_speed_limit_time = event.get('log_absolutetime').to_f
                 end               
               end

             elsif event.get('log_data_name') == 'lane_crossing_detection' and event.get('log_data_value') != 'lane_crossing' and event.get('log_model') == 'public_plugin_vehicle_data'
               new_event = event.clone
               new_event.set('log_data_name', 'lane_crossing');
               new_event.set('log_data_value', 'true')
               new_event.set('log_data_info1', @current_weather)
               new_event.set('log_data_info2', @current_car_mode)
               new_event_block.call(new_event)

             end"
  }

  mutate {
    convert => ["log_data_numerical","float"]
  }
}

I do all step for install & configure filebeat & elk.

I followed this topic https://discuss.elastic.co/t/index-problem-keyword/308112 but nothing.

i do the filebeat setup -e with elasticsearch port

and after i set the logstash port and i launched filebeat.

After i retry but with the logstash port and this command:

filebeat setup --index-management -E output.logstash.enabled=false -E 'output.elasticsearch.hosts=["192.168.66.214:9200"]'

and it's allways the same problem.

If someone have the answer that can help me a lot !

Hello @Dreinale !

Are you able to follow the setup instructions and getting filebeat -> ES -> Kibana working?

Per this comment: Index problem .keyword - #5 by stephenb

Hi @JLeysens ,

yes i that work but i need logstash for parsing my data.

Are you using a module that requires a pipeline to be applied?

In the first comment I shared the logstash configuration takes this into consideration

i remember i enabled 2 modules (system & nginx)
image

But i just disabled them and i's allways the same

Hmm, if you see the logstash output config here it is applying pipelines to the incoming data.

Try updating your logstash output to match? Specifically something like:

output {
  if [@metadata][pipeline] {
    elasticsearch {
      hosts => "<your-host>"
      manage_template => false
      index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
      pipeline => "%{[@metadata][pipeline]}" 
      user => "elastic"
      password => "secret"
    }
  } else {
    elasticsearch {
      hosts => "<your-host>"
      manage_template => false
      index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
      user => "elastic"
      password => "secret"
    }
  }
}
1 Like

@JLeysens @Dreinale

Actually if you are using modules + logstash + datastreams + 8.x the output is a little different our docs are wrong and there is a tiny bug

Please look at this thread

output {
  if [@metadata][pipeline] {
    elasticsearch {
      hosts => "http://localhost:9200"
      pipeline => "%{[@metadata][pipeline]}"
      user => "elastic"
      password => "password"
      manage_template => false
      index => "%{[@metadata][beat]}-%{[@metadata][version]}"
      action => "create" <!--- Important
    }
  } else {
    elasticsearch {
      hosts => "http://localhost:9200"
      user => "elastic"
      password => "password"
      manage_template => false
      index => "%{[@metadata][beat]}-%{[@metadata][version]}"
      action => "create"
    }
  }
} 

I think I explain why somewhere in there... :slight_smile:

2 Likes

Ohh my god you found the solution !

the problem was there

      index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"

we need to write it like this

      index => "%{[@metadata][beat]}-%{[@metadata][version]}"

Thank you so much @JLeysens & @stephenb !

2 Likes

Yes, you need to understand that you need to write to the datastream and in 7.x it was the write alias and that's how it works... Pretty common mistake.

Glad you got it working!!

2 Likes

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.