Hi,
I try to install & configure elk 8.3.3 without security on the docker "host" and filebeat on another docker.
All works, i have the live logs, i imported my dashboards, but i have this:
and the problem is because all of my logs are in field.keyword and not field.
there is my .conf of my logstash:
# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.
input {
beats {
port => 5044
id => "from_filebeat"
ssl => false
}
}
output {
elasticsearch {
hosts => ["http://elasticsearch:9200"]
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
}
}
filter {
dissect {
mapping => {
"message" => "%{log_time} %{log_absolutetime} %{log_model} %{log_zone} %{log_data_direction} %{log_data_name} %{log_data_value}"
}
}
date {
match => [ "log_time", "HH:mm:ss.SSS", "ISO8601" ]
}
ruby {
init => "@current_car_mode = ''
@transfer_car_mode = ''
@start_driving_time = 0
@current_weather = 'rain'
@current_speed_limit = 300.0
@current_speed = 0.0
@start_over_speed_limit_time = 0
@current_distraction = '0'
@start_distraction_time = 0
@start_reaction_time = 0
@start_transfer_time = 0
@current_driving_level = 'senior'
#@test_reaction_time = 0
#@test_transfer_time = 0
@first_time = 0"
code => "if @first_time == 0
@first_time = 1
new_event = event.clone()
new_event.set('log_data_name', 'distraction_time');
new_event.set('log_data_value', '0')
new_event.set('log_data_info1', @current_weather)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', 0)
new_event_block.call(new_event)
new_event = event.clone()
new_event.set('log_data_name', 'over_speed_limit_time');
new_event.set('log_data_value', '0')
new_event.set('log_data_info1', @current_weather)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', 0)
new_event_block.call(new_event)
@start_driving_time = event.get('log_absolutetime').to_f
@start_over_speed_limit_time = event.get('log_absolutetime').to_f
@start_distraction_time = event.get('log_absolutetime').to_f
@start_reaction_time = event.get('log_absolutetime').to_f
@start_transfer_time = event.get('log_absolutetime').to_f
end
event.set('log_data_numerical', 0.0)
if event.get('log_data_name') == 'car_mode' and event.get('log_model') == 'public_plugin_vehicle_data'
if @current_car_mode == ''
@start_driving_time = event.get('log_absolutetime').to_f
else
if event.get('log_data_value') != @current_car_mode
new_event = event.clone
new_event.set('log_data_name', 'driving_time');
new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_driving_time)
new_event.set('log_data_info1', @current_weather)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_driving_time)
new_event_block.call(new_event)
@start_driving_time = event.get('log_absolutetime').to_f
end
end
if @transfer_car_mode != ''
new_event = event.clone
new_event.set('log_data_name', 'transfer_time');
new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_transfer_time)
new_event.set('log_data_info1', @current_driving_level)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_transfer_time)
new_event_block.call(new_event)
@start_reaction_time = 0
@start_transfer_time = 0
@transfer_car_mode = ''
end
if event.get('log_data_value') == 'mrm'
new_event = event.clone
new_event.set('log_data_name', 'mrm_detection');
new_event.set('log_data_value', 'true')
new_event.set('log_data_info1', @current_driving_level)
new_event.set('log_data_info2', @current_car_mode)
new_event_block.call(new_event)
end
@current_car_mode = event.get('log_data_value')
elsif event.get('log_data_name') == 'authority_transfer_request' and event.get('log_model') == 'public_plugin_smart_cabin'
if event.get('log_data_value') == 'request_start'
@start_reaction_time = event.get('log_absolutetime').to_f
@start_transfer_time = event.get('log_absolutetime').to_f
@transfer_car_mode = ''
elsif event.get('log_data_value') == 'request_taken_into_account'
if @start_reaction_time != 0 and @transfer_car_mode == ''
new_event = event.clone
new_event.set('log_data_name', 'reaction_time');
new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_reaction_time)
new_event.set('log_data_info1', @current_driving_level)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_reaction_time)
new_event_block.call(new_event)
@start_transfer_time = event.get('log_absolutetime').to_f
@transfer_car_mode = @current_car_mode
end
elsif event.get('log_data_value') == 'request_cancel'
@start_reaction_time = 0
@start_transfer_time = 0
@transfer_car_mode = ''
elsif event.get('log_data_value') == 'request_end'
#new_event = event.clone
#new_event.set('log_data_name', 'transfer_time');
#new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_transfer_time)
#new_event.set('log_data_info1', @current_driving_level)
#new_event.set('log_data_info2', @transfer_car_mode)
#new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_transfer_time)
#new_event_block.call(new_event)
#@start_transfer_time = 0
#@start_reaction_time = 0
#@transfer_car_mode = ''
end
elsif event.get('log_data_name') == 'user.user_driving_level' and event.get('log_zone') == 'user_driver'
@current_driving_level = event.get('log_data_value')
elsif event.get('log_data_name') == 'weather' and event.get('log_model') == 'public_plugin_ros_driving_environment'
if @current_car_mode != ''
new_event = event.clone
new_event.set('log_data_name', 'driving_time');
new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_driving_time)
new_event.set('log_data_info1', @current_weather)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_driving_time)
new_event_block.call(new_event)
@start_driving_time = event.get('log_absolutetime').to_f
end
if @current_distraction == '1'
new_event = event.clone
new_event.set('log_data_name', 'distraction_time');
new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_distraction_time)
new_event.set('log_data_info1', @current_weather)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_distraction_time)
new_event_block.call(new_event)
@start_distraction_time = event.get('log_absolutetime').to_f
end
if (@current_speed > @current_speed_limit)
new_event = event.clone
new_event.set('log_data_name', 'over_speed_limit_time');
new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
new_event.set('log_data_info1', @current_weather)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
new_event_block.call(new_event)
@start_over_speed_limit_time = event.get('log_absolutetime').to_f
end
@current_weather = event.get('log_data_value')
elsif event.get('log_data_name') == 'distraction.distracted'
if event.get('log_data_value') == '1'
if @current_distraction == '0'
@start_distraction_time = event.get('log_absolutetime').to_f
end
event.set('log_data_numerical', '1');
@current_distraction = '1'
#new_event = event.clone
#new_event.set('log_data_name', 'reaction_time');
#new_event.set('log_data_value', @test_reaction_time)
#new_event.set('log_data_info1', @current_weather)
#new_event.set('log_data_info2', 'autonomous_driving')
#new_event.set('log_data_numerical', @test_reaction_time)
#@test_reaction_time = @test_reaction_time + 2
#new_event_block.call(new_event)
else
if @current_distraction == '1'
new_event = event.clone
new_event.set('log_data_name', 'distraction_time');
new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_distraction_time)
new_event.set('log_data_info1', @current_weather)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_distraction_time)
new_event_block.call(new_event)
end
event.set('log_data_numerical', '0');
@current_distraction = '0'
#new_event = event.clone
#new_event.set('log_data_name', 'transfer_time');
#new_event.set('log_data_value', @test_transfer_time)
#new_event.set('log_data_info1', @current_weather)
#new_event.set('log_data_info2', 'autonomous_driving')
#new_event.set('log_data_numerical', @test_transfer_time)
#@test_transfer_time = @test_transfer_time + 2
#new_event_block.call(new_event)
end
elsif event.get('log_data_name') == 'speed_limit' and event.get('log_model') == 'public_plugin_vehicle_data'
if (@current_speed > @current_speed_limit)
@current_speed_limit = event.get('log_data_value').to_f
if @current_speed_limit >= @current_speed
new_event = event.clone
new_event.set('log_data_name', 'over_speed_limit_time');
new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
new_event.set('log_data_info1', @current_weather)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
new_event_block.call(new_event)
end
else
@current_speed_limit = event.get('log_data_value').to_f
if @current_speed_limit < @current_speed
@start_over_speed_limit_time = event.get('log_absolutetime').to_f
end
end
elsif event.get('log_data_name') == 'speed' and event.get('log_model') == 'public_plugin_vehicle_data'
if @current_speed > @current_speed_limit
@current_speed = event.get('log_data_value').to_f
if @current_speed_limit >= @current_speed
new_event = event.clone
new_event.set('log_data_name', 'over_speed_limit_time');
new_event.set('log_data_value', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
new_event.set('log_data_info1', @current_weather)
new_event.set('log_data_info2', @current_car_mode)
new_event.set('log_data_numerical', event.get('log_absolutetime').to_f - @start_over_speed_limit_time)
new_event_block.call(new_event)
end
else
@current_speed = event.get('log_data_value').to_f
if @current_speed_limit < @current_speed
@start_over_speed_limit_time = event.get('log_absolutetime').to_f
end
end
elsif event.get('log_data_name') == 'lane_crossing_detection' and event.get('log_data_value') != 'lane_crossing' and event.get('log_model') == 'public_plugin_vehicle_data'
new_event = event.clone
new_event.set('log_data_name', 'lane_crossing');
new_event.set('log_data_value', 'true')
new_event.set('log_data_info1', @current_weather)
new_event.set('log_data_info2', @current_car_mode)
new_event_block.call(new_event)
end"
}
mutate {
convert => ["log_data_numerical","float"]
}
}
I do all step for install & configure filebeat & elk.
i do the filebeat setup -e with elasticsearch port
and after i set the logstash port and i launched filebeat.
If someone have the answer that can help me a lot !
