Hello ,
i have an issue concerning logstash
this is the log that i have,
Settings: Default pipeline workers: 4
Pipeline main started
my logstash log file
input {
http {
type => "test_http"
host => "0.0.0.0" # default: 0.0.0.0
port => 9200 # default: 8080
}
file {
type => "apache_access"
path => ["/var/log/remote/apache2//access.log"]
stat_interval => 2
start_position => "beginning"
}
file {
type => "visu_access"
path => ["/var/log/remote/visu-stat//stat.log"]
stat_interval => 2
start_position => "beginning"
}
file {
type => "visu_stat"
path => ["/var/log/remote/visu-stat/**/player.log"]
stat_interval => 2
start_position => "beginning"
}
}
filter {
if [type] == "test_http" {
grok {
match => { "message" => "%{GREEDYDATA:all}" }
}
}
else if [type] == "apache_access" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:day} %{GREEDYDATA:server_name} %
{GREEDYDATA:apache} %{IPORHOST:vhost}:%{NUMBER:vhost_port} %{COMBINEDAPACHELOG}" }
}
mutate {
remove_field =>["day","apache"]
}
}
else if [type] == "visu_access" {
grok {
match => {"message" =>"%{GREEDYDATA:timestamp} %{IP:ip} %{GREEDYDATA:apache}]: %
{GREEDYDATA:data}" }
}
ruby {
code => "
fields = event['data'].split(';;')
event['coutry_code'] = fields[fields.size-2]
event['video_type'] = fields[fields.size-1]
event['mail'] = fields[0]
event['client'] = fields[1]
event['user_ip'] = fields[2]
event['user_id'] = fields[3]
event['user_agent'] = fields[4]
event['video'] = fields[5]
event['tmp'] = fields[6]
event['is_newuser'] = fields[7]
event['referer'] = fields[8]
event['smil'] = fields[9]
event['smil_path'] = fields[10]
"
}
}
if [type] == "visu_stat" {
grok {
match => [ "message", "%{DATE:date} %{TIME:time} %{DATA:timezone} %{GREEDYDATA:inputFields}"]
}
mutate {
split => ["inputFields", " "]
add_field => {"x_event" => "%{inputFields[0]}"}
add_field => {"x_category" => "%{inputFields[1]}"}
add_field => {"x_severity" => "%{inputFields[2]}"}
add_field => {"x_status" => "%{inputFields[3]}"}
add_field => {"x_ctx" => "%{inputFields[4]}"}
add_field => {"x_comment" => "%{inputFields[5]}"}
add_field => {"x_vhost" => "%{inputFields[6]}"}
add_field => {"x_app" => "%{inputFields[7]}"}
add_field => {"x_appinst" => "%{inputFields[8]}"}
add_field => {"duration" => "%{inputFields[9]}"}
add_field => {"s_ip" => "%{inputFields[10]}"}
add_field => {"s_port" => "%{inputFields[11]}"}
add_field => {"s_uri" => "%{inputFields[12]}"}
add_field => {"c_ip" => "%{inputFields[13]}"}
add_field => {"c_proto" => "%{inputFields[14]}"}
add_field => {"c_referrer" => "%{inputFields[15]}"}
add_field => {"agent" => "%{inputFields[16]}"}
add_field => {"c_client_id" => "%{inputFields[17]}"}
add_field => {"cs_bytes" => "%{inputFields[18]}"}
add_field => {"vsc_bytes" => "%{inputFields[19]}"}
add_field => {"x_stream_id" => "%{inputFields[20]}"}
add_field => {"x_spos" => "%{inputFields[21]}"}
add_field => {"cs_stream_bytes" => "%{inputFields[22]}"}
add_field => {"sc_stream_bytes" => "%{inputFields[23]}"}
add_field => {"x_sname" => "%{inputFields[24]}"}
add_field => {"x_sname_query" => "%{inputFields[25]}"}
add_field => {"x_file_name" => "%{inputFields[26]}"}
add_field => {"x_file_ext" => "%{inputFields[27]}"}
add_field => {"x_file_size" => "%{inputFields[28]}"}
add_field => {"x_file_length" => "%{inputFields[29]}"}
add_field => {"x_suri" => "%{inputFields[30]}"}
add_field => {"x_suri_stem" => "%{inputFields[31]}"}
add_field => {"x_suri_query" => "%{inputFields[32]}"}
add_field => {"cs_uri_stem" => "%{inputFields[33]}"}
add_field => {"cs_uri_query" => "%{inputFields[34]}"}
remove_field => ["inputFields","date","time","timezone","logtime"]
}
grok {
match => { "c_referrer" => "%{URIPROTO}://%{URIHOST:url}" }
}
if ![url] {
mutate {
replace => { "url" => "%{c_referrer}" }
}
}
mutate {
split => ["cannaux", ","]
}
mutate {
split => ["themes", ","]
}
grok {
match => { "duration" => "%{NUMBER:x_duration:int}" }
}
date {
match => [ "fulltime","YY-MM-dd HH:mm:ss"]
target => "playtime"
}
useragent {
source => "agent"
target => "c_user_agent"
remove_field => ["agent"]
}
mutate {
remove_field => ["duration","logdate","fulltime"]
}
if [s_ip] and [s_ip] !~ "(^127.0.0.1)|(^10.)|(^172.1[6-9].)|(^172.2[0-9].)|(^172.3[0-1].)|(^192.168.)|
(^169.254.)" {
geoip {
source => "s_ip"
target => "s_geoip"
database => "/config-dir/GeoLite2-City.mmdb"
add_field => [ "[s_geoip][coordinates]", "%{[s_geoip][longitude]}" ]
add_field => [ "[s_geoip][coordinates]", "%{[s_geoip][latitude]}" ]
}
mutate {
convert => [ "[s_geoip][coordinates]", "float"]
}
}
if [c_ip] and [c_ip] !~ "(^127.0.0.1)|(^10.)|(^172.1[6-9].)|(^172.2[0-9].)|(^172.3[0-1].)|(^192.168.)|
(^169.254.)" {
geoip {
source => "c_ip"
target => "c_geoip"
database => "/config-dir/GeoLite2-City.mmdb"
add_field => [ "[c_geoip][coordinates]", "%{[c_geoip][longitude]}" ]
add_field => [ "[c_geoip][coordinates]", "%{[c_geoip][latitude]}" ]
}
mutate {
convert => [ "[c_geoip][coordinates]", "float"]
}
}
grok {
match => { "c_ip" => "%{IP:c_zone_ip}" }
}
}
}
output {
if [type] == "apache_access" {
elasticsearch {
hosts => ["elasticsearch:9200"]
}
}
else if [type] == "visu_access" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "visu-access-"
}
}
else if [type] == "visu_stat" {
elasticsearch {
codec => json {}
hosts => ["elasticsearch:9200"]
index => "visu-player"
}
}
else if [type] == "test_http" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "test-http-"
}
}
}