Losgtash running but there is no output


(marxper) #1

Hello ,
i have an issue concerning logstash
this is the log that i have,

  Settings: Default pipeline workers: 4
  Pipeline main started

my logstash log file

input {
http {
type => "test_http"
host => "0.0.0.0" # default: 0.0.0.0
port => 9200 # default: 8080
}

file {
type => "apache_access"
path => ["/var/log/remote/apache2//access.log"]
stat_interval => 2
start_position => "beginning"
}
file {
type => "visu_access"
path => ["/var/log/remote/visu-stat//stat.log"]
stat_interval => 2
start_position => "beginning"
}
file {
type => "visu_stat"
path => ["/var/log/remote/visu-stat/**/player.log"]
stat_interval => 2
start_position => "beginning"
}
}
filter {
if [type] == "test_http" {
grok {
match => { "message" => "%{GREEDYDATA:all}" }
}
}
else if [type] == "apache_access" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:day} %{GREEDYDATA:server_name} %
{GREEDYDATA:apache} %{IPORHOST:vhost}:%{NUMBER:vhost_port} %{COMBINEDAPACHELOG}" }
}
mutate {
remove_field =>["day","apache"]
}
}
 else if [type] == "visu_access" {
grok {
match => {"message" =>"%{GREEDYDATA:timestamp} %{IP:ip} %{GREEDYDATA:apache}]: %
{GREEDYDATA:data}" }
}
ruby {
code => "
fields = event['data'].split(';;')
event['coutry_code'] = fields[fields.size-2]
event['video_type'] = fields[fields.size-1]
event['mail'] = fields[0]
event['client'] = fields[1]
event['user_ip'] = fields[2]
event['user_id'] = fields[3]
event['user_agent'] = fields[4]
event['video'] = fields[5]
event['tmp'] = fields[6]
event['is_newuser'] = fields[7]
event['referer'] = fields[8]
event['smil'] = fields[9]
event['smil_path'] = fields[10]
"
}
}
if [type] == "visu_stat" {
grok {
match => [ "message", "%{DATE:date} %{TIME:time} %{DATA:timezone} %{GREEDYDATA:inputFields}"]
}
mutate {
split => ["inputFields", " "]
add_field => {"x_event" => "%{inputFields[0]}"}
add_field => {"x_category" => "%{inputFields[1]}"}
add_field => {"x_severity" => "%{inputFields[2]}"}
add_field => {"x_status" => "%{inputFields[3]}"}
add_field => {"x_ctx" => "%{inputFields[4]}"}
add_field => {"x_comment" => "%{inputFields[5]}"}
add_field => {"x_vhost" => "%{inputFields[6]}"}
add_field => {"x_app" => "%{inputFields[7]}"}
add_field => {"x_appinst" => "%{inputFields[8]}"}
add_field => {"duration" => "%{inputFields[9]}"}
add_field => {"s_ip" => "%{inputFields[10]}"}
add_field => {"s_port" => "%{inputFields[11]}"}
add_field => {"s_uri" => "%{inputFields[12]}"}
add_field => {"c_ip" => "%{inputFields[13]}"}
add_field => {"c_proto" => "%{inputFields[14]}"}
add_field => {"c_referrer" => "%{inputFields[15]}"}
add_field => {"agent" => "%{inputFields[16]}"}
add_field => {"c_client_id" => "%{inputFields[17]}"}
add_field => {"cs_bytes" => "%{inputFields[18]}"}
add_field => {"vsc_bytes" => "%{inputFields[19]}"}
add_field => {"x_stream_id" => "%{inputFields[20]}"}
add_field => {"x_spos" => "%{inputFields[21]}"}
add_field => {"cs_stream_bytes" => "%{inputFields[22]}"}
add_field => {"sc_stream_bytes" => "%{inputFields[23]}"}
add_field => {"x_sname" => "%{inputFields[24]}"}
add_field => {"x_sname_query" => "%{inputFields[25]}"}
add_field => {"x_file_name" => "%{inputFields[26]}"}
add_field => {"x_file_ext" => "%{inputFields[27]}"}
add_field => {"x_file_size" => "%{inputFields[28]}"}
add_field => {"x_file_length" => "%{inputFields[29]}"}
add_field => {"x_suri" => "%{inputFields[30]}"}
add_field => {"x_suri_stem" => "%{inputFields[31]}"}
add_field => {"x_suri_query" => "%{inputFields[32]}"}
add_field => {"cs_uri_stem" => "%{inputFields[33]}"}
add_field => {"cs_uri_query" => "%{inputFields[34]}"}
 remove_field => ["inputFields","date","time","timezone","logtime"]
}
grok {
match => { "c_referrer" => "%{URIPROTO}://%{URIHOST:url}" }
}
 if ![url] {
mutate {
replace => { "url" => "%{c_referrer}" }
}
}
 mutate {
split => ["cannaux", ","]
}
mutate {
split => ["themes", ","]
}
grok {
match => { "duration" => "%{NUMBER:x_duration:int}" }
}
date {
match => [ "fulltime","YY-MM-dd HH:mm:ss"]
 target => "playtime"
 }
useragent {
source => "agent"
target => "c_user_agent"
remove_field => ["agent"]
}
mutate {
remove_field => ["duration","logdate","fulltime"]
}
if [s_ip] and [s_ip] !~ "(^127.0.0.1)|(^10.)|(^172.1[6-9].)|(^172.2[0-9].)|(^172.3[0-1].)|(^192.168.)|
(^169.254.)" {
geoip {
source => "s_ip"
target => "s_geoip"
database => "/config-dir/GeoLite2-City.mmdb"
 add_field => [ "[s_geoip][coordinates]", "%{[s_geoip][longitude]}" ]
add_field => [ "[s_geoip][coordinates]", "%{[s_geoip][latitude]}" ]
 }
mutate {
convert => [ "[s_geoip][coordinates]", "float"]
}
}
if [c_ip] and [c_ip] !~ "(^127.0.0.1)|(^10.)|(^172.1[6-9].)|(^172.2[0-9].)|(^172.3[0-1].)|(^192.168.)|
(^169.254.)" {
geoip {
source => "c_ip"
target => "c_geoip"
database => "/config-dir/GeoLite2-City.mmdb"
add_field => [ "[c_geoip][coordinates]", "%{[c_geoip][longitude]}" ]
add_field => [ "[c_geoip][coordinates]", "%{[c_geoip][latitude]}" ]
}
mutate {
convert => [ "[c_geoip][coordinates]", "float"]
}
}
grok {
match => { "c_ip" => "%{IP:c_zone_ip}" }
}
}
}
output {
if [type] == "apache_access" {
elasticsearch {
hosts => ["elasticsearch:9200"]
}
}
else if [type] == "visu_access" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "visu-access-"
}
}
else if [type] == "visu_stat" {
elasticsearch {
codec => json {}
hosts => ["elasticsearch:9200"]
index => "visu-player"
}
}
 else if [type] == "test_http" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "test-http-"
}
}
}

(Magnus B├Ąck) #2

Use only a stdout { codec => rubydebug } output until you've verified that your inputs work correctly. Have you tried posting anything to port 9200 to see if the http input works? Regarding the file inputs do read up on sincedb in the file input documentation and search the archives of this site for more information.


(marxper) #3

Thank you for your reponse,
I do not understand you very much , but i will use only

 stdout { codec => rubydebug }

because i have the conf file all ready prepared , i just use it


(marxper) #4

I changed this part :

output {
 if [type] == "apache_access" {
 elasticsearch {
    hosts => ["elasticsearch:9200"]
  }
}
else if [type] == "visu_access" { 
 elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "visu-access-"
  }
}
else if [type] == "visu_stat" {
 elasticsearch {
    codec => json {}
    hosts => ["elasticsearch:9200"]
    index => "visu-player"
 }

}
else if [type] == "test_http" {
 elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "test-http-"
  }
}

}

to be like this :

output {
 stdout { codec => rubydebug }
}

It the same result


(system) #5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.