Trouble
I set "document_type" on filebeat, and set "if" and "type" on logstash conf.
I've confirmed these logs are on AWS S3 bucket, however all logs are combined into one bucket "web-app" without separation.
It seems to that "if [type]" is not correctly working on logstash.
filter{
if [type] == "web-app" {
Could you give me advice to solve this trouble?
Thanks.
Environment
- logstash 2.3.4
- filebeat 1.2.3
- s3-output-plugin
conf files are below.
#####################
clinet: filebeat
#####################
$cat /etc/filebeat/filebeat.yml
filebeat:
prospectors:
-
paths:
- /apphome/MetaData/log/*
document_type: web-app
input_type: log
multiline:
pattern: "^[[:digit:]]{4}"
negate: true
match: after
-
paths:
- /var/log/httpd/intage.access_log
document_type: http-access
input_type: log
-
paths:
- /var/log/httpd/intage.error_log
document_type: http-error
input_type: log
registry_file: /var/lib/filebeat/registry
output:
logstash:
hosts: ["logstash-srv-01:5044"]
shipper:
logging:
files:
#####################
server: logstash
#####################
$cat /etc/logstash/conf.d/logstash-conf.json
input {
beats {
port => 5044
}
}
filter {
if [type] == "web-app" {
grok {
patterns_dir => ["/opt/logstash/extra_patterns"]
match => ["message", "%{TIMESTAMP_4S:timestamp_4s} %{GREEDYDATA}"]
add_field => ["timestamp", "%{timestamp_4s} +0900"]
}
date {
match => ["timestamp", "yyyy/MM/dd HH:mm:ss Z"]
remove_field => ["timestamp", "timestamp_4s"]
}
}
}
output {
if [type] == "web-app" {
s3 {
access_key_id => "(your-key)"
secret_access_key => "(your-secret)"
region => "ap-northeast-1"
bucket => "web-app"
prefix => "log/"
codec => "json_lines"
time_file => 1
}
}
else if [type] == "http-access" {
s3 {
access_key_id => "(your-key)"
secret_access_key => "(your-secret)"
region => "ap-northeast-1"
bucket => "http-access"
prefix => "log/"
codec => "json_lines"
time_file => 1
}
}
else if [type] == "http-error" {
s3 {
access_key_id => "(your-key)"
secret_access_key => "(your-secret)"
region => "ap-northeast-1"
bucket => "http-error"
prefix => "log/"
codec => "json_lines"
time_file => 1
}
}
}