Server Side
#java -version
java version "1.8.0_101"
#elasticsearch version
"5.0.0-beta1",
#bin/kibana --version
5.0.0-beta1
#bin/logstash --version
logstash 5.0.0-beta1
#/etc/logstash/conf.d/02-beats-input.conf
input {
beats {
port => 5044
ssl => true
ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
}
}
#/etc/logstash/conf.d/10-syslog-filter.conf
filter {
if [type] == "syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:[%{POSINT:syslog_pid}])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}
#/etc/logstash/conf.d/30-elasticsearch-output.conf
output {
elasticsearch {
hosts => ["localhost:9200"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
#bin/logstash -f /etc/logstash/conf.d/ --config.test_and_exit
Configuration OK
#bin/logstash-plugin install logstash-input-beats
Installation successful
#bin/logstash-plugin update logstash-input-beats
Updated logstash-input-beats 3.1.4 to 3.1.6
#bin/logstash-plugin install logstash-output-elasticsearch
Installation successful
#bin/logstash-plugin update logstash-output-elasticsearch
Updated logstash-output-elasticsearch 5.1.1 to 5.1.2
#/var/log/logstash/logstash-plain.log
[2016-10-05T14:19:50,717][WARN ][logstash.outputs.elasticsearch] Elasticsearch output attempted to sniff for new connections but cannot. No living connections are detected. Pool contains the following current URLs {:url_info=>{}}
[2016-10-05T14:19:51,952][ERROR][org.logstash.beats.BeatsHandler] Exception: not an SSL/TLS record: 325700000001324300000..................f7000000ffffbf7b794e
[2016-10-05T14:19:55,718][WARN ][logstash.outputs.elasticsearch] UNEXPECTED POOL ERROR {:e=>#<LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError: No Available connections>}
client side
#bin/filebeat --version
filebeat version 5.0.0-beta1 (amd64), libbeat 5.0.0-beta1
#/etc/filebeat/filebeat.yml
filebeat:
prospectors:
-
paths:
- /var/log/auth.log
- /var/log/syslog
# - /var/log/*.log
input_type: log
document_type: syslog
registry_file: /var/lib/filebeat/registry
output:
logstash:
hosts: ["my_elk_server_ip:5044"]
bulk_max_size: 1024
tls:
certificate_authorities: ["/etc/pki/tls/certs/logstash-forwarder.crt"]
shipper:
logging:
files:
rotateeverybytes: 10485760 # = 10MB
#bin/filebeat -c /etc/filebeat/filebeat.yml -e -v
2016/10/05 08:24:23.710760 beat.go:204: INFO filebeat start running.
2016/10/05 08:24:23.710783 registrar.go:66: INFO Registry file set to: /var/lib/filebeat/registry
2016/10/05 08:24:23.710824 registrar.go:99: INFO Loading registrar data from /var/lib/filebeat/registry
2016/10/05 08:24:23.711077 prospector.go:106: INFO Starting prospector of type: log
2016/10/05 08:24:23.711312 log.go:60: INFO Harvester started for file: /var/log/syslog
2016/10/05 08:24:23.711406 spooler.go:64: INFO Starting spooler: spool_size: 2048; idle_timeout: 5s
2016/10/05 08:24:23.711432 registrar.go:178: INFO Starting Registrar
2016/10/05 08:24:23.711473 sync.go:41: INFO Start sending events to output
2016/10/05 08:24:23.716637 log.go:60: INFO Harvester started for file: /var/log/auth.log
2016/10/05 08:24:23.747492 sync.go:85: ERR Failed to publish events caused by: EOF
2016/10/05 08:24:23.747737 single.go:91: INFO Error publishing events (retrying): EOF
2016/10/05 08:24:53.710713 logp.go:230: INFO Non-zero metrics in the last 30s: filebeat.harvester.running=2 filebeat.harvester.open_files=2 filebeat.harvester.started=2 libbeat.logstash.published_but_not_acked_events=5120 libbeat.logstash.call_count.PublishEvents=5 libbeat.logstash.publish.write_bytes=2305 libbeat.publisher.published_events=2046 libbeat.logstash.publish.read_errors=5
Can anybody help me for this problem ?
Thanks in advance.