Hi!
I'm running Elastic stack in k8s and have no luck in pairing filebeat with logstash
logstash config:
input {
jdbc {
jdbc_driver_library => "/usr/share/logstash/logstash-core/lib/jars/postgresql-jdbc.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "${LOGSTASH_JDBC_URL_ORDERS}"
jdbc_user => "${LOGSTASH_JDBC_USERNAME}"
jdbc_password => "${LOGSTASH_JDBC_PASSWORD}"
jdbc_validate_connection => true
tracking_column_type => "timestamp"
tracking_column => "updated_at"
use_column_value => true
last_run_metadata_path => "/tmp/orders_logstash_jdbc_last_run"
statement => ''
schedule => "* * * * *"
tags => ["orders"]
}
jdbc {
jdbc_driver_library => "/usr/share/logstash/logstash-core/lib/jars/postgresql-jdbc.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "${LOGSTASH_JDBC_URL_BFF}"
jdbc_user => "${LOGSTASH_JDBC_USERNAME}"
jdbc_password => "${LOGSTASH_JDBC_PASSWORD}"
jdbc_validate_connection => true
last_run_metadata_path => "/tmp/bff_logstash_jdbc_last_run"
statement => ''
schedule => "* * * * *"
tags => ["partner_contexts"]
}
jdbc {
jdbc_driver_library => "/usr/share/logstash/logstash-core/lib/jars/postgresql-jdbc.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "${LOGSTASH_JDBC_URL_AUTH}"
jdbc_user => "${LOGSTASH_JDBC_USERNAME}"
jdbc_password => "${LOGSTASH_JDBC_PASSWORD}"
jdbc_validate_connection => true
last_run_metadata_path => "/tmp/users_logstash_jdbc_last_run"
statement => ''
schedule => "* * * * *"
tags => ["users"]
}
beats {
client_inactivity_timeout => 120
port => 5044
}
}
filter {
if "orders" in [tags] {
mutate {
add_field => { "[@metadata][type]" => "orders" }
}
}
else if "partner_contexts" in [tags] {
mutate {
add_field => { "[@metadata][type]" => "partner_contexts" }
}
}
else if "users" in [tags] {
mutate {
add_field => { "[@metadata][type]" => "users" }
}
}
mutate {
remove_field => "tags"
}
}
output {
if [@metadata][type] == "orders" {
elasticsearch {
hosts => ["${LOGSTASH_ELASTICSEARCH_HOST}"]
index => "orders"
document_type => "orders"
document_id => "%{uuid}"
}
}
else if [@metadata][type] == "partner_contexts" {
elasticsearch {
hosts => ["${LOGSTASH_ELASTICSEARCH_HOST}"]
index => "partner_contexts"
document_type => "partner_contexts"
document_id => "%{id}"
}
}
else if [@metadata][type] == "users" {
elasticsearch {
hosts => ["${LOGSTASH_ELASTICSEARCH_HOST}"]
index => "users"
document_type => "users"
document_id => "%{user_id}"
}
}
else if [@metadata][beat] == "filebeat" {
elasticsearch {
hosts => ["${LOGSTASH_ELASTICSEARCH_HOST}"]
index => "%{[@metadata][beat]}-%{[@metadata][version]}"
}
}
}
logstash service helm:
apiVersion: v1
kind: Service
metadata:
name: {{ .Chart.Name }}-service
namespace: {{ .Values.global.env }}
spec:
selector:
app: {{ .Chart.Name }}
ports:
- port: 5044
targetPort: 5044
name: filebeat
type: ClusterIP
filebeat config:
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: dev
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
filebeat.inputs:
- type: container
paths:
- /var/log/containers/gateway-*.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
output.logstash:
hosts: ['logstash:5044']
I get this error on almost every filebeat deployment:
`ERROR pipeline/output.go:100 Failed to connect to backoff(async(tcp://logstash:5044)): lookup logstash on 10.96.0.10:53: server misbehaving`
'almost' because 2 time everything worked as it should. No changes to the config or anything.
Logstash log has no errors.
Any ideas why this can be happening?
Filebeat can output directly to ES just file, and Logstash outputs postgres data to ES to. But filebeat somehow can't connect to logstash almost never.