sure, Thanks @ChrsMark !!
suspecting the issue with filebeat-config-ingress, now the pods are in error syncing state.
please check below all our deployments & let us know how to fix push only nginx-ingress-controller pods logs to our elk.
deamonset, we're using is:
{
"kind": "DaemonSet",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "filebeat-ingress",
"namespace": "logging",
"labels": {
"k8s-app": "filebeat",
"kubernetes.io/cluster-service": "true"
},
"finalizers": [
"foregroundDeletion"
]
},
"spec": {
"selector": {
"matchLabels": {
"k8s-app": "filebeat",
"kubernetes.io/cluster-service": "true"
}
},
"template": {
"metadata": {
"creationTimestamp": null,
"labels": {
"k8s-app": "filebeat",
"kubernetes.io/cluster-service": "true"
}
},
"spec": {
"volumes": [
{
"name": "config",
"configMap": {
"name": "filebeat-config-ingress",
"defaultMode": 384
}
},
{
"name": "varlibdockercontainers",
"hostPath": {
"path": "/var/lib/docker/containers",
"type": ""
}
},
{
"name": "logforwarderssl",
"secret": {
"secretName": "logforwarderssl",
"defaultMode": 384
}
},
{
"name": "prospectors",
"configMap": {
"name": "filebeat-prospectors-ingress",
"defaultMode": 384
}
},
{
"name": "data",
"emptyDir": {}
}
],
"containers": [
{
"name": "filebeat",
"image": "docker.elastic.co/beats/filebeat:6.3.0",
"args": [
"-c",
"/etc/filebeat.yml",
"-e"
],
"env": [
{
"name": "ELASTICSEARCH_HOST",
"value": "testelk.lk.com"
},
{
"name": "ELASTIC_CLOUD_ID"
},
{
"name": "ELASTIC_CLOUD_AUTH"
},
{
"name": "POD_NAMESPACE",
"valueFrom": {
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.namespace"
}
}
}
],
"resources": {
"limits": {
"memory": "200Mi"
},
"requests": {
"cpu": "100m",
"memory": "100Mi"
}
},
"volumeMounts": [
{
"name": "config",
"readOnly": true,
"mountPath": "/etc/filebeat.yml",
"subPath": "filebeat.yml"
},
{
"name": "prospectors",
"readOnly": true,
"mountPath": "/usr/share/filebeat/prospectors.d"
},
{
"name": "data",
"mountPath": "/usr/share/filebeat/data"
},
{
"name": "varlibdockercontainers",
"readOnly": true,
"mountPath": "/var/lib/docker/containers"
},
{
"name": "logforwarderssl",
"mountPath": "/certs/logforwarderssl"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent",
"securityContext": {
"runAsUser": 0
}
}
],
"restartPolicy": "Always",
"terminationGracePeriodSeconds": 30,
"dnsPolicy": "ClusterFirst",
"serviceAccountName": "filebeat",
"serviceAccount": "filebeat",
"securityContext": {},
"schedulerName": "default-scheduler"
}
},
"updateStrategy": {
"type": "OnDelete"
},
"templateGeneration": 6,
"revisionHistoryLimit": 10
}
}
2 configmaps, we're using are:
filebeat-config-ingress:
filebeat.config:
prospectors:
# Mounted `filebeat-prospectors` configmap:
path: ${path.config}/prospectors.d/*.yml
# Reload prospectors configs as they change:
reload.enabled: false
modules:
path: ${path.config}/modules.d/*.yml
# Reload module configs as they change:
reload.enabled: false
filebeat.autodiscover:
providers:
- type: kubernetes
templates:
- condition:
equals:
kubernetes.container.name: "nginx-ingress-controller"
config:
- module: nginx
access:
input:
type: docker
containers.ids:
- "${data.kubernetes.container.id}"
processors:
- add_cloud_metadata:
- add_docker_metadata:
- add_kubernetes_metadata:
cloud.id: ${ELASTIC_CLOUD_ID}
cloud.auth: ${ELASTIC_CLOUD_AUTH}
output:
logstash:
hosts: ["testelk.lk.com:5044"]
filebeat-prospectors-ingress:
kubernetes.yml:
- type: docker
containers:
ids: "*"
path: /var/lib/docker/containers
fields:
type: k8s_nginx_ingress_ctrls
registry_file: "/var/lib/filebeat/registry"
multiline:
pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
negate: true
match: after
processors:
- add_kubernetes_metadata:
in_cluster: true
namespace: ${POD_NAMESPACE}
the last error logs of pod given below:
2019-11-13T07:13:15.323Z INFO add_cloud_metadata/add_cloud_metadata.go:301 add_cloud_metadata: hosting provider type detected as ec2, metadata={"availability_zone":"ap-south-1b","instance_id":"i-0d5d5bba0be1e449e","machine_type":"r3.xlarge","provider":"ec2","region":"ap-south-1"}
2019-11-13T07:13:15.324Z INFO instance/beat.go:275 filebeat stopped.
2019-11-13T07:13:15.324Z ERROR instance/beat.go:691 Exiting: error initializing publisher: error initializing processors: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?
Exiting: error initializing publisher: error initializing processors: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?
