Hi,
We're running filebeat as a deamonset on all nodes of our on-premises kubernetes clusters.
(Filebeat sends logs to logstash, logstash does some processesing and sends them on to elasticsearch.)
On one of our clusters this is working fine. On the other one we have this issue:
Only four of the filebeats pick up logs and send them on. If I delete all filebeat pods, the four nodes whose logs are send are often four different ones, but never more than four.
I'm running filebeat in debug, which shows - for the working ones- logs like this:
Summary
2018-12-31T12:49:43.979Z DEBUG [publish] pipeline/processor.go:308 Publish event: {
"@timestamp": "2018-12-31T12:49:41.882Z",
"@metadata": {
"beat": "filebeat",
"type": "doc",
"version": "6.5.4"
},
"offset": 19062135,
"stream": "stdout",
"prospector": {
"type": "docker"
},
"input": {
"type": "docker"
},
"message": "2018-12-31 12:49:41.875 [debug] <0.24878.685> Supervisor {<0.24878.685>,rabbit_connection_sup} started rabbit_connection_helper_sup:start_link() at pid <0.24831.685>", "source": "/var/lib/docker/containers/b03f9c1b1b6f9c7811a680b7149ecb3397b9f1d03380a305d3bda46baf0e2712/b03f9c1b1b6f9c7811a680b7149ecb3397b9f1d03380a305d3bda46baf0e2712-json.log",
"kubernetes": {
"node": {
"name": "lcl-worker-005"
},
"container": {
"name": "rabbitmq"
},
"namespace": "development-1",
"statefulset": {
"name": "rabbitmq"
},
"labels": {
"controller-revision-hash": "rabbitmq-7d88bd67f",
"statefulset": {
"kubernetes": {
"io/pod-name": "rabbitmq-0"
}
},
"component": "rabbitmq"
},
"pod": {
"name": "rabbitmq-0"
}
},
"beat": {
"name": "filebeat-t8nfr",
"hostname": "filebeat-t8nfr",
"version": "6.5.4"
},
"host": {
"name": "filebeat-t8nfr"
}
}
And metrics:
Summary
{
"monitoring": {
"metrics": {
"beat": {
"cpu": {
"system": {
"ticks": 31630,
"time": {
"ms": 203
}
},
"total": {
"ticks": 249710,
"time": {
"ms": 1513
},
"value": 249710
},
"user": {
"ticks": 218080,
"time": {
"ms": 1310
}
}
},
"handles": {
"limit": {
"hard": 1048576,
"soft": 1048576
},
"open": 35
},
"info": {
"ephemeral_id": "935d404b-d5b9-4458-a8c9-80818c2b7107",
"uptime": {
"ms": 5132179
}
},
"memstats": {
"gc_next": 18202432,
"memory_alloc": 11776776,
"memory_total": 16818113864,
"rss": -491520
}
},
"filebeat": {
"events": {
"active": -184,
"added": 2953,
"done": 3137
},
"harvester": {
"open_files": 27,
"running": 27
}
},
"libbeat": {
"config": {
"module": {
"running": 0
}
},
"output": {
"events": {
"acked": 2970,
"active": -49,
"batches": 24,
"total": 2921
},
"read": {
"bytes": 144
},
"write": {
"bytes": 243393
}
},
"pipeline": {
"clients": 59,
"events": {
"active": 61,
"published": 2892,
"total": 2892
},
"queue": {
"acked": 2970
}
}
},
"registrar": {
"states": {
"current": 129,
"update": 3027
},
"writes": {
"success": 26,
"total": 25
}
},
"system": {
"load": {
"1": 5.43,
"15": 13.28,
"5": 10.84,
"norm": {
"1": 1.3575,
"15": 3.32,
"5": 2.71
}
}
}
}
}
}
continues in reply