Hello everyone,
I am currently facing an issue where I keep encountering errors in the connection between the Elastic Agents (Filebeat) and Logstash.
There are always two related errors, and the target is always Logstash on port 5044 (Beats Input).
These errors occur with both Elastic Agent versions 8.14.x and 8.16.x, as well as with Logstash.
Error messages from the Elastic Agent:
failed to publish events: write tcp XX.XXX.XX.X:42550->XX.XXX.XX.X:5044: write: connection reset by peer
failed to publish events caused by: write tcp XX.XXX.XX.X:42550->XX.XXX.XX.X:5044: write: connection reset by peer
Initially, there were a lot of these messages, but they were significantly reduced using the following parameters. Unfortunately, the errors have not been eliminated entirely.
In the Logstash output of the Elastic Agent, the following was configured:
pipelining: 0
On the Beats Input in Elastic Agent, the timeout was increased:
client_inactivity_timeout => 1200
The log contains an entry referencing Beats v7, which results in a 404 error:
"log.origin.function": ["github.com/elastic/beats/v7/libbeat/outputs/logstash.(*syncClient).Publish"]
Does anyone have any idea where these errors might be coming from and how I can resolve them?
Thank you and best regards.
Here is the complete log entry:
{
"_index": ".ds-logs-elastic_agent.filebeat-XXXX-2024.12.04-000093",
"_id": "3wx2lpMBlD3uJqTDHxEd",
"_version": 1,
"_score": 0,
"_source": {
"agent": {
"name": "XXXXXXX",
"id": "a34f3d82-fe99-41c9-8480-5f3cc82a8ba6",
"ephemeral_id": "2e51f012-5f8b-44d8-859a-f195276548cf",
"type": "filebeat",
"version": "8.16.1"
},
"service.name": "filebeat",
"log": {
"file": {
"inode": "393721",
"path": "/opt/Elastic/Agent/data/elastic-agent-8.16.1-b6da7f/logs/elastic-agent-20241205.ndjson",
"device_id": "64768"
},
"offset": 6656134,
"source": "log-d0a517d9-9b23-4313-aae9-132d953e5716"
},
"elastic_agent": {
"id": "a34f3d82-fe99-41c9-8480-5f3cc82a8ba6",
"version": "8.16.1",
"snapshot": false
},
"message": "Failed to publish events caused by: write tcp xx.xxx.xx.x:42550->xx.xxx.xx.x:5044: write: connection reset by peer",
"log.logger": "logstash",
"tags": ["beats_input_codec_plain_applied"],
"log.origin": {
"file.line": 162,
"function": "github.com/elastic/beats/v7/libbeat/outputs/logstash.(*syncClient).Publish",
"file.name": "logstash/sync.go"
},
"input": {
"type": "filestream"
},
"component": {
"binary": "filebeat",
"id": "log-d0a517d9-9b23-4313-aae9-132d953e5716",
"type": "log",
"dataset": "elastic_agent.filebeat"
},
"@timestamp": "2024-12-05T10:55:27.784Z",
"ecs": {
"version": "8.0.0"
},
"data_stream": {
"namespace": "XXXX",
"type": "logs",
"dataset": "elastic_agent.filebeat"
},
"@version": "1",
"host": {
"hostname": "XXXXXXX",
"os": {
"kernel": "5.15.0-116-generic",
"codename": "jammy",
"name": "Ubuntu",
"type": "linux",
"family": "debian",
"version": "22.04.5 LTS (Jammy Jellyfish)",
"platform": "ubuntu"
},
"ip": ["xx.xxx.xx.x", "fe80::250:56ff:fe99:2ad7"],
"containerized": false,
"name": "XXXXXXX",
"id": "f383c09d524c4d0f9cad10d61490b4af",
"mac": ["00-50-56-99-2A-D7"],
"architecture": "x86_64"
},
"log.level": "error",
"event": {
"original": "Failed to publish events caused by: write tcp xx.xxx.xx.x:42550->xx.xxx.xx.x:5044: write: connection reset by peer",
"dataset": "elastic_agent.filebeat"
},
"category": "other"
},
"fields": {
"elastic_agent.version": ["8.16.1"],
"component.binary": ["filebeat"],
"host.os.name.text": ["Ubuntu"],
"host.name.text": ["XXXXXXX"],
"host.hostname": ["XXXXXXX"],
"host.mac": ["00-50-56-99-2A-D7"],
"component.id": ["log-d0a517d9-9b23-4313-aae9-132d953e5716"],
"agent.name.text": ["XXXXXXX"],
"host.os.version": ["22.04.5 LTS (Jammy Jellyfish)"],
"host.os.name": ["Ubuntu"],
"log.level": ["error"],
"agent.name": ["XXXXXXX"],
"host.name": ["XXXXXXX"],
"event.original": ["Failed to publish events caused by: write tcp xx.xxx.xx.x:42550->xx.xxx.xx.x:5044: write: connection reset by peer"],
"host.os.type": ["linux"],
"input.type": ["filestream"],
"log.source": ["log-d0a517d9-9b23-4313-aae9-132d953e5716"],
"log.offset": [6656134],
"data_stream.type": ["logs"],
"tags": ["beats_input_codec_plain_applied"],
"host.architecture": ["x86_64"],
"log.origin.function": ["github.com/elastic/beats/v7/libbeat/outputs/logstash.(*syncClient).Publish"],
"agent.id": ["a34f3d82-fe99-41c9-8480-5f3cc82a8ba6"],
"ecs.version": ["8.0.0"],
"host.containerized": [false],
"service.name.text": ["filebeat"],
"agent.version": ["8.16.1"],
"host.os.family": ["debian"],
"log.origin.file.name.text": ["logstash/sync.go"],
"log.logger": ["logstash"],
"host.ip": ["xx.xxx.xx.x", "fe80::250:56ff:fe99:2ad7"],
"agent.type": ["filebeat"],
"host.os.kernel": ["5.15.0-116-generic"],
"component.dataset": ["elastic_agent.filebeat"],
"log.file.device_id": ["64768"],
"@version": ["1"],
"log.file.path.text": ["/opt/Elastic/Agent/data/elastic-agent-8.16.1-b6da7f/logs/elastic-agent-20241205.ndjson"],
"elastic_agent.snapshot": [false],
"host.id": ["f383c09d524c4d0f9cad10d61490b4af"],
"log.origin.file.line": [162],
"service.name": ["filebeat"],
"elastic_agent.id": ["a34f3d82-fe99-41c9-8480-5f3cc82a8ba6"],
"data_stream.namespace": ["XXXX"],
"host.os.codename": ["jammy"],
"message": ["Failed to publish events caused by: write tcp xx.xxx.xx.x:42550->xx.xxx.xx.x:5044: write: connection reset by peer"],
"component.type": ["log"],
"@timestamp": ["2024-12-05T10:55:27.784Z"],
"log.origin.file.name": ["logstash/sync.go"],
"host.os.platform": ["ubuntu"],
"log.file.inode": ["393721"],
"data_stream.dataset": ["elastic_agent.filebeat"],
"log.file.path": ["/opt/Elastic/Agent/data/elastic-agent-8.16.1-b6da7f/logs/elastic-agent-20241205.ndjson"],
"agent.ephemeral_id": ["2e51f012-5f8b-44d8-859a-f195276548cf"],
"category": ["other"],
"event.dataset": ["elastic_agent.filebeat"]
}
}