filebeat.yml
filebeat.inputs:
- type: container
enabled: true
paths:
- /var/lib/docker/containers/*/*.log
processors:
- add_docker_metadata: ~
output.elasticsearch:
hosts: ["172.20.111.199:9200"]
pipeline: "postfix-pipeline"
index: "mailcow-postfix-%{+yyyy.MM.dd}"
setup.template:
name: "postfix"
pattern: "postfix-*"
overwrite: true
# Logging configuration
logging.level: info
logging.to_files: true
logging.files:
path: /var/log/filebeat
name: filebeat
keepfiles: 2
permissions: 0644
# Required for filebeat modules
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
# Processors
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
Index Template:
{
"index_patterns": ["postfix-*"],
"data_stream": {},
"priority": 500,
"template": {
"mappings": {
"properties": {
"@timestamp": { "type": "date" },
"message": { "type": "text" },
"syslog_host": { "type": "keyword" },
"program": { "type": "keyword" },
"pid": { "type": "integer" },
"log": { "type": "text" }
}
}
}
}'
Ingest Pipeline:
curl -X PUT "http://172.20.111.199:9200/_ingest/pipeline/postfix-pipeline" -H 'Content-Type: application/json' -d'
{
"description": "Pipeline for parsing Postfix logs",
"processors": [
{
"grok": {
"field": "log",
"patterns": [
"%{SYSLOGTIMESTAMP:timestamp} %{SYSLOGHOST:syslog_host} %{DATA:program}(?:\\[%{POSINT:pid}\\])?: %{GREEDYDATA:message}"
]
}
},
{
"date": {
"field": "timestamp",
"formats": ["MMM d HH:mm:ss", "MMM dd HH:mm:ss"],
"timezone": "UTC"
}
}
]
}'
{"log.level":"error","@timestamp":"2024-07-12T15:35:54.530+0530","log.logger":"publisher_pipeline_output","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/publisher/pipeline.(*netClientWorker).run","file.name":"pipeline/client_worker.go","file.line":148},"message":"Failed to connect to backoff(elasticsearch(http://172.20.111.199:9200)): Connection marked as failed because the onConnect callback failed: error loading template: failed to put data stream: could not put data stream: 400 Bad Request: {\"error\":{\"root_cause\":[{\"type\":\"illegal_argument_exception\",\"reason\":\"no matching index template found for data stream [postfix]\"}],\"type\":\"illegal_argument_exception\",\"reason\":\"no matching index template found for data stream [postfix]\"},\"status\":400}. Response body: {\"error\":{\"root_cause\":[{\"type\":\"illegal_argument_exception\",\"reason\":\"no matching index template found for data stream [postfix]\"}],\"type\":\"illegal_argument_exception\",\"reason\":\"no matching index template found for data stream [postfix]\"},\"status\":400}","service.name":"filebeat","ecs.version":"1.6.0"}
Please help to sort the issue.
