After updating our ECK stack, filebeat indices grow out of control and eventually fill up the storage and crash the stack within a matter of hours. I literally only bumped up the version numbers and redeployed the stack. It wasn't an upgrade it was like a new install. The whole stack is deployed the exact same way we deployed 7.6.2. Can someone maybe point me in the right direction of what could be wrong or what has changed? Below is our filebeat deployment. The indices don't split after 5GB either (separate issue). The indices just continue to grow until the storage is full. Metricbeat and Apm seem to be working fine.
apiVersion: v1
kind: ConfigMap
metadata:
namespace: elastic-system
name: filebeat-indice-lifecycle
labels:
app: filebeat
data:
indice-lifecycle.json: |-
{
"policy": {
"phases": {
"hot": {
"actions": {
"rollover": {
"max_size": "5GB" ,
"max_age": "1d"
}
}
},
"delete": {
"min_age": "4d",
"actions": {
"delete": {}
}
}
}
}
}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: elastic-system
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
filebeat.modules:
- module: nginx
access:
enabled: true
var.paths: ["/var/log/nginx/access.log*"]
error:
enabled: true
var.paths: ["/var/log/nginx/access.log*"]
- module: system
filebeat.inputs:
- type: container
paths:
- /var/log/containers/*.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
processors:
- add_cloud_metadata:
- add_host_metadata:
setup.kibana:
host: 'kibana:5601'
# Import pre-existing dashboards
setup.dashboards.enabled: true
output.elasticsearch:
hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}']
username: ${ELASTICSEARCH_USERNAME}
password: ${ELASTICSEARCH_PASSWORD}
ssl.certificate_authorities:
- /etc/certificate/ca.crt
compression_level: 9
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: elastic-system
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
tolerations:
- key: elasticsearch
operator: Exists
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.7.0
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: ELASTICSEARCH_HOST
value: https://host
- name: ELASTICSEARCH_PORT
value: "9200"
- name: ELASTICSEARCH_USERNAME
value: elastic
- name: ELASTICSEARCH_PASSWORD
value: password
# - name: ELASTIC_CLOUD_ID
# value:
# - name: ELASTIC_CLOUD_AUTH
# value:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
resources:
limits:
memory: 1024Mi
requests:
cpu: 100m
memory: 256Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: certs
mountPath: /etc/certificate/ca.crt
readOnly: true
subPath: ca.crt
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
- name: filebeat-indice-lifecycle
mountPath: /etc/indice-lifecycle.json
readOnly: true
subPath: indice-lifecycle.json
volumes:
- name: filebeat-indice-lifecycle
configMap:
defaultMode: 0600
name: filebeat-indice-lifecycle
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: certs
secret:
secretName: eck-es-http-certs-public
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
- name: data
hostPath:
path: /path
type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
namespace: elastic-system
subjects:
- kind: ServiceAccount
name: filebeat
namespace: elastic-system
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: elastic-system
labels:
k8s-app: filebeat