Filebeat 7.7 collection size

After updating our ECK stack, filebeat indices grow out of control and eventually fill up the storage and crash the stack within a matter of hours. I literally only bumped up the version numbers and redeployed the stack. It wasn't an upgrade it was like a new install. The whole stack is deployed the exact same way we deployed 7.6.2. Can someone maybe point me in the right direction of what could be wrong or what has changed? Below is our filebeat deployment. The indices don't split after 5GB either (separate issue). The indices just continue to grow until the storage is full. Metricbeat and Apm seem to be working fine.

apiVersion: v1
kind: ConfigMap
metadata:
  namespace: elastic-system
  name: filebeat-indice-lifecycle
  labels:
    app: filebeat
data:
  indice-lifecycle.json: |-
    {
      "policy": {
        "phases": {
          "hot": {
            "actions": {
              "rollover": {
                "max_size": "5GB" ,
                "max_age": "1d"
              }
            }
          },
          "delete": {
            "min_age": "4d",
            "actions": {
              "delete": {}
            }
          }
        }
      }
    }
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: elastic-system
  labels:
    k8s-app: filebeat
data:
  filebeat.yml: |-
    filebeat.modules:
    - module: nginx
      access:
        enabled: true
        var.paths: ["/var/log/nginx/access.log*"]
      error:
        enabled: true
        var.paths: ["/var/log/nginx/access.log*"]
    - module: system

    filebeat.inputs:
    - type: container
      paths:
        - /var/log/containers/*.log
      processors:
        - add_kubernetes_metadata:
            host: ${NODE_NAME}
            matchers:
            - logs_path:
                logs_path: "/var/log/containers/"

    processors:
      - add_cloud_metadata:
      - add_host_metadata:

    setup.kibana:
      host: 'kibana:5601'

    # Import pre-existing dashboards
    setup.dashboards.enabled: true

    output.elasticsearch:
      hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}']
      username: ${ELASTICSEARCH_USERNAME}
      password: ${ELASTICSEARCH_PASSWORD}
      ssl.certificate_authorities:
        - /etc/certificate/ca.crt
      compression_level: 9
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: filebeat
  namespace: elastic-system
  labels:
    k8s-app: filebeat
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      tolerations:
      - key: elasticsearch
        operator: Exists    
      serviceAccountName: filebeat
      terminationGracePeriodSeconds: 30
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
      - name: filebeat
        image: docker.elastic.co/beats/filebeat:7.7.0
        args: [
          "-c", "/etc/filebeat.yml",
          "-e",
        ]
        env:
        - name: ELASTICSEARCH_HOST
          value: https://host
        - name: ELASTICSEARCH_PORT
          value: "9200"
        - name: ELASTICSEARCH_USERNAME
          value: elastic
        - name: ELASTICSEARCH_PASSWORD
          value: password
#        - name: ELASTIC_CLOUD_ID
#          value:
#        - name: ELASTIC_CLOUD_AUTH
#          value:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        securityContext:
          runAsUser: 0
        resources:
          limits:
            memory: 1024Mi
          requests:
            cpu: 100m
            memory: 256Mi
        volumeMounts:
        - name: config
          mountPath: /etc/filebeat.yml
          readOnly: true
          subPath: filebeat.yml
        - name: certs
          mountPath: /etc/certificate/ca.crt
          readOnly: true
          subPath: ca.crt
        - name: data
          mountPath: /usr/share/filebeat/data
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: varlog
          mountPath: /var/log
          readOnly: true
        - name: filebeat-indice-lifecycle
          mountPath: /etc/indice-lifecycle.json
          readOnly: true
          subPath: indice-lifecycle.json
      volumes:
      - name: filebeat-indice-lifecycle
        configMap:
          defaultMode: 0600
          name: filebeat-indice-lifecycle
      - name: config
        configMap:
          defaultMode: 0600
          name: filebeat-config
      - name: certs
        secret:
          secretName: eck-es-http-certs-public
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: varlog
        hostPath:
          path: /var/log
      - name: data
        hostPath:
          path: /path
          type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: filebeat
  namespace: elastic-system
subjects:
- kind: ServiceAccount
  name: filebeat
  namespace: elastic-system
roleRef:
  kind: ClusterRole
  name: filebeat
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: filebeat
  labels:
    k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
  resources:
  - namespaces
  - pods
  verbs:
  - get
  - watch
  - list
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat
  namespace: elastic-system
  labels:
    k8s-app: filebeat

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.