Competitive Filebeat access on local disk

Elastic Orchestration Elastic Cloud on Kubernetes (ECK)
4

Hi,

In fact, after some research of my own, I have resolved the concern in question. As you specify in your answer, the original problem is related to the (non-permitted) sharing of the /var/lib/filebeat-data directory.

So here are the changes made in the two filebeat files. With this, I have two Filebeat components that work in separate namespaces (one harvests Nginx logs and the other harvests Apache logs). See the added and modified lines below.

filebeat.yaml (namespace_a):

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: nsa
  labels:
    k8s-app: filebeat
data:
  filebeat.yml: |-
    tags: ["nsa"]

    path.data: ${path.home}/data-a ### <<=== Added line

    filebeat.config:
      modules:
        path: ${path.config}/modules.d/*.yml
        reload.enabled: false

    filebeat.autodiscover:

      providers:
        - type: kubernetes
          host: ${NODE_NAME}
          hints.enabled: true

          templates:
            - conditions.and:
                - contains.kubernetes.container.image: nginx
                - equals.kubernetes.namespace: nsa
              config:
                - module: nginx
                  access:
                    enabled: true
                    var.paths: ["/usr/share/filebeat/nginxlogs/access.log"]
                  error:
                    enabled: true
                    var.paths: ["/usr/share/filebeat/nginxlogs/error.log"]

    processors:
      - add_cloud_metadata:
      - add_host_metadata:
      - add_docker_metadata:

    output.logstash:
      hosts: ["logstash-nsa:5044"]

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: filebeat
  namespace: nsa
  labels:
    k8s-app: filebeat
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      serviceAccountName: filebeat
      terminationGracePeriodSeconds: 30
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
        - name: filebeat
          image: docker.elastic.co/beats/filebeat:7.8.0
          args: [
            "-c", "/etc/filebeat.yml",
            "-e",
          ]
          env:
            - name: ELASTICSEARCH_HOST
              value: elasticsearch-es-http.esk
            - name: ELASTICSEARCH_PORT
              value: "9200"
            - name: ELASTICSEARCH_USERNAME
              value: elastic
            - name: ELASTICSEARCH_PASSWORD
              valueFrom:
                secretKeyRef:
                  key: elastic
                  name: elasticsearch-es-elastic-user
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          securityContext:
            runAsUser: 0
          resources:
            limits:
              memory: 200Mi
            requests:
              cpu: 100m
              memory: 100Mi
          volumeMounts:
            - name: config
              mountPath: /etc/filebeat.yml
              subPath: filebeat.yml
              readOnly: true
            - name: data
              mountPath: /usr/share/filebeat/data-a ### <<=== Modified line
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: varlog
              mountPath: /var/log
              readOnly: true
            - name: es-certs
              mountPath: /mnt/elastic/tls.crt
              readOnly: true
              subPath: tls.crt
            - name: nginxlogs
              mountPath: /usr/share/filebeat/nginxlogs

      volumes:
        - name: config
          configMap:
            defaultMode: 0600
            name: filebeat-config
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: varlog
          hostPath:
            path: /var/log
        - name: data
          hostPath:
            path: /var/lib/filebeat-data-a ### <<=== Modified line
            type: DirectoryOrCreate
        - name: es-certs
          secret:
            secretName: elasticsearch-es-http-certs-public
        - name: nginxlogs
          hostPath:
            path: /c/nsa/nginx-data

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: filebeat
subjects:
  - kind: ServiceAccount
    name: filebeat
    namespace: nsa
roleRef:
  kind: ClusterRole
  name: filebeat
  apiGroup: rbac.authorization.k8s.io

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: filebeat
  labels:
    k8s-app: filebeat
rules:
  - apiGroups: [""]
    resources:
      - namespaces
      - pods
    verbs:
      - get
      - watch
      - list

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat
  namespace: nsa
  labels:
    k8s-app: filebeat

filebeat.yaml (namespace_b):

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: nsb
  labels:
    k8s-app: filebeat
data:
  filebeat.yml: |-
    tags: ["nsb"]

    path.data: ${path.home}/data-b ### <<=== Added line

    filebeat.config:
      modules:
        path: ${path.config}/modules.d/*.yml
        reload.enabled: false

    filebeat.autodiscover:

      providers:
        - type: kubernetes
          host: ${NODE_NAME}
          hints.enabled: true

          templates:
            - conditions.and:
                - contains.kubernetes.container.image: httpd
                - equals.kubernetes.namespace: nsb
              config:
                - module: apache2
                  access:
                    input:
                      type: docker
                      containers.ids:
                        - ${data.kubernetes.container.id}
                  error:
                    input:
                      type: docker
                      containers.ids:
                        - ${data.kubernetes.container.id}

    processors:
      - add_cloud_metadata:
      - add_host_metadata:
      - add_docker_metadata:

    output.logstash:
      hosts: ["logstash-nsb:5044"]

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: filebeat
  namespace: nsb
  labels:
    k8s-app: filebeat
spec:
  selector:
    matchLabels:
      k8s-app: filebeat
  template:
    metadata:
      labels:
        k8s-app: filebeat
    spec:
      serviceAccountName: filebeat
      terminationGracePeriodSeconds: 30
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
        - name: filebeat
          image: docker.elastic.co/beats/filebeat:7.8.0
          args: [
            "-c", "/etc/filebeat.yml",
            "-e",
          ]
          env:
            - name: ELASTICSEARCH_HOST
              value: elasticsearch-es-http.esk
            - name: ELASTICSEARCH_PORT
              value: "9200"
            - name: ELASTICSEARCH_USERNAME
              value: elastic
            - name: ELASTICSEARCH_PASSWORD
              valueFrom:
                secretKeyRef:
                  key: elastic
                  name: elasticsearch-es-elastic-user
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          securityContext:
            runAsUser: 0
          resources:
            limits:
              memory: 200Mi
            requests:
              cpu: 100m
              memory: 100Mi
          volumeMounts:
            - name: config
              mountPath: /etc/filebeat.yml
              subPath: filebeat.yml
              readOnly: true
            - name: data
              mountPath: /usr/share/filebeat/data-b ### <<=== Modified line
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: varlog
              mountPath: /var/log
              readOnly: true
            - name: es-certs
              mountPath: /mnt/elastic/tls.crt
              readOnly: true
              subPath: tls.crt

      volumes:
        - name: config
          configMap:
            defaultMode: 0600
            name: filebeat-config
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: varlog
          hostPath:
            path: /var/log
        - name: data
          hostPath:
            path: /var/lib/filebeat-data-b ### <<=== Modified line
            type: DirectoryOrCreate
        - name: es-certs
          secret:
            secretName: elasticsearch-es-http-certs-public

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: filebeat
subjects:
  - kind: ServiceAccount
    name: filebeat
    namespace: nsb
roleRef:
  kind: ClusterRole
  name: filebeat
  apiGroup: rbac.authorization.k8s.io

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: filebeat
  labels:
    k8s-app: filebeat
rules:
  - apiGroups: [""]
    resources:
      - namespaces
      - pods
    verbs:
      - get
      - watch
      - list

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: filebeat
  namespace: nsb
  labels:
    k8s-app: filebeat

---

Thank you again for your response!

Guillaume.