Hello,
Following my previous posts, I'm coming to the end of my project. To finalize all this, I would have liked this time to pass the logs from 2 Nginx (located in different namespaces) to ES then Kibana (both located in a 3rd namespace).
With only one Nginx in the namespace_a and ES/Kibana in the namespace_esk : no problem... it works.
On the other hand, if I launch the last namespace (with its own Nginx, Filebeat, Logstash), this new Filebeat crashes with this message :
Following my previous posts, I'm coming to the end of my project. To finalize all this, I would have liked this time to pass the logs from 2 Nginx (located in different namespaces) to ES then Kibana (both located in a 3rd namespace).
With only one Nginx in the namespace_a and ES/Kibana in the namespace_esk : no problem... it works.
On the other hand, if I launch the last namespace (with its own Nginx, FIlebeat, Logstash), this new Filebeat crashes with this message :
2020-08-04T10:38:39.183Z INFO instance/beat.go:404 filebeat stopped.
2020-08-04T10:38:39.184Z ERROR instance/beat.go:958 Exiting: data path already locked by another beat. Please make sure that multiple beats are not sharing the same data path (path.data).
Exiting: data path already locked by another beat. Please make sure that multiple beats are not sharing the same data path (path.data).
The namespace_b also works very well if it is started alone with the namespace_esk... so the problem appears well when both namespaces with Filebeat are started together.
The problem apparently comes from a competitive access on the Nginx PV path but I don't know if it's possible to handle this.
Here are my files:
filebeat.yaml (namespace_a):
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: nsa
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
tags: ["nsa"]
filebeat.config:
modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
filebeat.autodiscover:
providers:
- type: kubernetes
host: ${NODE_NAME}
hints.enabled: true
templates:
- conditions.and:
- contains.kubernetes.container.image: nginx
- equals.kubernetes.namespace: nsa
config:
- module: nginx
access:
enabled: true
var.paths: ["/usr/share/filebeat/nginxlogs/access.log"]
error:
enabled: true
var.paths: ["/usr/share/filebeat/nginxlogs/error.log"]
processors:
- add_cloud_metadata:
- add_host_metadata:
- add_docker_metadata:
output.logstash:
hosts: ["logstash-nsa:5044"]
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: nsa
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.8.0
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: ELASTICSEARCH_HOST
value: elasticsearch-es-http.esk
- name: ELASTICSEARCH_PORT
value: "9200"
- name: ELASTICSEARCH_USERNAME
value: elastic
- name: ELASTICSEARCH_PASSWORD
valueFrom:
secretKeyRef:
key: elastic
name: elasticsearch-es-elastic-user
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
subPath: filebeat.yml
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
- name: es-certs
mountPath: /mnt/elastic/tls.crt
readOnly: true
subPath: tls.crt
- name: nginxlogs
mountPath: /usr/share/filebeat/nginxlogs
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
- name: es-certs
secret:
secretName: elasticsearch-es-http-certs-public
- name: nginxlogs
hostPath:
path: /c/nsa/nginx-data
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: nsa
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""]
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: nsa
labels:
k8s-app: filebeat
volume.yaml (namespace_a):
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: nginx-data-pv-nsa
namespace: nsa
spec:
capacity:
storage: 5Gi
volumeMode: Filesystem
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
storageClassName: ""
hostPath:
path: /c/nsa/nginx-data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: nginx-data-pvc-nsa
namespace: nsa
spec:
storageClassName: ""
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
volumeName: nginx-data-pv-nsa
filebeat.yaml (namespace_b):
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: nsb
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
tags: ["nsb"]
filebeat.config:
modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
filebeat.autodiscover:
providers:
- type: kubernetes
host: ${NODE_NAME}
hints.enabled: true
templates:
- conditions.and:
- contains.kubernetes.container.image: nginx
- equals.kubernetes.namespace: nsb
config:
- module: nginx
access:
enabled: true
var.paths: ["/usr/share/filebeat/nginxlogs/access.log"]
error:
enabled: true
var.paths: ["/usr/share/filebeat/nginxlogs/error.log"]
processors:
- add_cloud_metadata:
- add_host_metadata:
- add_docker_metadata:
output.logstash:
hosts: ["logstash-nsb:5044"]
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: nsb
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.8.0
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: ELASTICSEARCH_HOST
value: elasticsearch-es-http.esk
- name: ELASTICSEARCH_PORT
value: "9200"
- name: ELASTICSEARCH_USERNAME
value: elastic
- name: ELASTICSEARCH_PASSWORD
valueFrom:
secretKeyRef:
key: elastic
name: elasticsearch-es-elastic-user
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
subPath: filebeat.yml
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
- name: es-certs
mountPath: /mnt/elastic/tls.crt
readOnly: true
subPath: tls.crt
- name: nginxlogs
mountPath: /usr/share/filebeat/nginxlogs
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
- name: es-certs
secret:
secretName: elasticsearch-es-http-certs-public
- name: nginxlogs
hostPath:
path: /c/nsb/nginx-data
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: nsb
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""]
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: nsb
labels:
k8s-app: filebeat
volume.yaml (namespace_b):
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: nginx-data-pv-nsb
namespace: nsb
spec:
capacity:
storage: 5Gi
volumeMode: Filesystem
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
storageClassName: ""
hostPath:
path: /c/nsb/nginx-data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: nginx-data-pvc-nsb
namespace: nsb
spec:
storageClassName: ""
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
volumeName: nginx-data-pv-nsb
...