Bonjour,
Suite à mes posts précédents, j'arrive au bout de mon projet. Pour finaliser tout ça, j'aurais aimé cette fois passer les logs de 2 Nginx (situés dans des namespaces différents) vers ES puis Kibana (placés tous deux dans un 3ème namespace).
Avec un seul Nginx dans le namespace_a et ES/Kibana dans le namespace_esk : aucun souci... ça marche.
Par contre, si je lance en plus, le dernier namespace (avec son propre Nginx, FIlebeat, Logstash), ce nouveau Filebeat plante avec ce message :
2020-08-04T10:38:39.183Z INFO instance/beat.go:404 filebeat stopped.
2020-08-04T10:38:39.184Z ERROR instance/beat.go:958 Exiting: data path already locked by another beat. Please make sure that multiple beats are not sharing the same data path (path.data).
Exiting: data path already locked by another beat. Please make sure that multiple beats are not sharing the same data path (path.data).
Le namespace_b fonctionne également très bien s'il est démarré seul avec le namespace_esk... le problème apparaît donc bien quand les deux namespaces avec Filebeat sont démarrés ensemble.
Le problème vient apparemment d'un accès concurrentiel sur le path des PV Nginx mais je ne sais s'il est possible de gérer cela.
Voici les fichiers que j'utilise :
filebeat.yaml (namespace_a) :
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: nsa
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
tags: ["nsa"]
filebeat.config:
modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
filebeat.autodiscover:
providers:
- type: kubernetes
host: ${NODE_NAME}
hints.enabled: true
templates:
- conditions.and:
- contains.kubernetes.container.image: nginx
- equals.kubernetes.namespace: nsa
config:
- module: nginx
access:
enabled: true
var.paths: ["/usr/share/filebeat/nginxlogs/access.log"]
error:
enabled: true
var.paths: ["/usr/share/filebeat/nginxlogs/error.log"]
processors:
- add_cloud_metadata:
- add_host_metadata:
- add_docker_metadata:
output.logstash:
hosts: ["logstash-nsa:5044"]
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: nsa
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.8.0
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: ELASTICSEARCH_HOST
value: elasticsearch-es-http.esk
- name: ELASTICSEARCH_PORT
value: "9200"
- name: ELASTICSEARCH_USERNAME
value: elastic
- name: ELASTICSEARCH_PASSWORD
valueFrom:
secretKeyRef:
key: elastic
name: elasticsearch-es-elastic-user
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
subPath: filebeat.yml
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
- name: es-certs
mountPath: /mnt/elastic/tls.crt
readOnly: true
subPath: tls.crt
- name: nginxlogs
mountPath: /usr/share/filebeat/nginxlogs
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
- name: es-certs
secret:
secretName: elasticsearch-es-http-certs-public
- name: nginxlogs
hostPath:
path: /c/nsa/nginx-data
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: nsa
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""]
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: nsa
labels:
k8s-app: filebeat
volume.yaml (namespace_a) :
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: nginx-data-pv-nsa
namespace: nsa
spec:
capacity:
storage: 5Gi
volumeMode: Filesystem
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
storageClassName: ""
hostPath:
path: /c/nsa/nginx-data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: nginx-data-pvc-nsa
namespace: nsa
spec:
storageClassName: ""
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
volumeName: nginx-data-pv-nsa
filebeat.yaml (namespace_b) :
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: nsb
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
tags: ["nsb"]
filebeat.config:
modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
filebeat.autodiscover:
providers:
- type: kubernetes
host: ${NODE_NAME}
hints.enabled: true
templates:
- conditions.and:
- contains.kubernetes.container.image: nginx
- equals.kubernetes.namespace: nsb
config:
- module: nginx
access:
enabled: true
var.paths: ["/usr/share/filebeat/nginxlogs/access.log"]
error:
enabled: true
var.paths: ["/usr/share/filebeat/nginxlogs/error.log"]
processors:
- add_cloud_metadata:
- add_host_metadata:
- add_docker_metadata:
output.logstash:
hosts: ["logstash-nsb:5044"]
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: nsb
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.8.0
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: ELASTICSEARCH_HOST
value: elasticsearch-es-http.esk
- name: ELASTICSEARCH_PORT
value: "9200"
- name: ELASTICSEARCH_USERNAME
value: elastic
- name: ELASTICSEARCH_PASSWORD
valueFrom:
secretKeyRef:
key: elastic
name: elasticsearch-es-elastic-user
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
subPath: filebeat.yml
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
- name: es-certs
mountPath: /mnt/elastic/tls.crt
readOnly: true
subPath: tls.crt
- name: nginxlogs
mountPath: /usr/share/filebeat/nginxlogs
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
- name: es-certs
secret:
secretName: elasticsearch-es-http-certs-public
- name: nginxlogs
hostPath:
path: /c/nsb/nginx-data
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: nsb
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""]
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: nsb
labels:
k8s-app: filebeat
volume.yaml (namespace_b) :
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: nginx-data-pv-nsb
namespace: nsb
spec:
capacity:
storage: 5Gi
volumeMode: Filesystem
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
storageClassName: ""
hostPath:
path: /c/nsb/nginx-data
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: nginx-data-pvc-nsb
namespace: nsb
spec:
storageClassName: ""
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
volumeName: nginx-data-pv-nsb
...