JournalBeat stopped shipping logs on all environments for us from 19th Dec, 2020. We configured our beats services to ship logs from aws instance to elasticsearch through logstash. Our filebeat and metricbeat logs are being shipped fine. It has issue only shipping the journalbeat logs.
We have verified the SELinuxOptions as well. We tried enabling the debug logs on elasticsearch, logstash and journalbeat as well. Not finding any errors but still no logs in the elasticsearch.
ELK Stack version: 7.10.1
JournalBeats version: 7.10.1
Kubernetes version: 1.19.4
Below is our kubernetes manifest for the journalbeat.
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
annotations:
seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
name: journalbeat
namespace: kube-system
spec:
allowedCapabilities:
- KILL
- CHOWN
- FSETID
- FOWNER
- SETGID
- SETUID
- SETFCAP
- SETPCAP
- AUDIT_WRITE
- NET_BIND_SERVICE
fsGroup:
rule: RunAsAny
hostIPC: false
hostNetwork: false
hostPID: false
privileged: false
requiredDropCapabilities:
- MKNOD
- DAC_OVERRIDE
- NET_RAW
- SYS_CHROOT
runAsUser:
rule: RunAsAny
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
volumes:
- secret
- configMap
- hostPath
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: journalbeat
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: journalbeat-config
namespace: kube-system
labels:
k8s-app: journalbeat-logging
version: v1
data:
journalbeat.yml: |
name: "${NODENAME}"
journalbeat.inputs:
- paths: []
seek: cursor
cursor_seek_fallback: tail
processors:
- add_kubernetes_metadata:
host: "${NODENAME}"
in_cluster: true
default_indexers.enabled: false
default_matchers.enabled: false
indexers:
- container:
matchers:
- fields:
lookup_fields: ["container.id"]
- decode_json_fields:
fields: ["message"]
process_array: false
max_depth: 1
target: ""
overwrite_keys: true
- drop_event.when:
or:
- regexp.kubernetes.pod.name: "filebeat-.*"
- regexp.kubernetes.pod.name: "journalbeat-.*"
- regexp.kubernetes.pod.name: "metrics-server-.*"
- equals.syslog.identifier: "audit"
- regexp.message: '^.*?\baudit\b.*?$'
- and:
- equals.systemd.unit: "docker.service"
- or:
- regexp.message: '^.*?\bJournalbeat is experimental.*?$'
- and:
- equals.systemd.unit: "init.scope"
- or:
- regexp.message: '^.*?\bStarted libcontainer container.*?$'
- equals.message: "Started rpm-ostree System Management Daemon."
- regexp.message: '^.*?\bSucceeded.*?$'
# Updated to ignore conflict of indices between environments
setup.ilm.enabled: false
setup.template.enabled: false
output.logstash:
hosts: '${LOGSTASH_HOSTS}'
compression_level: 7
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: journalbeat
rules:
- apiGroups:
- ""
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
- apiGroups:
- extensions
resourceNames:
- journalbeat
resources:
- podsecuritypolicies
verbs:
- use
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: journalbeat
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: journalbeat
subjects:
- kind: ServiceAccount
name: journalbeat
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: journalbeat
namespace: kube-system
labels:
k8s-app: journalbeat-logging
version: v1
spec:
selector:
matchLabels:
k8s-app: journalbeat-logging
version: v1
template:
metadata:
labels:
k8s-app: journalbeat-logging
version: v1
app: journalbeat
name: journalbeat
spec:
containers:
- args:
- -e
- -c
- /etc/journalbeat.yml
command:
- journalbeat
env:
- name: NODENAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: PODNAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: LOGSTASH_HOSTS
value: logstash.company.com
- name: LOGSTASH_PORT
value: "5044"
- name: ENVIRONMENT
value: prod
image: docker.elastic.co/beats/journalbeat:7.10.1
imagePullPolicy: Always
name: journalbeat
resources:
limits:
cpu: 600m
memory: 800Mi
requests:
cpu: 200m
memory: 400Mi
volumeMounts:
- mountPath: /usr/share/journalbeat/data
name: data
- mountPath: /var/log/journal
name: var-journal
- mountPath: /run/log/journal
name: run-journal
- mountPath: /etc/journalbeat.yml
name: config
subPath: journalbeat.yml
- mountPath: /etc/machine-id
name: machine-id
hostNetwork: true
nodeSelector: {}
securityContext:
seLinuxOptions:
user: system_u
role: system_r
type: spc_t
level: s0
fsGroup: 0
runAsUser: 0
serviceAccountName: journalbeat
terminationGracePeriodSeconds: 60
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /var/log/journal/journalbeat-data
name: data
- hostPath:
path: /var/log/journal
name: var-journal
- hostPath:
path: /run/log/journal
name: run-journal
- hostPath:
path: /etc/machine-id
name: machine-id
- configMap:
items:
- key: journalbeat.yml
path: journalbeat.yml
name: journalbeat-config
name: config