I downloaded the latest manifest file
curl -L -O https://raw.githubusercontent.com/elastic/beats/8.15/deploy/kubernetes/filebeat-kubernetes.yaml
I then copied it, renamed the copy, and edited the file with the idea of the DaemonSet running under its own ServiceAccount, ClusterRole, ConfigMap, etc. and thus avoiding conflicts with the Filebeat DaemonSet already running on the same Kubernetes node but collecting logs from other locations. The other Filebeat DaemonSet is running under the default names for ServiceAccount, ClusterRole, etc.
Here is the full content of the file
apiVersion: v1
kind: ServiceAccount
metadata:
name: ceo-filebeat
namespace: kube-system
labels:
k8s-app: ceo-filebeat
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: ceo-filebeat
labels:
k8s-app: ceo-filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
- nodes
verbs:
- get
- watch
- list
- apiGroups: ["apps"]
resources:
- replicasets
verbs: ["get", "list", "watch"]
- apiGroups: ["batch"]
resources:
- jobs
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: ceo-filebeat
# should be the namespace where filebeat is running
namespace: kube-system
labels:
k8s-app: ceo-filebeat
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs: ["get", "create", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: ceo-filebeat-kubeadm-config
namespace: kube-system
labels:
k8s-app: ceo-filebeat
rules:
- apiGroups: [""]
resources:
- configmaps
resourceNames:
- kubeadm-config
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: ceo-filebeat
subjects:
- kind: ServiceAccount
name: ceo-filebeat
namespace: kube-system
roleRef:
kind: ClusterRole
name: ceo-filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: ceo-filebeat
namespace: kube-system
subjects:
- kind: ServiceAccount
name: ceo-filebeat
namespace: kube-system
roleRef:
kind: Role
name: ceo-filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: ceo-filebeat-kubeadm-config
namespace: kube-system
subjects:
- kind: ServiceAccount
name: ceo-filebeat
namespace: kube-system
roleRef:
kind: Role
name: ceo-filebeat-kubeadm-config
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ConfigMap
metadata:
name: ceo-filebeat-config
namespace: kube-system
labels:
k8s-app: ceo-filebeat
data:
filebeat.yml: |-
filebeat.inputs:
- type: filestream
id: ceo-api-dev1-container-logs
paths:
- /var/log/containers/ceo-api-*.log
fields_under_root: true
fields:
data_stream.type: logs
data_stream.dataset: ceo
data_stream.namespace: api
app_id: ceo-api-dev1
parsers:
- container: ~
prospector:
scanner:
fingerprint.enabled: true
symlinks: true
file_identity.fingerprint: ~
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
namespace: ceo-dev1
matchers:
- logs_path:
logs_path: "/var/log/containers/"
processors:
- add_cloud_metadata:
- add_host_metadata:
cloud.id: ${ELASTIC_CLOUD_ID}
cloud.auth: ${ELASTIC_CLOUD_AUTH}
setup.ilm.enabled: false
setup.template.enabled: false
output.elasticsearch:
hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
username: ${ELASTICSEARCH_USERNAME}
password: ${ELASTICSEARCH_PASSWORD}
index: "%{[data_stream.type]}-%{[data_stream.dataset]}-%{[data_stream.namespace]}"
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: ceo-filebeat
namespace: kube-system
labels:
k8s-app: ceo-filebeat
spec:
selector:
matchLabels:
k8s-app: ceo-filebeat
template:
metadata:
labels:
k8s-app: ceo-filebeat
spec:
serviceAccountName: ceo-filebeat
terminationGracePeriodSeconds: 30
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:8.15.2
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: ELASTICSEARCH_HOST
value: "http://our.cluster.host"
- name: ELASTICSEARCH_PORT
value: "9200"
- name: ELASTICSEARCH_USERNAME
value: elastic
- name: ELASTICSEARCH_PASSWORD
value: changeme
- name: ELASTIC_CLOUD_ID
value:
- name: ELASTIC_CLOUD_AUTH
value:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
# If using Red Hat OpenShift uncomment this:
#privileged: true
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
volumes:
- name: config
configMap:
defaultMode: 0640
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
- name: data
hostPath:
# When filebeat runs as non-root user, this directory needs to be writable by group (g+w).
path: /var/lib/filebeat-data
type: DirectoryOrCreate
I opened an interactive shell on the pod whose log I want to collect
kubectl exec -it POD_NAME -n ceo-dev1 -- sh
Then inside the shell, I did a tail -f on the log file. I then did something on the application running in the pod to generate log output, watched the output appear in the log, then tried to create an index pattern in Kibana. Kibana says nothing matches the index logs-ceo-* as configured above. So it looks like the log content is not being shipped to Elasticsearch