Hi elastic team,
I am currently struggling with the following issue.
First I am trying to replace fluentd with filebeat in our kubernetes cluster for log collection.
So far first tests are looking promessing, since I was testing with a newly deployed namespace that I called filebeat...
However, when trying to get some exiting namespaces collected, no logs are being collected, while instead I get the following warning in the logs:
2020-07-30T12:48:17.560Z WARN [elasticsearch] elasticsearch/client.go:407 Cannot index event publisher.Event{Content:beat.Event{Timestamp:time.Time{wall:0xa506f86, ext:63731710088, loc:(*time.Location)(nil)}, Meta:null, Fields:{"agent":{"ephemeral_id":"8ba50000-da7f-4881-82d2-2871b478b197","hostname":"5-21-282-887-1-236d55eb","id":"4ad5f100-d441-42ae-8b7e-7a8c565f5e78","name":"5-21-282-887-1-236d55eb","type":"filebeat","version":"7.8.1"},"cloud":{"account":{"id":"118596554645"},"availability_zone":"eu-central-1a","image":{"id":"ami-093d166464cf5d246"},"instance":{"id":"i-0d6ca934e34ffbd3c"},"machine":{"type":"c5.2xlarge"},"provider":"aws","region":"eu-central-1"},"ecs":{"version":"1.5.0"},"host":{"architecture":"x86_64","containerized":true,"hostname":"5-21-282-887-1-236d55eb","id":"1a018e03a49f4bfc904c69b0d6c08959","name":"5-21-282-887-1-236d55eb","os":{"codename":"Core","family":"redhat","kernel":"5.6.2-1.el7.elrepo.x86_64","name":"CentOS Linux","platform":"centos","version":"7 (Core)"}},"input":{"type":"container"},"kubernetes":{"container":{"image":"vault:1.4.2","name":"vault"},"labels":{"app":"vault","pod-template-hash":"7547986f9b"},"namespace":"rzneo","node":{"name":"5-21-282-887-1-236d55eb"},"pod":{"name":"vault-7547986f9b-t9q8h","uid":"d2639f12-0bd9-47af-b66b-974cf32591d8"},"replicaset":{"name":"vault-7547986f9b"}},"log":{"file":{"path":"/var/log/containers/vault-7547986f9b-t9q8h_rzneo_vault-486fbec7c9538b6f3b2402268c6f294908f3386ccec198cc3d694c951a468385.log"},"offset":994990},"message":"2020-07-30T12:48:08.172Z [INFO] expiration: revoked lease: lease_id=auth/token/create/h267d90e4e7d736dd6804d96a30df2c766027f5c76147feee2f75ccd8e5d0e7f3","stream":"stderr"}, Private:file.State{Id:"", Finished:false, Fileinfo:(*os.fileStat)(0xc0001af040), Source:"/var/log/containers/vault-7547986f9b-t9q8h_rzneo_vault-486fbec7c9538b6f3b2402268c6f294908f3386ccec198cc3d694c951a468385.log", Offset:995212, Timestamp:time.Time{wall:0xbfc0ce00ce704feb, ext:73095592258, loc:(*time.Location)(0x5b261c0)}, TTL:-1, Type:"container", Meta:map[string]string(nil), FileStateOS:file.StateOS{Inode:0x3dd334, Device:0x10309}}, TimeSeries:false}, Flags:0x1, Cache:publisher.EventCache{m:common.MapStr(nil)}} (status=403): {"type":"security_exception","reason":"action [indices:admin/auto_create] is unauthorized for user [elastic-system-filebeat-beat-user]"}
This is a bit confusing to be honest as it is working fine for an just created namespace.
So right now I am not able to use filebeat for any namespace but the one that I created for test purposes.
I am using the following manifest for deployment:
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
name: filebeat
spec:
type: filebeat
version: 7.8.1
elasticsearchRef:
name: elastic
namespace: elastic-system
# Currently this setting requires Kibana to have TLS enabled.
# See https://github.com/elastic/cloud-on-k8s/issues/3523 for more information.
kibanaRef:
name: kibana
namespace: elastic-system
config:
output.elasticsearch.index: "%{[kubernetes.namespace]}-filebeat-%{+xxxx.ww}"
setup.template.name: "filebeat"
setup.template.pattern: "*-filebeat-*"
setup.template.overwrite: true
setup.template.settings:
index.number_of_shards: 3
index.number_of_replicas: 1
setup.ilm.enabled: false
setup.ilm.policy_name: "denic-index-policy"
setup.dashboards.enabled: true
filebeat.autodiscover.providers:
- type: kubernetes
node: ${HOSTNAME}
hints.default_config.enabled: "false"
templates:
- condition.equals.kubernetes.namespace: "filebeat"
config:
- type: container
paths: ["/var/log/containers/*-${data.kubernetes.container.id}.log"]
multiline.pattern: '^[[:space:]]'
multiline.negate: false
multiline.match: after
exclude_lines: ["^\\s+[\\-`('.|_]"] # drop asciiart lines
processors:
- add_cloud_metadata: {}
- add_host_metadata:
netinfo.enabled: false
- add_docker_metadata: {}
- add_kubernetes_metadata:
host: ${HOSTNAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
daemonSet:
podTemplate:
spec:
serviceAccountName: filebeat
automountServiceAccountToken: true
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true # Allows to provide richer host metadata
securityContext:
runAsUser: 0
# If using Red Hat OpenShift uncomment this:
#privileged: true
containers:
- name: filebeat
volumeMounts:
- name: varlogcontainers
mountPath: /var/log/containers
- name: varlogpods
mountPath: /var/log/pods
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
volumes:
- name: varlogcontainers
hostPath:
path: /var/log/containers
- name: varlogpods
hostPath:
path: /var/log/pods
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: elastic-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: elastic-system
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io