Hi!
Basically I'm trying to ship all logs from my k8s cluster, while adding kubernetes metadata to all containers and joining some multiline output on some of them.
I only get the kubernetes: {}-data on some log lines, most do not have them.
Here's my deployment yaml, including the filebeat-config:
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-system
labels:
k8s-app: filebeat
kubernetes.io/cluster-service: "true"
data:
filebeat.yml: |-
filebeat.inputs:
- type: docker
symlinks: true
containers:
path: "/var/lib/docker/containers"
ids:
- '*'
fields:
type: kubernetes_log
cluster: dev
processors:
- add_kubernetes_metadata:
in_cluster: true
namespace: ${POD_NAMESPACE}
filebeat.autodiscover:
providers:
- type: kubernetes
templates:
- condition:
or:
- equals:
kubernetes.container.name: "some-container"
- equals:
kubernetes.container.name: "some-other-container"
config:
- type: docker
containers.ids:
- "${data.kubernetes.container.id}"
multiline.pattern: '^2'
multiline.negate: true
multiline.match: after
multiline.max_lines: 500
multiline.timeout: 5s
processors:
- add_cloud_metadata:
output.kafka:
hosts:
- '1.2.3.4:9092'
- '1.2.3.5:9092'
- '1.2.3.6:9092'
topic: 'dev-logging'
partition.round_robin:
reachable_only: false
required_acks: 1
compression: gzip
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
kubernetes.io/cluster-service: "true"
spec:
template:
metadata:
labels:
k8s-app: filebeat
kubernetes.io/cluster-service: "true"
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:6.5.3
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
securityContext:
runAsUser: 0
resources:
limits:
memory: 200Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: data
mountPath: /usr/share/filebeat/data
- name: varlogpods
mountPath: /var/log/pods
readOnly: true
- name: varlogcontainers
mountPath: /var/log/containers
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlogpods
hostPath:
path: /var/log/pods
- name: varlogcontainers
hostPath:
path: /var/log/containers
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: kube-system
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: kube-system
---
Any hints are greatly appreciated