Hi,
I'm trying to deploy filebeats on Kubernetes cluster(EKS) using ECK stack as below(a). I want to split the output index by namespace, something like mentioned (b) which is not working. Can you suggest how can I pass the output to this below.
(a) Standard filebeat deployment config -
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
name: logs
spec:
type: filebeat
version: 8.12.1
elasticsearchRef:
name: logs
monitoring:
logs:
elasticsearchRefs:
- name: logs
metrics:
elasticsearchRefs:
- name: logs
config:
filebeat.inputs:
- type: container
paths:
- /var/log/containers/*.log
json.keys_under_root: true
json.add_error_key: true
json.ignore_decoding_error: true
json.expand_keys: true
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
- timestamp:
field: timestamp
ignore_missing: true
ignore_failure: true
layouts:
- '2006-01-02 15:04:05.999'
- '2006-01-02T15:04:05Z'
- '2006-01-02T15:04:05.999Z'
- '2006-01-02T15:04:05.999-07:00'
- drop_fields:
fields: ["host.ip","host.mac"]
setup:
ilm:
policy_file: /tmp/ilm-policy.json
template:
settings:
index:
number_of_shards: 3
number_of_replicas: 1
daemonSet:
podTemplate:
spec:
serviceAccountName: filebeat
automountServiceAccountToken: true
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
securityContext:
runAsUser: 0
containers:
- name: filebeat
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: varlogcontainers
mountPath: /var/log/containers
- name: varlogpods
mountPath: /var/log/pods
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
- name: ilm-policy
mountPath: /tmp/ilm-policy.json
subPath: ilm-policy.json
volumes:
- name: varlogcontainers
hostPath:
path: /var/log/containers
- name: varlogpods
hostPath:
path: /var/log/pods
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: ilm-policy
configMap:
name: filebeat-ilm
items:
- key: ilm-policy.json
path: ilm-policy.json
(b) output split index by namespace
config:
output.elasticsearch:
indices:
- index: "testing-ns-index"
when:
regexp:
kubernetes.namespace: "testing*"
- index: "other-ns-index"
when:
not.regexp:
kubernetes.namespace: "testing*"
Tried configuration which is giving error
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
name: logs
spec:
type: filebeat
version: 8.12.1
elasticsearchRef:
name: logs
monitoring:
logs:
elasticsearchRefs:
- name: logs
metrics:
elasticsearchRefs:
- name: logs
config:
filebeat.inputs:
- type: container
paths:
- /var/log/containers/*.log
json.keys_under_root: true
json.add_error_key: true
json.ignore_decoding_error: true
json.expand_keys: true
output.elasticsearch:
indices:
- index: "testing-ns-index"
when:
regexp:
kubernetes.namespace: "testing*"
- index: "other-ns-index"
when:
not.regexp:
kubernetes.namespace: "testing*"
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
- timestamp:
field: timestamp
ignore_missing: true
ignore_failure: true
layouts:
- '2006-01-02 15:04:05.999'
- '2006-01-02T15:04:05Z'
- '2006-01-02T15:04:05.999Z'
- '2006-01-02T15:04:05.999-07:00'
- drop_fields:
fields: ["host.ip","host.mac"]
setup:
ilm:
policy_file: /tmp/ilm-policy.json
template:
settings:
index:
number_of_shards: 3
number_of_replicas: 1
daemonSet:
podTemplate:
spec:
serviceAccountName: filebeat
automountServiceAccountToken: true
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
securityContext:
runAsUser: 0
containers:
- name: filebeat
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: varlogcontainers
mountPath: /var/log/containers
- name: varlogpods
mountPath: /var/log/pods
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
- name: ilm-policy
mountPath: /tmp/ilm-policy.json
subPath: ilm-policy.json
volumes:
- name: varlogcontainers
hostPath:
path: /var/log/containers
- name: varlogpods
hostPath:
path: /var/log/pods
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: ilm-policy
configMap:
name: filebeat-ilm
items:
- key: ilm-policy.json
path: ilm-policy.json
Suggest the above filebeat config to pass output with 'when' condition.