Hello, I am currently trying to get our Kubernetes metrics using metricbeat deployed on ECK. We are getting the metrics for the Kubernetes cluster, however, the metrics we are getting is incorrect.
We are seeing the issue described in this forum thread.
We tried using the solution mentioned in this forum thread by adding
**unique: true**
however we are not sure why it is not working as we can see in the logs that a leader is being assigned.
I have attached the metricbeat.yaml file below for reference
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
name: metricbeat
namespace: elasticsearch
spec:
type: metricbeat
version: 7.16.2
config:
metricbeat:
autodiscover:
providers:
- hints:
default_config: {}
enabled: "true"
node: ${NODE_NAME}
type: kubernetes
unique: true
identifier: leader-election-metricbeat
scope: cluster
modules:
- module: system
period: 10s
metricsets:
- cpu
- load
- memory
- network
- process
- process_summary
process:
include_top_n:
by_cpu: 5
by_memory: 5
processes:
- .*
- module: system
period: 1m
metricsets:
- filesystem
- fsstat
processors:
- drop_event:
when:
regexp:
system:
filesystem:
mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib)($|/)
- module: kubernetes
period: 10s
node: ${NODE_NAME}
hosts:
- https://${NODE_NAME}:10250
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
ssl:
verification_mode: none
metricsets:
- node
- system
- pod
- container
- volume
- module: kubernetes
hosts: ["kube-state-metrics.kube-system.svc.cluster.local:8080"]
period: 10s
add_metadata: true
metricsets:
- state_node
- state_deployment
- state_daemonset
- state_replicaset
- state_pod
- state_container
- state_job
- state_cronjob
- state_resourcequota
- state_statefulset
- state_service
processors:
- add_cloud_metadata: {}
- add_host_metadata: {}
- add_fields:
target: orchestrator.cluster
fields:
name: JPES-1
output.elasticsearch:
hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
username: ${ELASTICSEARCH_USERNAME}
password: ${ELASTICSEARCH_PASSWORD}
setup.kibana:
host: 'http://kibana-kb-http.elasticsearch:5601'
username: ${ELASTICSEARCH_USERNAME}
password: ${ELASTICSEARCH_PASSWORD}
setup.dashboards.enabled: true
setup.dashboards.retry.enabled: true
setup.ilm:
enabled: true
policy_name: "metricbeat-k8s-policy"
rollover_alias: "metricbeat-k8s-rollover"
policy_file: "ilm-policy.json"
pattern: "{now/d}-000001"
overwrite: true
daemonSet:
podTemplate:
spec:
serviceAccountName: metricbeat
automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context
containers:
- args:
- -e
- -c
- /etc/beat.yml
- -system.hostfs=/hostfs
name: metricbeat
resources:
requests:
memory: 200Mi
cpu: 100m
limits:
memory: 500Mi
cpu: 200m
volumeMounts:
- mountPath: /hostfs/sys/fs/cgroup
name: cgroup
- mountPath: /var/run/docker.sock
name: dockersock
- mountPath: /hostfs/proc
name: proc
- mountPath: /usr/share/metricbeat/ilm-policy.json
subPath: ilm-policy.json
name: config-ilm
readOnly: true
env:
- name: ELASTICSEARCH_HOSTS
value: elk-cluster-es-http.elasticsearch.svc.cluster.local
- name: ELASTICSEARCH_HOST
value: elk-cluster-es-http.elasticsearch.svc.cluster.local
- name: ELASTICSEARCH_PORT
value: "9200"
- name: ELASTICSEARCH_USERNAME
value: elastic
- name: ELASTICSEARCH_PASSWORD
value: #masked#
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
runAsUser: 0
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /sys/fs/cgroup
name: cgroup
- hostPath:
path: /var/run/docker.sock
name: dockersock
- hostPath:
path: /proc
name: proc
- name: config-ilm
secret:
secretName: metricbeat-ilm-policy-config
---
apiVersion: v1
kind: Secret
metadata:
name: metricbeat-ilm-policy-config
namespace: elasticsearch
stringData:
ilm-policy.json: |-
{
"policy": {
"_meta": {
"managed-by": "kubernets-ilm-policy",
"description": "Metricbeat ILM Policy"
},
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_size": "50gb",
"max_age": "14d"
}
}
},
"warm": {
"min_age": "0d",
"actions": {
"set_priority": {
"priority": 50
}
}
},
"cold": {
"min_age": "0d",
"actions": {
"set_priority": {
"priority": 0
}
}
},
"delete": {
"min_age": "14d",
"actions": {
"delete": {}
}
}
}
}
}
---
# permissions needed for metricbeat
# source: https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-kubernetes.html
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: metricbeat
labels:
k8s-app: metricbeat
rules:
- apiGroups: [""]
resources:
- nodes
- namespaces
- events
- pods
- services
verbs: ["get", "list", "watch"]
- apiGroups: ["batch"]
resources:
- jobs
verbs: ["get", "list", "watch"]
- apiGroups: ["extensions"]
resources:
- replicasets
verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
resources:
- statefulsets
- deployments
- replicasets
verbs: ["get", "list", "watch"]
- apiGroups:
- ""
resources:
- nodes/stats
verbs:
- get
- nonResourceURLs:
- /metrics
verbs:
- get
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: metricbeat
namespace: elasticsearch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: metricbeat
subjects:
- kind: ServiceAccount
name: metricbeat
namespace: elasticsearch
roleRef:
kind: ClusterRole
name: metricbeat
apiGroup: rbac.authorization.k8s.io
---
Please advise on a solution. Thanks!