Hi I used metricbeat to monitor my k8s cluster. i downloard metricbeat Download Metricbeat • Ship Metrics to Elasticsearch | Elastic site and used linux 64. i alredy create bear taken and used it. but still not working controllermanager and scheduler.
kubernate module
# metricbeat.modules:
# Node metrics, from kubelet:
- module: kubernetes
metricsets:
- container
- node
- pod
- system
- volume
period: 10s
enabled: true
hosts: ["https://10.9.0.152:10250/stats/summary",
"https://10.9.0.153:10250/stats/summary",
"https://10.9.0.154:10250/stats/summary",
"https://10.9.0.155:10250/stats/summary",
"https://10.9.0.156:10250/stats/summary",
"https://10.9.0.157:10250/stats/summary"]
# hosts: ["https://node2:10250/stats/summary"]
bearer_token_file: /etc/kubernetes/token
ssl.verification_mode: "none"
ssl.certificate_authorities:
- /etc/kubernetes/pki/ca.crt
# State metrics from kube-state-metrics service:
- module: kubernetes
enabled: true
metricsets:
- state_node
- state_daemonset
- state_deployment
- state_replicaset
- state_statefulset
- state_pod
- state_container
- state_cronjob
- state_resourcequota
- state_service
- state_persistentvolume
- state_persistentvolumeclaim
- state_storageclass
# Uncomment this to get k8s events:
#- event period: 10s
hosts: ["http://10.9.0.152:30241"]
# Enriching parameters:
add_metadata: true
# Kubernetes API server
# (when running metricbeat as a deployment)
- module: kubernetes
enabled: true
metricsets:
- apiserver
hosts: ["https://10.9.0.157:6443"]
# hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"]
bearer_token_file: /etc/kubernetes/token
# ssl.verification_mode: "none"
ssl.certificate_authorities:
- /etc/kubernetes/pki/ca.crt
period: 30s
# Kubernetes proxy server
# (when running metricbeat locally at hosts or as a daemonset + host network)
- module: kubernetes
enabled: true
metricsets:
- proxy
hosts: ["127.0.0.1:10249"]
# bearer_token_file: /etc/kubernetes/token
# ssl.verification_mode: "none"
# ssl.certificate_authorities:
# - /etc/kubernetes/pki/ca.crt
period: 10s
# Kubernetes controller manager
# (URL and deployment method should be adapted to match the controller manager deployment / service / endpoint)
- module: kubernetes
enabled: true
metricsets:
- controllermanager
hosts: ["https://10.9.0.157:10257"]
bearer_token_file: /etc/kubernetes/token
ssl.verification_mode: "none"
ssl.certificate_authorities:
- /etc/kubernetes/pki/ca.crt
period: 10s
# Kubernetes scheduler
# (URL and deployment method should be adapted to match scheduler deployment / service / endpoint)
- module: kubernetes
enabled: true
metricsets:
- scheduler
hosts: ["https://10.9.0.157:10259"]
bearer_token_file: /etc/kubernetes/token
ssl.verification_mode: "none"
ssl.certificate_authorities:
- /etc/kubernetes/pki/ca.crt
period: 10s
show bellow error
2021-07-21T17:34:08.638Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"cpuacct":{"total":{"ns":4916444968}},"memory":{"mem":{"usage":{"bytes":-8192}}}},"cpu":{"system":{"ticks":50510,"time":{"ms":1529}},"total":{"ticks":146680,"time":{"ms":4297},"value":146680},"user":{"ticks":96170,"time":{"ms":2768}}},"handles":{"limit":{"hard":262144,"soft":1024},"open":58},"info":{"ephemeral_id":"0490cf05-73ab-4374-a5e4-76deab2c59ad","uptime":{"ms":1050174}},"memstats":{"gc_next":63676176,"memory_alloc":57174528,"memory_total":14209766152,"rss":239529984},"runtime":{"goroutines":348}},"libbeat":{"config":{"module":{"running":9}},"output":{"events":{"acked":1536,"active":50,"batches":33,"total":1536},"read":{"bytes":357403},"write":{"bytes":4415946}},"pipeline":{"clients":57,"events":{"active":300,"published":1458,"total":1458},"queue":{"acked":1536}}},"metricbeat":{"kubernetes":{"apiserver":{"events":1,"failures":1},"container":{"events":162,"success":162},"controllermanager":{"events":3,"failures":3},"node":{"events":18,"success":18},"pod":{"events":153,"success":153},"proxy":{"events":9,"success":9},"scheduler":{"events":3,"failures":3},"state_container":{"events":177,"success":177},"state_daemonset":{"events":6,"success":6},"state_deployment":{"events":46,"success":46},"state_node":{"events":18,"success":18},"state_pod":{"events":168,"success":168},"state_replicaset":{"events":93,"success":93},"state_service":{"events":58,"success":58},"state_statefulset":{"events":18,"success":18},"state_storageclass":{"events":21,"success":21},"system":{"events":54,"success":54},"volume":{"events":300,"success":300}},"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":20,"success":20},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":84,"success":84},"process":{"events":30,"success":30},"process_summary":{"events":3,"success":3},"socket_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.83,"15":1,"5":0.94,"norm":{"1":0.2075,"15":0.25,"5":0.235}}}}}}
2021-07-21T17:34:12.327Z ERROR [kubernetes.state_deployment] state_deployment/state_deployment.go:98 decoding of metric family failed: context deadline exceeded (Client.Timeout or context cancellation while reading body)
2021-07-21T17:34:16.927Z INFO module/wrapper.go:259 Error fetching data for metricset kubernetes.controllermanager: error getting processed metrics: unexpected status code 403 from server
2021-07-21T17:34:17.027Z INFO module/wrapper.go:259 Error fetching data for metricset kubernetes.scheduler: error getting processed metrics: unexpected status code 403 from server
2021-07-21T17:34:22.327Z ERROR [kubernetes.state_deployment] state_deployment/state_deployment.go:98 decoding of metric family failed: context deadline exceeded (Client.Timeout or context cancellation while reading body)
2021-07-21T17:34:26.921Z INFO module/wrapper.go:259 Error fetching data for metricset kubernetes.controllermanager: error getting processed metrics: unexpected status code 403 from server
2021-07-21T17:34:27.030Z INFO module/wrapper.go:259 Error fetching data for metricset kubernetes.scheduler: error getting processed metrics: unexpected status code 403 from server
2021-07-21T17:34:27.103Z INFO module/wrapper.go:259 Error fetching data for metricset kubernetes.apiserver: error getting metrics: error making http request: Get "https://10.9.0.157:6443/metrics": x509: certificate signed by unknown authority
2021-07-21T17:34:36.991Z INFO module/wrapper.go:259 Error fetching data for metricset kubernetes.controllermanager: error getting processed metrics: unexpected status code 403 from server
2021-07-21T17:34:36.991Z INFO module/wrapper.go:259 Error fetching data for metricset kubernetes.scheduler: error getting processed metrics: unexpected status code 403 from server