Metricbeat for monitoring Openshift Horizontal Pod Autoscaler

Elastic Stack Beats
1

Hi, i wanted to know if there is a actually a support for monitoring Openshift HPA workload. Due to client request, i need to monitor the HPA workload on the cluster.

My Metricbeat collect data from kube-state-metrics installed on the cluster. But after looking into the source code, i cannot find metricset that fetch data from any of the HPA metric name. ( e.g kube_horizontalpodautoscaler_status_condition, kube_horizontalpodautoscaler_status_current_replicas, etc ) ref : HPA metricset

am i missing some configuration or it could be achieved using another way?

Here is my configmap for daemonset-config :

apiVersion: v1
kind: ConfigMap
metadata:
  name: metricbeat-daemonset-config
  namespace: elk-monitoring
  labels:
    k8s-app: metricbeat
data:
  metricbeat.yml: |-
    metricbeat.config.modules:
      # Mounted `metricbeat-daemonset-modules` configmap:
      path: ${path.config}/modules.d/*.yml
      # Reload module configs as they change:
      reload.enabled: false

    metricbeat.autodiscover:
      providers:
        - type: kubernetes
          scope: cluster
          node: ${NODE_NAME}
          unique: true
          templates:
            - config:
                - module: kubernetes
                  hosts: ["kube-state-metrics.elk-monitoring.svc.cluster.local:8080"]
                  period: 30s
                  add_metadata: true
                  metricsets:
                    - state_namespace
                    - state_node
                    - state_deployment
                    - state_daemonset
                    - state_replicaset
                    - state_pod
                    - state_container
                    - state_job
                    - state_cronjob
                    - state_resourcequota
                    - state_statefulset
                    - state_service
                    - state_persistentvolume
                    - state_persistentvolumeclaim
                    - state_storageclass
                - module: kubernetes
                  metricsets:
                    - apiserver
                  hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"]
                  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
                  ssl.certificate_authorities:
                    - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                  period: 30s
                - module: kubernetes
                  metricsets:
                    - event
        - type: kubernetes
          node: ${NODE_NAME}
          hints.enabled: true

    processors:
      - add_cloud_metadata:

    output.elasticsearch:
      hosts: ["http://mynode-1:9200","http://mynode-2:9200","http://mynode-3:9200"]
      username: "myusername"
      password: "mypassword"

and here is my configmap for daemonset-module :

apiVersion: v1
kind: ConfigMap
metadata:
  name: metricbeat-daemonset-modules
  namespace: elk-monitoring
  labels:
    k8s-app: metricbeat
data:
  system.yml: |-
    - module: system
      period: 30s
      metricsets:
        - cpu
        - load
        - memory
        - network
        - process
        - process_summary
        - core
      processes: ['.*']
      process.include_top_n:
        by_cpu: 5      # include top 5 processes by CPU
        by_memory: 5   # include top 5 processes by memory

    - module: system
      period: 1m
      metricsets:
        - filesystem
        - fsstat
      processors:
      - drop_event.when.regexp:
          system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)'
  kubernetes.yml: |-
    - module: kubernetes
      metricsets:
        - node
        - system
        - pod
        - container
        - volume
      period: 30s
      host: ${NODE_NAME}
      hosts: ["https://${NODE_NAME}:10250"]
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      ssl.verification_mode: "none"
    - module: kubernetes
      metricsets:
        - proxy
      period: 30s
      host: ${NODE_NAME}
      hosts: ["localhost:29101"]
2

i decided to use a logstash to get the HPA data using http_poller plugin since i can't get any answer.

if anyone stumbled into my post and looking for the solution. here is my configuration

input {
	http_poller {
		urls => {
			cluster_url => { 
				url => 'https://your-kube-state-metric:<yourKSMport>/metrics'
				method => "GET"
                                ssl => true
                                cacert => "/your/path/to/ca.crt"
                                client_cert => "/your/path/to/ca.crt"
                                client_key => "/your/path/to/ca.key"
                                ssl_truststore_path => "/your/path/to/keystore.jks"
                                ssl_truststore_password => "yourtruststorepassword"
                                ssl_certificate_verification => true
                                headers => {
                                       Accept => "text/plain"
				    }
				}
			}
	codec => plain
	schedule => { every => "10s" }
 }
}

# split the message so logstash can parse properly for each line
filter {
  split {
    field => "message"
  }

# this grok only match for the HPA information exposed by KSM
  grok {
    match => {
      "message" => [
        'kube_horizontalpodautoscaler_status_desired_replicas{namespace="%{DATA:namespace}",horizontalpodautoscaler="%{DATA:hpa_name}"}\s+%{NUMBER:desired_replica}',
        'kube_horizontalpodautoscaler_status_current_replicas{namespace="%{DATA:namespace}",horizontalpodautoscaler="%{DATA:hpa_name}"}\s+%{NUMBER:current_replica}',
        'kube_horizontalpodautoscaler_spec_max_replicas{namespace="%{DATA:namespace}",horizontalpodautoscaler="%{DATA:hpa_name}"}\s+%{NUMBER:max_replica}',
        'kube_horizontalpodautoscaler_spec_min_replicas{namespace="%{DATA:namespace}",horizontalpodautoscaler="%{DATA:hpa_name}"}\s+%{NUMBER:min_replica}'
      ]
    }
  }

# drop everything else other than HPA to reduce index size
  if "_grokparsefailure" in [tags] {
    drop { }
  }

# change the field name for readability
  mutate {
    rename => {
      "namespace" => "kubernetes.namespace"
      "hpa_name" => "kubernetes.hpa_name"
      "desired_replica" => "kubernetes.hpa.replicas.desired_replica"
      "current_replica" => "kubernetes.hpa.replicas.current_replica"
      "max_replica" => "kubernetes.hpa.replicas.max_replica"
      "min_replica" => "kubernetes.hpa.replicas.min_replica"
    }

# convert the field into integer type and delete the original message, it can be endless and goint to take a lot of storage
   convert => {
      "kubernetes.hpa.replicas.desired_replica" => "integer"
      "kubernetes.hpa.replicas.current_replica" => "integer"
      "kubernetes.hpa.replicas.max_replica" => "integer"
      "kubernetes.hpa.replicas.min_replica" => "integer"
    }
  remove_field => ["message", "[event][original]"]
  }
 
}

logstash require you to recognize the certificate so you need to add the truststore in the jvm.option

-Djavax.net.ssl.trustStore=/your/path/to/truststore.jks
-Djavax.net.ssl.trustStorePassword=yourtruststorepassword

if anyone has more optimized way to get the data, feel free to reply this post

cheers!