Heartbeat doesn't discover anything on Kubernetes

Elastic Stack Beats
1

Hello,
I've Elastic Stack 8.1 installed.
This is my heartbeat config below:

heartbeat.yml: |-
    heartbeat.autodiscover:
     # Autodiscover pods
     providers:
       - type: kubernetes
         resource: pod
         scope: cluster
         node: ${NODE_NAME}
         hints.enabled: true
   
     # Autodiscover services
     providers:
       - type: kubernetes
         resource: service
         scope: cluster
         node: ${NODE_NAME}
         hints.enabled: true
   
     # Autodiscover nodes
     providers:
       - type: kubernetes
         resource: node
         node: ${NODE_NAME}
         scope: cluster
         templates:
           # Example, check SSH port of all cluster nodes:
           - condition: ~
             config:
               - hosts:
                   - ${data.host}:22
                 name: ${data.kubernetes.node.name}
                 schedule: '@every 10s'
                 timeout: 5s
                 type: tcp
    processors:
      - add_cloud_metadata:
    cloud.id: ${ELASTIC_CLOUD_ID}
    cloud.auth: ${ELASTIC_CLOUD_AUTH}
    output.elasticsearch:
      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
      username: ${ELASTICSEARCH_USERNAME}
      password: ${ELASTICSEARCH_PASSWORD}
      ssl.verification_mode: none
      protocol: 'https'

...and these are logs of heartbeat:

{"log.level":"info","@timestamp":"2022-04-07T02:56:46.440Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":184},"message":"Non-zero metrics in the last 30s","service.name":"heartbeat","monitoring":{"metrics":{"beat":{"cgroup":{"cpuacct":{"total":{"ns":16366903}},"memory":{"mem":{"usage":{"bytes":4096}}}},"cpu":{"system":{"ticks":450,"time":{"ms":3}},"total":{"ticks":1070,"time":{"ms":15},"value":1070},"user":{"ticks":620,"time":{"ms":12}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":10},"info":{"ephemeral_id":"6cc118d9-5e12-4a45-b2f9-dc2e5eb6a0b9","uptime":{"ms":1590080},"version":"8.1.2"},"memstats":{"gc_next":11372544,"memory_alloc":7873120,"memory_total":87443368,"rss":106016768},"runtime":{"goroutines":31}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":9,"active":0,"batches":3,"total":9},"read":{"bytes":2368},"write":{"bytes":18884}},"pipeline":{"clients":3,"events":{"active":0,"published":9,"total":9},"queue":{"acked":9}}},"system":{"load":{"1":1.53,"15":1.54,"5":1.76,"norm":{"1":0.3825,"15":0.385,"5":0.44}}}},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2022-04-07T02:57:16.441Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":184},"message":"Non-zero metrics in the last 30s","service.name":"heartbeat","monitoring":{"metrics":{"beat":{"cgroup":{"cpuacct":{"total":{"ns":14937570}},"memory":{"mem":{"usage":{"bytes":8192}}}},"cpu":{"system":{"ticks":450,"time":{"ms":5}},"total":{"ticks":1080,"time":{"ms":15},"value":1080},"user":{"ticks":630,"time":{"ms":10}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":10},"info":{"ephemeral_id":"6cc118d9-5e12-4a45-b2f9-dc2e5eb6a0b9","uptime":{"ms":1620079},"version":"8.1.2"},"memstats":{"gc_next":11372544,"memory_alloc":8923952,"memory_total":88494200,"rss":106016768},"runtime":{"goroutines":31}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":9,"active":0,"batches":3,"total":9},"read":{"bytes":2369},"write":{"bytes":18877}},"pipeline":{"clients":3,"events":{"active":0,"published":9,"total":9},"queue":{"acked":9}}},"system":{"load":{"1":1.49,"15":1.53,"5":1.73,"norm":{"1":0.3725,"15":0.3825,"5":0.4325}}}},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2022-04-07T02:57:46.445Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":184},"message":"Non-zero metrics in the last 30s","service.name":"heartbeat","monitoring":{"metrics":{"beat":{"cgroup":{"cpuacct":{"total":{"ns":15525206}},"memory":{"mem":{"usage":{"bytes":24576}}}},"cpu":{"system":{"ticks":460,"time":{"ms":5}},"total":{"ticks":1100,"time":{"ms":15},"value":1100},"user":{"ticks":640,"time":{"ms":10}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":10},"info":{"ephemeral_id":"6cc118d9-5e12-4a45-b2f9-dc2e5eb6a0b9","uptime":{"ms":1650085},"version":"8.1.2"},"memstats":{"gc_next":11372544,"memory_alloc":10270632,"memory_total":89840880,"rss":106016768},"runtime":{"goroutines":31}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":9,"active":0,"batches":3,"total":9},"read":{"bytes":2369},"write":{"bytes":18883}},"pipeline":{"clients":3,"events":{"active":0,"published":9,"total":9},"queue":{"acked":9}}},"system":{"load":{"1":1.32,"15":1.52,"5":1.66,"norm":{"1":0.33,"15":0.38,"5":0.415}}}},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2022-04-07T02:58:16.443Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":184},"message":"Non-zero metrics in the last 30s","service.name":"heartbeat","monitoring":{"metrics":{"beat":{"cgroup":{"cpuacct":{"total":{"ns":18962386}},"memory":{"mem":{"usage":{"bytes":352256}}}},"cpu":{"system":{"ticks":470,"time":{"ms":10}},"total":{"ticks":1120,"time":{"ms":18},"value":1120},"user":{"ticks":650,"time":{"ms":8}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":10},"info":{"ephemeral_id":"6cc118d9-5e12-4a45-b2f9-dc2e5eb6a0b9","uptime":{"ms":1680078},"version":"8.1.2"},"memstats":{"gc_next":11621888,"memory_alloc":6408400,"memory_total":90909288,"rss":106033152},"runtime":{"goroutines":31}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":9,"active":0,"batches":3,"total":9},"read":{"bytes":2369},"write":{"bytes":18878}},"pipeline":{"clients":3,"events":{"active":0,"published":9,"total":9},"queue":{"acked":9}}},"system":{"load":{"1":1.34,"15":1.51,"5":1.63,"norm":{"1":0.335,"15":0.3775,"5":0.4075}}}},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2022-04-07T02:58:46.442Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":184},"message":"Non-zero metrics in the last 30s","service.name":"heartbeat","monitoring":{"metrics":{"beat":{"cgroup":{"cpuacct":{"total":{"ns":17033375}},"memory":{"mem":{"usage":{"bytes":28672}}}},"cpu":{"system":{"ticks":470,"time":{"ms":4}},"total":{"ticks":1130,"time":{"ms":16},"value":1130},"user":{"ticks":660,"time":{"ms":12}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":10},"info":{"ephemeral_id":"6cc118d9-5e12-4a45-b2f9-dc2e5eb6a0b9","uptime":{"ms":1710082},"version":"8.1.2"},"memstats":{"gc_next":11621888,"memory_alloc":8085368,"memory_total":92586256,"rss":106033152},"runtime":{"goroutines":31}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":9,"active":0,"batches":3,"total":9},"read":{"bytes":2368},"write":{"bytes":18877}},"pipeline":{"clients":3,"events":{"active":0,"published":9,"total":9},"queue":{"acked":9}}},"system":{"load":{"1":1.49,"15":1.52,"5":1.64,"norm":{"1":0.3725,"15":0.38,"5":0.41}}}},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2022-04-07T02:59:16.441Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":184},"message":"Non-zero metrics in the last 30s","service.name":"heartbeat","monitoring":{"metrics":{"beat":{"cgroup":{"cpuacct":{"total":{"ns":15132943}},"memory":{"mem":{"usage":{"bytes":131072}}}},"cpu":{"system":{"ticks":480,"time":{"ms":6}},"total":{"ticks":1150,"time":{"ms":15},"value":1150},"user":{"ticks":670,"time":{"ms":9}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":10},"info":{"ephemeral_id":"6cc118d9-5e12-4a45-b2f9-dc2e5eb6a0b9","uptime":{"ms":1740079},"version":"8.1.2"},"memstats":{"gc_next":11621888,"memory_alloc":9151792,"memory_total":93652680,"rss":106033152},"runtime":{"goroutines":31}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":9,"active":0,"batches":3,"total":9},"read":{"bytes":2369},"write":{"bytes":18880}},"pipeline":{"clients":3,"events":{"active":0,"published":9,"total":9},"queue":{"acked":9}}},"system":{"load":{"1":2.07,"15":1.56,"5":1.77,"norm":{"1":0.5175,"15":0.39,"5":0.4425}}}},"ecs.version":"1.6.0"}}

...but Heartbeat doesn't discover any pods, services and nodes. Am I missing something?

Thanks & Regards

2

Hello @tirelibirefe,

There might be a problem with the yaml syntax here so heartbeat can not parse the config properly. Could you try removing the second and third providers keys and see if it helps?

It would look like this after removing the duplicated keys:

heartbeat.yml: |-
    heartbeat.autodiscover:

     # Autodiscover pods
     providers:
       - type: kubernetes
         resource: pod
         scope: cluster
         node: ${NODE_NAME}
         hints.enabled: true
   
     # Autodiscover services
       - type: kubernetes
         resource: service
         scope: cluster
         node: ${NODE_NAME}
         hints.enabled: true
   
     # Autodiscover nodes
       - type: kubernetes
         resource: node
         node: ${NODE_NAME}
         scope: cluster
         templates:
           # Example, check SSH port of all cluster nodes:
           - condition: ~
             config:
               - hosts:
                   - ${data.host}:22
                 name: ${data.kubernetes.node.name}
                 schedule: '@every 10s'
                 timeout: 5s
                 type: tcp
    processors:
      - add_cloud_metadata:
    cloud.id: ${ELASTIC_CLOUD_ID}
    cloud.auth: ${ELASTIC_CLOUD_AUTH}
    output.elasticsearch:
      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
      username: ${ELASTICSEARCH_USERNAME}
      password: ${ELASTICSEARCH_PASSWORD}
      ssl.verification_mode: none
      protocol: 'https'

Thanks.

3

Hello @kyungeunni Thanks for the response.

I tried what you advised but nothing changed. Pods & Services aren't discovered. :cry:

image
image1130×706 70 KB 

data:
  heartbeat.yml: |-
    heartbeat.autodiscover:
     # Autodiscover pods
      providers:
        - type: kubernetes
          resource: pod
          scope: cluster
          node: ${NODE_NAME}
          hints.enabled: true
             
      # Autodiscover services
        - type: kubernetes
          resource: service
          scope: cluster
          node: ${NODE_NAME}
          hints.enabled: true
             
      # Autodiscover nodes
        - type: kubernetes
          resource: node
          node: ${NODE_NAME}
          scope: cluster
          templates:
            # Example, check SSH port of all cluster nodes:
            - condition: ~
              config:
                - hosts:
                    - ${data.host}:22
                  name: ${data.kubernetes.node.name}
                  schedule: '@every 10s'
                  timeout: 5s
                  type: tcp
    processors:
      - add_cloud_metadata:
    output.elasticsearch:
      hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
      username: ${ELASTICSEARCH_USERNAME}
      password: ${ELASTICSEARCH_PASSWORD}
4

Hi again!

The config looks correct and I can see the autodiscover for nodes works. It might be the hint-based providers are not able to find the hints in Kubernetes Pod annotations. Could you please review your pods and annotate them accordingly to one of the hints?

Let me know how it goes :slight_smile:

5

Thanks, it's ok now.

Actually I suggested it would discover services, pods and nodes. Tainting all k8s artifacts, teaching/defining everything to Heartbeat/Metricbeat one by one... I don't know, not seem practical/useful.

...anyway, thanks again.

1 Like
6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.