Connect: connection refused

I have a fresh elasticsearch cluster deployed on kubernetes. I have deployed metricbeat 8.7.0 and i get the following error in the logs of each metricbeat pod. Does anyone know how to resolve this issue?

{"log.level":"error","@timestamp":"2023-04-17T11:53:47.058Z","log.origin":{"file.name":"module/wrapper.go","file.line":256},"message":"Error fetching data for metricset beat.state: error making http request: Get "http://ip-10-0-6-193.eu-west-2.compute.internal:5066/state\": dial tcp 10.0.6.193:5066: connect: connection refused","service.name":"metricbeat","ecs.version":"1.6.0"}

kubectl get elastic -n elastic-system
NAME                                                            HEALTH   NODES   VERSION   PHASE   AGE
elasticsearch.elasticsearch.k8s.elastic.co/elasticsearch-prod   green    6       8.7.0     Ready   169m

NAME                                       HEALTH   NODES   VERSION   AGE
kibana.kibana.k8s.elastic.co/kibana-prod   green    1       8.7.0     161m

NAME                                  HEALTH   AVAILABLE   EXPECTED   TYPE         VERSION   AGE
beat.beat.k8s.elastic.co/filebeat     green    6           6          filebeat     8.7.0     150m
beat.beat.k8s.elastic.co/metricbeat   green    6           6          metricbeat   8.7.0     77m

metricbeat.yml

apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
  name: metricbeat
  namespace: elastic-system
spec:
  type: metricbeat
  version: 8.7.0
  elasticsearchRef:
    name: elasticsearch-prod
  kibanaRef:
    name: kibana-prod
  config:
    http.enabled: true
    setup.ilm:
      enabled: true
      policy_name: metricbeat-custom
      policy_file: /etc/indice-lifecycle.json
    setup.template.settings:
      index:
        routing.allocation.require.type: "monitoring"
    metricbeat:
      autodiscover:
        providers:
          - type: kubernetes
            scope: cluster
            hints.enabled: true
            templates:
              - condition:
                  contains:
                    kubernetes.labels.scrape: es
                config:
                  - module: elasticsearch
                    metricsets:
                      - ccr
                      - cluster_stats
                      - enrich
                      - index
                      - index_recovery
                      - index_summary
                      - ml_job
                      - node_stats
                      - shard
                    period: 10s
                    hosts: "https://${data.host}:${data.ports.https}"
                    username: ${MONITORED_ES_USERNAME}
                    password: ${MONITORED_ES_PASSWORD}
                    # WARNING: disables TLS as the default certificate is not valid for the pod FQDN
                    # TODO: switch this to "certificate" when available: https://github.com/elastic/beats/issues/8164
                    ssl.verification_mode: "none"
                    xpack.enabled: true
              - condition:
                  contains:
                    kubernetes.labels.scrape: kb
                config:
                  - module: kibana
                    metricsets:
                      - stats
                    period: 10s
                    hosts: "https://${data.host}:${data.ports.https}"
                    username: ${MONITORED_ES_USERNAME}
                    password: ${MONITORED_ES_PASSWORD}
                    # WARNING: disables TLS as the default certificate is not valid for the pod FQDN
                    # TODO: switch this to "certificate" when available: https://github.com/elastic/beats/issues/8164
                    ssl.verification_mode: "none"
                    xpack.enabled: true
              - condition:
                  contains:
                    kubernetes.labels.common.k8s.elastic.co/type: beat
                config:
                  - module: beat
                    metricsets:
                      - stats
                      - state
                    period: 10s
                    hosts: "http://${NODE_NAME}:5066"
                    # WARNING: disables TLS as the default certificate is not valid for the pod FQDN
                    # TODO: switch this to "certificate" when available: https://github.com/elastic/beats/issues/8164
                    ssl.verification_mode: "none"
                    xpack.enabled: true
      modules:
      - module: system
        period: 10s
        metricsets:
        - cpu
        - load
        - memory
        - network
        - process
        - process_summary
        process:
          include_top_n:
            by_cpu: 5
            by_memory: 5
        processes:
        - .*
      - module: system
        period: 1m
        metricsets:
        - filesystem
        - fsstat
        processors:
        - drop_event:
            when:
              regexp:
                system:
                  filesystem:
                    mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib)($|/)
                      #- module: docker
                      #  period: 10s
                      #  hosts: ["unix:///var/run/docker.sock"]
                      #  metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"]
      - module: kubernetes  # kubelet 
        period: 10s
        node: ${NODE_NAME}
        hosts: ["https://${NODE_NAME}:10250"]
        add_metadata: true
        metricsets: ["node", "system", "pod", "container", "volume"]
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        ssl.verification_mode: "none"
          #      - module: kubernetes # kube-state-metrics
          #        period: 10s
          #        node: ${NODE_NAME}
          #        hosts:
          #        - "kube-prometheus-stack-kube-state-metrics.monitoring.svc.cluster.local:8080"
          #        add_metadata: true
          #        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
          #        ssl:
          #          verification_mode: none
          #        metricsets:
          #        - state_node
          #        - state_daemonset
          #        - state_deployment
          #        - state_replicaset
          #        - state_statefulset
          #        - state_pod
          #        - state_container
          #        - state_job
          #        - state_cronjob
          #        - state_resourcequota
          #        - state_service
          #        - state_persistentvolume
          #        - state_persistentvolumeclaim
          #        - state_storageclass
          ## causes pods to restart 
          ##      - module: aws
          ##        period: 60s
          ##        metricsets:
          ##          - ec2
          ##          - lambda
          ##          - rds
          ##          - ebs
          ##          - elb
          ##        access_key_id: ${AWS_ACCESS_KEY_ID} 
          ##        secret_access_key: ${AWS_SECRET_ACCESS_KEY}
    processors:
    - add_cloud_metadata: {}
    - add_host_metadata: {}
  daemonSet:
    podTemplate:
      spec:
        serviceAccountName: metricbeat
        automountServiceAccountToken: true # some older Beat versions are depending on this settings presence in k8s context
        containers:
        - args:
          - -e
          - -c
          - /etc/beat.yml
          - -system.hostfs=/hostfs
          name: metricbeat
          volumeMounts:
          - mountPath: /hostfs/sys/fs/cgroup
            name: cgroup
          - mountPath: /var/run/docker.sock
            name: dockersock
          - mountPath: /hostfs/proc
            name: proc
          - name: indice-lifecycle
            mountPath: /etc/indice-lifecycle.json
            readOnly: true
            subPath: indice-lifecycle.json
          env:
          - name: NODE_NAME
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
          - name: MONITORED_ES_USERNAME
            value: elastic
          - name: MONITORED_ES_PASSWORD
            valueFrom:
              secretKeyRef:
                key: elastic
                name: elasticsearch-prod-es-elastic-user
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                name: prod-aws
                key: SERVERLESS_AWS_ACCESS_KEY_ID
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: prod-aws
                key: SERVERLESS_AWS_SECRET_ACCESS_KEY
          resources:
            limits:
              cpu: 200m
              memory: 300Mi
            requests:
              cpu: 100m
              memory: 200Mi
        dnsPolicy: ClusterFirstWithHostNet
        hostNetwork: true # Allows to provide richer host metadata
        securityContext:
          runAsUser: 0
        terminationGracePeriodSeconds: 30
        volumes:
        - hostPath:
            path: /sys/fs/cgroup
          name: cgroup
        - hostPath:
            path: /var/run/docker.sock
          name: dockersock
        - hostPath:
            path: /proc
          name: proc
        - name: indice-lifecycle
          configMap:
            defaultMode: 0600
            name: metricbeat-indice-lifecycle


---
# permissions needed for metricbeat
# source: https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-kubernetes.html
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: metricbeat
  namespace: elastic-system
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - namespaces
  - events
  - pods
  - services
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - "extensions"
  resources:
  - replicasets
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - apps
  resources:
  - statefulsets
  - deployments
  - replicasets
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/stats
  verbs:
  - get
- nonResourceURLs:
  - /metrics
  verbs:
  - get
- apiGroups: ["batch"]
  resources: ["*"]
  verbs: ["get", "list", "watch"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: metricbeat
  namespace: elastic-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: metricbeat
  namespace: elastic-system
subjects:
- kind: ServiceAccount
  name: metricbeat
  namespace: elastic-system
roleRef:
  kind: ClusterRole
  name: metricbeat
  apiGroup: rbac.authorization.k8s.io
---
# metricbeat.indice-lifecycle.configmap.yml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: elastic-system
  name: metricbeat-indice-lifecycle
  labels:
    app: metricbeat
data:
  indice-lifecycle.json: |-
    {
      "policy": {
        "phases": {
          "hot": {
            "actions": {
              "rollover": {
                "max_primary_shard_size": "50GB",
                "max_size": "50GB",
                "max_age": "1d"
              }
            }
          },
          "delete": {
            "min_age": "7d",
            "actions": {
              "delete": {}
            }
          }
        }
      }
    }    
---

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.