Kibana unable to revive connection to Elasticsearch in Kubernetes

[rtemotfuorco123]

Hi,

I am getting Unable to revive connection: http://elasticsearch:9200/ in Kibana service logs in Kubernetes. What I don't understand is that, I am explicitly setting the Kibana env vars as listed below (commented out for demonstration purposes for now) but it is still looking for http://elasticsearch:9200/ rather than http://es-svc:9200/. Does anyone know what I am missing here please? My ES nodes work fine, see cluster info below.

Thanks

curl http://127.0.0.1:9200/_cluster/state?pretty

{
  "cluster_name" : "k8s-observer",
  "cluster_uuid" : "ATSb-P-YSymNNeE_bgxs1g",
  "version" : 17,
  "state_uuid" : "uohfqb6cSCKJD1hAX9qNqQ",
  "master_node" : "Le1biwxHR4yX8UUn-6ljpQ",
  "blocks" : { },
  "nodes" : {
    "Le1biwxHR4yX8UUn-6ljpQ" : {
      "name" : "es-sts-2",
      "ephemeral_id" : "vIM9K7aiRdGgh3WL2x_0Kg",
      "transport_address" : "172.17.0.8:9300",
      "attributes" : {
        "ml.machine_memory" : "1073741824",
        "xpack.installed" : "true",
        "ml.max_open_jobs" : "20"
      }
    },
    "-mX7KFqdThWizFua7AoySw" : {
      "name" : "es-sts-0",
      "ephemeral_id" : "dxarv9ExQGmXA1a8ma6tbA",
      "transport_address" : "172.17.0.6:9300",
      "attributes" : {
        "ml.machine_memory" : "1073741824",
        "ml.max_open_jobs" : "20",
        "xpack.installed" : "true"
      }
    },
    "OWTHLfUaQrmuKAF2XIVOWg" : {
      "name" : "es-sts-1",
      "ephemeral_id" : "9G0MZq7CQp2_4dJMWFyF9g",
      "transport_address" : "172.17.0.7:9300",
      "attributes" : {
        "ml.machine_memory" : "1073741824",
        "ml.max_open_jobs" : "20",
        "xpack.installed" : "true"
      }
    }
  }
}

ns.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: observer

es.yaml

apiVersion: v1
kind: Service
metadata:
  name: es-svc
  namespace: observer
  labels:
    app: elasticsearch
spec:
  clusterIP: None
  selector:
    app: elasticsearch
  ports:
    - name: http
      protocol: TCP
      port: 9200
    - name: node
      protocol: TCP
      port: 9300

---

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: es-sts
  namespace: observer
spec:
  serviceName: es-svc
  replicas: 3
  selector:
    matchLabels:
      app: elasticsearch
  template:
    metadata:
      labels:
        app: elasticsearch
    spec:
      containers:
        - name: elasticsearch
          image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0
          resources:
            limits:
              cpu: 1000m
              memory: 1Gi
            requests:
              cpu: 500m
              memory: 1Gi
          ports:
            - name: http
              protocol: TCP
              containerPort: 9200
            - name: node
              protocol: TCP
              containerPort: 9300
          volumeMounts:
            - name: es-volume
              mountPath: /usr/share/elasticsearch/data
          env:
            - name: cluster.name
              value: k8s-observer
            - name: node.name
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: discovery.seed_hosts
              value: "es-sts-0.es-svc,es-sts-1.es-svc,es-sts-2.es-svc"
            - name: cluster.initial_master_nodes
              value: "es-sts-0,es-sts-1,es-sts-2"
            - name: ES_JAVA_OPTS
              value: "-Xms512m -Xmx512m"
      initContainers:
        - name: chown
          image: busybox
          command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
          securityContext:
            privileged: true
          volumeMounts:
            - name: es-volume
              mountPath: /usr/share/elasticsearch/data
        - name: sysctl
          image: busybox
          command: ["sysctl", "-w", "vm.max_map_count=262144"]
          securityContext:
            privileged: true
        - name: ulimit
          image: busybox
          command: ["sh", "-c", "ulimit -n 65536"]
          securityContext:
            privileged: true
  volumeClaimTemplates:
    - metadata:
        name: es-volume
      spec:
        storageClassName: standard
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: 10Gi

kb.yaml

apiVersion: v1
kind: Service
metadata:
  name: kb-svc
  namespace: observer
  labels:
    app: kibana
spec:
  selector:
    app: kibana
  ports:
    - name: http
      protocol: TCP
      port: 5601

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kb-dep
  namespace: observer
  labels:
    app: kibana
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kibana
  template:
    metadata:
      labels:
        app: kibana
    spec:
      containers:
        - name: kibana
          image: docker.elastic.co/kibana/kibana:7.2.0
          resources:
            limits:
              cpu: 1000m
              memory: 1Gi
            requests:
              cpu: 500m
              memory: 1Gi
          ports:
            - name: http
              protocol: TCP
              containerPort: 5601
          env:
#            - name: ELASTICSEARCH_HOSTS
#              value: http://es-svc:9200

#            - name: elasticsearch.hosts
#              value: http://es-svc:9200
#            - name: ELASTICSEARCH_URL
#              value: http://es-svc:9200

#            - name: elasticsearch.url
#              value: http://es-svc:9200

#            - name: ELASTICSEARCH_URL
#              value: http://elasticsearch:9200

#            - name: elasticsearch.url
#              value: http://elasticsearch:9200

#            - name: elasticsearch.hosts
#              value: "es-sts-0.es-svc,es-sts-1.es-svc,es-sts-2.es-svc"