Elasticsearch unable to form a cluster

Elastic Stack Elasticsearch
1

this is my docker-compose.yml

version: '3.8'

services:

      els01:
        image: docker.elastic.co/elasticsearch/elasticsearch:8.11.3
        hostname: els01
        volumes:
          - /mnt/data/els01:/usr/share/elasticsearch/data
        ulimits:
          memlock:
            soft: -1
            hard: -1
        environment:
          - node.name=els01
          - node.roles=master
          - xpack.security.transport.ssl.enabled=false
          - xpack.security.enabled=false
          - xpack.license.self_generated.type=basic
          - cluster.initial_master_nodes=els01,els02,els03
          - cluster.name=elk-cluster
          - discovery.seed_hosts=els02,els03
          - network.host=0.0.0.0
          - bootstrap.memory_lock=true
          - ES_JAVA_OPTS=-Xms2g -Xmx2g
        ports:
          - "9200:9200"
          - "9300:9300"
        deploy:
          replicas: 1
        healthcheck:
          test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health"]
          interval: 30s
          timeout: 10s
          retries: 5
        networks:
          - elk

      els02:
        image: docker.elastic.co/elasticsearch/elasticsearch:8.11.3
        hostname: els02
        volumes:
          - /mnt/data/els02:/usr/share/elasticsearch/data
        ulimits:
          memlock:
            soft: -1
            hard: -1
        environment:
          - node.name=els02
          - xpack.security.transport.ssl.enabled=false
          - node.roles=master
          - xpack.security.enabled=false
          - xpack.license.self_generated.type=basic
          - network.host=0.0.0.0
          - cluster.initial_master_nodes=els01,els02,els03
          - cluster.name=elk-cluster
          - discovery.seed_hosts=els01,els03
          - bootstrap.memory_lock=true
          - ES_JAVA_OPTS=-Xms2g -Xmx2g
        ports:
          - "9201:9200"
          - "9301:9300"
        deploy:
          replicas: 1
        healthcheck:
          test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health"]
          interval: 30s
          timeout: 10s
          retries: 5
        networks:
          - elk

      els03:
        image: docker.elastic.co/elasticsearch/elasticsearch:8.11.3
        hostname: els03
        volumes:
          - /mnt/data/els03:/usr/share/elasticsearch/data
        ulimits:
          memlock:
            soft: -1
            hard: -1
        environment:
          - node.name=els03
          - xpack.security.transport.ssl.enabled=false
          - node.roles=master
          - network.host=0.0.0.0
          - xpack.security.enabled=false
          - xpack.license.self_generated.type=basic
          - cluster.initial_master_nodes=els01,els02,els03
          - cluster.name=elk-cluster
          - discovery.seed_hosts=els01,els02
          - bootstrap.memory_lock=true
          - ES_JAVA_OPTS=-Xms2g -Xmx2g
        ports:
          - "9202:9200"
          - "9302:9300"
        deploy:
          replicas: 1
        healthcheck:
          test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health"]
          interval: 30s
          timeout: 10s
          retries: 5
        networks:
          - elk

      kibana:
        image: docker.elastic.co/kibana/kibana:8.11.3
        hostname: kibana
        ports:
          - "5601:5601"
        environment:
          - node.name=kibana
          - ES_HOSTS=http://els01:9200,http://els02:9201,http://els03:9202
          - elasticsearch.username=elastic
          - elasticsearch.password=changeme
        deploy:
          replicas: 1
        depends_on:
          - els01
          - els02
          - els03
        networks:
          - elk

      logstash:
        image: docker.elastic.co/logstash/logstash:8.11.3
        hostname: logstash
        ports:
          - "5044:5044"
          - "9600:9600"
          - "5000:5000/tcp"
          - "5000:5000/udp"
        environment:
          - node.name=logstash
        deploy:
          replicas: 1
        depends_on:
          - els01
          - els02
          - els03
        networks:
          - elk

networks:
  elk:
    driver: overlay
    internal: true

volumes:
  els01:
    driver: local
  els02:
    driver: local
  els03:
    driver: local

can anyone point on something false?
all of them "els01,els02,els03" says fail to resolve, but kibana container resolve logstash :confused:

log of els01 :

rch.cluster.name":"elk-cluster","error.type":"java.net.UnknownHostException","error.message":"els03","error.stack_trace":"java.net.UnknownHostException: els03\n\tat java.base/java.net.InetAddress$CachedLookup.get(InetAddress.java:988)\n\tat java.base/java.net.InetAddress.getAllByName0(InetAddress.java:1818)\n\tat java.base/java.net.InetAddress.getAllByName(InetAddress.java:1688)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:659)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:601)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:1089)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.discovery.SeedHostsResolver.lambda$resolveHosts$0(SeedHostsResolver.java:92)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:916)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\n"}
elk-cluster_els01.1.n1eep7o6442d@node03    | {"@timestamp":"2024-01-11T00:01:47.185Z", "log.level": "WARN", "message":"failed to resolve host [els02]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[els01][cluster_coordination][T#1]","log.logger":"org.elasticsearch.discovery.SeedHostsResolver","elasticsearch.node.name":"els01","elasticsearch.cluster.name":"elk-cluster","error.type":"java.net.UnknownHostException","error.message":"els02: Name or service not known","error.stack_trace":"java.net.UnknownHostException: els02: Name or service not known\n\tat java.base/java.net.Inet4AddressImpl.lookupAllHostAddr(Native Method)\n\tat java.base/java.net.Inet4AddressImpl.lookupAllHostAddr(Inet4AddressImpl.java:43)\n\tat java.base/java.net.InetAddress$PlatformResolver.lookupByName(InetAddress.java:1211)\n\tat java.base/java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1828)\n\tat java.base/java.net.InetAddress$NameServiceAddresses.get(InetAddress.java:1139)\n\tat java.base/java.net.InetAddress.getAllByName0(InetAddress.java:1818)\n\tat java.base/java.net.InetAddress.getAllByName(InetAddress.java:1688)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:659)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:601)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:1089)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.discovery.SeedHostsResolver.lambda$resolveHosts$0(SeedHostsResolver.java:92)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:916)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\n"}
elk-cluster_els01.1.n1eep7o6442d@node03    | {"@timestamp":"2024-01-11T00:01:47.185Z", "log.level": "WARN", "message":"failed to resolve host [els03]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[els01][cluster_coordination][T#1]","log.logger":"org.elasticsearch.discovery.SeedHostsResolver","elasticsearch.node.name":"els01","elasticsearch.cluster.name":"elk-cluster","error.type":"java.net.UnknownHostException","error.message":"els03: Name or service not known","error.stack_trace":"java.net.UnknownHostException: els03: Name or service not known\n\tat java.base/java.net.Inet4AddressImpl.lookupAllHostAddr(Native Method)\n\tat java.base/java.net.Inet4AddressImpl.lookupAllHostAddr(Inet4AddressImpl.java:43)\n\tat java.base/java.net.InetAddress$PlatformResolver.lookupByName(InetAddress.java:1211)\n\tat java.base/java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1828)\n\tat java.base/java.net.InetAddress$NameServiceAddresses.get(InetAddress.java:1139)\n\tat java.base/java.net.InetAddress.getAllByName0(InetAddress.java:1818)\n\tat java.base/java.net.InetAddress.getAllByName(InetAddress.java:1688)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:659)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:601)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:1089)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.discovery.SeedHostsResolver.lambda$resolveHosts$0(SeedHostsResolver.java:92)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:916)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\n"}
elk-cluster_els01.1.n1eep7o6442d@node03    | {"@timestamp":"2024-01-11T00:01:47.446Z", "log.level": "WARN", "message":"path: /_cluster/health, params: {}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[els01][generic][T#3]","log.logger":"rest.suppressed","elasticsearch.node.name":"els01","elasticsearch.cluster.name":"elk-cluster","error.type":"org.elasticsearch.discovery.MasterNotDiscoveredException","error.stack_trace":"org.elasticsearch.discovery.MasterNotDiscoveredException\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:317)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:355)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:293)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:645)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:916)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\n"}
elk-cluster_els01.1.n1eep7o6442d@node03    | {"@timestamp":"2024-01-11T00:01:48.134Z", "log.level": "WARN", "message":"failed to resolve host [els02]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[els01][cluster_coordination][T#1]","log.logger":"org.elasticsearch.discovery.SeedHostsResolver","elasticsearch.node.name":"els01","elasticsearch.cluster.name":"elk-cluster","error.type":"java.net.UnknownHostException","error.message":"els02","error.stack_trace":"java.net.UnknownHostException: els02\n\tat java.base/java.net.InetAddress$CachedLookup.get(InetAddress.java:988)\n\tat java.base/java.net.InetAddress.getAllByName0(InetAddress.java:1818)\n\tat java.base/java.net.InetAddress.getAllByName(InetAddress.java:1688)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:659)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:601)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:1089)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.discovery.SeedHostsResolver.lambda$resolveHosts$0(SeedHostsResolver.java:92)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:916)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\n"}
elk-cluster_els01.1.n1eep7o6442d@node03    | {"@timestamp":"2024-01-11T00:01:48.134Z", "log.level": "WARN", "message":"failed to resolve host [els03]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[els01][cluster_coordination][T#1]","log.logger":"org.elasticsearch.discovery.SeedHostsResolver","elasticsearch.node.name":"els01","elasticsearch.cluster.name":"elk-cluster","error.type":"java.net.UnknownHostException","error.message":"els03","error.stack_trace":"java.net.UnknownHostException: els03\n\tat java.base/java.net.InetAddress$CachedLookup.get(InetAddress.java:988)\n\tat java.base/java.net.InetAddress.getAllByName0(InetAddress.java:1818)\n\tat java.base/java.net.InetAddress.getAllByName(InetAddress.java:1688)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TcpTransport.parse(TcpTransport.java:659)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TcpTransport.addressesFromString(TcpTransport.java:601)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.transport.TransportService.addressesFromString(TransportService.java:1089)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.discovery.SeedHostsResolver.lambda$resolveHosts$0(SeedHostsResolver.java:92)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)\n\tat org.elasticsearch.server@8.11.3/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:916)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\n"}
2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.