2 server 2 node each architecture for elastic not working

Hi all,

server 1- 10.40.7.134

----------------docker-compose.yml file----------------

version: '2.2'
services:
  elasticsearch-1:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-1
    environment:
      - node.name=node-1
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.134:9300,10.40.7.135:9300,10.40.7.135:9301
      - cluster.initial_master_nodes=node-1,node-2,node-3,node-4
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
    ports:
      - 9200:9200
    volumes:
      - /var/elasticsearch/elasticsearch-1.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata1:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144

  elasticsearch-2:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-2
    environment:
      - node.name=node-2
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.134:9300,10.40.7.135:9300,10.40.7.135:9301
      - cluster.initial_master_nodes=node-1,node-2,node-3,node-4
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
    ports:
      - 9201:9201
    volumes:
      - /var/elasticsearch/elasticsearch-2.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata2:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144

volumes:
  esdata1:
    driver: local
  esdata2:
    driver: local

----------------elasticserach-1.yml----------------

cluster.name: my-elastic-cluster
node.name: node-1
node.master: true
node.data: true
network.host: 0.0.0.0
http.port: 9200
transport.port: 9300
discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.135:9300", "10.40.7.135:9301"]
cluster.initial_master_nodes: ["node-1", "node-2", "node-3", "node-4"]
path.data: /usr/share/elasticsearch/data/node1
path.logs: /usr/share/elasticsearch/logs
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

----------------elasticserach-2.yml----------------

cluster.name: my-elastic-cluster
node.name: node-2
node.master: true
node.data: true
network.host: 0.0.0.0
http.port: 9201
transport.port: 9301
discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.135:9300", "10.40.7.135:9301"]
cluster.initial_master_nodes: ["node-1", "node-2", "node-3", "node-4"]
path.data: /usr/share/elasticsearch/data/node2
path.logs: /usr/share/elasticsearch/logs
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

server 2- 10.40.7.135

----------------docker-compose.yml file----------------

version: '2.2'
services:
  elasticsearch-3:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-3
    environment:
      - node.name=node-3
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.134:9300,10.40.7.134:9301,10.40.7.135:9300
      - cluster.initial_master_nodes=node-1,node-2,node-3,node-4
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
    ports:
      - 9200:9200
    volumes:
      - /var/elasticsearch/elasticsearch-3.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata3:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144
  elasticsearch-4:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-4
    environment:
      - node.name=node-4
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.134:9300,10.40.7.134:9301,10.40.7.135:9300
      - cluster.initial_master_nodes=node-1,node-2,node-3,node-4
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
    ports:
      - 9201:9201
    volumes:
      - /var/elasticsearch/elasticsearch-4.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata4:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144
volumes:
  esdata3:
    driver: local
  esdata4:
    driver: local

----------------elasticserach-3.yml----------------

cluster.name: my-elastic-cluster
node.name: node-3
node.master: true
node.data: true
network.host: 0.0.0.0
http.port: 9200
transport.port: 9300
discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.134:9301", "10.40.7.135:9300"]
cluster.initial_master_nodes: ["node-1", "node-2", "node-3", "node-4"]
path.data: /usr/share/elasticsearch/data/node3
path.logs: /usr/share/elasticsearch/logs
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

----------------elasticserach-4.yml----------------

cluster.name: my-elastic-cluster
node.name: node-4
node.master: true
node.data: true
network.host: 0.0.0.0
http.port: 9201
transport.port: 9301
discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.134:9301", "10.40.7.135:9300"]
cluster.initial_master_nodes: ["node-1", "node-2", "node-3", "node-4"]
path.data: /usr/share/elasticsearch/data/node4
path.logs: /usr/share/elasticsearch/logs
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

getting error->

{"type": "server", "timestamp": "2024-10-21T12:18:53,461Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "path: /.kibana_task_manager/_search, params: {ignore_unavailable=true, index=.kibana_task_manager, track_total_hits=true}", 
"stacktrace": ["org.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];",

not getting where is the exact issue? what changes i need to do?

Thank in advance!

Hello,

Can you edit your post and format your configurations? It is pretty hard to read and understand this way.

Use the Preformatted text button in each configuration or put then between ``` marks, like this:

```
your configuration
```

Thank you for quick reply sir. Please check now.

You need to share more log lines, the one you shared is not enough to troubleshoot.

When you start Elasticsearch it logs what it is doing, please try to start it to get fresh logs.

Also, in your docker you are not exposing the 9300 and 9301 port, you need to expose them or the nodes will not be able to join in a cluster.

Hi sir,

as suggested by you i exposed both the ports.

now i am not getting error mentioned above.
Error on both the server previously was ->

{"type": "server", "timestamp": "2024-10-23T13:34:13,313Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and this node must discover master-eligible nodes [node-1, node-2, node-3, node-4] to bootstrap a cluster: have discovered [{node-1}{Ha2tk0A3SS-EQMuGB9tXLg}{NYkjTiQ7TUecmwBtEVV6nA}{127.0.0.1}{127.0.0.1:9300}{ilmr}]; discovery will continue using [10.40.7.134:9300, 10.40.7.135:9300, 10.40.7.135:9301] from hosts providers and [{node-1}{Ha2tk0A3SS-EQMuGB9tXLg}{NYkjTiQ7TUecmwBtEVV6nA}{127.0.0.1}{127.0.0.1:9300}{ilmr}] from last-known cluster state; node term 0, last-accepted version 0 in term 0" }

now it is ->

{"type": "server", "timestamp": "2024-10-23T13:38:19,544Z", "level": "WARN", "component": "o.e.d.PeerFinder", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "address [10.40.7.134:9300], node [null], requesting [false] connection failed: [][10.40.7.134:9300] general node connection failure: handshake failed because connection reset" }

thank you in advance.

Are these the only logs? You should have more error and warning logs, you need to provide all error and warning logs.

You have 4 nodes, you shared logs of just one of them.

Please get the warning/error logs of all nodes and share them.

Also, check this similar question here.

If I'm not wrong in this scenario you need to set docker to use network-moder as host.

And you will probably need to set network.publish_host to the ip of your docker host specific for each container.

Hi sir,

as suggested by you i have set network.publish_host in all 4 elasticsearch-X.yml file
basically i want to setup. 2 nodes each on 2 servers.
i wanted server 1(master, node-1), server 2(node-2, node-3), but dont know will it work or not. But currently i am following above setup, where all nodes are master and data also. Please check below logs. do you think is it something related to network issue ?

Created elasticsearch keystore in /usr/share/elasticsearch/config/elasticsearch.keystore
{"type": "server", "timestamp": "2024-10-24T10:38:33,465Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "version[7.17.1], pid[7], build[default/docker/e5acb99f822233d62d6444ce45a4543dc1c8059a/2022-02-23T22:20:54.153567231Z], OS[Linux/5.10.0-32-amd64/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/17.0.2/17.0.2+8]" }
{"type": "server", "timestamp": "2024-10-24T10:38:33,470Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2024-10-24T10:38:33,471Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j2.formatMsgNoLookups=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-850068476061202831, -XX:+HeapDumpOnOutOfMemoryError, -XX:+ExitOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,473Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,474Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,474Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,474Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,474Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,475Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,475Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,475Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,475Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,476Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,476Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,476Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [legacy-geo]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,476Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,476Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,477Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,477Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,477Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,477Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,477Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,478Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,478Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,478Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,478Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,479Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,479Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,479Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,479Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,479Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,480Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,480Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [vector-tile]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,480Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,480Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,480Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,481Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,481Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,481Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,481Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,481Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,482Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,482Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,482Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,482Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,482Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,483Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,483Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,483Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,483Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,483Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,484Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,484Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,484Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,484Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,484Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,485Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,485Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,485Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,485Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,485Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,486Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,486Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "no plugins loaded" }
{"type": "deprecation.elasticsearch", "timestamp": "2024-10-24T10:38:36,517Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "[node.data] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.data", "category": "settings" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,536Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/sda4)]], net usable_space [110.9gb], net total_space [133.7gb], types [xfs]" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,536Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "heap size [512mb], compressed ordinary object pointers [true]" }
{"type": "deprecation.elasticsearch", "timestamp": "2024-10-24T10:38:36,538Z", "level": "CRITICAL", "component": "o.e.d.c.s.Settings", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "[node.master] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", "key": "node.master", "category": "settings" }
{"type": "server", "timestamp": "2024-10-24T10:38:36,614Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "node name [node-1], node ID [Ha2tk0A3SS-EQMuGB9tXLg], cluster name [my-elastic-cluster], roles [transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]" }
{"type": "deprecation.elasticsearch", "timestamp": "2024-10-24T10:38:36,617Z", "level": "CRITICAL", "component": "o.e.d.n.Node", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "legacy role settings [node.data, node.master] are deprecated, use [node.roles=[transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]]", "key": "legacy role settings", "category": "settings" }
{"type": "server", "timestamp": "2024-10-24T10:38:42,950Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "[controller/527] [Main.cc@122] controller (64 bit): Version 7.17.1 (Build 6d8a28b39bf223) Copyright (c) 2022 Elasticsearch BV" }
{"type": "server", "timestamp": "2024-10-24T10:38:43,402Z", "level": "INFO", "component": "o.e.x.s.a.Realms", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "license mode is [trial], currently licensed security realms are [reserved/reserved,file/default_file,native/default_native]" }
{"type": "server", "timestamp": "2024-10-24T10:38:43,426Z", "level": "INFO", "component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]" }
{"type": "server", "timestamp": "2024-10-24T10:38:44,667Z", "level": "INFO", "component": "o.e.i.g.ConfigDatabases", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "initialized default databases [[GeoLite2-Country.mmdb, GeoLite2-City.mmdb, GeoLite2-ASN.mmdb]], config databases [[]] and watching [/usr/share/elasticsearch/config/ingest-geoip] for changes" }
{"type": "server", "timestamp": "2024-10-24T10:38:44,668Z", "level": "INFO", "component": "o.e.i.g.DatabaseNodeService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "initialized database registry, using geoip-databases directory [/tmp/elasticsearch-850068476061202831/geoip-databases/Ha2tk0A3SS-EQMuGB9tXLg]" }
{"type": "server", "timestamp": "2024-10-24T10:38:45,739Z", "level": "INFO", "component": "o.e.t.NettyAllocator", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "creating NettyAllocator with the following configs: [name=unpooled, suggested_max_allocation_size=1mb, factors={es.unsafe.use_unpooled_allocator=null, g1gc_enabled=true, g1gc_region_size=4mb, heap_size=512mb}]" }
{"type": "server", "timestamp": "2024-10-24T10:38:45,792Z", "level": "INFO", "component": "o.e.i.r.RecoverySettings", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "using rate limit [40mb] with [default=40mb, read=0b, write=0b, max=0b]" }
{"type": "server", "timestamp": "2024-10-24T10:38:45,861Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "using discovery type [zen] and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2024-10-24T10:38:46,501Z", "level": "INFO", "component": "o.e.g.DanglingIndicesState", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually" }
{"type": "server", "timestamp": "2024-10-24T10:38:47,267Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "initialized" }
{"type": "server", "timestamp": "2024-10-24T10:38:47,268Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "starting ..." }
{"type": "server", "timestamp": "2024-10-24T10:38:47,287Z", "level": "INFO", "component": "o.e.x.s.c.f.PersistentCache", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "persistent cache index loaded" }
{"type": "server", "timestamp": "2024-10-24T10:38:47,288Z", "level": "INFO", "component": "o.e.x.d.l.DeprecationIndexingComponent", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "deprecation component started" }
{"type": "server", "timestamp": "2024-10-24T10:38:47,481Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "publish_address {10.40.7.134:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300}" }
{"type": "server", "timestamp": "2024-10-24T10:38:47,899Z", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks" }
{"type": "server", "timestamp": "2024-10-24T10:38:57,926Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and this node must discover master-eligible nodes [node-1, node-2, node-3, node-4] to bootstrap a cluster: have discovered [{node-1}{Ha2tk0A3SS-EQMuGB9tXLg}{VNOdVnITRrWky3yswM0bjg}{10.40.7.134}{10.40.7.134:9300}{cdfhilmrstw}]; discovery will continue using [10.40.7.135:9300, 10.40.7.135:9301] from hosts providers and [{node-1}{Ha2tk0A3SS-EQMuGB9tXLg}{VNOdVnITRrWky3yswM0bjg}{10.40.7.134}{10.40.7.134:9300}{cdfhilmrstw}] from last-known cluster state; node term 0, last-accepted version 0 in term 0" }

and after some interval on server 1

{"type": "server", "timestamp": "2024-10-24T10:47:14,237Z", "level": "WARN", "component": "o.e.d.PeerFinder", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "address [10.40.7.135:9301], node [null], requesting [false] connection failed: [][10.40.7.135:9301] connect_exception: Connection refused: /10.40.7.135:9301: Connection refused" }
{"type": "server", "timestamp": "2024-10-24T10:47:14,247Z", "level": "WARN", "component": "o.e.d.PeerFinder", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "address [10.40.7.135:9300], node [null], requesting [false] connection failed: [][10.40.7.135:9300] general node connection failure: handshake failed because connection reset" }

and on server 2

{"type": "server", "timestamp": "2024-10-24T10:46:30,588Z", "level": "WARN", "component": "o.e.d.PeerFinder", "cluster.name": "my-elastic-cluster", "node.name": "node-3", "message": "address [10.40.7.134:9301], node [null], requesting [false] connection failed: [][10.40.7.134:9301] connect_exception: Connection refused: /10.40.7.134:9301: Connection refused" }
{"type": "server", "timestamp": "2024-10-24T10:46:30,593Z", "level": "WARN", "component": "o.e.d.PeerFinder", "cluster.name": "my-elastic-cluster", "node.name": "node-3", "message": "address [10.40.7.134:9300], node [null], requesting [false] connection failed: [][10.40.7.134:9300] general node connection failure: handshake failed because connection reset" }

please check. Thank you.

This seems like a network issue, the node cannot connect to this IP on this port, there is not much to do until this is solved.

Please share the current compose you are using.

Keep in mind that in the configuration you are trying to use both your nodes are master elegible, but your configuration is not resilient, if one of the machines goes down, the entire cluster will go down.

2 Likes

solved that error by adding network.host: 0.0.0.0 in elasticserach-X.yml file.
my server 1 is up and running with Green state. for server 2 ... i am getting error

{"type": "server", "timestamp": "2024-10-24T15:35:04,983Z", "level": "WARN", "component": "o.e.d.HandshakingTransportAddressConnector", "cluster.name": "my-elastic-cluster", "node.name": "node-3", "message": "[connectToRemoteMasterNode[10.40.7.134:9301]] completed handshake with [{node-2}{gvunxcqzTjKyRvrsnIAH9Q}{LlzmBg7PQxWkffsa0dhTbg}{10.4.2.132}{10.4.2.132:9301}{cdfhilmrstw}{ml.machine_memory=10430627840, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=536870912, transform.node=true}] but followup connection failed", 
"stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [node-2][10.4.2.132:9301] connect_exception",
"at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:1047) ~[elasticsearch-7.17.1.jar:7.17.1]",
"at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:279) ~[elasticsearch-7.17.1.jar:7.17.1]",
"at org.elasticsearch.core.CompletableContext.lambda$addListener$0(CompletableContext.java:31) ~[elasticsearch-core-7.17.1.jar:7.17.1]",
"at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?]",
"at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?]",
"at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]",
"at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]",
"at org.elasticsearch.core.CompletableContext.completeExceptionally(CompletableContext.java:46) ~[elasticsearch-core-7.17.1.jar:7.17.1]",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:58) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:578) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:571) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:550) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:491) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:616) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:609) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:707) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:620) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:583) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) ~[?:?]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
"at java.lang.Thread.run(Thread.java:833) [?:?]",
"Caused by: io.netty.channel.AbstractChannel$AnnotatedNoRouteToHostException: No route to host: 10.4.2.132/10.4.2.132:9301",
"Caused by: java.net.NoRouteToHostException: No route to host",
"at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
"at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]",
"at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]",
"at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]",

this basically means After the handshake, the connection failed. not getting the exact reason.

Please check. Thank you.

I have added network.publish_host: 10.40.7.134 in server 1 elasticsearch 1 & 2 .yml file &
network.publish_host: 10.40.7.135 in server 2 elasticsearch 3 & 4 .yml file
No errors now.

server 1 health-

{
  "cluster_name" : "my-elastic-cluster",
  "status" : "green",
  "timed_out" : false,
  "number_of_nodes" : 2,
  "number_of_data_nodes" : 2,
  "active_primary_shards" : 16,
  "active_shards" : 32,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 0,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 100.0
}

server 2 health

{
  "cluster_name" : "my-elastic-cluster",
  "status" : "green",
  "timed_out" : false,
  "number_of_nodes" : 2,
  "number_of_data_nodes" : 2,
  "active_primary_shards" : 2,
  "active_shards" : 4,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 0,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 100.0
}

how "active_shards" : 32 in server 1 and "active_shards" : 4 server 2 ??
plus how can i check this to verify my all configurations are correct? how can i delete 1 node and how election happen, how new node will appear again.

Please help. thank you.

From what you shared your nodes formed 2 different clusters, not one cluster.

You didn´t share the current docker-compose you are using as asked, it is not possible to know what could be the issue without looking at the configuration you are using.

Please find below details

-----server 1(10.40.7.134)-----
docker-compose.yml-----

version: '2.2'
services:
  elasticsearch-1:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-1
    environment:
      - node.name=node-1
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.134:9300,10.40.7.134:9301,10.40.7.135:9300,10.40.7.135:9301
      - cluster.initial_master_nodes=node-1,node-2,node-3,node-4
      - bootstrap.memory_lock=true
      - cluster.routing.allocation.allow_rebalance=always
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
      - ingest.geoip.downloader.enabled=false
    ports:
      - 9200:9200
      - 9300:9300
    volumes:
      - /var/elasticsearch/elasticsearch-1.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata1:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144
  elasticsearch-2:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-2
    environment:
      - node.name=node-2
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.134:9300,10.40.7.134:9301,10.40.7.135:9300,10.40.7.135:9301
      - cluster.initial_master_nodes=node-1,node-2,node-3,node-4
      - bootstrap.memory_lock=true
      - cluster.routing.allocation.allow_rebalance=always
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
      - ingest.geoip.downloader.enabled=false
    ports:
      - 9201:9201
      - 9301:9301
    volumes:
      - /var/elasticsearch/elasticsearch-2.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata2:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144
volumes:
  esdata1:
    driver: local
  esdata2:
    driver: local

----elasticserach-1.yml----

cluster.name: my-elastic-cluster
node.name: node-1
node.master: true
node.data: true
network.host: 0.0.0.0
#http.host: 0.0.0.0
http.port: 9200
  #transport.host: 0.0.0.0
transport.port: 9300

network.publish_host: 10.40.7.134

  #discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.134:9301","10.40.7.135:9300", "10.40.7.135:9301"]
  #cluster.initial_master_nodes: ["node-1","node-2", "node-3", "node-4"]


path.data: /usr/share/elasticsearch/data/node1
path.logs: /usr/share/elasticsearch/logs

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

----elasticserach-2.yml----

cluster.name: my-elastic-cluster
node.name: node-2
node.master: true
node.data: true

network.host: 0.0.0.0
#http.host: 0.0.0.0
http.port: 9201
  #transport.host: 0.0.0.0
transport.port: 9301

network.publish_host: 10.40.7.134

  #discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.134:9301", "10.40.7.135:9300", "10.40.7.135:9301"]
  #cluster.initial_master_nodes: ["node-1", "node-2","node-3", "node-4"]

path.data: /usr/share/elasticsearch/data/node2
path.logs: /usr/share/elasticsearch/logs

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

-----server 2(10.40.7.135)-----
docker-compose.yml-----

version: '2.2'
services:
  elasticsearch-3:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-3
    environment:
      - node.name=node-3
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.135:9300,10.40.7.135:9301,10.40.7.134:9300,10.40.7.134:9301
      - cluster.initial_master_nodes=node-3,node-4,node-1,node-2
      - bootstrap.memory_lock=true
      - cluster.routing.allocation.allow_rebalance=always
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
      - ingest.geoip.downloader.enabled=false
    ports:
      - 9200:9200
      - 9300:9300
    volumes:
      - /var/elasticsearch/elasticsearch-3.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata3:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144
  elasticsearch-4:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-4
    environment:
      - node.name=node-4
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.135:9300,10.40.7.135:9301,10.40.7.134:9300,10.40.7.134:9301
      - cluster.initial_master_nodes=node-3,node-4,node-1,node-2
      - bootstrap.memory_lock=true
      - cluster.routing.allocation.allow_rebalance=always
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
      - ingest.geoip.downloader.enabled=false
    ports:
      - 9201:9201
      - 9301:9301
    volumes:
      - /var/elasticsearch/elasticsearch-4.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata4:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144
volumes:
  esdata3:
    driver: local
  esdata4:
    driver: local

----elasticserach-3.yml----

cluster.name: my-elastic-cluster
node.name: node-3
node.master: false
node.data: true

network.host: 0.0.0.0
#http.host: 0.0.0.0
http.port: 9200
  #transport.host: 0.0.0.0
transport.port: 9300

network.publish_host: 10.40.7.135

  #discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.134:9301", "10.40.7.135:9300", "10.40.7.135:9301"]
  #cluster.initial_master_nodes: ["node-1", "node-2", "node-3", "node-4"]

path.data: /usr/share/elasticsearch/data/node3
path.logs: /usr/share/elasticsearch/logs

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

----elasticserach-4.yml----

cluster.name: my-elastic-cluster
node.name: node-4
node.master: false
node.data: true

network.host: 0.0.0.0
#http.host: 0.0.0.0
http.port: 9201
  #transport.host: 0.0.0.0
transport.port: 9301

network.publish_host: 10.40.7.135

  #discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.134:9301", "10.40.7.135:9300", "10.40.7.135:9301"]
  #cluster.initial_master_nodes: ["node-1", "node-2", "node-3", "node-4"]

path.data: /usr/share/elasticsearch/data/node4
path.logs: /usr/share/elasticsearch/logs

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
  1. my cluster name is same. so do u mean to say if they are on different server , then they will form different cluster, is it so?
    my both server ips are in same subnet.
  2. i already shared health result of 2 servers/clusters.
    Now kibana error
{"statusCode":500,"error":"Internal Server Error","message":"[illegal_argument_exception: [illegal_argument_exception] Reason: application privileges must refer to at least one resource]: application privileges must refer to at least one resource"}
  1. plus i want to know how can i check this to verify my all configurations are correct? how can i delete 1 node and how election happen, how new node will appear again

Please help. Thank you.