2 server 2 node each architecture for elastic not working

Elastic Stack Elasticsearch
,
1

Hi all,

server 1- 10.40.7.134

----------------docker-compose.yml file----------------

version: '2.2'
services:
  elasticsearch-1:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-1
    environment:
      - node.name=node-1
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.134:9300,10.40.7.135:9300,10.40.7.135:9301
      - cluster.initial_master_nodes=node-1,node-2,node-3,node-4
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
    ports:
      - 9200:9200
    volumes:
      - /var/elasticsearch/elasticsearch-1.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata1:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144

  elasticsearch-2:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-2
    environment:
      - node.name=node-2
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.134:9300,10.40.7.135:9300,10.40.7.135:9301
      - cluster.initial_master_nodes=node-1,node-2,node-3,node-4
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
    ports:
      - 9201:9201
    volumes:
      - /var/elasticsearch/elasticsearch-2.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata2:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144

volumes:
  esdata1:
    driver: local
  esdata2:
    driver: local

----------------elasticserach-1.yml----------------

cluster.name: my-elastic-cluster
node.name: node-1
node.master: true
node.data: true
network.host: 0.0.0.0
http.port: 9200
transport.port: 9300
discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.135:9300", "10.40.7.135:9301"]
cluster.initial_master_nodes: ["node-1", "node-2", "node-3", "node-4"]
path.data: /usr/share/elasticsearch/data/node1
path.logs: /usr/share/elasticsearch/logs
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

----------------elasticserach-2.yml----------------

cluster.name: my-elastic-cluster
node.name: node-2
node.master: true
node.data: true
network.host: 0.0.0.0
http.port: 9201
transport.port: 9301
discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.135:9300", "10.40.7.135:9301"]
cluster.initial_master_nodes: ["node-1", "node-2", "node-3", "node-4"]
path.data: /usr/share/elasticsearch/data/node2
path.logs: /usr/share/elasticsearch/logs
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

server 2- 10.40.7.135

----------------docker-compose.yml file----------------

version: '2.2'
services:
  elasticsearch-3:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-3
    environment:
      - node.name=node-3
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.134:9300,10.40.7.134:9301,10.40.7.135:9300
      - cluster.initial_master_nodes=node-1,node-2,node-3,node-4
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
    ports:
      - 9200:9200
    volumes:
      - /var/elasticsearch/elasticsearch-3.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata3:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144
  elasticsearch-4:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
    container_name: elasticsearch-4
    environment:
      - node.name=node-4
      - cluster.name=my-elastic-cluster
      - discovery.seed_hosts=10.40.7.134:9300,10.40.7.134:9301,10.40.7.135:9300
      - cluster.initial_master_nodes=node-1,node-2,node-3,node-4
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_USERNAME=elastic
      - ELASTIC_PASSWORD=password
    ports:
      - 9201:9201
    volumes:
      - /var/elasticsearch/elasticsearch-4.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /var/ssl:/usr/share/elasticsearch/config/certificates
      - /var/esdata4:/usr/share/elasticsearch/data
    extra_hosts:
      - "elastic.helo.ai:127.0.0.1"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 262144
        hard: 262144
volumes:
  esdata3:
    driver: local
  esdata4:
    driver: local

----------------elasticserach-3.yml----------------

cluster.name: my-elastic-cluster
node.name: node-3
node.master: true
node.data: true
network.host: 0.0.0.0
http.port: 9200
transport.port: 9300
discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.134:9301", "10.40.7.135:9300"]
cluster.initial_master_nodes: ["node-1", "node-2", "node-3", "node-4"]
path.data: /usr/share/elasticsearch/data/node3
path.logs: /usr/share/elasticsearch/logs
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

----------------elasticserach-4.yml----------------

cluster.name: my-elastic-cluster
node.name: node-4
node.master: true
node.data: true
network.host: 0.0.0.0
http.port: 9201
transport.port: 9301
discovery.seed_hosts: ["10.40.7.134:9300", "10.40.7.134:9301", "10.40.7.135:9300"]
cluster.initial_master_nodes: ["node-1", "node-2", "node-3", "node-4"]
path.data: /usr/share/elasticsearch/data/node4
path.logs: /usr/share/elasticsearch/logs
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.bundle.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.crt
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/helo.ai/helo.ai.key

getting error->

{"type": "server", "timestamp": "2024-10-21T12:18:53,461Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "path: /.kibana_task_manager/_search, params: {ignore_unavailable=true, index=.kibana_task_manager, track_total_hits=true}", 
"stacktrace": ["org.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];",

not getting where is the exact issue? what changes i need to do?

Thank in advance!

2

Hello,

Can you edit your post and format your configurations? It is pretty hard to read and understand this way.

Use the Preformatted text button in each configuration or put then between ``` marks, like this:

```
your configuration
```

3

Thank you for quick reply sir. Please check now.

4

You need to share more log lines, the one you shared is not enough to troubleshoot.

When you start Elasticsearch it logs what it is doing, please try to start it to get fresh logs.

Also, in your docker you are not exposing the 9300 and 9301 port, you need to expose them or the nodes will not be able to join in a cluster.

5

Hi sir,

as suggested by you i exposed both the ports.

now i am not getting error mentioned above.
Error on both the server previously was ->

{"type": "server", "timestamp": "2024-10-23T13:34:13,313Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and this node must discover master-eligible nodes [node-1, node-2, node-3, node-4] to bootstrap a cluster: have discovered [{node-1}{Ha2tk0A3SS-EQMuGB9tXLg}{NYkjTiQ7TUecmwBtEVV6nA}{127.0.0.1}{127.0.0.1:9300}{ilmr}]; discovery will continue using [10.40.7.134:9300, 10.40.7.135:9300, 10.40.7.135:9301] from hosts providers and [{node-1}{Ha2tk0A3SS-EQMuGB9tXLg}{NYkjTiQ7TUecmwBtEVV6nA}{127.0.0.1}{127.0.0.1:9300}{ilmr}] from last-known cluster state; node term 0, last-accepted version 0 in term 0" }

now it is ->

{"type": "server", "timestamp": "2024-10-23T13:38:19,544Z", "level": "WARN", "component": "o.e.d.PeerFinder", "cluster.name": "my-elastic-cluster", "node.name": "node-1", "message": "address [10.40.7.134:9300], node [null], requesting [false] connection failed: [][10.40.7.134:9300] general node connection failure: handshake failed because connection reset" }

thank you in advance.

6

Are these the only logs? You should have more error and warning logs, you need to provide all error and warning logs.

You have 4 nodes, you shared logs of just one of them.

Please get the warning/error logs of all nodes and share them.

7

Also, check this similar question here.

If I'm not wrong in this scenario you need to set docker to use network-moder as host.

And you will probably need to set network.publish_host to the ip of your docker host specific for each container.