Does Elasticsearch have capability to form ES cluster with only one ES docker service running multiple replicas of it in Docker-Swarm

I wanna run only one elasticsearch service in docker-swarm cluster having multiple replicas or at least one replica of this service in each node of docker-swarm cluster.

Does Elasticsearch have capability to form ES cluster with only one ES docker service running multiple replicas of it in Docker-Swarm. Below is my elasticsearch.yml config and docker-compose.yml files

Is it possible or not . If yes what tweaking needs to be done in elasticsearch.yml file to get desired results.

version: '3.5'
services:
    elasticsearch:
    image: elasticsearch:5
    deploy:
      mode: global
#      replicas: 3
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 3
        window: 120s
#      placement:
#        constraints: [node.role == manager]
    labels:
      - com.docker.lb.sticky_session_cookie=session
      - com.docker.lb.port=9300
      - com.docker.lb.hosts=demo.local

    environment:
      - ES_JAVA_OPTS=${ELASTICSEARCH_MEM}
      - METADATA="demo-sticky"
    volumes:
      - esdata:/usr/share/elasticsearch/data
    configs:
      - source: elasticsearch.yml
        target: /usr/share/elasticsearch/config/elasticsearch.yml
    logging:
      driver: syslog
      options:
        tag: "{{.Name}}/{{.ID}}"
    ports:
      - 9200:9200
      - 9300:9300
volumes:
  esdata:
configs:
  elasticsearch.yml:
    external: true
#elasticsearch.yml file
http.host: 0.0.0.0
transport.host: 0.0.0.0

network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 1
node.name: elasticsearch-node1
discovery.zen.ping.unicast.hosts: ["elasticsearch"]
cluster.name: elasticsearch
transport.publish_port: 9300
transport.publish_host: elasticsearch
node.master: true

#node.data: true

yes it is possible to achieve it.
just add endpoint_mode as dnsrr like below in docker-compose.yaml file.
for deploying one instance/container across all docker-swarm nodes use "mode: global" in docker-compose.yaml file.
For deploying all ES cluster instance/containers in one machine, use "mode: replicated" and specify replicas to deploy as say "replicas: 3" in docker-compose.yaml file.

With usage of "endpoint_mode: dnsrr" we won't able to expose service ports.
For this purpose we should use an external load balancer like nginx or haproxy.
Like below nginx docker-compose.yaml file

version: '3.5'
services:
    
    elasticsearch:
       image: elasticsearch:5
       deploy:
#          mode: replicated
#          replicas: 3
          mode: global
          endpoint_mode: dnsrr
          restart_policy:
             condition: on-failure
             delay: 5s
             max_attempts: 3
             window: 120s
#       placement:
#           constraints: [node.role == manager]
       environment:
         - ES_JAVA_OPTS=${ELASTICSEARCH_MEM} 
       volumes:
         - esdata:/usr/share/elasticsearch/data
       configs:
         - source: elasticsearch.yml
           target: /usr/share/elasticsearch/config/elasticsearch.yml
         
#     ports:
#       - 9200:9200
#       - 9300:9300
    nginx-lb:
       image: "nginx:1.15.4"
       deploy:
           restart_policy:
              condition: on-failure
             delay: 5s
             max_attempts: 3
             window: 120s
           placement:
                constraints: [node.role == manager]
      volumes:
          - ./nginx.conf:/etc/nginx/nginx.conf
      ports:
          - 9200:9200
          - 9300:9300
  
volumes:
  esdata:
configs:
  elasticsearch.yml:
    external: true

elasticsearch.yaml file

transport.host: 0.0.0.0
http.host: 0.0.0.0
network.host: 0.0.0.0
cluster.name: elasticsearch
discovery.zen.ping.unicast.hosts: elasticsearch  
discovery.zen.minimum_master_nodes: 1 
node.max_local_storage_nodes: 20

nginx.conf file

events {}
stream {
    upstream stream_backend {
        server elasticsearch:9300;

    }
    
    
    server {
        listen        9300;
        proxy_pass    stream_backend;
        proxy_timeout 3s;
        proxy_connect_timeout 1s;
    }
    

}

http {

  upstream elasticsearch9200 {
      server elasticsearch:9200;


  }

  server {
      listen 9200;
      server_name _;

      location / {
        proxy_pass http://elasticsearch9200;
      }

  }

}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.