Master not discovered yet

Elastic Stack Elasticsearch
1

Trying to figure out what is missing from this configuration. I've exec'd into the containers and all hosts can see each other by name and all ports are open and working yet I'm still getting master node not discovered yet.

I'm on a completely clean box running the following config:

version: '2.2'
services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
    container_name: es01
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02,es03,es04
      - cluster.initial_master_nodes=es01,es02,es03,es04
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms28g -Xmx28g"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /var/elasticsearch-data/data01:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
    networks:
      - elastic
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
    container_name: es02
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02,es03,es04
      - cluster.initial_master_nodes=es01,es02,es03,es04
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms28g -Xmx28g"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /var/elasticsearch-data/data02:/usr/share/elasticsearch/data
    networks:
      - elastic
  es03:
    container_name: es03
    image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
    privileged: true
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02,es03,es04
      - cluster.initial_master_nodes=es01,es02,es03,es04
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms28g -Xmx28g"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /var/elasticsearch-data/data03:/usr/share/elasticsearch/data
    networks:
      - elastic
  es04:
    container_name: es04
    image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
    privileged: true
    environment:
      - node.name=es04
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02,es03,es04
      - cluster.initial_master_nodes=es01,es02,es03,es04
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms28g -Xmx28g"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /var/elasticsearch-data/data04:/usr/share/elasticsearch/data
    networks:
      - elastic
  kibana:
    image: docker.elastic.co/kibana/kibana:7.7.0
    privileged: true
    container_name: kib01
    ports:
      - 5601:5601
    environment:
      ELASTICSEARCH_HOSTS: http://es01:9200 http://es02:9200 http://es03:9200 http://es04:9200
    networks:
      - elastic

volumes:
  data01:
    driver: local
  data02:
    driver: local
  data03:
    driver: local
  data04:
    driver: local

networks:
  elastic:
    driver: bridge

Node names match, initial master nodes are the same, and I'm still getting, " es03 | {"type": "server", "timestamp": "2020-05-25T21:44:18,155Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "es-docker-cluster", "node.name": "es03", "message": "master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and this node must discover master-eligible nodes [es01, es02, es03, es04] to bootstrap a cluster: have discovered [{es03}{wa9ZXVugT1eHwsaENKTd6Q}{-0gcBjoVSa609OhY-iM-pQ}{172.19.0.6}{172.19.0.6:9300}{dilmrt}{ml.machine_memory=1080309936128, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}]; discovery will continue using [172.19.0.3:9300, 172.19.0.4:9300, 172.19.0.5:9300] from hosts providers and [{es03}{wa9ZXVugT1eHwsaENKTd6Q}{-0gcBjoVSa609OhY-iM-pQ}{172.19.0.6}{172.19.0.6:9300}{dilmrt}{ml.machine_memory=1080309936128, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}] from last-known cluster state; node term 0, last-accepted version 0 in term 0" }"

Feature request: It would be ultra helpful if there were some indication of why it isn't discovering the other hosts. While it's helpful that it tells me it couldn't join, it is very difficult to troubleshoot because I have no clue why it didn't join. Could it not see the port? Can it not ping? It's telling me it knows to look for the other hosts and since I can ping them and nmap the port from inside the container it just leaves me at a loss as to what it wants.

2

It's worth noting that the compose file in Elasticsearch's documentation does not work and presents the same error.

I ran it completely unchanged.

version: '2.2'
services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
    container_name: es01
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data01:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
    networks:
      - elastic

  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
    container_name: es02
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data02:/usr/share/elasticsearch/data
    ports:
      - 9201:9201
    networks:
      - elastic

  es03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
    container_name: es03
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - data03:/usr/share/elasticsearch/data
    ports:
      - 9202:9202
    networks:
      - elastic

  kib01:
    image: docker.elastic.co/kibana/kibana:7.7.0
    container_name: kib01
    ports:
      - 5601:5601
    environment:
      ELASTICSEARCH_URL: http://es01:9200
      ELASTICSEARCH_HOSTS: http://es01:9200
    networks:
      - elastic

volumes:
  data01:
    driver: local
  data02:
    driver: local
  data03:
    driver: local

networks:
  elastic:
    driver: bridge
3

Turns out RHEL8 + Docker + backend networking = does not work.

I fired up exactly the same config with exactly the same commands ran on the box in CENTOS7. Worked no problem. Came up immediately.

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.