Dockerized ES Cluster multi node with one container/VM Problems

DBond · May 7, 2020, 5:00pm

Dear Experts,

Anybody has experience on how to have secured ES cluster with multi nodes, that each node is running on a docker container, and each docker container runs on a VM ?
For example:
VM1/DC1/N1 : ES1 and Kibana
VM1/DC2/N2 : ES2
VM2/DC3/N3 : ES3

Elasticsearch 7.6
Kibana 7.6

All three nodes are master-eligible, data and ingest Node.
Issues:

They are not see each other, so cannot form a cluster (the same for all nodes)

{"type": "server", "timestamp": "2020-05-07T16:42:06,945Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "clues01", "node.name": "dev.net", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [ThXscQCwR6q2tRq0tGDlug, tgypkDqVS_-x5_1_DXmp9Q], have discovered [{dev..net}{ThXscQCwR6q2tRq0tGDlug}{3eWMA9Q7QVy14Joe_RMwVA}{XX.YY.ZZ.162}{XX.YY.ZZ.162:9300}{dim}{xpack.installed=true}] which is not a quorum; discovery will continue using [XX.YY.ZZ.189:9300, XX.YY.ZZ.189:9300] from hosts providers and [{dev.net}{ThXscQCwR6q2tRq0tGDlug}{3eWMA9Q7QVy14Joe_RMwVA}{XX.YY.ZZ.162}{XX.YY.ZZ.162:9300}{dim}{xpack.installed=true}] from last-known cluster state; node term 0, last-accepted version 0 in term 0" }

Kibana doesn't start ()

 {"type":"log","@timestamp":"2020-05-07T16:44:58Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"No living connections"}
        {"type":"log","@timestamp":"2020-05-07T16:44:59Z","tags":["warning","elasticsearch","data"],"pid":6,"message":"Unable to revive connection: https://playground.net:9200/"}

I can get the es status, seems okay, but with cluster_uuid : "na"

{
      "name" : "playground.net",
      "cluster_name" : "clues01",
      "cluster_uuid" : "_na_",
      "version" : {
        "number" : "7.6.0",
        "build_flavor" : "default",
        "build_type" : "docker",
        "build_hash" : "7f634e9f44834fbc12724506cc1da681b0c3b1e3",
        "build_date" : "2020-02-06T00:09:00.449973Z",
        "build_snapshot" : false,
        "lucene_version" : "8.4.0",
        "minimum_wire_compatibility_version" : "6.8.0",
        "minimum_index_compatibility_version" : "6.0.0-beta1"
      },
      "tagline" : "You Know, for Search"
    }

Does it make sense at all to hope that this config will work? lost a lot of hair already

Configs:
es1.yml & es2.yml:

    bootstrap.memory_lock: true
    cluster.initial_master_nodes: playground.net,dev.net
    cluster.name: clues01
    discovery.seed_hosts: XX.YY.ZZ.162,XX.YY.ZZ.189
    network.bind_host: 0.0.0.0
    network.publish_host: XX.YY.ZZ.189
    node.name: playground.net
    xpack.license.self_generated.type: basic
    xpack.ml.enabled: false
    xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca.crt
    xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/es.crt
    xpack.security.http.ssl.enabled: true
    xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/es.key
    xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca.crt
    xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/es.crt
    xpack.security.transport.ssl.enabled: true
    xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/es.key
    xpack.security.transport.ssl.verification_mode: certificate

docker-composeVM1.yml

version: '2.2'

services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
    container_name: es01
    environment:
        - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esvol01:/usr/share/elasticsearch/data
      - ./certs:$CERTS_DIR
      - ./config/es1.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
    ports:
      - 9200:9200
    networks:
      - esnet

    healthcheck:
      test: curl --cacert /usr/share/elasticsearch/config/certificates/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5

  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
    container_name: es02
    environment:
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esvol02:/usr/share/elasticsearch/data
      - ./certs:$CERTS_DIR
      - ./config/es2.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
    networks:
      - esnet

  kib01:
    image: docker.elastic.co/kibana/kibana:${VERSION}
    container_name: kib01
    depends_on:
      es01:
        condition: service_healthy
    ports:
      - 5601:5601
    environment:
      SERVERNAME: playground.net
      ELASTICSEARCH_URL: https://playground.net:9200
      ELASTICSEARCH_HOSTS: https://playground.net:9200
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca.crt
      SERVER_SSL_ENABLED: "true"
      SERVER_SSL_CERTIFICATE: $CERTS_DIR/kib.crt
      SERVER_SSL_KEY: $CERTS_DIR/kib.key
      SERVER_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca.crt
    volumes:
      - ./certs:$CERTS_DIR
    networks:
      - esnet

volumes:
  esvol01:
    driver: local
  esvol02:
    driver: local
  certs:
    driver: local

networks:
   esnet:
    driver: bridge

es3.yml

bootstrap.memory_lock: true
cluster.initial_master_nodes: playground.net,dev.net
cluster.name: clues01
discovery.seed_hosts: XX.YY.ZZ.189,XX.YY.ZZ.189
network.bind_host: 0.0.0.0
network.publish_host: XX.YY.ZZ.162
node.name: dev.net
xpack.license.self_generated.type: basic
xpack.ml.enabled: false
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca.crt
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/es.crt
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/es.key
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/es.crt
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/es.key
xpack.security.transport.ssl.verification_mode: certificate

docker-composeVM2.yml

version: "2.2"

services:
   es03:
     image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
     container_name: es03
     environment:
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     networks:
       - esnet
     ports:
       - 9300:9300
     volumes:
       - esvol03:/usr/share/elasticsearch/data
       - ./elasticsearch/certs:/usr/share/elasticsearch/config/certificates
       - ./elasticsearch/config/es-3.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
     ulimits:
      memlock:
        soft: -1
        hard: -1
volumes:
   esvol03:
     driver: local

networks:
   esnet:
     driver: bridge

DBond · May 7, 2020, 10:18pm

Solve it!!

Following configuration works for me...
I'll share it here, so the experts can comment & throw their opinion...

docker-composeVM1.yml

version: '2.2'

services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
    container_name: es01
    environment:
        - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esvol01:/usr/share/elasticsearch/data
      - ./certs:$CERTS_DIR
      - ./config/es01.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
    ports:
      - 9200:9200
    network_mode: host

    healthcheck:
      test: curl --cacert /usr/share/elasticsearch/config/certificates/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5

  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
    container_name: es02
    environment:
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esvol02:/usr/share/elasticsearch/data
      - ./certs:$CERTS_DIR
      - ./config/es02.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
    network_mode: host

  kib01:
    image: docker.elastic.co/kibana/kibana:${VERSION}
    container_name: kib01
    depends_on:
      es01:
        condition: service_healthy
    ports:
      - 5601:5601
    environment:
      SERVERNAME: localhost
      ELASTICSEARCH_URL: https://localhost:9200
      ELASTICSEARCH_HOSTS: https://localhost:9200
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca.crt
      SERVER_SSL_ENABLED: "true"
      SERVER_SSL_CERTIFICATE: $CERTS_DIR/kib.crt
      SERVER_SSL_KEY: $CERTS_DIR/kib.key
      SERVER_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca.crt
      ELASTICSEARCH_SSL_VERIFICATIONMODE: "certificate"
    volumes:
      - ./certs:$CERTS_DIR
    network_mode: "host"

volumes:
  esvol01:
    driver: local
  esvol02:
    driver: local
  certs:
    driver: local

es01.yml & es02.yml

bootstrap.memory_lock: true
cluster.name: clues01
discovery.seed_hosts: XX.YY.ZZ.162,XX.YY.ZZ.189
network.bind_host: 0.0.0.0
network.host: _site_
node.name: "${HOSTNAME}_es01" # --> "es02" for es02.yml
xpack.license.self_generated.type: basic
xpack.ml.enabled: false
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca.crt
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/es.crt
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/es.key
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/es.crt
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/es.key
xpack.security.transport.ssl.verification_mode: certificate

docker-compose-VM2.yml

version: "2.2"

services:
   es03:
     image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
     container_name: es03
     environment:
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     network_mode: host
     ports:
       - 9300:9300
     volumes:
       - esvol03:/usr/share/elasticsearch/data
       - ./elasticsearch/certs:/usr/share/elasticsearch/config/certificates
       - ./elasticsearch/config/es03.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
     ulimits:
      memlock:
        soft: -1
        hard: -1
volumes:
   esvol03:
     driver: local

es03.yml

bootstrap.memory_lock: true
cluster.name: clues01
discovery.seed_hosts: XX.YY.ZZ.189,XX.YY.ZZ.189
network.bind_host: 0.0.0.0
network.host: _site_
node.name: "${HOSTNAME}_es03"
xpack.license.self_generated.type: basic
xpack.ml.enabled: false
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca.crt
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/es.crt
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/es.key
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca.crt
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/es.crt
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/es.key
xpack.security.transport.ssl.verification_mode: certificate

system · June 4, 2020, 10:18pm

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.